^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * kvm nested virtualization support for s390x
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Copyright IBM Corp. 2016, 2018
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * Author(s): David Hildenbrand <dahi@linux.vnet.ibm.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include <linux/vmalloc.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <linux/kvm_host.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/bug.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/list.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/bitmap.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/sched/signal.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <asm/gmap.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <asm/mmu_context.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <asm/sclp.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <asm/nmi.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <asm/dis.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include "kvm-s390.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #include "gaccess.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) struct vsie_page {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) struct kvm_s390_sie_block scb_s; /* 0x0000 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) * the backup info for machine check. ensure it's at
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) * the same offset as that in struct sie_page!
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) struct mcck_volatile_info mcck_info; /* 0x0200 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) * The pinned original scb. Be aware that other VCPUs can modify
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) * it while we read from it. Values that are used for conditions or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) * are reused conditionally, should be accessed via READ_ONCE.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) struct kvm_s390_sie_block *scb_o; /* 0x0218 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) /* the shadow gmap in use by the vsie_page */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) struct gmap *gmap; /* 0x0220 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) /* address of the last reported fault to guest2 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) unsigned long fault_addr; /* 0x0228 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) /* calculated guest addresses of satellite control blocks */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) gpa_t sca_gpa; /* 0x0230 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) gpa_t itdba_gpa; /* 0x0238 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) gpa_t gvrd_gpa; /* 0x0240 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) gpa_t riccbd_gpa; /* 0x0248 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) gpa_t sdnx_gpa; /* 0x0250 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) __u8 reserved[0x0700 - 0x0258]; /* 0x0258 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) struct kvm_s390_crypto_cb crycb; /* 0x0700 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) __u8 fac[S390_ARCH_FAC_LIST_SIZE_BYTE]; /* 0x0800 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) /* trigger a validity icpt for the given scb */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) static int set_validity_icpt(struct kvm_s390_sie_block *scb,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) __u16 reason_code)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) scb->ipa = 0x1000;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) scb->ipb = ((__u32) reason_code) << 16;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) scb->icptcode = ICPT_VALIDITY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) /* mark the prefix as unmapped, this will block the VSIE */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) static void prefix_unmapped(struct vsie_page *vsie_page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) atomic_or(PROG_REQUEST, &vsie_page->scb_s.prog20);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) /* mark the prefix as unmapped and wait until the VSIE has been left */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) static void prefix_unmapped_sync(struct vsie_page *vsie_page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) prefix_unmapped(vsie_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) if (vsie_page->scb_s.prog0c & PROG_IN_SIE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) atomic_or(CPUSTAT_STOP_INT, &vsie_page->scb_s.cpuflags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) while (vsie_page->scb_s.prog0c & PROG_IN_SIE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) cpu_relax();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) /* mark the prefix as mapped, this will allow the VSIE to run */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) static void prefix_mapped(struct vsie_page *vsie_page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) atomic_andnot(PROG_REQUEST, &vsie_page->scb_s.prog20);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) /* test if the prefix is mapped into the gmap shadow */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) static int prefix_is_mapped(struct vsie_page *vsie_page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) return !(atomic_read(&vsie_page->scb_s.prog20) & PROG_REQUEST);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) /* copy the updated intervention request bits into the shadow scb */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) static void update_intervention_requests(struct vsie_page *vsie_page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) const int bits = CPUSTAT_STOP_INT | CPUSTAT_IO_INT | CPUSTAT_EXT_INT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) int cpuflags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) cpuflags = atomic_read(&vsie_page->scb_o->cpuflags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) atomic_andnot(bits, &vsie_page->scb_s.cpuflags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) atomic_or(cpuflags & bits, &vsie_page->scb_s.cpuflags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) /* shadow (filter and validate) the cpuflags */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) static int prepare_cpuflags(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) struct kvm_s390_sie_block *scb_o = vsie_page->scb_o;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) int newflags, cpuflags = atomic_read(&scb_o->cpuflags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) /* we don't allow ESA/390 guests */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) if (!(cpuflags & CPUSTAT_ZARCH))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) return set_validity_icpt(scb_s, 0x0001U);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) if (cpuflags & (CPUSTAT_RRF | CPUSTAT_MCDS))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) return set_validity_icpt(scb_s, 0x0001U);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) else if (cpuflags & (CPUSTAT_SLSV | CPUSTAT_SLSR))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) return set_validity_icpt(scb_s, 0x0007U);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) /* intervention requests will be set later */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) newflags = CPUSTAT_ZARCH;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) if (cpuflags & CPUSTAT_GED && test_kvm_facility(vcpu->kvm, 8))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) newflags |= CPUSTAT_GED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) if (cpuflags & CPUSTAT_GED2 && test_kvm_facility(vcpu->kvm, 78)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) if (cpuflags & CPUSTAT_GED)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) return set_validity_icpt(scb_s, 0x0001U);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) newflags |= CPUSTAT_GED2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) if (test_kvm_cpu_feat(vcpu->kvm, KVM_S390_VM_CPU_FEAT_GPERE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) newflags |= cpuflags & CPUSTAT_P;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) if (test_kvm_cpu_feat(vcpu->kvm, KVM_S390_VM_CPU_FEAT_GSLS))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) newflags |= cpuflags & CPUSTAT_SM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) if (test_kvm_cpu_feat(vcpu->kvm, KVM_S390_VM_CPU_FEAT_IBS))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) newflags |= cpuflags & CPUSTAT_IBS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) if (test_kvm_cpu_feat(vcpu->kvm, KVM_S390_VM_CPU_FEAT_KSS))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) newflags |= cpuflags & CPUSTAT_KSS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) atomic_set(&scb_s->cpuflags, newflags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) /* Copy to APCB FORMAT1 from APCB FORMAT0 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) static int setup_apcb10(struct kvm_vcpu *vcpu, struct kvm_s390_apcb1 *apcb_s,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) unsigned long apcb_o, struct kvm_s390_apcb1 *apcb_h)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) struct kvm_s390_apcb0 tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) if (read_guest_real(vcpu, apcb_o, &tmp, sizeof(struct kvm_s390_apcb0)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) apcb_s->apm[0] = apcb_h->apm[0] & tmp.apm[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) apcb_s->aqm[0] = apcb_h->aqm[0] & tmp.aqm[0] & 0xffff000000000000UL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) apcb_s->adm[0] = apcb_h->adm[0] & tmp.adm[0] & 0xffff000000000000UL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) * setup_apcb00 - Copy to APCB FORMAT0 from APCB FORMAT0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) * @vcpu: pointer to the virtual CPU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) * @apcb_s: pointer to start of apcb in the shadow crycb
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) * @apcb_o: pointer to start of original apcb in the guest2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) * @apcb_h: pointer to start of apcb in the guest1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) * Returns 0 and -EFAULT on error reading guest apcb
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) static int setup_apcb00(struct kvm_vcpu *vcpu, unsigned long *apcb_s,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) unsigned long apcb_o, unsigned long *apcb_h)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) if (read_guest_real(vcpu, apcb_o, apcb_s,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) sizeof(struct kvm_s390_apcb0)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) bitmap_and(apcb_s, apcb_s, apcb_h, sizeof(struct kvm_s390_apcb0));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) * setup_apcb11 - Copy the FORMAT1 APCB from the guest to the shadow CRYCB
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) * @vcpu: pointer to the virtual CPU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) * @apcb_s: pointer to start of apcb in the shadow crycb
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) * @apcb_o: pointer to start of original guest apcb
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) * @apcb_h: pointer to start of apcb in the host
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) * Returns 0 and -EFAULT on error reading guest apcb
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) static int setup_apcb11(struct kvm_vcpu *vcpu, unsigned long *apcb_s,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) unsigned long apcb_o,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) unsigned long *apcb_h)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) if (read_guest_real(vcpu, apcb_o, apcb_s,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) sizeof(struct kvm_s390_apcb1)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) bitmap_and(apcb_s, apcb_s, apcb_h, sizeof(struct kvm_s390_apcb1));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) * setup_apcb - Create a shadow copy of the apcb.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) * @vcpu: pointer to the virtual CPU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) * @crycb_s: pointer to shadow crycb
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) * @crycb_o: pointer to original guest crycb
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) * @crycb_h: pointer to the host crycb
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) * @fmt_o: format of the original guest crycb.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) * @fmt_h: format of the host crycb.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) * Checks the compatibility between the guest and host crycb and calls the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) * appropriate copy function.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) * Return 0 or an error number if the guest and host crycb are incompatible.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) static int setup_apcb(struct kvm_vcpu *vcpu, struct kvm_s390_crypto_cb *crycb_s,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) const u32 crycb_o,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) struct kvm_s390_crypto_cb *crycb_h,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) int fmt_o, int fmt_h)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) struct kvm_s390_crypto_cb *crycb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) crycb = (struct kvm_s390_crypto_cb *) (unsigned long)crycb_o;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) switch (fmt_o) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) case CRYCB_FORMAT2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) if ((crycb_o & PAGE_MASK) != ((crycb_o + 256) & PAGE_MASK))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) return -EACCES;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) if (fmt_h != CRYCB_FORMAT2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) return setup_apcb11(vcpu, (unsigned long *)&crycb_s->apcb1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) (unsigned long) &crycb->apcb1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) (unsigned long *)&crycb_h->apcb1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) case CRYCB_FORMAT1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) switch (fmt_h) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) case CRYCB_FORMAT2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) return setup_apcb10(vcpu, &crycb_s->apcb1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) (unsigned long) &crycb->apcb0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) &crycb_h->apcb1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) case CRYCB_FORMAT1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) return setup_apcb00(vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) (unsigned long *) &crycb_s->apcb0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) (unsigned long) &crycb->apcb0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) (unsigned long *) &crycb_h->apcb0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) case CRYCB_FORMAT0:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) if ((crycb_o & PAGE_MASK) != ((crycb_o + 32) & PAGE_MASK))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) return -EACCES;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) switch (fmt_h) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) case CRYCB_FORMAT2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) return setup_apcb10(vcpu, &crycb_s->apcb1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) (unsigned long) &crycb->apcb0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) &crycb_h->apcb1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) case CRYCB_FORMAT1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) case CRYCB_FORMAT0:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) return setup_apcb00(vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) (unsigned long *) &crycb_s->apcb0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) (unsigned long) &crycb->apcb0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) (unsigned long *) &crycb_h->apcb0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) * shadow_crycb - Create a shadow copy of the crycb block
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) * @vcpu: a pointer to the virtual CPU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) * @vsie_page: a pointer to internal date used for the vSIE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) * Create a shadow copy of the crycb block and setup key wrapping, if
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) * requested for guest 3 and enabled for guest 2.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) * We accept format-1 or format-2, but we convert format-1 into format-2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) * in the shadow CRYCB.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) * Using format-2 enables the firmware to choose the right format when
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) * scheduling the SIE.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) * There is nothing to do for format-0.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) * This function centralize the issuing of set_validity_icpt() for all
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) * the subfunctions working on the crycb.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) * Returns: - 0 if shadowed or nothing to do
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) * - > 0 if control has to be given to guest 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) static int shadow_crycb(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) struct kvm_s390_sie_block *scb_o = vsie_page->scb_o;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) const uint32_t crycbd_o = READ_ONCE(scb_o->crycbd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) const u32 crycb_addr = crycbd_o & 0x7ffffff8U;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) unsigned long *b1, *b2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) u8 ecb3_flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) u32 ecd_flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) int apie_h;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) int apie_s;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) int key_msk = test_kvm_facility(vcpu->kvm, 76);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) int fmt_o = crycbd_o & CRYCB_FORMAT_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) int fmt_h = vcpu->arch.sie_block->crycbd & CRYCB_FORMAT_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) scb_s->crycbd = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) apie_h = vcpu->arch.sie_block->eca & ECA_APIE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) apie_s = apie_h & scb_o->eca;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) if (!apie_s && (!key_msk || (fmt_o == CRYCB_FORMAT0)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) if (!crycb_addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) return set_validity_icpt(scb_s, 0x0039U);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) if (fmt_o == CRYCB_FORMAT1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) if ((crycb_addr & PAGE_MASK) !=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) ((crycb_addr + 128) & PAGE_MASK))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) return set_validity_icpt(scb_s, 0x003CU);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) if (apie_s) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) ret = setup_apcb(vcpu, &vsie_page->crycb, crycb_addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) vcpu->kvm->arch.crypto.crycb,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) fmt_o, fmt_h);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) goto end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) scb_s->eca |= scb_o->eca & ECA_APIE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) /* we may only allow it if enabled for guest 2 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) ecb3_flags = scb_o->ecb3 & vcpu->arch.sie_block->ecb3 &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) (ECB3_AES | ECB3_DEA);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) ecd_flags = scb_o->ecd & vcpu->arch.sie_block->ecd & ECD_ECC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) if (!ecb3_flags && !ecd_flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) goto end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) /* copy only the wrapping keys */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) if (read_guest_real(vcpu, crycb_addr + 72,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) vsie_page->crycb.dea_wrapping_key_mask, 56))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) return set_validity_icpt(scb_s, 0x0035U);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) scb_s->ecb3 |= ecb3_flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) scb_s->ecd |= ecd_flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) /* xor both blocks in one run */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) b1 = (unsigned long *) vsie_page->crycb.dea_wrapping_key_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) b2 = (unsigned long *)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) vcpu->kvm->arch.crypto.crycb->dea_wrapping_key_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) /* as 56%8 == 0, bitmap_xor won't overwrite any data */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) bitmap_xor(b1, b1, b2, BITS_PER_BYTE * 56);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) end:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) switch (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) case -EINVAL:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) return set_validity_icpt(scb_s, 0x0022U);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) case -EFAULT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) return set_validity_icpt(scb_s, 0x0035U);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) case -EACCES:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) return set_validity_icpt(scb_s, 0x003CU);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) scb_s->crycbd = ((__u32)(__u64) &vsie_page->crycb) | CRYCB_FORMAT2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) /* shadow (round up/down) the ibc to avoid validity icpt */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) static void prepare_ibc(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) struct kvm_s390_sie_block *scb_o = vsie_page->scb_o;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) /* READ_ONCE does not work on bitfields - use a temporary variable */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) const uint32_t __new_ibc = scb_o->ibc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) const uint32_t new_ibc = READ_ONCE(__new_ibc) & 0x0fffU;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) __u64 min_ibc = (sclp.ibc >> 16) & 0x0fffU;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) scb_s->ibc = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) /* ibc installed in g2 and requested for g3 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) if (vcpu->kvm->arch.model.ibc && new_ibc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) scb_s->ibc = new_ibc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) /* takte care of the minimum ibc level of the machine */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) if (scb_s->ibc < min_ibc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) scb_s->ibc = min_ibc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) /* take care of the maximum ibc level set for the guest */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) if (scb_s->ibc > vcpu->kvm->arch.model.ibc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) scb_s->ibc = vcpu->kvm->arch.model.ibc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) /* unshadow the scb, copying parameters back to the real scb */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) static void unshadow_scb(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) struct kvm_s390_sie_block *scb_o = vsie_page->scb_o;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) /* interception */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) scb_o->icptcode = scb_s->icptcode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) scb_o->icptstatus = scb_s->icptstatus;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) scb_o->ipa = scb_s->ipa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) scb_o->ipb = scb_s->ipb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) scb_o->gbea = scb_s->gbea;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) /* timer */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) scb_o->cputm = scb_s->cputm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) scb_o->ckc = scb_s->ckc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) scb_o->todpr = scb_s->todpr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) /* guest state */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) scb_o->gpsw = scb_s->gpsw;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) scb_o->gg14 = scb_s->gg14;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) scb_o->gg15 = scb_s->gg15;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) memcpy(scb_o->gcr, scb_s->gcr, 128);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) scb_o->pp = scb_s->pp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) /* branch prediction */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) if (test_kvm_facility(vcpu->kvm, 82)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) scb_o->fpf &= ~FPF_BPBC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) scb_o->fpf |= scb_s->fpf & FPF_BPBC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) /* interrupt intercept */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) switch (scb_s->icptcode) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) case ICPT_PROGI:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) case ICPT_INSTPROGI:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) case ICPT_EXTINT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) memcpy((void *)((u64)scb_o + 0xc0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) (void *)((u64)scb_s + 0xc0), 0xf0 - 0xc0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) if (scb_s->ihcpu != 0xffffU)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) scb_o->ihcpu = scb_s->ihcpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) * Setup the shadow scb by copying and checking the relevant parts of the g2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) * provided scb.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) * Returns: - 0 if the scb has been shadowed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) * - > 0 if control has to be given to guest 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) static int shadow_scb(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) struct kvm_s390_sie_block *scb_o = vsie_page->scb_o;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) /* READ_ONCE does not work on bitfields - use a temporary variable */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) const uint32_t __new_prefix = scb_o->prefix;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) const uint32_t new_prefix = READ_ONCE(__new_prefix);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) const bool wants_tx = READ_ONCE(scb_o->ecb) & ECB_TE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) bool had_tx = scb_s->ecb & ECB_TE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) unsigned long new_mso = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) int rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) /* make sure we don't have any leftovers when reusing the scb */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) scb_s->icptcode = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) scb_s->eca = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) scb_s->ecb = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) scb_s->ecb2 = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) scb_s->ecb3 = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) scb_s->ecd = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) scb_s->fac = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) scb_s->fpf = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) rc = prepare_cpuflags(vcpu, vsie_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) if (rc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) /* timer */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) scb_s->cputm = scb_o->cputm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) scb_s->ckc = scb_o->ckc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) scb_s->todpr = scb_o->todpr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) scb_s->epoch = scb_o->epoch;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) /* guest state */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) scb_s->gpsw = scb_o->gpsw;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) scb_s->gg14 = scb_o->gg14;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) scb_s->gg15 = scb_o->gg15;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) memcpy(scb_s->gcr, scb_o->gcr, 128);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) scb_s->pp = scb_o->pp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) /* interception / execution handling */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) scb_s->gbea = scb_o->gbea;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) scb_s->lctl = scb_o->lctl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) scb_s->svcc = scb_o->svcc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) scb_s->ictl = scb_o->ictl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) * SKEY handling functions can't deal with false setting of PTE invalid
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) * bits. Therefore we cannot provide interpretation and would later
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) * have to provide own emulation handlers.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) if (!(atomic_read(&scb_s->cpuflags) & CPUSTAT_KSS))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) scb_s->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) scb_s->icpua = scb_o->icpua;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) if (!(atomic_read(&scb_s->cpuflags) & CPUSTAT_SM))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) new_mso = READ_ONCE(scb_o->mso) & 0xfffffffffff00000UL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) /* if the hva of the prefix changes, we have to remap the prefix */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) if (scb_s->mso != new_mso || scb_s->prefix != new_prefix)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) prefix_unmapped(vsie_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) /* SIE will do mso/msl validity and exception checks for us */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) scb_s->msl = scb_o->msl & 0xfffffffffff00000UL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) scb_s->mso = new_mso;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) scb_s->prefix = new_prefix;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) /* We have to definetly flush the tlb if this scb never ran */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) if (scb_s->ihcpu != 0xffffU)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) scb_s->ihcpu = scb_o->ihcpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) /* MVPG and Protection Exception Interpretation are always available */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) scb_s->eca |= scb_o->eca & (ECA_MVPGI | ECA_PROTEXCI);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) /* Host-protection-interruption introduced with ESOP */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) if (test_kvm_cpu_feat(vcpu->kvm, KVM_S390_VM_CPU_FEAT_ESOP))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) scb_s->ecb |= scb_o->ecb & ECB_HOSTPROTINT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) /* transactional execution */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) if (test_kvm_facility(vcpu->kvm, 73) && wants_tx) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) /* remap the prefix is tx is toggled on */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) if (!had_tx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) prefix_unmapped(vsie_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) scb_s->ecb |= ECB_TE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) /* branch prediction */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) if (test_kvm_facility(vcpu->kvm, 82))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) scb_s->fpf |= scb_o->fpf & FPF_BPBC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) /* SIMD */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) if (test_kvm_facility(vcpu->kvm, 129)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) scb_s->eca |= scb_o->eca & ECA_VX;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) scb_s->ecd |= scb_o->ecd & ECD_HOSTREGMGMT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) /* Run-time-Instrumentation */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) if (test_kvm_facility(vcpu->kvm, 64))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) scb_s->ecb3 |= scb_o->ecb3 & ECB3_RI;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) /* Instruction Execution Prevention */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) if (test_kvm_facility(vcpu->kvm, 130))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) scb_s->ecb2 |= scb_o->ecb2 & ECB2_IEP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) /* Guarded Storage */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) if (test_kvm_facility(vcpu->kvm, 133)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) scb_s->ecb |= scb_o->ecb & ECB_GS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) scb_s->ecd |= scb_o->ecd & ECD_HOSTREGMGMT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) if (test_kvm_cpu_feat(vcpu->kvm, KVM_S390_VM_CPU_FEAT_SIIF))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) scb_s->eca |= scb_o->eca & ECA_SII;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) if (test_kvm_cpu_feat(vcpu->kvm, KVM_S390_VM_CPU_FEAT_IB))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) scb_s->eca |= scb_o->eca & ECA_IB;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) if (test_kvm_cpu_feat(vcpu->kvm, KVM_S390_VM_CPU_FEAT_CEI))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) scb_s->eca |= scb_o->eca & ECA_CEI;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) /* Epoch Extension */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) if (test_kvm_facility(vcpu->kvm, 139))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) scb_s->ecd |= scb_o->ecd & ECD_MEF;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) /* etoken */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) if (test_kvm_facility(vcpu->kvm, 156))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) scb_s->ecd |= scb_o->ecd & ECD_ETOKENF;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) scb_s->hpid = HPID_VSIE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) scb_s->cpnc = scb_o->cpnc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) prepare_ibc(vcpu, vsie_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) rc = shadow_crycb(vcpu, vsie_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) if (rc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) unshadow_scb(vcpu, vsie_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) void kvm_s390_vsie_gmap_notifier(struct gmap *gmap, unsigned long start,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) unsigned long end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) struct kvm *kvm = gmap->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) struct vsie_page *cur;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) unsigned long prefix;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) if (!gmap_is_shadow(gmap))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) if (start >= 1UL << 31)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) /* We are only interested in prefix pages */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) * Only new shadow blocks are added to the list during runtime,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) * therefore we can safely reference them all the time.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) for (i = 0; i < kvm->arch.vsie.page_count; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) page = READ_ONCE(kvm->arch.vsie.pages[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) if (!page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) cur = page_to_virt(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) if (READ_ONCE(cur->gmap) != gmap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) prefix = cur->scb_s.prefix << GUEST_PREFIX_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) /* with mso/msl, the prefix lies at an offset */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) prefix += cur->scb_s.mso;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) if (prefix <= end && start <= prefix + 2 * PAGE_SIZE - 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) prefix_unmapped_sync(cur);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) * Map the first prefix page and if tx is enabled also the second prefix page.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) * The prefix will be protected, a gmap notifier will inform about unmaps.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) * The shadow scb must not be executed until the prefix is remapped, this is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) * guaranteed by properly handling PROG_REQUEST.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) * Returns: - 0 on if successfully mapped or already mapped
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) * - > 0 if control has to be given to guest 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) * - -EAGAIN if the caller can retry immediately
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) * - -ENOMEM if out of memory
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) static int map_prefix(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) u64 prefix = scb_s->prefix << GUEST_PREFIX_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) int rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) if (prefix_is_mapped(vsie_page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) /* mark it as mapped so we can catch any concurrent unmappers */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) prefix_mapped(vsie_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) /* with mso/msl, the prefix lies at offset *mso* */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) prefix += scb_s->mso;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) rc = kvm_s390_shadow_fault(vcpu, vsie_page->gmap, prefix, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) if (!rc && (scb_s->ecb & ECB_TE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) rc = kvm_s390_shadow_fault(vcpu, vsie_page->gmap,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) prefix + PAGE_SIZE, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) * We don't have to mprotect, we will be called for all unshadows.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) * SIE will detect if protection applies and trigger a validity.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) if (rc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) prefix_unmapped(vsie_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) if (rc > 0 || rc == -EFAULT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) rc = set_validity_icpt(scb_s, 0x0037U);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) * Pin the guest page given by gpa and set hpa to the pinned host address.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) * Will always be pinned writable.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) * Returns: - 0 on success
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) * - -EINVAL if the gpa is not valid guest storage
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) static int pin_guest_page(struct kvm *kvm, gpa_t gpa, hpa_t *hpa)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) page = gfn_to_page(kvm, gpa_to_gfn(gpa));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) if (is_error_page(page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) *hpa = (hpa_t) page_to_virt(page) + (gpa & ~PAGE_MASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) /* Unpins a page previously pinned via pin_guest_page, marking it as dirty. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) static void unpin_guest_page(struct kvm *kvm, gpa_t gpa, hpa_t hpa)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) kvm_release_pfn_dirty(hpa >> PAGE_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) /* mark the page always as dirty for migration */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) mark_page_dirty(kvm, gpa_to_gfn(gpa));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) /* unpin all blocks previously pinned by pin_blocks(), marking them dirty */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) static void unpin_blocks(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) hpa_t hpa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) hpa = (u64) scb_s->scaoh << 32 | scb_s->scaol;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) if (hpa) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) unpin_guest_page(vcpu->kvm, vsie_page->sca_gpa, hpa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) vsie_page->sca_gpa = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) scb_s->scaol = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) scb_s->scaoh = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) hpa = scb_s->itdba;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) if (hpa) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) unpin_guest_page(vcpu->kvm, vsie_page->itdba_gpa, hpa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) vsie_page->itdba_gpa = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) scb_s->itdba = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) hpa = scb_s->gvrd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) if (hpa) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) unpin_guest_page(vcpu->kvm, vsie_page->gvrd_gpa, hpa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) vsie_page->gvrd_gpa = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) scb_s->gvrd = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) hpa = scb_s->riccbd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) if (hpa) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) unpin_guest_page(vcpu->kvm, vsie_page->riccbd_gpa, hpa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) vsie_page->riccbd_gpa = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) scb_s->riccbd = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) hpa = scb_s->sdnxo;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) if (hpa) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) unpin_guest_page(vcpu->kvm, vsie_page->sdnx_gpa, hpa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) vsie_page->sdnx_gpa = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) scb_s->sdnxo = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) * Instead of shadowing some blocks, we can simply forward them because the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) * addresses in the scb are 64 bit long.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) * This works as long as the data lies in one page. If blocks ever exceed one
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) * page, we have to fall back to shadowing.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) * As we reuse the sca, the vcpu pointers contained in it are invalid. We must
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) * therefore not enable any facilities that access these pointers (e.g. SIGPIF).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) * Returns: - 0 if all blocks were pinned.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) * - > 0 if control has to be given to guest 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) * - -ENOMEM if out of memory
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) static int pin_blocks(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) struct kvm_s390_sie_block *scb_o = vsie_page->scb_o;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) hpa_t hpa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) gpa_t gpa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) int rc = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) gpa = READ_ONCE(scb_o->scaol) & ~0xfUL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) if (test_kvm_cpu_feat(vcpu->kvm, KVM_S390_VM_CPU_FEAT_64BSCAO))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) gpa |= (u64) READ_ONCE(scb_o->scaoh) << 32;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) if (gpa) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) if (gpa < 2 * PAGE_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) rc = set_validity_icpt(scb_s, 0x0038U);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) else if ((gpa & ~0x1fffUL) == kvm_s390_get_prefix(vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) rc = set_validity_icpt(scb_s, 0x0011U);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) else if ((gpa & PAGE_MASK) !=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) ((gpa + sizeof(struct bsca_block) - 1) & PAGE_MASK))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) rc = set_validity_icpt(scb_s, 0x003bU);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) if (!rc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) rc = pin_guest_page(vcpu->kvm, gpa, &hpa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) if (rc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) rc = set_validity_icpt(scb_s, 0x0034U);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) if (rc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) goto unpin;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) vsie_page->sca_gpa = gpa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) scb_s->scaoh = (u32)((u64)hpa >> 32);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) scb_s->scaol = (u32)(u64)hpa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) gpa = READ_ONCE(scb_o->itdba) & ~0xffUL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) if (gpa && (scb_s->ecb & ECB_TE)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) if (gpa < 2 * PAGE_SIZE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) rc = set_validity_icpt(scb_s, 0x0080U);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) goto unpin;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) /* 256 bytes cannot cross page boundaries */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) rc = pin_guest_page(vcpu->kvm, gpa, &hpa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) if (rc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) rc = set_validity_icpt(scb_s, 0x0080U);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) goto unpin;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) vsie_page->itdba_gpa = gpa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) scb_s->itdba = hpa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) gpa = READ_ONCE(scb_o->gvrd) & ~0x1ffUL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) if (gpa && (scb_s->eca & ECA_VX) && !(scb_s->ecd & ECD_HOSTREGMGMT)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) if (gpa < 2 * PAGE_SIZE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) rc = set_validity_icpt(scb_s, 0x1310U);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) goto unpin;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) * 512 bytes vector registers cannot cross page boundaries
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) * if this block gets bigger, we have to shadow it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) rc = pin_guest_page(vcpu->kvm, gpa, &hpa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) if (rc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) rc = set_validity_icpt(scb_s, 0x1310U);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) goto unpin;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) vsie_page->gvrd_gpa = gpa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) scb_s->gvrd = hpa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) gpa = READ_ONCE(scb_o->riccbd) & ~0x3fUL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) if (gpa && (scb_s->ecb3 & ECB3_RI)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) if (gpa < 2 * PAGE_SIZE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) rc = set_validity_icpt(scb_s, 0x0043U);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) goto unpin;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) /* 64 bytes cannot cross page boundaries */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) rc = pin_guest_page(vcpu->kvm, gpa, &hpa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) if (rc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) rc = set_validity_icpt(scb_s, 0x0043U);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) goto unpin;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) /* Validity 0x0044 will be checked by SIE */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) vsie_page->riccbd_gpa = gpa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) scb_s->riccbd = hpa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) if (((scb_s->ecb & ECB_GS) && !(scb_s->ecd & ECD_HOSTREGMGMT)) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) (scb_s->ecd & ECD_ETOKENF)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) unsigned long sdnxc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) gpa = READ_ONCE(scb_o->sdnxo) & ~0xfUL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) sdnxc = READ_ONCE(scb_o->sdnxo) & 0xfUL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) if (!gpa || gpa < 2 * PAGE_SIZE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) rc = set_validity_icpt(scb_s, 0x10b0U);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) goto unpin;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) if (sdnxc < 6 || sdnxc > 12) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) rc = set_validity_icpt(scb_s, 0x10b1U);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) goto unpin;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811) if (gpa & ((1 << sdnxc) - 1)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) rc = set_validity_icpt(scb_s, 0x10b2U);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) goto unpin;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) /* Due to alignment rules (checked above) this cannot
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) * cross page boundaries
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) rc = pin_guest_page(vcpu->kvm, gpa, &hpa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) if (rc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) rc = set_validity_icpt(scb_s, 0x10b0U);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) goto unpin;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) vsie_page->sdnx_gpa = gpa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) scb_s->sdnxo = hpa | sdnxc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) unpin:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) unpin_blocks(vcpu, vsie_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) /* unpin the scb provided by guest 2, marking it as dirty */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) static void unpin_scb(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834) gpa_t gpa)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) hpa_t hpa = (hpa_t) vsie_page->scb_o;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) if (hpa)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) unpin_guest_page(vcpu->kvm, gpa, hpa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) vsie_page->scb_o = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) * Pin the scb at gpa provided by guest 2 at vsie_page->scb_o.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846) * Returns: - 0 if the scb was pinned.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) * - > 0 if control has to be given to guest 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) static int pin_scb(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) gpa_t gpa)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) hpa_t hpa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) int rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855) rc = pin_guest_page(vcpu->kvm, gpa, &hpa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) if (rc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857) rc = kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858) WARN_ON_ONCE(rc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) vsie_page->scb_o = (struct kvm_s390_sie_block *) hpa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) * Inject a fault into guest 2.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) * Returns: - > 0 if control has to be given to guest 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869) * < 0 if an error occurred during injection.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) static int inject_fault(struct kvm_vcpu *vcpu, __u16 code, __u64 vaddr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872) bool write_flag)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) struct kvm_s390_pgm_info pgm = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875) .code = code,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) .trans_exc_code =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) /* 0-51: virtual address */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878) (vaddr & 0xfffffffffffff000UL) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879) /* 52-53: store / fetch */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880) (((unsigned int) !write_flag) + 1) << 10,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) /* 62-63: asce id (alway primary == 0) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) .exc_access_id = 0, /* always primary */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) .op_access_id = 0, /* not MVPG */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885) int rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) if (code == PGM_PROTECTION)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888) pgm.trans_exc_code |= 0x4UL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890) rc = kvm_s390_inject_prog_irq(vcpu, &pgm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891) return rc ? rc : 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895) * Handle a fault during vsie execution on a gmap shadow.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897) * Returns: - 0 if the fault was resolved
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898) * - > 0 if control has to be given to guest 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899) * - < 0 if an error occurred
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901) static int handle_fault(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903) int rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905) if (current->thread.gmap_int_code == PGM_PROTECTION)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906) /* we can directly forward all protection exceptions */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907) return inject_fault(vcpu, PGM_PROTECTION,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908) current->thread.gmap_addr, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910) rc = kvm_s390_shadow_fault(vcpu, vsie_page->gmap,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911) current->thread.gmap_addr, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) if (rc > 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913) rc = inject_fault(vcpu, rc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914) current->thread.gmap_addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915) current->thread.gmap_write_flag);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916) if (rc >= 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917) vsie_page->fault_addr = current->thread.gmap_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923) * Retry the previous fault that required guest 2 intervention. This avoids
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924) * one superfluous SIE re-entry and direct exit.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926) * Will ignore any errors. The next SIE fault will do proper fault handling.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928) static void handle_last_fault(struct kvm_vcpu *vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929) struct vsie_page *vsie_page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) if (vsie_page->fault_addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932) kvm_s390_shadow_fault(vcpu, vsie_page->gmap,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) vsie_page->fault_addr, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) vsie_page->fault_addr = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937) static inline void clear_vsie_icpt(struct vsie_page *vsie_page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939) vsie_page->scb_s.icptcode = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942) /* rewind the psw and clear the vsie icpt, so we can retry execution */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943) static void retry_vsie_icpt(struct vsie_page *vsie_page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945) struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946) int ilen = insn_length(scb_s->ipa >> 8);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948) /* take care of EXECUTE instructions */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949) if (scb_s->icptstatus & 1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950) ilen = (scb_s->icptstatus >> 4) & 0x6;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951) if (!ilen)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952) ilen = 4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954) scb_s->gpsw.addr = __rewind_psw(scb_s->gpsw, ilen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955) clear_vsie_icpt(vsie_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959) * Try to shadow + enable the guest 2 provided facility list.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960) * Retry instruction execution if enabled for and provided by guest 2.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962) * Returns: - 0 if handled (retry or guest 2 icpt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963) * - > 0 if control has to be given to guest 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965) static int handle_stfle(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967) struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968) __u32 fac = READ_ONCE(vsie_page->scb_o->fac) & 0x7ffffff8U;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970) if (fac && test_kvm_facility(vcpu->kvm, 7)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971) retry_vsie_icpt(vsie_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972) if (read_guest_real(vcpu, fac, &vsie_page->fac,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973) sizeof(vsie_page->fac)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974) return set_validity_icpt(scb_s, 0x1090U);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975) scb_s->fac = (__u32)(__u64) &vsie_page->fac;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981) * Get a register for a nested guest.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982) * @vcpu the vcpu of the guest
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983) * @vsie_page the vsie_page for the nested guest
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984) * @reg the register number, the upper 4 bits are ignored.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985) * returns: the value of the register.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987) static u64 vsie_get_register(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page, u8 reg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989) /* no need to validate the parameter and/or perform error handling */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990) reg &= 0xf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991) switch (reg) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992) case 15:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 993) return vsie_page->scb_s.gg15;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 994) case 14:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 995) return vsie_page->scb_s.gg14;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 996) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 997) return vcpu->run->s.regs.gprs[reg];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 998) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 999) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1000)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1001) static int vsie_handle_mvpg(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1002) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1003) struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1004) unsigned long pei_dest, pei_src, src, dest, mask, prefix;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1005) u64 *pei_block = &vsie_page->scb_o->mcic;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1006) int edat, rc_dest, rc_src;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1007) union ctlreg0 cr0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1008)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1009) cr0.val = vcpu->arch.sie_block->gcr[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1010) edat = cr0.edat && test_kvm_facility(vcpu->kvm, 8);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1011) mask = _kvm_s390_logical_to_effective(&scb_s->gpsw, PAGE_MASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1012) prefix = scb_s->prefix << GUEST_PREFIX_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1013)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1014) dest = vsie_get_register(vcpu, vsie_page, scb_s->ipb >> 20) & mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1015) dest = _kvm_s390_real_to_abs(prefix, dest) + scb_s->mso;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1016) src = vsie_get_register(vcpu, vsie_page, scb_s->ipb >> 16) & mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1017) src = _kvm_s390_real_to_abs(prefix, src) + scb_s->mso;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1018)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1019) rc_dest = kvm_s390_shadow_fault(vcpu, vsie_page->gmap, dest, &pei_dest);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1020) rc_src = kvm_s390_shadow_fault(vcpu, vsie_page->gmap, src, &pei_src);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1021) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1022) * Either everything went well, or something non-critical went wrong
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1023) * e.g. because of a race. In either case, simply retry.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1024) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1025) if (rc_dest == -EAGAIN || rc_src == -EAGAIN || (!rc_dest && !rc_src)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1026) retry_vsie_icpt(vsie_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1027) return -EAGAIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1028) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1029) /* Something more serious went wrong, propagate the error */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1030) if (rc_dest < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1031) return rc_dest;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1032) if (rc_src < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1033) return rc_src;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1034)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1035) /* The only possible suppressing exception: just deliver it */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1036) if (rc_dest == PGM_TRANSLATION_SPEC || rc_src == PGM_TRANSLATION_SPEC) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1037) clear_vsie_icpt(vsie_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1038) rc_dest = kvm_s390_inject_program_int(vcpu, PGM_TRANSLATION_SPEC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1039) WARN_ON_ONCE(rc_dest);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1040) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1041) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1042)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1043) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1044) * Forward the PEI intercept to the guest if it was a page fault, or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1045) * also for segment and region table faults if EDAT applies.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1046) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1047) if (edat) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1048) rc_dest = rc_dest == PGM_ASCE_TYPE ? rc_dest : 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1049) rc_src = rc_src == PGM_ASCE_TYPE ? rc_src : 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1050) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1051) rc_dest = rc_dest != PGM_PAGE_TRANSLATION ? rc_dest : 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1052) rc_src = rc_src != PGM_PAGE_TRANSLATION ? rc_src : 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1053) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1054) if (!rc_dest && !rc_src) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1055) pei_block[0] = pei_dest;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1056) pei_block[1] = pei_src;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1057) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1058) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1059)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1060) retry_vsie_icpt(vsie_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1061)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1062) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1063) * The host has edat, and the guest does not, or it was an ASCE type
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1064) * exception. The host needs to inject the appropriate DAT interrupts
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1065) * into the guest.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1066) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1067) if (rc_dest)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1068) return inject_fault(vcpu, rc_dest, dest, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1069) return inject_fault(vcpu, rc_src, src, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1070) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1071)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1072) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1073) * Run the vsie on a shadow scb and a shadow gmap, without any further
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1074) * sanity checks, handling SIE faults.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1075) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1076) * Returns: - 0 everything went fine
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1077) * - > 0 if control has to be given to guest 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1078) * - < 0 if an error occurred
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1079) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1080) static int do_vsie_run(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1081) __releases(vcpu->kvm->srcu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1082) __acquires(vcpu->kvm->srcu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1083) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1084) struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1085) struct kvm_s390_sie_block *scb_o = vsie_page->scb_o;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1086) int guest_bp_isolation;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1087) int rc = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1088)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1089) handle_last_fault(vcpu, vsie_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1090)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1091) srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1092)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1093) /* save current guest state of bp isolation override */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1094) guest_bp_isolation = test_thread_flag(TIF_ISOLATE_BP_GUEST);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1095)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1096) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1097) * The guest is running with BPBC, so we have to force it on for our
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1098) * nested guest. This is done by enabling BPBC globally, so the BPBC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1099) * control in the SCB (which the nested guest can modify) is simply
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1100) * ignored.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1101) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1102) if (test_kvm_facility(vcpu->kvm, 82) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1103) vcpu->arch.sie_block->fpf & FPF_BPBC)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1104) set_thread_flag(TIF_ISOLATE_BP_GUEST);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1105)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1106) local_irq_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1107) guest_enter_irqoff();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1108) local_irq_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1109)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1110) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1111) * Simulate a SIE entry of the VCPU (see sie64a), so VCPU blocking
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1112) * and VCPU requests also hinder the vSIE from running and lead
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1113) * to an immediate exit. kvm_s390_vsie_kick() has to be used to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1114) * also kick the vSIE.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1115) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1116) vcpu->arch.sie_block->prog0c |= PROG_IN_SIE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1117) barrier();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1118) if (!kvm_s390_vcpu_sie_inhibited(vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1119) rc = sie64a(scb_s, vcpu->run->s.regs.gprs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1120) barrier();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1121) vcpu->arch.sie_block->prog0c &= ~PROG_IN_SIE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1122)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1123) local_irq_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1124) guest_exit_irqoff();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1125) local_irq_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1126)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1127) /* restore guest state for bp isolation override */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1128) if (!guest_bp_isolation)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1129) clear_thread_flag(TIF_ISOLATE_BP_GUEST);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1130)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1131) vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1132)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1133) if (rc == -EINTR) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1134) VCPU_EVENT(vcpu, 3, "%s", "machine check");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1135) kvm_s390_reinject_machine_check(vcpu, &vsie_page->mcck_info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1136) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1137) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1138)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1139) if (rc > 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1140) rc = 0; /* we could still have an icpt */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1141) else if (rc == -EFAULT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1142) return handle_fault(vcpu, vsie_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1143)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1144) switch (scb_s->icptcode) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1145) case ICPT_INST:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1146) if (scb_s->ipa == 0xb2b0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1147) rc = handle_stfle(vcpu, vsie_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1148) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1149) case ICPT_STOP:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1150) /* stop not requested by g2 - must have been a kick */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1151) if (!(atomic_read(&scb_o->cpuflags) & CPUSTAT_STOP_INT))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1152) clear_vsie_icpt(vsie_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1153) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1154) case ICPT_VALIDITY:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1155) if ((scb_s->ipa & 0xf000) != 0xf000)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1156) scb_s->ipa += 0x1000;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1157) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1158) case ICPT_PARTEXEC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1159) if (scb_s->ipa == 0xb254)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1160) rc = vsie_handle_mvpg(vcpu, vsie_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1161) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1162) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1163) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1164) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1165)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1166) static void release_gmap_shadow(struct vsie_page *vsie_page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1167) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1168) if (vsie_page->gmap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1169) gmap_put(vsie_page->gmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1170) WRITE_ONCE(vsie_page->gmap, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1171) prefix_unmapped(vsie_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1172) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1173)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1174) static int acquire_gmap_shadow(struct kvm_vcpu *vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1175) struct vsie_page *vsie_page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1176) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1177) unsigned long asce;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1178) union ctlreg0 cr0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1179) struct gmap *gmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1180) int edat;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1181)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1182) asce = vcpu->arch.sie_block->gcr[1];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1183) cr0.val = vcpu->arch.sie_block->gcr[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1184) edat = cr0.edat && test_kvm_facility(vcpu->kvm, 8);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1185) edat += edat && test_kvm_facility(vcpu->kvm, 78);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1186)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1187) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1188) * ASCE or EDAT could have changed since last icpt, or the gmap
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1189) * we're holding has been unshadowed. If the gmap is still valid,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1190) * we can safely reuse it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1191) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1192) if (vsie_page->gmap && gmap_shadow_valid(vsie_page->gmap, asce, edat))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1193) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1194)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1195) /* release the old shadow - if any, and mark the prefix as unmapped */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1196) release_gmap_shadow(vsie_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1197) gmap = gmap_shadow(vcpu->arch.gmap, asce, edat);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1198) if (IS_ERR(gmap))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1199) return PTR_ERR(gmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1200) gmap->private = vcpu->kvm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1201) WRITE_ONCE(vsie_page->gmap, gmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1202) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1203) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1204)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1205) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1206) * Register the shadow scb at the VCPU, e.g. for kicking out of vsie.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1207) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1208) static void register_shadow_scb(struct kvm_vcpu *vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1209) struct vsie_page *vsie_page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1210) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1211) struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1212)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1213) WRITE_ONCE(vcpu->arch.vsie_block, &vsie_page->scb_s);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1214) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1215) * External calls have to lead to a kick of the vcpu and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1216) * therefore the vsie -> Simulate Wait state.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1217) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1218) kvm_s390_set_cpuflags(vcpu, CPUSTAT_WAIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1219) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1220) * We have to adjust the g3 epoch by the g2 epoch. The epoch will
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1221) * automatically be adjusted on tod clock changes via kvm_sync_clock.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1222) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1223) preempt_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1224) scb_s->epoch += vcpu->kvm->arch.epoch;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1225)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1226) if (scb_s->ecd & ECD_MEF) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1227) scb_s->epdx += vcpu->kvm->arch.epdx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1228) if (scb_s->epoch < vcpu->kvm->arch.epoch)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1229) scb_s->epdx += 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1230) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1231)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1232) preempt_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1233) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1234)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1235) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1236) * Unregister a shadow scb from a VCPU.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1237) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1238) static void unregister_shadow_scb(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1239) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1240) kvm_s390_clear_cpuflags(vcpu, CPUSTAT_WAIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1241) WRITE_ONCE(vcpu->arch.vsie_block, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1242) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1243)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1244) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1245) * Run the vsie on a shadowed scb, managing the gmap shadow, handling
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1246) * prefix pages and faults.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1247) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1248) * Returns: - 0 if no errors occurred
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1249) * - > 0 if control has to be given to guest 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1250) * - -ENOMEM if out of memory
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1251) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1252) static int vsie_run(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1253) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1254) struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1255) int rc = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1256)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1257) while (1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1258) rc = acquire_gmap_shadow(vcpu, vsie_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1259) if (!rc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1260) rc = map_prefix(vcpu, vsie_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1261) if (!rc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1262) gmap_enable(vsie_page->gmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1263) update_intervention_requests(vsie_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1264) rc = do_vsie_run(vcpu, vsie_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1265) gmap_enable(vcpu->arch.gmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1266) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1267) atomic_andnot(PROG_BLOCK_SIE, &scb_s->prog20);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1268)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1269) if (rc == -EAGAIN)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1270) rc = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1271) if (rc || scb_s->icptcode || signal_pending(current) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1272) kvm_s390_vcpu_has_irq(vcpu, 0) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1273) kvm_s390_vcpu_sie_inhibited(vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1274) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1275) cond_resched();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1276) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1277)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1278) if (rc == -EFAULT) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1279) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1280) * Addressing exceptions are always presentes as intercepts.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1281) * As addressing exceptions are suppressing and our guest 3 PSW
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1282) * points at the responsible instruction, we have to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1283) * forward the PSW and set the ilc. If we can't read guest 3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1284) * instruction, we can use an arbitrary ilc. Let's always use
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1285) * ilen = 4 for now, so we can avoid reading in guest 3 virtual
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1286) * memory. (we could also fake the shadow so the hardware
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1287) * handles it).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1288) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1289) scb_s->icptcode = ICPT_PROGI;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1290) scb_s->iprcc = PGM_ADDRESSING;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1291) scb_s->pgmilc = 4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1292) scb_s->gpsw.addr = __rewind_psw(scb_s->gpsw, 4);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1293) rc = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1294) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1295) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1296) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1297)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1298) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1299) * Get or create a vsie page for a scb address.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1300) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1301) * Returns: - address of a vsie page (cached or new one)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1302) * - NULL if the same scb address is already used by another VCPU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1303) * - ERR_PTR(-ENOMEM) if out of memory
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1304) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1305) static struct vsie_page *get_vsie_page(struct kvm *kvm, unsigned long addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1306) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1307) struct vsie_page *vsie_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1308) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1309) int nr_vcpus;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1310)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1311) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1312) page = radix_tree_lookup(&kvm->arch.vsie.addr_to_page, addr >> 9);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1313) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1314) if (page) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1315) if (page_ref_inc_return(page) == 2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1316) return page_to_virt(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1317) page_ref_dec(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1318) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1319)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1320) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1321) * We want at least #online_vcpus shadows, so every VCPU can execute
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1322) * the VSIE in parallel.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1323) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1324) nr_vcpus = atomic_read(&kvm->online_vcpus);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1325)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1326) mutex_lock(&kvm->arch.vsie.mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1327) if (kvm->arch.vsie.page_count < nr_vcpus) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1328) page = alloc_page(GFP_KERNEL | __GFP_ZERO | GFP_DMA);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1329) if (!page) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1330) mutex_unlock(&kvm->arch.vsie.mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1331) return ERR_PTR(-ENOMEM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1332) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1333) page_ref_inc(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1334) kvm->arch.vsie.pages[kvm->arch.vsie.page_count] = page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1335) kvm->arch.vsie.page_count++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1336) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1337) /* reuse an existing entry that belongs to nobody */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1338) while (true) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1339) page = kvm->arch.vsie.pages[kvm->arch.vsie.next];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1340) if (page_ref_inc_return(page) == 2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1341) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1342) page_ref_dec(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1343) kvm->arch.vsie.next++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1344) kvm->arch.vsie.next %= nr_vcpus;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1345) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1346) radix_tree_delete(&kvm->arch.vsie.addr_to_page, page->index >> 9);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1347) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1348) page->index = addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1349) /* double use of the same address */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1350) if (radix_tree_insert(&kvm->arch.vsie.addr_to_page, addr >> 9, page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1351) page_ref_dec(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1352) mutex_unlock(&kvm->arch.vsie.mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1353) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1354) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1355) mutex_unlock(&kvm->arch.vsie.mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1356)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1357) vsie_page = page_to_virt(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1358) memset(&vsie_page->scb_s, 0, sizeof(struct kvm_s390_sie_block));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1359) release_gmap_shadow(vsie_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1360) vsie_page->fault_addr = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1361) vsie_page->scb_s.ihcpu = 0xffffU;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1362) return vsie_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1363) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1364)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1365) /* put a vsie page acquired via get_vsie_page */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1366) static void put_vsie_page(struct kvm *kvm, struct vsie_page *vsie_page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1367) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1368) struct page *page = pfn_to_page(__pa(vsie_page) >> PAGE_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1369)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1370) page_ref_dec(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1371) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1372)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1373) int kvm_s390_handle_vsie(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1374) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1375) struct vsie_page *vsie_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1376) unsigned long scb_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1377) int rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1378)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1379) vcpu->stat.instruction_sie++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1380) if (!test_kvm_cpu_feat(vcpu->kvm, KVM_S390_VM_CPU_FEAT_SIEF2))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1381) return -EOPNOTSUPP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1382) if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1383) return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1384)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1385) BUILD_BUG_ON(sizeof(struct vsie_page) != PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1386) scb_addr = kvm_s390_get_base_disp_s(vcpu, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1387)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1388) /* 512 byte alignment */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1389) if (unlikely(scb_addr & 0x1ffUL))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1390) return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1391)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1392) if (signal_pending(current) || kvm_s390_vcpu_has_irq(vcpu, 0) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1393) kvm_s390_vcpu_sie_inhibited(vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1394) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1395)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1396) vsie_page = get_vsie_page(vcpu->kvm, scb_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1397) if (IS_ERR(vsie_page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1398) return PTR_ERR(vsie_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1399) else if (!vsie_page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1400) /* double use of sie control block - simply do nothing */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1401) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1402)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1403) rc = pin_scb(vcpu, vsie_page, scb_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1404) if (rc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1405) goto out_put;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1406) rc = shadow_scb(vcpu, vsie_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1407) if (rc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1408) goto out_unpin_scb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1409) rc = pin_blocks(vcpu, vsie_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1410) if (rc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1411) goto out_unshadow;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1412) register_shadow_scb(vcpu, vsie_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1413) rc = vsie_run(vcpu, vsie_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1414) unregister_shadow_scb(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1415) unpin_blocks(vcpu, vsie_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1416) out_unshadow:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1417) unshadow_scb(vcpu, vsie_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1418) out_unpin_scb:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1419) unpin_scb(vcpu, vsie_page, scb_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1420) out_put:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1421) put_vsie_page(vcpu->kvm, vsie_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1422)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1423) return rc < 0 ? rc : 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1424) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1425)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1426) /* Init the vsie data structures. To be called when a vm is initialized. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1427) void kvm_s390_vsie_init(struct kvm *kvm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1428) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1429) mutex_init(&kvm->arch.vsie.mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1430) INIT_RADIX_TREE(&kvm->arch.vsie.addr_to_page, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1431) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1432)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1433) /* Destroy the vsie data structures. To be called when a vm is destroyed. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1434) void kvm_s390_vsie_destroy(struct kvm *kvm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1435) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1436) struct vsie_page *vsie_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1437) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1438) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1439)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1440) mutex_lock(&kvm->arch.vsie.mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1441) for (i = 0; i < kvm->arch.vsie.page_count; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1442) page = kvm->arch.vsie.pages[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1443) kvm->arch.vsie.pages[i] = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1444) vsie_page = page_to_virt(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1445) release_gmap_shadow(vsie_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1446) /* free the radix tree entry */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1447) radix_tree_delete(&kvm->arch.vsie.addr_to_page, page->index >> 9);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1448) __free_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1449) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1450) kvm->arch.vsie.page_count = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1451) mutex_unlock(&kvm->arch.vsie.mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1452) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1453)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1454) void kvm_s390_vsie_kick(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1455) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1456) struct kvm_s390_sie_block *scb = READ_ONCE(vcpu->arch.vsie_block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1457)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1458) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1459) * Even if the VCPU lets go of the shadow sie block reference, it is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1460) * still valid in the cache. So we can safely kick it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1461) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1462) if (scb) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1463) atomic_or(PROG_BLOCK_SIE, &scb->prog20);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1464) if (scb->prog0c & PROG_IN_SIE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1465) atomic_or(CPUSTAT_STOP_INT, &scb->cpuflags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1466) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1467) }
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags