Voltage control on a kvm passed through 1080

Software & Operating Systems VFIO
, ,
1

sys info

I can control the core clocks and memory clocks fine with afterburner but changing the voltage causes the vm to crash (reboots the vm) on bare metal running the same windows install I can control voltage just fine

1 Like
2

Voltage control now seems to be applying correctly according to afterburner , however I am still getting VM crashes that I can’t reproduce on bare metal here is all the information relating to my VM

sudo virsh dumpxml "win11" 
<domain type='kvm'>
  <name>win11</name>
  <uuid>6e4baa04-efd3-4c57-a352-650f84d3c4cf</uuid>
  <metadata>
    <libosinfo:libosinfo xmlns:libosinfo="http://libosinfo.org/xmlns/libvirt/domain/1.0">
      <libosinfo:os id="http://microsoft.com/win/10"/>
    </libosinfo:libosinfo>
  </metadata>
  <memory unit='KiB'>8388608</memory>
  <currentMemory unit='KiB'>8388608</currentMemory>
  <vcpu placement='static'>8</vcpu>
  <iothreads>1</iothreads>
  <cputune>
    <vcpupin vcpu='0' cpuset='2'/>
    <vcpupin vcpu='1' cpuset='8'/>
    <vcpupin vcpu='2' cpuset='3'/>
    <vcpupin vcpu='3' cpuset='9'/>
    <vcpupin vcpu='4' cpuset='4'/>
    <vcpupin vcpu='5' cpuset='10'/>
    <vcpupin vcpu='6' cpuset='5'/>
    <vcpupin vcpu='7' cpuset='11'/>
    <emulatorpin cpuset='0-6'/>
    <iothreadpin iothread='1' cpuset='1-7'/>
  </cputune>
  <os>
    <type arch='x86_64' machine='pc-q35-6.2'>hvm</type>
    <loader readonly='yes' type='pflash'>/usr/share/edk2-ovmf/x64/OVMF_CODE.fd</loader>
    <nvram>/var/lib/libvirt/qemu/nvram/win11_VARS.fd</nvram>
    <bootmenu enable='no'/>
  </os>
  <features>
    <acpi/>
    <apic/>
    <hyperv mode='custom'>
      <relaxed state='on'/>
      <vapic state='on'/>
      <spinlocks state='on' retries='8191'/>
      <vpindex state='on'/>
      <runtime state='on'/>
      <synic state='on'/>
      <stimer state='on'/>
      <reset state='off'/>
      <frequencies state='on'/>
      <reenlightenment state='off'/>
      <tlbflush state='on'/>
      <ipi state='on'/>
      <evmcs state='off'/>
    </hyperv>
    <vmport state='off'/>
    <ioapic driver='kvm'/>
  </features>
  <cpu mode='host-passthrough' check='none' migratable='on'>
    <topology sockets='1' dies='1' cores='4' threads='2'/>
    <cache mode='passthrough'/>
    <feature policy='require' name='topoext'/>
  </cpu>
  <clock offset='localtime'>
    <timer name='rtc' tickpolicy='catchup'/>
    <timer name='pit' tickpolicy='delay'/>
    <timer name='hpet' present='no'/>
    <timer name='hypervclock' present='yes'/>
    <timer name='tsc' present='yes' mode='native'/>
  </clock>
  <on_poweroff>destroy</on_poweroff>
  <on_reboot>restart</on_reboot>
  <on_crash>destroy</on_crash>
  <pm>
    <suspend-to-mem enabled='no'/>
    <suspend-to-disk enabled='no'/>
  </pm>
  <devices>
    <emulator>/usr/bin/qemu-system-x86_64</emulator>
    <controller type='usb' index='0' model='qemu-xhci' ports='15'>
      <address type='pci' domain='0x0000' bus='0x02' slot='0x00' function='0x0'/>
    </controller>
    <controller type='sata' index='0'>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x1f' function='0x2'/>
    </controller>
    <controller type='pci' index='0' model='pcie-root'/>
    <controller type='pci' index='1' model='pcie-root-port'>
      <model name='pcie-root-port'/>
      <target chassis='1' port='0x10'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0' multifunction='on'/>
    </controller>
    <controller type='pci' index='2' model='pcie-root-port'>
      <model name='pcie-root-port'/>
      <target chassis='2' port='0x11'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x1'/>
    </controller>
    <controller type='pci' index='3' model='pcie-root-port'>
      <model name='pcie-root-port'/>
      <target chassis='3' port='0x12'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x2'/>
    </controller>
    <controller type='pci' index='4' model='pcie-root-port'>
      <model name='pcie-root-port'/>
      <target chassis='4' port='0x13'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x3'/>
    </controller>
    <controller type='pci' index='5' model='pcie-root-port'>
      <model name='pcie-root-port'/>
      <target chassis='5' port='0x14'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x4'/>
    </controller>
    <controller type='pci' index='6' model='pcie-root-port'>
      <model name='pcie-root-port'/>
      <target chassis='6' port='0x15'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x5'/>
    </controller>
    <controller type='pci' index='7' model='pcie-root-port'>
      <model name='pcie-root-port'/>
      <target chassis='7' port='0x8'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x0'/>
    </controller>
    <controller type='virtio-serial' index='0'>
      <address type='pci' domain='0x0000' bus='0x03' slot='0x00' function='0x0'/>
    </controller>
    <interface type='network'>
      <mac address='52:54:00:95:9d:fe'/>
      <source network='default'/>
      <model type='e1000e'/>
      <address type='pci' domain='0x0000' bus='0x01' slot='0x00' function='0x0'/>
    </interface>
    <serial type='pty'>
      <target type='isa-serial' port='0'>
        <model name='isa-serial'/>
      </target>
    </serial>
    <console type='pty'>
      <target type='serial' port='0'/>
    </console>
    <input type='mouse' bus='ps2'/>
    <input type='keyboard' bus='ps2'/>
    <sound model='ich9'>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x1b' function='0x0'/>
    </sound>
    <audio id='1' type='spice'/>
    <hostdev mode='subsystem' type='pci' managed='yes'>
      <source>
        <address domain='0x0000' bus='0x01' slot='0x00' function='0x0'/>
      </source>
      <boot order='1'/>
      <address type='pci' domain='0x0000' bus='0x04' slot='0x00' function='0x0'/>
    </hostdev>
    <hostdev mode='subsystem' type='pci' managed='yes'>
      <source>
        <address domain='0x0000' bus='0x0d' slot='0x00' function='0x0'/>
      </source>
      <address type='pci' domain='0x0000' bus='0x05' slot='0x00' function='0x0'/>
    </hostdev>
    <hostdev mode='subsystem' type='pci' managed='yes'>
      <source>
        <address domain='0x0000' bus='0x0d' slot='0x00' function='0x1'/>
      </source>
      <address type='pci' domain='0x0000' bus='0x06' slot='0x00' function='0x0'/>
    </hostdev>
    <hostdev mode='subsystem' type='pci' managed='yes'>
      <source>
        <address domain='0x0000' bus='0x0e' slot='0x00' function='0x3'/>
      </source>
      <address type='pci' domain='0x0000' bus='0x07' slot='0x00' function='0x0'/>
    </hostdev>
    <redirdev bus='usb' type='spicevmc'>
      <address type='usb' bus='0' port='2'/>
    </redirdev>
    <redirdev bus='usb' type='spicevmc'>
      <address type='usb' bus='0' port='3'/>
    </redirdev>
    <memballoon model='none'/>
  </devices>
</domain>
journalctl -k -b

https://clbin.com/nPpjt

systemctl status libvirtd 
● libvirtd.service - Virtualization daemon
     Loaded: loaded (/usr/lib/systemd/system/libvirtd.service; enabled; vendor preset: disabled)
     Active: active (running) since Sun 2022-02-13 05:35:14 EST; 15h ago
TriggeredBy: ● libvirtd-ro.socket
             ● libvirtd-admin.socket
             ● libvirtd.socket
       Docs: man:libvirtd(8)
             https://libvirt.org
   Main PID: 823 (libvirtd)
      Tasks: 22 (limit: 32768)
     Memory: 39.8M
        CPU: 16.411s
     CGroup: /system.slice/libvirtd.service
             β”œβ”€ 823 /usr/bin/libvirtd --timeout 120
             β”œβ”€1147 /usr/bin/dnsmasq --conf-file=/var/lib/libvirt/dnsmasq/default.conf --leasefile-ro --dhcp-script=/usr/lib/l>
             └─1148 /usr/bin/dnsmasq --conf-file=/var/lib/libvirt/dnsmasq/default.conf --leasefile-ro --dhcp-script=/usr/lib/l>

Feb 13 11:01:11 Kiseki dnsmasq-dhcp[1147]: DHCPACK(virbr0) 192.168.122.28 52:54:00:95:9d:fe DESKTOP-3RP8NM3
Feb 13 20:26:36 Kiseki libvirtd[823]: End of file while reading data: Input/output error
Feb 13 20:27:12 Kiseki dnsmasq-dhcp[1147]: DHCPDISCOVER(virbr0) 192.168.122.28 52:54:00:95:9d:fe
Feb 13 20:27:12 Kiseki dnsmasq-dhcp[1147]: DHCPOFFER(virbr0) 192.168.122.28 52:54:00:95:9d:fe
Feb 13 20:27:12 Kiseki dnsmasq-dhcp[1147]: DHCPREQUEST(virbr0) 192.168.122.28 52:54:00:95:9d:fe
Feb 13 20:27:12 Kiseki dnsmasq-dhcp[1147]: DHCPACK(virbr0) 192.168.122.28 52:54:00:95:9d:fe DESKTOP-3RP8NM3
Feb 13 20:28:20 Kiseki dnsmasq-dhcp[1147]: DHCPDISCOVER(virbr0) 192.168.122.28 52:54:00:95:9d:fe
Feb 13 20:28:20 Kiseki dnsmasq-dhcp[1147]: DHCPOFFER(virbr0) 192.168.122.28 52:54:00:95:9d:fe
Feb 13 20:28:20 Kiseki dnsmasq-dhcp[1147]: DHCPREQUEST(virbr0) 192.168.122.28 52:54:00:95:9d:fe
Feb 13 20:28:20 Kiseki dnsmasq-dhcp[1147]: DHCPACK(virbr0) 192.168.122.28 52:54:00:95:9d:fe DESKTOP-3RP8NM3
journalctl -b -u libvirtd 
Feb 13 05:35:14 Kiseki systemd[1]: Starting Virtualization daemon...
Feb 13 05:35:14 Kiseki systemd[1]: Started Virtualization daemon.
Feb 13 05:35:14 Kiseki dnsmasq[1147]: started, version 2.86 cachesize 150
Feb 13 05:35:14 Kiseki dnsmasq[1147]: compile time options: IPv6 GNU-getopt DBus no-UBus i18n IDN2 DHCP DHCPv6 no-Lua TFTP con>
Feb 13 05:35:14 Kiseki dnsmasq-dhcp[1147]: DHCP, IP range 192.168.122.2 -- 192.168.122.254, lease time 1h
Feb 13 05:35:14 Kiseki dnsmasq-dhcp[1147]: DHCP, sockets bound exclusively to interface virbr0
Feb 13 05:35:14 Kiseki dnsmasq[1147]: reading /etc/resolv.conf
Feb 13 05:35:14 Kiseki dnsmasq[1147]: using nameserver 1.1.1.3#53
Feb 13 05:35:16 Kiseki dnsmasq[1147]: read /etc/hosts - 202182 addresses
Feb 13 05:35:16 Kiseki dnsmasq[1147]: read /var/lib/libvirt/dnsmasq/default.addnhosts - 0 addresses
Feb 13 05:35:16 Kiseki dnsmasq-dhcp[1147]: read /var/lib/libvirt/dnsmasq/default.hostsfile
Feb 13 10:37:55 Kiseki libvirtd[823]: libvirt version: 8.0.0
Feb 13 10:37:55 Kiseki libvirtd[823]: hostname: Kiseki
Feb 13 10:37:55 Kiseki libvirtd[823]: End of file while reading data: Input/output error
Feb 13 10:38:24 Kiseki dnsmasq-dhcp[1147]: DHCPDISCOVER(virbr0) 192.168.122.28 52:54:00:95:9d:fe
Feb 13 10:38:24 Kiseki dnsmasq-dhcp[1147]: DHCPOFFER(virbr0) 192.168.122.28 52:54:00:95:9d:fe
Feb 13 10:38:24 Kiseki dnsmasq-dhcp[1147]: DHCPREQUEST(virbr0) 192.168.122.28 52:54:00:95:9d:fe
Feb 13 10:38:24 Kiseki dnsmasq-dhcp[1147]: DHCPACK(virbr0) 192.168.122.28 52:54:00:95:9d:fe DESKTOP-3RP8NM3
Feb 13 10:49:42 Kiseki dnsmasq-dhcp[1147]: DHCPREQUEST(virbr0) 192.168.122.28 52:54:00:95:9d:fe
Feb 13 10:49:42 Kiseki dnsmasq-dhcp[1147]: DHCPACK(virbr0) 192.168.122.28 52:54:00:95:9d:fe DESKTOP-3RP8NM3
Feb 13 11:01:11 Kiseki dnsmasq-dhcp[1147]: DHCPREQUEST(virbr0) 192.168.122.28 52:54:00:95:9d:fe
Feb 13 11:01:11 Kiseki dnsmasq-dhcp[1147]: DHCPACK(virbr0) 192.168.122.28 52:54:00:95:9d:fe DESKTOP-3RP8NM3
Feb 13 20:26:36 Kiseki libvirtd[823]: End of file while reading data: Input/output error
Feb 13 20:27:12 Kiseki dnsmasq-dhcp[1147]: DHCPDISCOVER(virbr0) 192.168.122.28 52:54:00:95:9d:fe
Feb 13 20:27:12 Kiseki dnsmasq-dhcp[1147]: DHCPOFFER(virbr0) 192.168.122.28 52:54:00:95:9d:fe
Feb 13 20:27:12 Kiseki dnsmasq-dhcp[1147]: DHCPREQUEST(virbr0) 192.168.122.28 52:54:00:95:9d:fe
Feb 13 20:27:12 Kiseki dnsmasq-dhcp[1147]: DHCPACK(virbr0) 192.168.122.28 52:54:00:95:9d:fe DESKTOP-3RP8NM3
Feb 13 20:28:20 Kiseki dnsmasq-dhcp[1147]: DHCPDISCOVER(virbr0) 192.168.122.28 52:54:00:95:9d:fe
Feb 13 20:28:20 Kiseki dnsmasq-dhcp[1147]: DHCPOFFER(virbr0) 192.168.122.28 52:54:00:95:9d:fe
Feb 13 20:28:20 Kiseki dnsmasq-dhcp[1147]: DHCPREQUEST(virbr0) 192.168.122.28 52:54:00:95:9d:fe
Feb 13 20:28:20 Kiseki dnsmasq-dhcp[1147]: DHCPACK(virbr0) 192.168.122.28 52:54:00:95:9d:fe DESKTOP-3RP8NM3
sudo cat /var/log/libvirt/qemu/win11.log 
2022-02-13 15:37:59.830+0000: starting up libvirt version: 8.0.0, qemu version: 6.2.0, kernel: 5.16.8-246-tkg-pds, hostname: Kiseki
LC_ALL=C \
PATH=/usr/local/sbin:/usr/local/bin:/usr/bin \
HOME=/var/lib/libvirt/qemu/domain-1-win11 \
XDG_DATA_HOME=/var/lib/libvirt/qemu/domain-1-win11/.local/share \
XDG_CACHE_HOME=/var/lib/libvirt/qemu/domain-1-win11/.cache \
XDG_CONFIG_HOME=/var/lib/libvirt/qemu/domain-1-win11/.config \
/usr/bin/qemu-system-x86_64 \
-name guest=win11,debug-threads=on \
-S \
-object '{"qom-type":"secret","id":"masterKey0","format":"raw","file":"/var/lib/libvirt/qemu/domain-1-win11/master-key.aes"}' \
-blockdev '{"driver":"file","filename":"/usr/share/edk2-ovmf/x64/OVMF_CODE.fd","node-name":"libvirt-pflash0-storage","auto-read-only":true,"discard":"unmap"}' \
-blockdev '{"node-name":"libvirt-pflash0-format","read-only":true,"driver":"raw","file":"libvirt-pflash0-storage"}' \
-blockdev '{"driver":"file","filename":"/var/lib/libvirt/qemu/nvram/win11_VARS.fd","node-name":"libvirt-pflash1-storage","auto-read-only":true,"discard":"unmap"}' \
-blockdev '{"node-name":"libvirt-pflash1-format","read-only":false,"driver":"raw","file":"libvirt-pflash1-storage"}' \
-machine pc-q35-6.2,usb=off,vmport=off,dump-guest-core=off,kernel_irqchip=on,pflash0=libvirt-pflash0-format,pflash1=libvirt-pflash1-format,memory-backend=pc.ram \
-accel kvm \
-cpu host,migratable=on,topoext=on,hv-time=on,hv-relaxed=on,hv-vapic=on,hv-spinlocks=0x1fff,hv-vpindex=on,hv-runtime=on,hv-synic=on,hv-stimer=on,hv-frequencies=on,hv-tlbflush=on,hv-ipi=on,host-cache-info=on,l3-cache=off \
-m 8192 \
-object '{"qom-type":"memory-backend-ram","id":"pc.ram","size":8589934592}' \
-overcommit mem-lock=off \
-smp 8,sockets=1,dies=1,cores=4,threads=2 \
-object '{"qom-type":"iothread","id":"iothread1"}' \
-uuid 6e4baa04-efd3-4c57-a352-650f84d3c4cf \
-display none \
-no-user-config \
-nodefaults \
-chardev socket,id=charmonitor,fd=30,server=on,wait=off \
-mon chardev=charmonitor,id=monitor,mode=control \
-rtc base=localtime,driftfix=slew \
-global kvm-pit.lost_tick_policy=delay \
-no-hpet \
-no-shutdown \
-global ICH9-LPC.disable_s3=1 \
-global ICH9-LPC.disable_s4=1 \
-boot menu=off,strict=on \
-device pcie-root-port,port=16,chassis=1,id=pci.1,bus=pcie.0,multifunction=on,addr=0x2 \
-device pcie-root-port,port=17,chassis=2,id=pci.2,bus=pcie.0,addr=0x2.0x1 \
-device pcie-root-port,port=18,chassis=3,id=pci.3,bus=pcie.0,addr=0x2.0x2 \
-device pcie-root-port,port=19,chassis=4,id=pci.4,bus=pcie.0,addr=0x2.0x3 \
-device pcie-root-port,port=20,chassis=5,id=pci.5,bus=pcie.0,addr=0x2.0x4 \
-device pcie-root-port,port=21,chassis=6,id=pci.6,bus=pcie.0,addr=0x2.0x5 \
-device pcie-root-port,port=8,chassis=7,id=pci.7,bus=pcie.0,addr=0x1 \
-device qemu-xhci,p2=15,p3=15,id=usb,bus=pci.2,addr=0x0 \
-device virtio-serial-pci,id=virtio-serial0,bus=pci.3,addr=0x0 \
-netdev tap,fd=32,id=hostnet0 \
-device e1000e,netdev=hostnet0,id=net0,mac=52:54:00:95:9d:fe,bus=pci.1,addr=0x0 \
-chardev pty,id=charserial0 \
-device isa-serial,chardev=charserial0,id=serial0 \
-audiodev '{"id":"audio1","driver":"spice"}' \
-device ich9-intel-hda,id=sound0,bus=pcie.0,addr=0x1b \
-device hda-duplex,id=sound0-codec0,bus=sound0.0,cad=0,audiodev=audio1 \
-chardev spicevmc,id=charredir0,name=usbredir \
-device usb-redir,chardev=charredir0,id=redir0,bus=usb.0,port=2 \
-chardev spicevmc,id=charredir1,name=usbredir \
-device usb-redir,chardev=charredir1,id=redir1,bus=usb.0,port=3 \
-device vfio-pci,host=0000:01:00.0,id=hostdev0,bootindex=1,bus=pci.4,addr=0x0 \
-device vfio-pci,host=0000:0d:00.0,id=hostdev1,bus=pci.5,addr=0x0 \
-device vfio-pci,host=0000:0d:00.1,id=hostdev2,bus=pci.6,addr=0x0 \
-device vfio-pci,host=0000:0e:00.3,id=hostdev3,bus=pci.7,addr=0x0 \
-sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny \
-msg timestamp=on
char device redirected to /dev/pts/0 (label charserial0)
audio: Could not init `spice' audio driver
audio: warning: Using timer based audio emulation
2022-02-13T16:07:40.540702Z qemu-system-x86_64: terminating on signal 15 from pid 823 (/usr/bin/libvirtd)
2022-02-13 16:07:42.170+0000: shutting down, reason=shutdown
2022-02-14 01:26:40.915+0000: starting up libvirt version: 8.0.0, qemu version: 6.2.0, kernel: 5.16.8-246-tkg-pds, hostname: Kiseki
LC_ALL=C \
PATH=/usr/local/sbin:/usr/local/bin:/usr/bin \
HOME=/var/lib/libvirt/qemu/domain-2-win11 \
XDG_DATA_HOME=/var/lib/libvirt/qemu/domain-2-win11/.local/share \
XDG_CACHE_HOME=/var/lib/libvirt/qemu/domain-2-win11/.cache \
XDG_CONFIG_HOME=/var/lib/libvirt/qemu/domain-2-win11/.config \
/usr/bin/qemu-system-x86_64 \
-name guest=win11,debug-threads=on \
-S \
-object '{"qom-type":"secret","id":"masterKey0","format":"raw","file":"/var/lib/libvirt/qemu/domain-2-win11/master-key.aes"}' \
-blockdev '{"driver":"file","filename":"/usr/share/edk2-ovmf/x64/OVMF_CODE.fd","node-name":"libvirt-pflash0-storage","auto-read-only":true,"discard":"unmap"}' \
-blockdev '{"node-name":"libvirt-pflash0-format","read-only":true,"driver":"raw","file":"libvirt-pflash0-storage"}' \
-blockdev '{"driver":"file","filename":"/var/lib/libvirt/qemu/nvram/win11_VARS.fd","node-name":"libvirt-pflash1-storage","auto-read-only":true,"discard":"unmap"}' \
-blockdev '{"node-name":"libvirt-pflash1-format","read-only":false,"driver":"raw","file":"libvirt-pflash1-storage"}' \
-machine pc-q35-6.2,usb=off,vmport=off,dump-guest-core=off,kernel_irqchip=on,pflash0=libvirt-pflash0-format,pflash1=libvirt-pflash1-format,memory-backend=pc.ram \
-accel kvm \
-cpu host,migratable=on,topoext=on,hv-time=on,hv-relaxed=on,hv-vapic=on,hv-spinlocks=0x1fff,hv-vpindex=on,hv-runtime=on,hv-synic=on,hv-stimer=on,hv-frequencies=on,hv-tlbflush=on,hv-ipi=on,host-cache-info=on,l3-cache=off \
-m 8192 \
-object '{"qom-type":"memory-backend-ram","id":"pc.ram","size":8589934592}' \
-overcommit mem-lock=off \
-smp 8,sockets=1,dies=1,cores=4,threads=2 \
-object '{"qom-type":"iothread","id":"iothread1"}' \
-uuid 6e4baa04-efd3-4c57-a352-650f84d3c4cf \
-display none \
-no-user-config \
-nodefaults \
-chardev socket,id=charmonitor,fd=30,server=on,wait=off \
-mon chardev=charmonitor,id=monitor,mode=control \
-rtc base=localtime,driftfix=slew \
-global kvm-pit.lost_tick_policy=delay \
-no-hpet \
-no-shutdown \
-global ICH9-LPC.disable_s3=1 \
-global ICH9-LPC.disable_s4=1 \
-boot menu=off,strict=on \
-device pcie-root-port,port=16,chassis=1,id=pci.1,bus=pcie.0,multifunction=on,addr=0x2 \
-device pcie-root-port,port=17,chassis=2,id=pci.2,bus=pcie.0,addr=0x2.0x1 \
-device pcie-root-port,port=18,chassis=3,id=pci.3,bus=pcie.0,addr=0x2.0x2 \
-device pcie-root-port,port=19,chassis=4,id=pci.4,bus=pcie.0,addr=0x2.0x3 \
-device pcie-root-port,port=20,chassis=5,id=pci.5,bus=pcie.0,addr=0x2.0x4 \
-device pcie-root-port,port=21,chassis=6,id=pci.6,bus=pcie.0,addr=0x2.0x5 \
-device pcie-root-port,port=8,chassis=7,id=pci.7,bus=pcie.0,addr=0x1 \
-device qemu-xhci,p2=15,p3=15,id=usb,bus=pci.2,addr=0x0 \
-device virtio-serial-pci,id=virtio-serial0,bus=pci.3,addr=0x0 \
-netdev tap,fd=32,id=hostnet0 \
-device e1000e,netdev=hostnet0,id=net0,mac=52:54:00:95:9d:fe,bus=pci.1,addr=0x0 \
-chardev pty,id=charserial0 \
-device isa-serial,chardev=charserial0,id=serial0 \
-audiodev '{"id":"audio1","driver":"spice"}' \
-device ich9-intel-hda,id=sound0,bus=pcie.0,addr=0x1b \
-device hda-duplex,id=sound0-codec0,bus=sound0.0,cad=0,audiodev=audio1 \
-chardev spicevmc,id=charredir0,name=usbredir \
-device usb-redir,chardev=charredir0,id=redir0,bus=usb.0,port=2 \
-chardev spicevmc,id=charredir1,name=usbredir \
-device usb-redir,chardev=charredir1,id=redir1,bus=usb.0,port=3 \
-device vfio-pci,host=0000:01:00.0,id=hostdev0,bootindex=1,bus=pci.4,addr=0x0 \
-device vfio-pci,host=0000:0d:00.0,id=hostdev1,bus=pci.5,addr=0x0 \
-device vfio-pci,host=0000:0d:00.1,id=hostdev2,bus=pci.6,addr=0x0 \
-device vfio-pci,host=0000:0e:00.3,id=hostdev3,bus=pci.7,addr=0x0 \
-sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny \
-msg timestamp=on
char device redirected to /dev/pts/0 (label charserial0)
audio: Could not init `spice' audio driver
audio: warning: Using timer based audio emulation
2022-02-14T01:28:44.538950Z qemu-system-x86_64: terminating on signal 15 from pid 823 (/usr/bin/libvirtd)
2022-02-14 01:28:46.376+0000: shutting down, reason=shutdown

this is a single GPU passtrough for now ( until I get back my 1060)
so these files may also be relevant

 ╭─zany130@Kiseki in /etc/libvirt/hooksπŸ”’ took 8ms
 ╰─λ tree
.
β”œβ”€β”€ kvm.conf
β”œβ”€β”€ qemu
└── qemu.d
    └── win11
        β”œβ”€β”€ prepare
        β”‚   └── begin
        β”‚       └── start.sh
        └── release
            └── end
                └── revert.sh

6 directories, 4 files
kvm.conf 
File: kvm.conf
#GPU to passthrough
VIRSH_GPU_VIDEO=pci_0000_0d_00_0
VIRSH_GPU_AUDIO=pci_0000_0d_00_1
start.sh 
File: qemu.d/win11/prepare/begin/start.sh
#!/bin/bash
# Helpful to read output when debugging
set -x

# Load variables we defined
source "/etc/libvirt/hooks/kvm.conf"

# Stop display manager and all other applications using the GPU
systemctl stop sddm.service
killall  kioslave5
killall  latte-dock

# Unbind VTconsoles
echo 0 > /sys/class/vtconsole/vtcon0/bind
echo 0 > /sys/class/vtconsole/vtcon1/bind

# Unbind EFI-Framebuffer
echo efi-framebuffer.0 > /sys/bus/platform/drivers/efi-framebuffer/unbind

# Avoid a Race condition by waiting 2 seconds. This can be calibrated to be shorter or longer if required for your system
sleep 3

# Unload Nvidia Driver
modprobe -r nvidia_drm
modprobe -r nvidia_modeset
modprobe -r nvida
modprobe -r nvidia_uvm

# Unbind GPU
virsh nodedev-detach $VIRSH_GPU_VIDEO
virsh nodedev-detach $VIRSH_GPU_AUDIO


# Load VFIO Kernel Module
modprobe vfio
modprobe vfio-pci
modprobe vfio_iommu_type1
revert.sh 
File: qemu.d/win11/release/end/revert.sh
#!/bin/bash
set -x

#Load variables
source "/etc/libvirt/hooks/kvm.conf"

# unload vfio-pci
modprobe -r vfio_pci
modprobe -r vfio_iommu_type1
modprobe -r vfio

# Re-Bind GPU to Nvidia Driver
virsh nodedev-reattach $VIRSH_GPU_VIDEO
virsh nodedev-reattach $VIRSH_GPU_AUDIO

# Reload nvidia modules
modprobe nvidia
modprobe nvidia_modeset
modprobe nvidia_uvm
modprobe nvidia_drm

# Rebind VT consoles
echo 1 > /sys/class/vtconsole/vtcon0/bind
echo 1 > /sys/class/vtconsole/vtcon1/bind
# Some machines might have more than 1 virtual console. Add a line for each corresponding VTConsole
#echo 1 > /sys/class/vtconsole/vtcon1/bind

# Read nvidia x config
nvidia-xconfig --query-gpu-info > /dev/null 2>&1

#Bind EFI-framebuffer
echo "efi-framebuffer.0" > /sys/bus/platform/drivers/efi-framebuffer/bind

# Restart Display Manager
systemctl start sddm.service
2 Likes
3

So while troubleshooting I found out that some nvidia modprobe procses spawn while the vm is running

image
image1920Γ—1080 234 KB

so i modifed my start script in my hooks to add a killall nvidia and killall nvidia-modeset which seems to fix this

New start.sh 
source "/etc/libvirt/hooks/kvm.conf"

# Stop display manager and all other applications using the GPU and some other apps that use up resorcses
systemctl stop sddm.service
systemctl stop ananicy-cpp.service
systemctl stop preload.service
systemctl stop prelockd.service
systemctl stop memavaild.service
systemctl stop uresourced.service
systemctl stop irqbalance.service

killall nvidia
killall nvidia-modeset
killall kioslave5
killall latte-dock
killall ulauncher
killall urserver
killall Discord
killall python
killall steam
killall vivaldi-bin
killall wine

# Unbind VTconsoles
echo 0 > /sys/class/vtconsole/vtcon0/bind
echo 0 > /sys/class/vtconsole/vtcon1/bind

# Unbind EFI-Framebuffer
echo efi-framebuffer.0 > /sys/bus/platform/drivers/efi-framebuffer/unbind

# Avoid a Race condition by waiting 2 seconds. This can be calibrated to be shorter or longer if required for your system
sleep 3

# Unload Nvidia Driver
modprobe -r nvidia_drm
modprobe -r nvidia_modeset
modprobe -r nvidia_uvm
modprobe -r nvidia

# Unbind GPU
virsh nodedev-detach $VIRSH_GPU_VIDEO
virsh nodedev-detach $VIRSH_GPU_AUDIO


# Load VFIO Kernel Module
modprobe vfio
modprobe vfio-pci
modprobe vfio_iommu_type1

I went ahead and also add a few more kill commands to stop some other process that where still running and using resources such as discord.
This didn’t fix my crashes however and then I realized I forgot to pass a β€œpatched” vbios in my vm xml.

win11.xml 
<domain type='kvm'>
  <name>win11</name>
  <uuid>6e4baa04-efd3-4c57-a352-650f84d3c4cf</uuid>
  <metadata>
    <libosinfo:libosinfo xmlns:libosinfo="http://libosinfo.org/xmlns/libvirt/domain/1.0">
      <libosinfo:os id="http://microsoft.com/win/10"/>
    </libosinfo:libosinfo>
  </metadata>
  <memory unit='KiB'>8388608</memory>
  <currentMemory unit='KiB'>8388608</currentMemory>
  <vcpu placement='static'>8</vcpu>
  <iothreads>1</iothreads>
  <cputune>
    <vcpupin vcpu='0' cpuset='2'/>
    <vcpupin vcpu='1' cpuset='8'/>
    <vcpupin vcpu='2' cpuset='3'/>
    <vcpupin vcpu='3' cpuset='9'/>
    <vcpupin vcpu='4' cpuset='4'/>
    <vcpupin vcpu='5' cpuset='10'/>
    <vcpupin vcpu='6' cpuset='5'/>
    <vcpupin vcpu='7' cpuset='11'/>
    <emulatorpin cpuset='0-6'/>
    <iothreadpin iothread='1' cpuset='1-7'/>
  </cputune>
  <os>
    <type arch='x86_64' machine='pc-q35-6.2'>hvm</type>
    <loader readonly='yes' type='pflash'>/usr/share/edk2-ovmf/x64/OVMF_CODE.fd</loader>
    <nvram>/var/lib/libvirt/qemu/nvram/win11_VARS.fd</nvram>
    <bootmenu enable='no'/>
  </os>
  <features>
    <acpi/>
    <apic/>
    <hyperv mode='custom'>
      <relaxed state='on'/>
      <vapic state='on'/>
      <spinlocks state='on' retries='8191'/>
      <vpindex state='on'/>
      <runtime state='on'/>
      <synic state='on'/>
      <stimer state='on'/>
      <reset state='off'/>
      <frequencies state='on'/>
      <reenlightenment state='off'/>
      <tlbflush state='on'/>
      <ipi state='on'/>
      <evmcs state='off'/>
    </hyperv>
    <vmport state='off'/>
    <ioapic driver='kvm'/>
  </features>
  <cpu mode='host-passthrough' check='none' migratable='on'>
    <topology sockets='1' dies='1' cores='4' threads='2'/>
    <cache mode='passthrough'/>
    <feature policy='require' name='topoext'/>
  </cpu>
  <clock offset='localtime'>
    <timer name='rtc' tickpolicy='catchup'/>
    <timer name='pit' tickpolicy='delay'/>
    <timer name='hpet' present='no'/>
    <timer name='hypervclock' present='yes'/>
    <timer name='tsc' present='yes' mode='native'/>
  </clock>
  <on_poweroff>destroy</on_poweroff>
  <on_reboot>restart</on_reboot>
  <on_crash>destroy</on_crash>
  <pm>
    <suspend-to-mem enabled='no'/>
    <suspend-to-disk enabled='no'/>
  </pm>
  <devices>
    <emulator>/usr/bin/qemu-system-x86_64</emulator>
    <controller type='usb' index='0' model='qemu-xhci' ports='15'>
      <address type='pci' domain='0x0000' bus='0x02' slot='0x00' function='0x0'/>
    </controller>
    <controller type='sata' index='0'>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x1f' function='0x2'/>
    </controller>
    <controller type='pci' index='0' model='pcie-root'/>
    <controller type='pci' index='1' model='pcie-root-port'>
      <model name='pcie-root-port'/>
      <target chassis='1' port='0x10'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0' multifunction='on'/>
    </controller>
    <controller type='pci' index='2' model='pcie-root-port'>
      <model name='pcie-root-port'/>
      <target chassis='2' port='0x11'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x1'/>
    </controller>
    <controller type='pci' index='3' model='pcie-root-port'>
      <model name='pcie-root-port'/>
      <target chassis='3' port='0x12'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x2'/>
    </controller>
    <controller type='pci' index='4' model='pcie-root-port'>
      <model name='pcie-root-port'/>
      <target chassis='4' port='0x13'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x3'/>
    </controller>
    <controller type='pci' index='5' model='pcie-root-port'>
      <model name='pcie-root-port'/>
      <target chassis='5' port='0x14'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x4'/>
    </controller>
    <controller type='pci' index='6' model='pcie-root-port'>
      <model name='pcie-root-port'/>
      <target chassis='6' port='0x15'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x5'/>
    </controller>
    <controller type='pci' index='7' model='pcie-root-port'>
      <model name='pcie-root-port'/>
      <target chassis='7' port='0x8'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x0'/>
    </controller>
    <controller type='virtio-serial' index='0'>
      <address type='pci' domain='0x0000' bus='0x03' slot='0x00' function='0x0'/>
    </controller>
    <interface type='network'>
      <mac address='52:54:00:95:9d:fe'/>
      <source network='default'/>
      <model type='virtio'/>
      <address type='pci' domain='0x0000' bus='0x01' slot='0x00' function='0x0'/>
    </interface>
    <serial type='pty'>
      <target type='isa-serial' port='0'>
        <model name='isa-serial'/>
      </target>
    </serial>
    <console type='pty'>
      <target type='serial' port='0'/>
    </console>
    <input type='mouse' bus='ps2'/>
    <input type='keyboard' bus='ps2'/>
    <sound model='ich9'>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x1b' function='0x0'/>
    </sound>
    <audio id='1' type='spice'/>
    <hostdev mode='subsystem' type='pci' managed='yes'>
      <source>
        <address domain='0x0000' bus='0x01' slot='0x00' function='0x0'/>
      </source>
      <boot order='1'/>
      <address type='pci' domain='0x0000' bus='0x04' slot='0x00' function='0x0'/>
    </hostdev>
    <hostdev mode='subsystem' type='pci' managed='yes'>
      <source>
        <address domain='0x0000' bus='0x0d' slot='0x00' function='0x0'/>
      </source>
      <rom file='/var/lib/libvirt/patch.rom'/>
      <address type='pci' domain='0x0000' bus='0x05' slot='0x00' function='0x0'/>
    </hostdev>
    <hostdev mode='subsystem' type='pci' managed='yes'>
      <source>
        <address domain='0x0000' bus='0x0d' slot='0x00' function='0x1'/>
      </source>
      <address type='pci' domain='0x0000' bus='0x06' slot='0x00' function='0x0'/>
    </hostdev>
    <hostdev mode='subsystem' type='pci' managed='yes'>
      <source>
        <address domain='0x0000' bus='0x0e' slot='0x00' function='0x3'/>
      </source>
      <address type='pci' domain='0x0000' bus='0x07' slot='0x00' function='0x0'/>
    </hostdev>
    <redirdev bus='usb' type='spicevmc'>
      <address type='usb' bus='0' port='2'/>
    </redirdev>
    <redirdev bus='usb' type='spicevmc'>
      <address type='usb' bus='0' port='3'/>
    </redirdev>
    <memballoon model='none'/>
  </devices>
</domain>

this did fix my vm crashes ,but haven still crashes after a while where as on bare metal it does not. Tried removing the overclock and I still get the crashes. I have also been able to get the crash on other GPU intensive apps so I am at a loss as to what to do next…

1 Like
4

image
image1920Γ—1440 112 KB

got this blue screen now with no core dump checking the event viewer shows this Dump file creation failed due to error during dump creation
so i thought i might have a corrupted install or that i was having issues passing trough my 960 nvme to the vm so I did a new install with a 970 sdd and same result

5

Found out my blue screens were caused my the low-level afterburner driver so fixed that by disabling that option. Also ended up deciding that to stick to a single gpu passtrough(no longer have an extra gpu and its not worth it buying a new one for the host plus the extra ram i would need)

here is my final configs

sudo virsh dumpxml "win11" 
<domain type='kvm'>
  <name>win11</name>
  <uuid>d6070c27-f4b9-45a6-a5fa-edc82f91808a</uuid>
  <metadata>
    <libosinfo:libosinfo xmlns:libosinfo="http://libosinfo.org/xmlns/libvirt/domain/1.0">
      <libosinfo:os id="http://microsoft.com/win/10"/>
    </libosinfo:libosinfo>
  </metadata>
  <memory unit='KiB'>14680064</memory>
  <currentMemory unit='KiB'>14680064</currentMemory>
  <vcpu placement='static'>10</vcpu>
  <iothreads>1</iothreads>
  <cputune>
    <vcpupin vcpu='0' cpuset='1'/>
    <vcpupin vcpu='1' cpuset='7'/>
    <vcpupin vcpu='2' cpuset='2'/>
    <vcpupin vcpu='3' cpuset='8'/>
    <vcpupin vcpu='4' cpuset='3'/>
    <vcpupin vcpu='5' cpuset='9'/>
    <vcpupin vcpu='6' cpuset='4'/>
    <vcpupin vcpu='7' cpuset='10'/>
    <vcpupin vcpu='8' cpuset='5'/>
    <vcpupin vcpu='9' cpuset='11'/>
    <emulatorpin cpuset='0,6'/>
    <iothreadpin iothread='1' cpuset='0,6'/>
  </cputune>
  <os>
    <type arch='x86_64' machine='pc-q35-6.2'>hvm</type>
    <loader readonly='yes' secure='yes' type='pflash'>/usr/share/edk2-ovmf/x64/OVMF_CODE.secboot.fd</loader>
    <nvram>/var/lib/libvirt/qemu/nvram/win11_VARS.fd</nvram>
  </os>
  <features>
    <acpi/>
    <apic/>
    <hyperv mode='custom'>
      <relaxed state='on'/>
      <vapic state='on'/>
      <spinlocks state='on' retries='8191'/>
      <vpindex state='on'/>
      <runtime state='on'/>
      <synic state='on'/>
      <stimer state='on'/>
      <reset state='off'/>
      <frequencies state='on'/>
      <reenlightenment state='off'/>
      <tlbflush state='on'/>
      <ipi state='on'/>
      <evmcs state='off'/>
    </hyperv>
    <vmport state='off'/>
    <smm state='on'/>
  </features>
  <cpu mode='host-passthrough' check='none' migratable='on'>
    <topology sockets='1' dies='1' cores='5' threads='2'/>
    <cache mode='passthrough'/>
    <feature policy='require' name='topoext'/>
  </cpu>
  <clock offset='localtime'>
    <timer name='rtc' tickpolicy='catchup'/>
    <timer name='pit' tickpolicy='delay'/>
    <timer name='hpet' present='no'/>
    <timer name='hypervclock' present='yes'/>
    <timer name='tsc' present='yes' mode='native'/>
  </clock>
  <on_poweroff>destroy</on_poweroff>
  <on_reboot>restart</on_reboot>
  <on_crash>destroy</on_crash>
  <pm>
    <suspend-to-mem enabled='no'/>
    <suspend-to-disk enabled='no'/>
  </pm>
  <devices>
    <emulator>/usr/bin/qemu-system-x86_64</emulator>
    <disk type='block' device='disk'>
      <driver name='qemu' type='raw' cache='none' io='native'/>
      <source dev='/dev/disk/by-id/ata-ST2000DX002-2DV164_Z4ZC832K'/>
      <target dev='vda' bus='virtio'/>
      <address type='pci' domain='0x0000' bus='0x09' slot='0x00' function='0x0'/>
    </disk>
    <controller type='usb' index='0' model='qemu-xhci' ports='15'>
      <address type='pci' domain='0x0000' bus='0x01' slot='0x00' function='0x0'/>
    </controller>
    <controller type='sata' index='0'>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x1f' function='0x2'/>
    </controller>
    <controller type='pci' index='0' model='pcie-root'/>
    <controller type='pci' index='1' model='pcie-root-port'>
      <model name='pcie-root-port'/>
      <target chassis='1' port='0x10'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0' multifunction='on'/>
    </controller>
    <controller type='pci' index='2' model='pcie-root-port'>
      <model name='pcie-root-port'/>
      <target chassis='2' port='0x11'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x1'/>
    </controller>
    <controller type='pci' index='3' model='pcie-root-port'>
      <model name='pcie-root-port'/>
      <target chassis='3' port='0x12'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x2'/>
    </controller>
    <controller type='pci' index='4' model='pcie-root-port'>
      <model name='pcie-root-port'/>
      <target chassis='4' port='0x13'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x3'/>
    </controller>
    <controller type='pci' index='5' model='pcie-root-port'>
      <model name='pcie-root-port'/>
      <target chassis='5' port='0x8'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x0' multifunction='on'/>
    </controller>
    <controller type='pci' index='6' model='pcie-root-port'>
      <model name='pcie-root-port'/>
      <target chassis='6' port='0x9'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x1'/>
    </controller>
    <controller type='pci' index='7' model='pcie-root-port'>
      <model name='pcie-root-port'/>
      <target chassis='7' port='0xa'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x2'/>
    </controller>
    <controller type='pci' index='8' model='pcie-root-port'>
      <model name='pcie-root-port'/>
      <target chassis='8' port='0xb'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x3'/>
    </controller>
    <controller type='pci' index='9' model='pcie-root-port'>
      <model name='pcie-root-port'/>
      <target chassis='9' port='0xc'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x4'/>
    </controller>
    <controller type='pci' index='10' model='pcie-to-pci-bridge'>
      <model name='pcie-pci-bridge'/>
      <address type='pci' domain='0x0000' bus='0x03' slot='0x00' function='0x0'/>
    </controller>
    <controller type='pci' index='11' model='pcie-root-port'>
      <model name='pcie-root-port'/>
      <target chassis='11' port='0xd'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x5'/>
    </controller>
    <controller type='virtio-serial' index='0'>
      <address type='pci' domain='0x0000' bus='0x02' slot='0x00' function='0x0'/>
    </controller>
    <interface type='direct'>
      <mac address='52:54:00:92:63:bb'/>
      <source dev='enp9s0' mode='bridge'/>
      <model type='virtio'/>
      <address type='pci' domain='0x0000' bus='0x04' slot='0x00' function='0x0'/>
    </interface>
    <serial type='pty'>
      <target type='isa-serial' port='0'>
        <model name='isa-serial'/>
      </target>
    </serial>
    <console type='pty'>
      <target type='serial' port='0'/>
    </console>
    <channel type='spicevmc'>
      <target type='virtio' name='com.redhat.spice.0'/>
      <address type='virtio-serial' controller='0' bus='0' port='1'/>
    </channel>
    <input type='mouse' bus='ps2'/>
    <input type='keyboard' bus='ps2'/>
    <tpm model='tpm-tis'>
      <backend type='passthrough'>
        <device path='/dev/tpm0'/>
      </backend>
    </tpm>
    <sound model='ich9'>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x1b' function='0x0'/>
    </sound>
    <audio id='1' type='spice'/>
    <hostdev mode='subsystem' type='pci' managed='yes'>
      <source>
        <address domain='0x0000' bus='0x01' slot='0x00' function='0x0'/>
      </source>
      <boot order='1'/>
      <address type='pci' domain='0x0000' bus='0x05' slot='0x00' function='0x0'/>
    </hostdev>
    <hostdev mode='subsystem' type='pci' managed='yes'>
      <source>
        <address domain='0x0000' bus='0x0d' slot='0x00' function='0x0'/>
      </source>
      <address type='pci' domain='0x0000' bus='0x06' slot='0x00' function='0x0'/>
    </hostdev>
    <hostdev mode='subsystem' type='pci' managed='yes'>
      <source>
        <address domain='0x0000' bus='0x0d' slot='0x00' function='0x1'/>
      </source>
      <address type='pci' domain='0x0000' bus='0x07' slot='0x00' function='0x0'/>
    </hostdev>
    <hostdev mode='subsystem' type='pci' managed='yes'>
      <source>
        <address domain='0x0000' bus='0x0e' slot='0x00' function='0x3'/>
      </source>
      <address type='pci' domain='0x0000' bus='0x08' slot='0x00' function='0x0'/>
    </hostdev>
    <redirdev bus='usb' type='spicevmc'>
      <address type='usb' bus='0' port='2'/>
    </redirdev>
    <redirdev bus='usb' type='spicevmc'>
      <address type='usb' bus='0' port='3'/>
    </redirdev>
    <memballoon model='none'/>
  </devices>
</domain>
lscpu -e 
lscpu -e
CPU NODE SOCKET CORE L1d:L1i:L2:L3 ONLINE    MAXMHZ    MINMHZ      MHZ
  0    0      0    0 0:0:0:0          yes 3600.0000 2200.0000 4099.295
  1    0      0    1 1:1:1:0          yes 3600.0000 2200.0000 3600.000
  2    0      0    2 2:2:2:0          yes 3600.0000 2200.0000 3600.000
  3    0      0    3 4:4:4:1          yes 3600.0000 2200.0000 3600.000
  4    0      0    4 5:5:5:1          yes 3600.0000 2200.0000 4126.429
  5    0      0    5 6:6:6:1          yes 3600.0000 2200.0000 3600.000
  6    0      0    0 0:0:0:0          yes 3600.0000 2200.0000 3600.000
  7    0      0    1 1:1:1:0          yes 3600.0000 2200.0000 3600.000
  8    0      0    2 2:2:2:0          yes 3600.0000 2200.0000 3600.000
  9    0      0    3 4:4:4:1          yes 3600.0000 2200.0000 3600.000
 10    0      0    4 5:5:5:1          yes 3600.0000 2200.0000 3600.000
 11    0      0    5 6:6:6:1          yes 3600.0000 2200.0000 3600.000
cat /etc/libvirt/hooks/kvm.conf 
File: /etc/libvirt/hooks/kvm.conf
#GPU to passthrough
VIRSH_GPU_VIDEO=pci_0000_0d_00_0
VIRSH_GPU_AUDIO=pci_0000_0d_00_1
cat /etc/libvirt/hooks/qemu.d/win11/prepare/begin/start.sh 
source "/etc/libvirt/hooks/kvm.conf"

# Stop display manager and all other applications using the GPU and some other apps that use up resorcses
systemctl stop sddm.service
systemctl stop ananicy-cpp.service
systemctl stop preload.service
systemctl stop prelockd.service
systemctl stop memavaild.service
systemctl stop uresourced.service
systemctl stop irqbalance.service

killall nvidia
killall nvidia-modeset
killall kioslave5
killall -s KILL latte-dock
killall ulauncher
killall urserver
killall Discord
killall python
killall steam
killall vivaldi-bin
killall wine

# Unbind VTconsoles
echo 0 > /sys/class/vtconsole/vtcon0/bind
echo 0 > /sys/class/vtconsole/vtcon1/bind

# Unbind EFI-Framebuffer
echo efi-framebuffer.0 > /sys/bus/platform/drivers/efi-framebuffer/unbind

# Avoid a Race condition by waiting 2 seconds. This can be calibrated to be shorter or longer if required for your system
sleep 3

# Unload Nvidia Driver
modprobe -r nvidia_drm
modprobe -r nvidia_modeset
modprobe -r nvidia_uvm
modprobe -r nvidia

# Unbind GPU
virsh nodedev-detach $VIRSH_GPU_VIDEO
virsh nodedev-detach $VIRSH_GPU_AUDIO


# Load VFIO Kernel Module
modprobe vfio
modprobe vfio-pci
modprobe vfio_iommu_type1
cat /etc/libvirt/hooks/qemu.d/win11/release/end/revert.sh 
#!/bin/bash
set -x

#Load variables
source "/etc/libvirt/hooks/kvm.conf"

# unload vfio-pci
modprobe -r vfio_pci
modprobe -r vfio_iommu_type1
modprobe -r vfio

# Re-Bind GPU to Nvidia Driver
virsh nodedev-reattach $VIRSH_GPU_VIDEO
virsh nodedev-reattach $VIRSH_GPU_AUDIO

# Reload nvidia modules
modprobe nvidia
modprobe nvidia_modeset
modprobe nvidia_uvm
modprobe nvidia_drm

# Rebind VT consoles
echo 1 > /sys/class/vtconsole/vtcon0/bind
echo 1 > /sys/class/vtconsole/vtcon1/bind
# Some machines might have more than 1 virtual console. Add a line for each corresponding VTConsole
#echo 1 > /sys/class/vtconsole/vtcon1/bind

# Read nvidia x config
nvidia-xconfig --query-gpu-info > /dev/null 2>&1

#Bind EFI-framebuffer
echo "efi-framebuffer.0" > /sys/bus/platform/drivers/efi-framebuffer/bind

# Restart Display Manager and stopped services
systemctl start sddm.service
systemctl start ananicy-cpp.service
systemctl start preload.service
systemctl start prelockd.service
systemctl start memavaild.service
systemctl start uresourced.service
systemctl start irqbalance.service

please comment if anything could be done better. One thing I’m not sure of is the cpu pinning as im not sure its better to have the emulator thread and io thread pinned to the same (hyperthreaded) cpu core and then the 5 remaining (hyperthreaded)cores pinned for the vm or have 2 (hyperthreaded) cpu core for the emulator and io threads.

1 Like