Besides the xml, some boot paratmeters are important, too. “mitigations=off” plays a bit performance lift.
Below are the parameters that I use.
GRUB_CMDLINE_LINUX_DEFAULT="amd_iommu=on iommu=pt kvm.ignore_msrs=1 amd_pstate=passive mitigations=off pci=assign-busses vfio-pci.ids=10de:2782,10de:22bc"
For the VM. Here are things you need to look for.
- Enable “hugepages”, google it if you don’t know how to enable it.
- CPU configuration, use mode=“host-passthrough”.
- CPU pinning. Every vcpu actually represents a thread, not a core. The configuration below is how I passthrough 1st CCD. Note that, cpuset 0 and 16 actually are 2 threads on the 1st (same) core.
NOTE: I use “i440fx” because I migrated this VM from an old machine. If you create a new vm, using “q35” platform is highly recommended.
<domain type="kvm">
<name>win10gpu2</name>
<uuid>e3b05049-8201-4447-a4b3-b310220cc5bd</uuid>
<memory unit="KiB">40960000</memory>
<currentMemory unit="KiB">40960000</currentMemory>
<memoryBacking>
<hugepages/>
</memoryBacking>
<vcpu placement="static">16</vcpu>
<iothreads>2</iothreads>
<cputune>
<vcpupin vcpu="0" cpuset="0"/>
<vcpupin vcpu="1" cpuset="16"/>
<vcpupin vcpu="2" cpuset="1"/>
<vcpupin vcpu="3" cpuset="17"/>
<vcpupin vcpu="4" cpuset="2"/>
<vcpupin vcpu="5" cpuset="18"/>
<vcpupin vcpu="6" cpuset="3"/>
<vcpupin vcpu="7" cpuset="19"/>
<vcpupin vcpu="8" cpuset="4"/>
<vcpupin vcpu="9" cpuset="20"/>
<vcpupin vcpu="10" cpuset="5"/>
<vcpupin vcpu="11" cpuset="21"/>
<vcpupin vcpu="12" cpuset="6"/>
<vcpupin vcpu="13" cpuset="22"/>
<vcpupin vcpu="14" cpuset="7"/>
<vcpupin vcpu="15" cpuset="23"/>
<iothreadpin iothread="1" cpuset="8,24"/>
<iothreadpin iothread="2" cpuset="9,25"/>
</cputune>
<os>
<type arch="x86_64" machine="pc-i440fx-jammy">hvm</type>
<boot dev="hd"/>
</os>
<features>
<acpi/>
<apic/>
<hyperv mode="custom">
<relaxed state="on"/>
<vapic state="on"/>
<spinlocks state="on" retries="8191"/>
<vpindex state="on"/>
<synic state="on"/>
<stimer state="on"/>
<reset state="on"/>
<frequencies state="on"/>
</hyperv>
<vmport state="off"/>
</features>
<cpu mode="host-passthrough" check="none" migratable="off">
<topology sockets="1" dies="1" cores="8" threads="2"/>
<cache mode="passthrough"/>
<feature policy="require" name="topoext"/>
</cpu>
<clock offset="utc">
<timer name="rtc" tickpolicy="catchup"/>
<timer name="pit" tickpolicy="delay"/>
<timer name="hpet" present="no"/>
<timer name="hypervclock" present="yes"/>
<timer name="tsc" present="yes" mode="native"/>
</clock>
<on_poweroff>destroy</on_poweroff>
<on_reboot>restart</on_reboot>
<on_crash>destroy</on_crash>
<pm>
<suspend-to-mem enabled="no"/>
<suspend-to-disk enabled="no"/>
</pm>
<devices>
<emulator>/usr/bin/qemu-system-x86_64</emulator>
<disk type="block" device="disk">
<driver name="qemu" type="raw" cache="none" io="native" discard="unmap"/>
<source dev="/var/lib/libvirt/images/win10-gpu.img"/>
<target dev="vda" bus="virtio"/>
<address type="pci" domain="0x0000" bus="0x00" slot="0x08" function="0x0"/>
</disk>
<controller type="usb" index="0" model="ich9-ehci1">
<address type="pci" domain="0x0000" bus="0x00" slot="0x05" function="0x7"/>
</controller>
<controller type="usb" index="0" model="ich9-uhci1">
<master startport="0"/>
<address type="pci" domain="0x0000" bus="0x00" slot="0x05" function="0x0" multifunction="on"/>
</controller>
<controller type="usb" index="0" model="ich9-uhci2">
<master startport="2"/>
<address type="pci" domain="0x0000" bus="0x00" slot="0x05" function="0x1"/>
</controller>
<controller type="usb" index="0" model="ich9-uhci3">
<master startport="4"/>
<address type="pci" domain="0x0000" bus="0x00" slot="0x05" function="0x2"/>
</controller>
<controller type="pci" index="0" model="pci-root"/>
<controller type="pci" index="1" model="pci-bridge">
<model name="pci-bridge"/>
<target chassisNr="1"/>
<address type="pci" domain="0x0000" bus="0x00" slot="0x0b" function="0x0"/>
</controller>
<controller type="virtio-serial" index="0">
<address type="pci" domain="0x0000" bus="0x00" slot="0x06" function="0x0"/>
</controller>
<interface type="direct">
<mac address="52:54:00:60:42:55"/>
<source dev="enp15s0f1v7" mode="passthrough"/>
<model type="virtio"/>
<driver name="vhost"/>
<address type="pci" domain="0x0000" bus="0x00" slot="0x09" function="0x0"/>
</interface>
<serial type="pty">
<target type="isa-serial" port="0">
<model name="isa-serial"/>
</target>
</serial>
<console type="pty">
<target type="serial" port="0"/>
</console>
<input type="tablet" bus="usb">
<address type="usb" bus="0" port="1"/>
</input>
<input type="mouse" bus="ps2"/>
<input type="keyboard" bus="ps2"/>
<graphics type="spice" autoport="yes">
<listen type="address"/>
<gl enable="no"/>
</graphics>
<sound model="ich6">
<address type="pci" domain="0x0000" bus="0x00" slot="0x04" function="0x0"/>
</sound>
<audio id="1" type="spice"/>
<video>
<model type="qxl" ram="65536" vram="65536" vgamem="16384" heads="1" primary="yes"/>
<address type="pci" domain="0x0000" bus="0x00" slot="0x02" function="0x0"/>
</video>
<hostdev mode="subsystem" type="pci" managed="yes">
<source>
<address domain="0x0000" bus="0x01" slot="0x00" function="0x0"/>
</source>
<address type="pci" domain="0x0000" bus="0x00" slot="0x03" function="0x0" multifunction="on"/>
</hostdev>
<hostdev mode="subsystem" type="pci" managed="yes">
<source>
<address domain="0x0000" bus="0x01" slot="0x00" function="0x1"/>
</source>
<address type="pci" domain="0x0000" bus="0x00" slot="0x03" function="0x1"/>
</hostdev>
<hostdev mode="subsystem" type="pci" managed="yes">
<source>
<address domain="0x0000" bus="0x13" slot="0x00" function="0x0"/>
</source>
<address type="pci" domain="0x0000" bus="0x00" slot="0x0a" function="0x0"/>
</hostdev>
<redirdev bus="usb" type="spicevmc">
<address type="usb" bus="0" port="2"/>
</redirdev>
<redirdev bus="usb" type="spicevmc">
<address type="usb" bus="0" port="3"/>
</redirdev>
<memballoon model="virtio">
<address type="pci" domain="0x0000" bus="0x00" slot="0x07" function="0x0"/>
</memballoon>
</devices>
</domain>