I recently set up Windows to run under Linux following the amazing guide posted here. I have played a few games, and everything was great, Doom : Eternal give me within 5% of bare metal(Over 150fps) Until I tried Metro: Exodus. This game runs at between 0 and 30 FPS and is basically a stuttery slide-slow even in the menus.
I’ve tried to play around with CPUPinning with some success. It seems once I specified the IOThread the performance doubled. However, this is still unplayable with dips to 0 and spikes to 50.
All game files are stored on a nvme drive that was passed through completely to the guest. I’ve set up LookingGlass as well but I see the same stutters using a physical display connected to the guest GPU.
I’ve tried searching but the info I’ve found is above my ability to understand at this point. I’m not sure how to improve from here.
Side note, The Mobo does not support x8/x8 and I’ve been unable to get the main GPU to successfully pass through (I get stuck at the mobo splash screen when I try). This means the guest GPU is running at x4. However, this is Gen 4 x4 and since I can push 150fps in other games and that is what this GPU does in this game normally I would assume that is not the issue. I do have another mobo coming that will support x8/x8
-----The Specs------
Ryzen 5900X 12C 24T
32GB Ram
Host GPU - 3080
Guest GPU - 3080ti
Asus Tuf Pro X570 (Soon to be replaced with Asus Pro WS X570-ACE
Target resolution and quality - 1440p 144hz, High/Ultra
—Host Info—
Ubuntu 21.04
Linux ubu 5.11.0-31-generic #33-Ubuntu SMP Wed Aug 11 13:19:04 UTC 2021 x86_64 x86_64 x86_64 GNU/Linux
—Guest Info—
Windows 10 21H1
6 C 12T
16GB Ram
—Virt Manager Config—
<domain type="kvm">
<name>win10</name>
<uuid>76c29c2e-5d4b-4938-8d66-00449a60b6b1</uuid>
<metadata>
<libosinfo:libosinfo xmlns:libosinfo="http://libosinfo.org/xmlns/libvirt/domain/1.0">
<libosinfo:os id="http://microsoft.com/win/10"/>
</libosinfo:libosinfo>
</metadata>
<memory unit="KiB">15624192</memory>
<currentMemory unit="KiB">15624192</currentMemory>
<vcpu placement="static">12</vcpu>
<iothreads>1</iothreads>
<cputune>
<vcpupin vcpu="0" cpuset="6"/>
<vcpupin vcpu="1" cpuset="7"/>
<vcpupin vcpu="2" cpuset="8"/>
<vcpupin vcpu="3" cpuset="9"/>
<vcpupin vcpu="4" cpuset="10"/>
<vcpupin vcpu="5" cpuset="11"/>
<vcpupin vcpu="6" cpuset="18"/>
<vcpupin vcpu="7" cpuset="19"/>
<vcpupin vcpu="8" cpuset="20"/>
<vcpupin vcpu="9" cpuset="21"/>
<vcpupin vcpu="10" cpuset="22"/>
<vcpupin vcpu="11" cpuset="23"/>
<emulatorpin cpuset="12-17"/>
<iothreadpin iothread="1" cpuset="0-3"/>
</cputune>
<os>
<type arch="x86_64" machine="pc-q35-5.2">hvm</type>
<loader readonly="yes" type="pflash">/usr/share/OVMF/OVMF_CODE_4M.ms.fd</loader>
<nvram>/var/lib/libvirt/qemu/nvram/ubu10_VARS.fd</nvram>
</os>
<features>
<acpi/>
<apic/>
<hyperv>
<relaxed state="on"/>
<vapic state="on"/>
<spinlocks state="on" retries="8191"/>
<vpindex state="on"/>
<synic state="on"/>
<stimer state="on"/>
<reset state="on"/>
<vendor_id state="on" value="1234567890ab"/>
<frequencies state="on"/>
</hyperv>
</features>
<cpu mode="custom" match="exact" check="partial">
<model fallback="allow">EPYC</model>
<topology sockets="1" dies="1" cores="6" threads="2"/>
<feature policy="require" name="topoext"/>
</cpu>
<clock offset="localtime">
<timer name="rtc" tickpolicy="catchup"/>
<timer name="pit" tickpolicy="delay"/>
<timer name="hpet" present="no"/>
<timer name="hypervclock" present="yes"/>
</clock>
<on_poweroff>destroy</on_poweroff>
<on_reboot>restart</on_reboot>
<on_crash>destroy</on_crash>
<pm>
<suspend-to-mem enabled="no"/>
<suspend-to-disk enabled="no"/>
</pm>
<devices>
<emulator>/usr/bin/qemu-system-x86_64</emulator>
<disk type="block" device="disk">
<driver name="qemu" type="raw" cache="directsync" io="native"/>
<source dev="/dev/md0"/>
<backingStore/>
<target dev="sdb" bus="sata"/>
<boot order="1"/>
<address type="drive" controller="0" bus="0" target="0" unit="1"/>
</disk>
<controller type="usb" index="0" model="qemu-xhci">
<address type="pci" domain="0x0000" bus="0x02" slot="0x00" function="0x0"/>
</controller>
<controller type="sata" index="0">
<address type="pci" domain="0x0000" bus="0x00" slot="0x1f" function="0x2"/>
</controller>
<controller type="pci" index="0" model="pcie-root"/>
<controller type="pci" index="1" model="pcie-root-port">
<model name="pcie-root-port"/>
<target chassis="1" port="0x10"/>
<address type="pci" domain="0x0000" bus="0x00" slot="0x02" function="0x0" multifunction="on"/>
</controller>
<controller type="pci" index="2" model="pcie-root-port">
<model name="pcie-root-port"/>
<target chassis="2" port="0x11"/>
<address type="pci" domain="0x0000" bus="0x00" slot="0x02" function="0x1"/>
</controller>
<controller type="pci" index="3" model="pcie-root-port">
<model name="pcie-root-port"/>
<target chassis="3" port="0x12"/>
<address type="pci" domain="0x0000" bus="0x00" slot="0x02" function="0x2"/>
</controller>
<controller type="pci" index="4" model="pcie-root-port">
<model name="pcie-root-port"/>
<target chassis="4" port="0x13"/>
<address type="pci" domain="0x0000" bus="0x00" slot="0x02" function="0x3"/>
</controller>
<controller type="pci" index="5" model="pcie-root-port">
<model name="pcie-root-port"/>
<target chassis="5" port="0x14"/>
<address type="pci" domain="0x0000" bus="0x00" slot="0x02" function="0x4"/>
</controller>
<controller type="pci" index="6" model="pcie-root-port">
<model name="pcie-root-port"/>
<target chassis="6" port="0x15"/>
<address type="pci" domain="0x0000" bus="0x00" slot="0x02" function="0x5"/>
</controller>
<controller type="pci" index="7" model="pcie-root-port">
<model name="pcie-root-port"/>
<target chassis="7" port="0x8"/>
<address type="pci" domain="0x0000" bus="0x00" slot="0x01" function="0x0" multifunction="on"/>
</controller>
<controller type="pci" index="8" model="pcie-to-pci-bridge">
<model name="pcie-pci-bridge"/>
<address type="pci" domain="0x0000" bus="0x07" slot="0x00" function="0x0"/>
</controller>
<controller type="pci" index="9" model="pcie-root-port">
<model name="pcie-root-port"/>
<target chassis="9" port="0x9"/>
<address type="pci" domain="0x0000" bus="0x00" slot="0x01" function="0x1"/>
</controller>
<controller type="pci" index="10" model="pcie-root-port">
<model name="pcie-root-port"/>
<target chassis="10" port="0xa"/>
<address type="pci" domain="0x0000" bus="0x00" slot="0x01" function="0x2"/>
</controller>
<controller type="virtio-serial" index="0">
<address type="pci" domain="0x0000" bus="0x03" slot="0x00" function="0x0"/>
</controller>
<interface type="bridge">
<mac address="aa:aa:aa:aa:aa:aa"/>
<source bridge="virbr0"/>
<model type="virtio-net-pci"/>
<address type="pci" domain="0x0000" bus="0x01" slot="0x00" function="0x0"/>
</interface>
<serial type="pty">
<target type="isa-serial" port="0">
<model name="isa-serial"/>
</target>
</serial>
<console type="pty">
<target type="serial" port="0"/>
</console>
<channel type="spicevmc">
<target type="virtio" name="com.redhat.spice.0"/>
<address type="virtio-serial" controller="0" bus="0" port="1"/>
</channel>
<input type="mouse" bus="ps2"/>
<input type="keyboard" bus="ps2"/>
<input type="keyboard" bus="virtio">
<address type="pci" domain="0x0000" bus="0x09" slot="0x00" function="0x0"/>
</input>
<graphics type="spice" autoport="yes">
<listen type="address"/>
<image compression="off"/>
</graphics>
<sound model="ich9">
<address type="pci" domain="0x0000" bus="0x00" slot="0x1b" function="0x0"/>
</sound>
<video>
<model type="none"/>
</video>
<hostdev mode="subsystem" type="pci" managed="yes">
<source>
<address domain="0x0000" bus="0x05" slot="0x00" function="0x0"/>
</source>
<address type="pci" domain="0x0000" bus="0x05" slot="0x00" function="0x0"/>
</hostdev>
<hostdev mode="subsystem" type="pci" managed="yes">
<source>
<address domain="0x0000" bus="0x05" slot="0x00" function="0x1"/>
</source>
<address type="pci" domain="0x0000" bus="0x06" slot="0x00" function="0x0"/>
</hostdev>
<hostdev mode="subsystem" type="pci" managed="yes">
<source>
<address domain="0x0000" bus="0x01" slot="0x00" function="0x0"/>
</source>
<address type="pci" domain="0x0000" bus="0x0a" slot="0x00" function="0x0"/>
</hostdev>
<redirdev bus="usb" type="spicevmc">
<address type="usb" bus="0" port="2"/>
</redirdev>
<redirdev bus="usb" type="spicevmc">
<address type="usb" bus="0" port="3"/>
</redirdev>
<memballoon model="virtio">
<address type="pci" domain="0x0000" bus="0x04" slot="0x00" function="0x0"/>
</memballoon>
<shmem name="looking-glass">
<model type="ivshmem-plain"/>
<size unit="M">64</size>
<address type="pci" domain="0x0000" bus="0x08" slot="0x01" function="0x0"/>
</shmem>
</devices>
</domain>
> cat /proc/meminfo | grep Huge
AnonHugePages: 15607808 kB
ShmemHugePages: 0 kB
FileHugePages: 0 kB
HugePages_Total: 4096
HugePages_Free: 4096
HugePages_Rsvd: 0
HugePages_Surp: 0
Hugepagesize: 2048 kB
Hugetlb: 8388608 kB
Scripts associated with KVM
bind_vfio.sh
#!/bin/sh
PREREQS=""
DEVS="0000:05:00.0 0000:05:00.1"
for DEV in $DEVS;
do echo "vfio-pci" > /sys/bus/pci/devices/$DEV/driver_override
done
modprobe -i vfio-pci
vmup.bash
#!/bin/bash
cat /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
for file in /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; do echo "performance" > $file; done
cat /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
vmdown.bash
#!/bin/bash
cat /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
for file in /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; do echo "schedutil" > $file; done
cat /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
rebuild_kvm.bash
#!/bin/bash
# steps to rebuild the Disk
#confirm drive doesn't already exist
if [ ! -f /dev/md0 ]; then
sudo modprobe loop
sudo modprobe linear
LOOP1=$(sudo losetup -f)
sudo losetup ${LOOP1} /opt/kvm/efi1
LOOP2=$(sudo losetup -f)
sudo losetup ${LOOP2} /opt/kvm/efi2
sudo mdadm --build --verbose /dev/md0 --chunk=512 --level=linear --raid-devices=3 ${LOOP1} /dev/nvme1n1p4 ${LOOP2}
exit 0
fi
—Guides I’ve used—
Main setup guide - VFIO in 2019 – Pop!_OS How-To (General Guide though)
Optimization guide - Performance optimizations for gaming on virtual machines
Guide to Boot physical Windows install - Boot volume KVM
(I wanted to use my existing dual boot windows install. With that in mind, I followed the guide listed above to just boot that install.)
I have done everything listed in the guides above except- Enable HugePages, Enable MSI Interrupts, adjust any cache settings inside KVM.
Again, If any additional info is needed/helpful please let me know. Thank you for your time and knowledge.
------------Edit1----
I noticed something off with the reported clock speeds inside the VM. Only Core 0 is hitting it’s normal speed 3.7. All other cores are reporting under 1ghz for the most part. I’ve attached 2 pics that were taken a couple seconds after minimizing the game. Also not all of the cores are showing in task manager or HWinfo.
The first is from Doom, you can see good utilization of the GPU and CPU.
The Second is from Metro, The CPU, and GPU are barely being used. This time Metro didn’t go above 10FPS.
If those reported clock speeds are correct that would most likely be either all or most of the issue. I have PBO enabled now but, I’m going to set a manual OC and see if that helps.
------------Edit 2-----------
I’ve been able to get the VM to display the correct number of cores/threads by setting the multiplier to 42(4.2ghz) did not help performance. The reported clock speeds of all cores other than 1 are still below 1ghz. I imagine this is just reporting incorrectly as I ran CineBench and it scores about 1/2 of the expected score. This makes sense since I’ve only passed through half the CPU. I’ve tested the Disk Speeds vis crystal disk mark, CPU via Cinebench(Both single and multi-core) and the GPU via games and they are all as expected when compared to Bare metal. I can’t find any bottleneck with the system and so far this is the only game I’ve found that shows this behavior.
----------------------Solution-------------------------------
The issue was with using the physical windows install and drive. After removing that from the setup it’s working great. I was never able to get Core isolation working correctly but after finding this issue it was not needed. The additional settings regarding “CPUTune” I don’t really think are needed. The performance was great without them but hey.
I still use the noted scripts to remove devices from the host when the VM starts and return them when it stops. These scripts also create/destroy huge pages and adjust the CPU governor. Again, I don’t really think this is needed but I had already done the work.
–Hardware passed to the guest
12 cores (6 cores x 2 threads per)
16gb of memory (Huge pages enabled)
3080ti
–Other
I’m using Scream for audio when not gaming and just passing through a headset when gaming
Looking-glass for video(Still testing if this impacts gaming)
—Virt Manager Config
<domain type="kvm">
<name>Windows10</name>
<uuid>99703d6f-909d-445d-b574-1fd80b2c066d</uuid>
<description>WIndows 10 for gaming</description>
<metadata>
<libosinfo:libosinfo xmlns:libosinfo="http://libosinfo.org/xmlns/libvirt/domain/1.0">
<libosinfo:os id="http://microsoft.com/win/10"/>
</libosinfo:libosinfo>
</metadata>
<memory unit="KiB">15624192</memory>
<currentMemory unit="KiB">15624192</currentMemory>
<memoryBacking>
<hugepages/>
</memoryBacking>
<vcpu placement="static">12</vcpu>
<iothreads>1</iothreads>
<cputune>
<vcpupin vcpu="0" cpuset="6"/>
<vcpupin vcpu="1" cpuset="18"/>
<vcpupin vcpu="2" cpuset="7"/>
<vcpupin vcpu="3" cpuset="19"/>
<vcpupin vcpu="4" cpuset="8"/>
<vcpupin vcpu="5" cpuset="20"/>
<vcpupin vcpu="6" cpuset="9"/>
<vcpupin vcpu="7" cpuset="21"/>
<vcpupin vcpu="8" cpuset="10"/>
<vcpupin vcpu="9" cpuset="22"/>
<vcpupin vcpu="10" cpuset="11"/>
<vcpupin vcpu="11" cpuset="23"/>
<emulatorpin cpuset="5"/>
<iothreadpin iothread="1" cpuset="17"/>
<vcpusched vcpus="0" scheduler="rr" priority="1"/>
<vcpusched vcpus="1" scheduler="rr" priority="1"/>
<vcpusched vcpus="2" scheduler="rr" priority="1"/>
<vcpusched vcpus="3" scheduler="rr" priority="1"/>
<vcpusched vcpus="4" scheduler="rr" priority="1"/>
<vcpusched vcpus="5" scheduler="rr" priority="1"/>
<vcpusched vcpus="6" scheduler="rr" priority="1"/>
<vcpusched vcpus="7" scheduler="rr" priority="1"/>
<vcpusched vcpus="8" scheduler="rr" priority="1"/>
<vcpusched vcpus="9" scheduler="rr" priority="1"/>
<vcpusched vcpus="10" scheduler="rr" priority="1"/>
<vcpusched vcpus="11" scheduler="rr" priority="1"/>
<iothreadsched iothreads="1" scheduler="fifo" priority="98"/>
</cputune>
<os>
<type arch="x86_64" machine="pc-q35-5.2">hvm</type>
<loader readonly="yes" type="pflash">/usr/share/OVMF/OVMF_CODE_4M.ms.fd</loader>
<nvram>/var/lib/libvirt/qemu/nvram/Windows10_VARS.fd</nvram>
</os>
<features>
<acpi/>
<apic/>
<hyperv>
<relaxed state="on"/>
<vapic state="on"/>
<spinlocks state="on" retries="8191"/>
</hyperv>
<vmport state="off"/>
</features>
<cpu mode="host-passthrough" check="partial" migratable="on">
<topology sockets="1" dies="1" cores="6" threads="2"/>
</cpu>
<clock offset="localtime">
<timer name="hpet" present="yes"/>
<timer name="hypervclock" present="yes"/>
</clock>
<on_poweroff>destroy</on_poweroff>
<on_reboot>restart</on_reboot>
<on_crash>destroy</on_crash>
<pm>
<suspend-to-mem enabled="no"/>
<suspend-to-disk enabled="no"/>
</pm>
<devices>
<emulator>/usr/bin/qemu-system-x86_64</emulator>
<disk type="file" device="disk">
<driver name="qemu" type="qcow2"/>
<source file="/home/pop/Documents/personal/Windows10.qcow2"/>
<target dev="vda" bus="virtio"/>
<boot order="1"/>
<address type="pci" domain="0x0000" bus="0x04" slot="0x00" function="0x0"/>
</disk>
<controller type="usb" index="0" model="qemu-xhci" ports="15">
<address type="pci" domain="0x0000" bus="0x02" slot="0x00" function="0x0"/>
</controller>
<controller type="sata" index="0">
<address type="pci" domain="0x0000" bus="0x00" slot="0x1f" function="0x2"/>
</controller>
<controller type="pci" index="0" model="pcie-root"/>
<controller type="pci" index="1" model="pcie-root-port">
<model name="pcie-root-port"/>
<target chassis="1" port="0x10"/>
<address type="pci" domain="0x0000" bus="0x00" slot="0x02" function="0x0" multifunction="on"/>
</controller>
<controller type="pci" index="2" model="pcie-root-port">
<model name="pcie-root-port"/>
<target chassis="2" port="0x11"/>
<address type="pci" domain="0x0000" bus="0x00" slot="0x02" function="0x1"/>
</controller>
<controller type="pci" index="3" model="pcie-root-port">
<model name="pcie-root-port"/>
<target chassis="3" port="0x12"/>
<address type="pci" domain="0x0000" bus="0x00" slot="0x02" function="0x2"/>
</controller>
<controller type="pci" index="4" model="pcie-root-port">
<model name="pcie-root-port"/>
<target chassis="4" port="0x13"/>
<address type="pci" domain="0x0000" bus="0x00" slot="0x02" function="0x3"/>
</controller>
<controller type="pci" index="5" model="pcie-root-port">
<model name="pcie-root-port"/>
<target chassis="5" port="0x14"/>
<address type="pci" domain="0x0000" bus="0x00" slot="0x02" function="0x4"/>
</controller>
<controller type="pci" index="6" model="pcie-root-port">
<model name="pcie-root-port"/>
<target chassis="6" port="0x15"/>
<address type="pci" domain="0x0000" bus="0x00" slot="0x02" function="0x5"/>
</controller>
<controller type="pci" index="7" model="pcie-root-port">
<model name="pcie-root-port"/>
<target chassis="7" port="0x16"/>
<address type="pci" domain="0x0000" bus="0x00" slot="0x02" function="0x6"/>
</controller>
<controller type="pci" index="8" model="pcie-root-port">
<model name="pcie-root-port"/>
<target chassis="8" port="0x17"/>
<address type="pci" domain="0x0000" bus="0x00" slot="0x02" function="0x7"/>
</controller>
<controller type="pci" index="9" model="pcie-root-port">
<model name="pcie-root-port"/>
<target chassis="9" port="0x8"/>
<address type="pci" domain="0x0000" bus="0x00" slot="0x01" function="0x0" multifunction="on"/>
</controller>
<controller type="pci" index="10" model="pcie-to-pci-bridge">
<model name="pcie-pci-bridge"/>
<address type="pci" domain="0x0000" bus="0x09" slot="0x00" function="0x0"/>
</controller>
<controller type="pci" index="11" model="pcie-root-port">
<model name="pcie-root-port"/>
<target chassis="11" port="0x9"/>
<address type="pci" domain="0x0000" bus="0x00" slot="0x01" function="0x1"/>
</controller>
<controller type="virtio-serial" index="0">
<address type="pci" domain="0x0000" bus="0x03" slot="0x00" function="0x0"/>
</controller>
<interface type="bridge">
<mac address="52:54:00:f3:87:c2"/>
<source bridge="virbr0"/>
<model type="virtio"/>
<address type="pci" domain="0x0000" bus="0x01" slot="0x00" function="0x0"/>
</interface>
<serial type="pty">
<target type="isa-serial" port="0">
<model name="isa-serial"/>
</target>
</serial>
<console type="pty">
<target type="serial" port="0"/>
</console>
<channel type="spicevmc">
<target type="virtio" name="com.redhat.spice.0"/>
<address type="virtio-serial" controller="0" bus="0" port="1"/>
</channel>
<input type="mouse" bus="ps2"/>
<input type="keyboard" bus="virtio">
<address type="pci" domain="0x0000" bus="0x0b" slot="0x00" function="0x0"/>
</input>
<input type="keyboard" bus="ps2"/>
<graphics type="spice" autoport="yes">
<listen type="address"/>
<image compression="off"/>
</graphics>
<sound model="ich9">
<address type="pci" domain="0x0000" bus="0x00" slot="0x1b" function="0x0"/>
</sound>
<video>
<model type="none"/>
</video>
<hostdev mode="subsystem" type="pci" managed="yes">
<source>
<address domain="0x0000" bus="0x0a" slot="0x00" function="0x0"/>
</source>
<address type="pci" domain="0x0000" bus="0x06" slot="0x00" function="0x0"/>
</hostdev>
<hostdev mode="subsystem" type="pci" managed="yes">
<source>
<address domain="0x0000" bus="0x0a" slot="0x00" function="0x1"/>
</source>
<address type="pci" domain="0x0000" bus="0x07" slot="0x00" function="0x0"/>
</hostdev>
<hostdev mode="subsystem" type="pci" managed="yes">
<source>
<address domain="0x0000" bus="0x05" slot="0x00" function="0x0"/>
</source>
<address type="pci" domain="0x0000" bus="0x08" slot="0x00" function="0x0"/>
</hostdev>
<hostdev mode="subsystem" type="usb" managed="yes">
<source>
<vendor id="0x046d"/>
<product id="0xc33f"/>
</source>
<address type="usb" bus="0" port="1"/>
</hostdev>
<hostdev mode="subsystem" type="usb" managed="yes">
<source>
<vendor id="0x046d"/>
<product id="0xc539"/>
</source>
<address type="usb" bus="0" port="4"/>
</hostdev>
<hostdev mode="subsystem" type="usb" managed="yes">
<source>
<vendor id="0x0b0e"/>
<product id="0x2465"/>
</source>
<address type="usb" bus="0" port="2"/>
</hostdev>
<memballoon model="virtio">
<address type="pci" domain="0x0000" bus="0x05" slot="0x00" function="0x0"/>
</memballoon>
<shmem name="looking-glass">
<model type="ivshmem-plain"/>
<size unit="M">64</size>
<address type="pci" domain="0x0000" bus="0x0a" slot="0x01" function="0x0"/>
</shmem>
</devices>
</domain>
Thanks to everyone who helped. In my case, the solution was to just use a normal qcow2 disk image rather than trying to boot my physical windows install.
