Solution: I had to add a dedicated 40GB NTFS partition that housed nothing but a massive 40GB pagefile for the system to use, and that seemed to do the trick. TLOU commits 35GB of virtual memory (RAM+pagefile) post shader compilation and an eye-watering 50GB during shader compilation on my 12-threaded system.
Hi folks,
Recently got a proper VFIO set up with storage and GPU passthrough, and it has mostly been running fine. However, there is a significant setback: some games, such as The Last of Us Part I and Alan Wake 2, do not run at all, stating there is not enough video memory in the system, even though the VM has 16GB RAM and 16GB VRAM. Other games run without issues, such as Control and RDR2. The problem persists regardless of whether both host and guest have ReBAR enabled or disabled.
TLOU 1 Error:
VM XML:
<domain xmlns:qemu="http://libvirt.org/schemas/domain/qemu/1.0" type="kvm">
<name>win11VFIO</name>
<uuid>eab573af-3d10-429c-8d54-d52bc73f3a99</uuid>
<metadata>
<libosinfo:libosinfo xmlns:libosinfo="http://libosinfo.org/xmlns/libvirt/domain/1.0">
<libosinfo:os id="http://microsoft.com/win/11"/>
</libosinfo:libosinfo>
</metadata>
<memory unit="KiB">16777216</memory>
<currentMemory unit="KiB">16777216</currentMemory>
<vcpu placement="static">12</vcpu>
<iothreads>1</iothreads>
<cputune>
<vcpupin vcpu="0" cpuset="0"/>
<vcpupin vcpu="1" cpuset="6"/>
<vcpupin vcpu="2" cpuset="1"/>
<vcpupin vcpu="3" cpuset="7"/>
<vcpupin vcpu="4" cpuset="2"/>
<vcpupin vcpu="5" cpuset="8"/>
<vcpupin vcpu="6" cpuset="3"/>
<vcpupin vcpu="7" cpuset="9"/>
<vcpupin vcpu="8" cpuset="4"/>
<vcpupin vcpu="9" cpuset="10"/>
<vcpupin vcpu="10" cpuset="5"/>
<vcpupin vcpu="11" cpuset="11"/>
<emulatorpin cpuset="0,6"/>
<iothreadpin iothread="1" cpuset="0,6"/>
</cputune>
<os firmware="efi">
<type arch="x86_64" machine="pc-q35-10.0">hvm</type>
<firmware>
<feature enabled="no" name="enrolled-keys"/>
<feature enabled="yes" name="secure-boot"/>
</firmware>
<loader readonly="yes" secure="yes" type="pflash" format="raw">/usr/share/edk2/x64/OVMF_CODE.secboot.4m.fd</loader>
<nvram template="/usr/share/edk2/x64/OVMF_VARS.4m.fd" templateFormat="raw" format="raw">/var/lib/libvirt/qemu/nvram/win11VFIO_VARS.fd</nvram>
<boot dev="hd"/>
<bootmenu enable="yes"/>
<smbios mode="host"/>
</os>
<features>
<acpi/>
<apic/>
<hyperv mode="custom">
<relaxed state="on"/>
<vapic state="on"/>
<spinlocks state="on" retries="8191"/>
<vpindex state="on"/>
<runtime state="on"/>
<synic state="on"/>
<stimer state="on"/>
<vendor_id state="on" value="0123456789ab"/>
<frequencies state="on"/>
<tlbflush state="on"/>
<ipi state="on"/>
<avic state="on"/>
</hyperv>
<vmport state="off"/>
<smm state="on"/>
</features>
<cpu mode="host-passthrough" check="none" migratable="on">
<topology sockets="1" dies="1" clusters="1" cores="6" threads="2"/>
<cache mode="passthrough"/>
<feature policy="require" name="topoext"/>
</cpu>
<clock offset="localtime">
<timer name="rtc" tickpolicy="catchup"/>
<timer name="pit" tickpolicy="delay"/>
<timer name="hpet" present="no"/>
<timer name="hypervclock" present="yes"/>
</clock>
<on_poweroff>destroy</on_poweroff>
<on_reboot>restart</on_reboot>
<on_crash>destroy</on_crash>
<pm>
<suspend-to-mem enabled="no"/>
<suspend-to-disk enabled="no"/>
</pm>
<devices>
<emulator>/usr/bin/qemu-system-x86_64</emulator>
<disk type="file" device="cdrom">
<driver name="qemu" type="raw"/>
<source file="/home/stratz/Documents/ISO/Win11_24H2_EnglishInternational_x64.iso"/>
<target dev="sda" bus="sata"/>
<readonly/>
<address type="drive" controller="0" bus="0" target="0" unit="0"/>
</disk>
<disk type="file" device="cdrom">
<driver name="qemu" type="raw"/>
<source file="/home/stratz/Documents/ISO/virtio-win-0.1.262.iso"/>
<target dev="sdb" bus="sata"/>
<readonly/>
<address type="drive" controller="0" bus="0" target="0" unit="1"/>
</disk>
<disk type="file" device="disk">
<driver name="qemu" type="raw"/>
<source file="/home/stratz/VM/win11virtio.img"/>
<target dev="vda" bus="virtio"/>
<address type="pci" domain="0x0000" bus="0x08" slot="0x00" function="0x0"/>
</disk>
<controller type="usb" index="0" model="qemu-xhci" ports="15">
<address type="pci" domain="0x0000" bus="0x02" slot="0x00" function="0x0"/>
</controller>
<controller type="pci" index="0" model="pcie-root"/>
<controller type="pci" index="1" model="pcie-root-port">
<model name="pcie-root-port"/>
<target chassis="1" port="0x10"/>
<address type="pci" domain="0x0000" bus="0x00" slot="0x02" function="0x0" multifunction="on"/>
</controller>
<controller type="pci" index="2" model="pcie-root-port">
<model name="pcie-root-port"/>
<target chassis="2" port="0x11"/>
<address type="pci" domain="0x0000" bus="0x00" slot="0x02" function="0x1"/>
</controller>
<controller type="pci" index="3" model="pcie-root-port">
<model name="pcie-root-port"/>
<target chassis="3" port="0x12"/>
<address type="pci" domain="0x0000" bus="0x00" slot="0x02" function="0x2"/>
</controller>
<controller type="pci" index="4" model="pcie-root-port">
<model name="pcie-root-port"/>
<target chassis="4" port="0x13"/>
<address type="pci" domain="0x0000" bus="0x00" slot="0x02" function="0x3"/>
</controller>
<controller type="pci" index="5" model="pcie-root-port">
<model name="pcie-root-port"/>
<target chassis="5" port="0x14"/>
<address type="pci" domain="0x0000" bus="0x00" slot="0x02" function="0x4"/>
</controller>
<controller type="pci" index="6" model="pcie-root-port">
<model name="pcie-root-port"/>
<target chassis="6" port="0x15"/>
<address type="pci" domain="0x0000" bus="0x00" slot="0x02" function="0x5"/>
</controller>
<controller type="pci" index="7" model="pcie-root-port">
<model name="pcie-root-port"/>
<target chassis="7" port="0x16"/>
<address type="pci" domain="0x0000" bus="0x00" slot="0x02" function="0x6"/>
</controller>
<controller type="pci" index="8" model="pcie-root-port">
<model name="pcie-root-port"/>
<target chassis="8" port="0x17"/>
<address type="pci" domain="0x0000" bus="0x00" slot="0x02" function="0x7"/>
</controller>
<controller type="pci" index="9" model="pcie-root-port">
<model name="pcie-root-port"/>
<target chassis="9" port="0x18"/>
<address type="pci" domain="0x0000" bus="0x00" slot="0x03" function="0x0" multifunction="on"/>
</controller>
<controller type="pci" index="10" model="pcie-root-port">
<model name="pcie-root-port"/>
<target chassis="10" port="0x19"/>
<address type="pci" domain="0x0000" bus="0x00" slot="0x03" function="0x1"/>
</controller>
<controller type="pci" index="11" model="pcie-root-port">
<model name="pcie-root-port"/>
<target chassis="11" port="0x1a"/>
<address type="pci" domain="0x0000" bus="0x00" slot="0x03" function="0x2"/>
</controller>
<controller type="pci" index="12" model="pcie-root-port">
<model name="pcie-root-port"/>
<target chassis="12" port="0x1b"/>
<address type="pci" domain="0x0000" bus="0x00" slot="0x03" function="0x3"/>
</controller>
<controller type="pci" index="13" model="pcie-root-port">
<model name="pcie-root-port"/>
<target chassis="13" port="0x1c"/>
<address type="pci" domain="0x0000" bus="0x00" slot="0x03" function="0x4"/>
</controller>
<controller type="pci" index="14" model="pcie-root-port">
<model name="pcie-root-port"/>
<target chassis="14" port="0x1d"/>
<address type="pci" domain="0x0000" bus="0x00" slot="0x03" function="0x5"/>
</controller>
<controller type="pci" index="15" model="pcie-root-port">
<model name="pcie-root-port"/>
<target chassis="15" port="0x1e"/>
<address type="pci" domain="0x0000" bus="0x00" slot="0x03" function="0x6"/>
</controller>
<controller type="pci" index="16" model="pcie-to-pci-bridge">
<model name="pcie-pci-bridge"/>
<address type="pci" domain="0x0000" bus="0x06" slot="0x00" function="0x0"/>
</controller>
<controller type="sata" index="0">
<address type="pci" domain="0x0000" bus="0x00" slot="0x1f" function="0x2"/>
</controller>
<controller type="virtio-serial" index="0">
<address type="pci" domain="0x0000" bus="0x03" slot="0x00" function="0x0"/>
</controller>
<interface type="network">
<mac address="52:54:00:3c:00:ab"/>
<source network="default"/>
<model type="virtio"/>
<address type="pci" domain="0x0000" bus="0x01" slot="0x00" function="0x0"/>
</interface>
<serial type="pty">
<target type="isa-serial" port="0">
<model name="isa-serial"/>
</target>
</serial>
<console type="pty">
<target type="serial" port="0"/>
</console>
<input type="mouse" bus="virtio">
<address type="pci" domain="0x0000" bus="0x0b" slot="0x00" function="0x0"/>
</input>
<input type="keyboard" bus="ps2"/>
<input type="keyboard" bus="virtio">
<address type="pci" domain="0x0000" bus="0x0a" slot="0x00" function="0x0"/>
</input>
<input type="mouse" bus="ps2"/>
<graphics type="spice" port="-1" autoport="no">
<listen type="address"/>
<image compression="off"/>
<gl enable="no"/>
</graphics>
<sound model="ich9">
<address type="pci" domain="0x0000" bus="0x00" slot="0x1b" function="0x0"/>
</sound>
<audio id="1" type="spice"/>
<video>
<model type="none"/>
</video>
<hostdev mode="subsystem" type="pci" managed="yes">
<source>
<address domain="0x0000" bus="0x04" slot="0x00" function="0x0"/>
</source>
<address type="pci" domain="0x0000" bus="0x09" slot="0x00" function="0x0"/>
</hostdev>
<hostdev mode="subsystem" type="pci" managed="yes">
<source>
<address domain="0x0000" bus="0x03" slot="0x00" function="0x0"/>
</source>
<rom bar="on" file="/dummy.rom"/>
<address type="pci" domain="0x0000" bus="0x04" slot="0x00" function="0x0"/>
</hostdev>
<hostdev mode="subsystem" type="pci" managed="yes">
<source>
<address domain="0x0000" bus="0x03" slot="0x00" function="0x1"/>
</source>
<address type="pci" domain="0x0000" bus="0x07" slot="0x00" function="0x0"/>
</hostdev>
<hostdev mode="subsystem" type="pci" managed="yes">
<source>
<address domain="0x0000" bus="0x0d" slot="0x00" function="0x0"/>
</source>
<address type="pci" domain="0x0000" bus="0x05" slot="0x00" function="0x0"/>
</hostdev>
<hostdev mode="subsystem" type="pci" managed="yes">
<source>
<address domain="0x0000" bus="0x0b" slot="0x00" function="0x0"/>
</source>
<address type="pci" domain="0x0000" bus="0x0c" slot="0x00" function="0x0"/>
</hostdev>
<watchdog model="itco" action="reset"/>
<memballoon model="none"/>
</devices>
<qemu:commandline>
<qemu:arg value="-fw_cfg"/>
<qemu:arg value="opt/ovmf/X-PciMmio64Mb,string=65536"/>
<qemu:arg value="-device"/>
<qemu:arg value="{'driver':'ivshmem-plain','id':'shmem0','memdev':'looking-glass'}"/>
<qemu:arg value="-object"/>
<qemu:arg value="{'qom-type':'memory-backend-file','id':'looking-glass','mem-path':'/dev/kvmfr0','size':67108864,'share':true}"/>
</qemu:commandline>
</domain>
System specs:
OS: Arch Linux x86_64
Kernel: Linux 6.16.8-2-cachyos
Uptime: 28 mins
Packages: 1882 (pacman), 13 (flatpak)
Shell: fish 4.0.8
Display (PX277PRO): 2560x1440 in 32", 165 Hz [External]
DE: KDE Plasma 6.4.5
WM: KWin (Wayland)
WM Theme: Breeze
Theme: Breeze (MaterialYouDark) [Qt], adw-gtk3-dark [GTK2/3/4]
Icons: Papirus-Dark [Qt], Papirus-Dark [GTK2/3/4]
Font: Noto Sans (10pt) [Qt], Noto Sans (10pt) [GTK2/3/4]
Cursor: WhiteSur (24px)
Terminal: konsole 25.8.1
CPU: AMD Ryzen 5 9600X (12) @ 5.68 GHz
GPU 1: AMD Radeon RX 9070 XT
GPU 2: AMD Radeon Graphics [Integrated]
Memory: 20.96 GiB / 30.51 GiB (69%)
Swap: 4.00 KiB / 23.44 GiB (0%)
Disk (/): 65.03 GiB / 68.36 GiB (95%) - btrfs
Disk (/home): 302.65 GiB / 377.31 GiB (80%) - ext4
Script that passes through GPU:
# #!/bin/bash
# #qemu start script with 9070xt Passthrough
#
# #set variables
VIRSH_GPU_VIDEO="0000:03:00.0"
VIRSH_GPU_AUDIO="0000:03:00.1"
#
#
# #Prevent processes from accessing the gpu, nuke any that already have through fuser.
chmod 0 /dev/dri/by-path/pci-$VIRSH_GPU_VIDEO-render
chmod 0 /dev/dri/by-path/pci-$VIRSH_GPU_VIDEO-card
fuser -k /dev/dri/by-path/pci-$VIRSH_GPU_VIDEO-render
fuser -k /dev/dri/by-path/pci-$VIRSH_GPU_VIDEO-card
# #Manually unbind
echo ''$VIRSH_GPU_VIDEO > "/sys/bus/pci/devices/${VIRSH_GPU_VIDEO}/driver/unbind"
echo ''$VIRSH_GPU_AUDIO > "/sys/bus/pci/devices/${VIRSH_GPU_AUDIO}/driver/unbind"
Let me know if any other info is gonna be useful.