Hey all, wanted to put some feelers out there because I am experiencing an odd issue while trying to passthrough my RTX 5080 to a windows VM using CachyOS with single gpu passthrough. Basically, when the VM is starting, it takes a really long time to see the tianocore logo and for the VM to boot. It takes up to 10 minutes to post and there don’t appear to be any libvirt or dmesg errors of note… What is odd is also that when I installed CachyOS and was just messing around before, the VM seemed to boot faster than it does when I re-installed CachyOS again for reasons…..
Hope I just have something misconfigured moreso than a bug.
Might have self resolved my issue… I am running a usb c type switcher for my work laptop dock and my PC… I noticed that if I did not bind the keyboard via USB to the VM it booted faster… But apparently if I unplug my usb devices with the usb c type switcher button toggle, it boots faster… I am confused? Maybe not, it just takes longer to boot sporadically.
Blargh: I think my script pre-hook script was not completely killing things using the GPU. I manage to get it to boot a bit more consistently. Also, cachyos-lts vs ther egular kernel seems to have some differences too.
Final edit: I don’t know. All I do know is that if a poll the qemu virsh state for the VM it results in drastically faster boots and shutdowns.
Full system details and config below:
Sysinfo:
Kernel: Linux 6.19.5-3-cachyos
DE: KDE Plasma 6.6.1
WM: KWin (Wayland)
CPU: Intel(R) Core(TM) i9-14900K
GPU: NVIDIA GeForce RTX 5080
Packages:
qemu-base 10.2.0-3
libvirt 1:12.0.0-2.1
Hooks:
startup script:
#!/bin/bash
set -x
exec >> /tmp/gpupass-start.log 2>&1
echo "=== Hook start: $(date) ==="
# Huge pages
echo 1 > /proc/sys/vm/compact_memory
echo 8192 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages
# Stop the display manager
systemctl stop display-manager.service
# SIGTERM kwin first for a graceful DRM master release
pkill -15 kwin_wayland 2>/dev/null || true
sleep 1
pkill -9 kwin_wayland 2>/dev/null || true
fuser -k /dev/nvidia0 /dev/nvidiactl /dev/nvidia-modeset \
/dev/dri/card0 /dev/dri/renderD128 2>/dev/null || true
# Unbind vtconsoles and EFI framebuffer
echo 0 > /sys/class/vtconsole/vtcon0/bind
[ -w /sys/class/vtconsole/vtcon1/bind ] && echo 0 > /sys/class/vtconsole/vtcon1/bind
echo efi-framebuffer.0 > /sys/bus/platform/drivers/efi-framebuffer/unbind 2>/dev/null || true
# Wait until no userspace process holds any DRM/nvidia fd open.
for i in $(seq 1 60); do
remaining=$(fuser /dev/dri/card0 /dev/dri/renderD128 \
/dev/nvidia0 /dev/nvidiactl 2>/dev/null)
[ -z "$remaining" ] && break
fuser -k /dev/nvidia0 /dev/nvidiactl /dev/nvidia-modeset \
/dev/dri/card0 /dev/dri/renderD128 2>/dev/null || true
sleep 0.5
done
sleep 1
# Unload nvidia modules in dependency order.
# nvidia_drm uses module refcount as a safe gate: succeeds only when all DRM
# fd cleanup is complete. Retry so transient post-kill cleanup can finish.
for i in $(seq 1 30); do
modprobe -r nvidia_drm 2>/dev/null && break
echo "nvidia_drm still busy (attempt $i/30), retrying in 1s..."
sleep 1
done
modprobe -r nvidia_modeset 2>/dev/null || true
modprobe -r nvidia_uvm 2>/dev/null || true
# modprobe -r nvidia implicitly unbinds 0000:01:00.0 from the nvidia driver
# via pci_unregister_driver() — no explicit driver/unbind needed or wanted.
modprobe -r nvidia 2>/dev/null || true
# Unbind audio from snd_hda_intel (nvidia module removal does not handle this)
echo "0000:01:00.1" > /sys/bus/pci/devices/0000:01:00.1/driver/unbind 2>/dev/null || true
sleep 1
# Load vfio
modprobe vfio
modprobe vfio_iommu_type1
modprobe vfio-pci
# Bind GPU and audio to vfio-pci
echo "10de 2c02" > /sys/bus/pci/drivers/vfio-pci/new_id
echo "10de 22e9" > /sys/bus/pci/drivers/vfio-pci/new_id
echo "=== Hook end: $(date) ==="
Stop script:
#!/bin/bash
set -x
# Unbind devices from vfio-pci and remove IDs
echo "0000:01:00.0" > /sys/bus/pci/devices/0000:01:00.0/driver/unbind 2>/dev/null || true
echo "0000:01:00.1" > /sys/bus/pci/devices/0000:01:00.1/driver/unbind 2>/dev/null || true
echo "10de 2c02" > /sys/bus/pci/drivers/vfio-pci/remove_id 2>/dev/null || true
echo "10de 22e9" > /sys/bus/pci/drivers/vfio-pci/remove_id 2>/dev/null || true
# 2. Unload vfio BEFORE loading nvidia
modprobe -r vfio-pci
modprobe -r vfio_iommu_type1
modprobe -r vfio
# 3. Now load nvidia modules in correct order
modprobe nvidia
modprobe nvidia_modeset
modprobe nvidia_uvm
modprobe nvidia_drm
# Rescan PCI bus so GPU and audio rebind to nvidia/snd_hda_intel
echo 1 > /sys/bus/pci/rescan
sleep 1
# 4. Rebind framebuffers
echo 1 > /sys/class/vtconsole/vtcon0/bind
[ -w /sys/class/vtconsole/vtcon1/bind ] && echo 1 > /sys/class/vtconsole/vtcon1/bind
echo efi-framebuffer.0 > /sys/bus/platform/drivers/efi-framebuffer/bind 2>/dev/null
# 5. Wait for nvidia to fully initialize before starting DM
sleep 10
# 6. Start display manager
systemctl start display-manager.service
echo 0 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages
begin monitor hook (poll qemu)
#!/bin/bash
VM="win11"
SHUTDOWN_TIMEOUT=120
while true; do
state=$(virsh domstate "$VM" 2>/dev/null)
case "$state" in
"running")
virsh qemu-monitor-command "$VM" --hmp "info status" >/dev/null 2>&1 || true
sleep 3
;;
"shut off"|"")
exit 0
;;
"paused (shutdown)"|"in shutdown"|"shutdown"|"paused")
# Poll via QMP monitor — this is what nudges libvirtd's event loop
for i in $(seq 1 $SHUTDOWN_TIMEOUT); do
state=$(virsh domstate "$VM" 2>/dev/null)
if [ "$state" = "shut off" ] || [ -z "$state" ]; then
exit 0
fi
virsh qemu-monitor-command "$VM" --hmp "info status" >/dev/null 2>&1 || true
sleep 1
done
virsh destroy "$VM" 2>/dev/null
exit 0
;;
*)
for i in $(seq 1 $SHUTDOWN_TIMEOUT); do
state=$(virsh domstate "$VM" 2>/dev/null)
if [ "$state" = "shut off" ] || [ -z "$state" ]; then
exit 0
fi
virsh qemu-monitor-command "$VM" --hmp "info status" >/dev/null 2>&1 || true
sleep 1
done
virsh destroy "$VM" 2>/dev/null
exit 0
;;
esac
done
libvirt config:
<domain type='kvm'>
<name>win11</name>
<uuid>85005f95-8c99-46ff-94d3-ffdc600e90f7</uuid>
<metadata>
<libosinfo:libosinfo xmlns:libosinfo="http://libosinfo.org/xmlns/libvirt/domain/1.0">
<libosinfo:os id="http://microsoft.com/win/11"/>
</libosinfo:libosinfo>
</metadata>
<memory unit='KiB'>16777216</memory>
<currentMemory unit='KiB'>16777216</currentMemory>
<memoryBacking>
<hugepages>
<page size='2048' unit='KiB'/>
</hugepages>
</memoryBacking>
<vcpu placement='static'>20</vcpu>
<cputune>
<vcpupin vcpu='0' cpuset='0'/>
<vcpupin vcpu='1' cpuset='8'/>
<vcpupin vcpu='2' cpuset='1'/>
<vcpupin vcpu='3' cpuset='9'/>
<vcpupin vcpu='4' cpuset='2'/>
<vcpupin vcpu='5' cpuset='10'/>
<vcpupin vcpu='6' cpuset='3'/>
<vcpupin vcpu='7' cpuset='11'/>
<vcpupin vcpu='8' cpuset='4'/>
<vcpupin vcpu='9' cpuset='12'/>
<vcpupin vcpu='10' cpuset='5'/>
<vcpupin vcpu='11' cpuset='13'/>
<vcpupin vcpu='12' cpuset='6'/>
<vcpupin vcpu='13' cpuset='14'/>
<vcpupin vcpu='14' cpuset='7'/>
<vcpupin vcpu='15' cpuset='15'/>
<vcpupin vcpu='16' cpuset='16'/>
<vcpupin vcpu='17' cpuset='17'/>
<emulatorpin cpuset='18-19'/>
</cputune>
<os firmware='efi'>
<type arch='x86_64' machine='pc-q35-10.2'>hvm</type>
<firmware>
<feature enabled='no' name='enrolled-keys'/>
<feature enabled='yes' name='secure-boot'/>
</firmware>
<loader readonly='yes' secure='yes' type='pflash' format='raw'>/usr/share/edk2/x64/OVMF_CODE.secboot.4m.fd</loader>
<nvram template='/usr/share/edk2/x64/OVMF_VARS.4m.fd' templateFormat='raw' format='raw'>/var/lib/libvirt/qemu/nvram/win11_VARS.fd</nvram>
<boot dev='hd'/>
</os>
<features>
<acpi/>
<apic/>
<hyperv mode='custom'>
<relaxed state='on'/>
<vapic state='on'/>
<spinlocks state='on' retries='8191'/>
<vpindex state='on'/>
<runtime state='on'/>
<synic state='on'/>
<stimer state='on'/>
<vendor_id state='on' value='nx1326'/>
<frequencies state='on'/>
<tlbflush state='on'/>
<ipi state='on'/>
<evmcs state='on'/>
<avic state='on'/>
</hyperv>
<kvm>
<hidden state='on'/>
</kvm>
<vmport state='off'/>
<smm state='on'/>
</features>
<cpu mode='host-passthrough' check='none' migratable='off'>
<topology sockets='1' dies='1' clusters='1' cores='10' threads='2'/>
<feature policy='require' name='topoext'/>
</cpu>
<clock offset='localtime'>
<timer name='rtc' tickpolicy='catchup'/>
<timer name='pit' tickpolicy='delay'/>
<timer name='hpet' present='no'/>
<timer name='hypervclock' present='yes'/>
</clock>
<on_poweroff>destroy</on_poweroff>
<on_reboot>restart</on_reboot>
<on_crash>destroy</on_crash>
<pm>
<suspend-to-mem enabled='no'/>
<suspend-to-disk enabled='no'/>
</pm>
<devices>
<emulator>/usr/bin/qemu-system-x86_64</emulator>
<disk type='file' device='disk'>
<driver name='qemu' type='qcow2'/>
<source file='/var/lib/libvirt/images/win11.qcow2'/>
<target dev='sdc' bus='scsi'/>
<address type='drive' controller='0' bus='0' target='0' unit='2'/>
</disk>
<controller type='usb' index='0' model='qemu-xhci' ports='15'>
<address type='pci' domain='0x0000' bus='0x02' slot='0x00' function='0x0'/>
</controller>
<controller type='pci' index='0' model='pcie-root'/>
<controller type='pci' index='1' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='1' port='0x10'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0' multifunction='on'/>
</controller>
<controller type='pci' index='2' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='2' port='0x11'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x1'/>
</controller>
<controller type='pci' index='3' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='3' port='0x12'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x2'/>
</controller>
<controller type='pci' index='4' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='4' port='0x13'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x3'/>
</controller>
<controller type='pci' index='5' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='5' port='0x14'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x4'/>
</controller>
<controller type='pci' index='6' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='6' port='0x15'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x5'/>
</controller>
<controller type='pci' index='7' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='7' port='0x16'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x6'/>
</controller>
<controller type='pci' index='8' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='8' port='0x17'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x7'/>
</controller>
<controller type='pci' index='9' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='9' port='0x18'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0' multifunction='on'/>
</controller>
<controller type='pci' index='10' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='10' port='0x19'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x1'/>
</controller>
<controller type='pci' index='11' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='11' port='0x1a'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x2'/>
</controller>
<controller type='pci' index='12' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='12' port='0x1b'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x3'/>
</controller>
<controller type='pci' index='13' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='13' port='0x1c'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x4'/>
</controller>
<controller type='pci' index='14' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='14' port='0x1d'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x5'/>
</controller>
<controller type='pci' index='15' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='15' port='0x1e'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x6'/>
</controller>
<controller type='pci' index='16' model='pcie-to-pci-bridge'>
<model name='pcie-pci-bridge'/>
<address type='pci' domain='0x0000' bus='0x05' slot='0x00' function='0x0'/>
</controller>
<controller type='scsi' index='0' model='virtio-scsi'>
<address type='pci' domain='0x0000' bus='0x06' slot='0x00' function='0x0'/>
</controller>
<controller type='sata' index='0'>
<address type='pci' domain='0x0000' bus='0x00' slot='0x1f' function='0x2'/>
</controller>
<interface type='network'>
<mac address='52:54:00:fa:fd:5a'/>
<source network='default'/>
<model type='virtio'/>
<address type='pci' domain='0x0000' bus='0x01' slot='0x00' function='0x0'/>
</interface>
<input type='mouse' bus='ps2'/>
<input type='keyboard' bus='ps2'/>
<tpm model='tpm-crb'>
<backend type='emulator' version='2.0'/>
</tpm>
<audio id='1' type='none'/>
<hostdev mode='subsystem' type='pci' managed='no'>
<source>
<address domain='0x0000' bus='0x01' slot='0x00' function='0x0'/>
</source>
<address type='pci' domain='0x0000' bus='0x03' slot='0x00' function='0x0'/>
</hostdev>
<hostdev mode='subsystem' type='pci' managed='no'>
<source>
<address domain='0x0000' bus='0x01' slot='0x00' function='0x1'/>
</source>
<address type='pci' domain='0x0000' bus='0x07' slot='0x00' function='0x0'/>
</hostdev>
<hostdev mode='subsystem' type='pci' managed='yes'>
<source>
<address domain='0x0000' bus='0x00' slot='0x14' function='0x0'/>
</source>
<address type='pci' domain='0x0000' bus='0x10' slot='0x01' function='0x0'/>
</hostdev>
<hostdev mode='subsystem' type='pci' managed='yes'>
<source>
<address domain='0x0000' bus='0x00' slot='0x14' function='0x2'/>
</source>
<address type='pci' domain='0x0000' bus='0x10' slot='0x02' function='0x0'/>
</hostdev>
<watchdog model='itco' action='reset'/>
<memballoon model='virtio'>
<address type='pci' domain='0x0000' bus='0x04' slot='0x00' function='0x0'/>
</memballoon>
</devices>
</domain>