Hey there,
I am Matthias and new to this forum. I found a lot of useful stuff in this forum and also like the Youtubecontent so I thought I’ll give it a try.
My objective:
Forwarding my AMD Radeon RX 7700S of my Framework 16 running Ubuntu 22.04. with the recommended OEM-Kernel to my Windows 11 virtual machine. Others already successfully passed through the card in Proxxmox and Arch.
My problem:
I always get error 43 and am limited to a resolution of 800x600 pixels. If i boot the VM into Ubuntu there are no problems and benchmarks results are fine.
One strange thing i noticed is, that I cannot reboot the VM. If I do, I loose the disyplay output. I have to shutdown the VM and then start it again in order to get display output. Rebooting with Ubuntu on the VM on the other hand works just fine.
What I have done so far:
-
I installed KVM and created a VM following the guides on sysyguides.
-
I forwarded the GPU (both devices) to the VM using virtmanager
-
I solved the host from crashing by isolating the GPU from the host. I created a file
/etc/modprobe.d/vfio.conf
and filled it with
options vfio-pci ids=1002:7480,1002:ab30
softdep amdgpu pre: vfio-pci
and updated initramfs. After that i got:
03:00.0 VGA compatible controller: Advanced Micro Devices, Inc. [AMD/ATI] Device 7480 (rev c1) (prog-if 00 [VGA controller])
Subsystem: Device f111:0007
Flags: fast devsel, IRQ 255, IOMMU group 16
Memory at 8400000000 (64-bit, prefetchable) [disabled] [size=8G]
Memory at 8600000000 (64-bit, prefetchable) [disabled] [size=256M]
I/O ports at a000 [disabled] [size=256]
Memory at 90d00000 (32-bit, non-prefetchable) [disabled] [size=1M]
Expansion ROM at 90e20000 [disabled] [size=128K]
Capabilities: <access denied>
Kernel driver in use: vfio-pci
Kernel modules: amdgpu
03:00.1 Audio device: Advanced Micro Devices, Inc. [AMD/ATI] Device ab30
Subsystem: Device f111:0005
Flags: fast devsel, IRQ 255, IOMMU group 17
Memory at 90e00000 (32-bit, non-prefetchable) [disabled] [size=16K]
Capabilities: <access denied>
Kernel driver in use: vfio-pci
Kernel modules: snd_hda_intel
- I changed the firmware from
<os firmware="efi">
<type arch="x86_64" machine="pc-q35-6.2">hvm</type>
<boot dev="hd"/>
</os>
to
<os>
<type arch="x86_64" machine="pc-q35-6.2">hvm</type>
<loader readonly="yes" type="pflash">/usr/share/OVMF/OVMF_CODE.fd</loader>
<nvram template="/usr/share/OVMF/OVMF_VARS.fd">/var/lib/libvirt/qemu/nvram/Windows11_VARS.fd</nvram>
<boot dev="hd"/>
</os>
and added
<hyperv mode="custom">
<vendor_id state="on" value="someRandomBits"/>
</hyperv>
<kvm> <hidden state="on"/> </kvm>
to the features-section.
-
I changed the PCI-Assignments so both devices of the GPU are on the same bus.
-
i deactivated QXL on the VM.
-
I reinstalled Windows several times and installed the driver with the Adrenaline-Software and also extracted the driver from the file and installed it manually - no difference.
-
I provided a dummy ROM to the GPU and a ROM that i extracted using GPUZ. In both cases i got no display output at all.
Here is my current XML:
<domain type="kvm">
<name>ubuntu22.04</name>
<uuid>557c47b4-9f8a-40a0-b08b-44571d2d82ba</uuid>
<metadata>
<libosinfo:libosinfo xmlns:libosinfo="http://libosinfo.org/xmlns/libvirt/domain/1.0">
<libosinfo:os id="http://ubuntu.com/ubuntu/22.04"/>
</libosinfo:libosinfo>
</metadata>
<memory unit="KiB">16384000</memory>
<currentMemory unit="KiB">16384000</currentMemory>
<vcpu placement="static">16</vcpu>
<os>
<type arch="x86_64" machine="pc-q35-6.2">hvm</type>
<loader readonly="yes" type="pflash">/usr/share/OVMF/OVMF_CODE.fd</loader>
<nvram template="/usr/share/OVMF/OVMF_VARS.fd">/var/lib/libvirt/qemu/nvram/Ubuntu_VARS.fd</nvram>
<smbios mode="host"/>
</os>
<features>
<acpi/>
<apic/>
<kvm>
<hidden state="on"/>
</kvm>
<vmport state="off"/>
</features>
<cpu mode="host-passthrough" check="none" migratable="on"/>
<clock offset="utc">
<timer name="rtc" tickpolicy="catchup"/>
<timer name="pit" tickpolicy="delay"/>
<timer name="hpet" present="no"/>
</clock>
<on_poweroff>destroy</on_poweroff>
<on_reboot>restart</on_reboot>
<on_crash>destroy</on_crash>
<pm>
<suspend-to-mem enabled="no"/>
<suspend-to-disk enabled="no"/>
</pm>
<devices>
<emulator>/usr/bin/qemu-system-x86_64</emulator>
<disk type="file" device="disk">
<driver name="qemu" type="qcow2" discard="unmap"/>
<source file="/var/lib/libvirt/images/ubuntu22.04.qcow2"/>
<target dev="vda" bus="virtio"/>
<address type="pci" domain="0x0000" bus="0x04" slot="0x00" function="0x0"/>
</disk>
<disk type="file" device="cdrom">
<driver name="qemu" type="raw"/>
<source file="/home/matthias/Schreibtisch/Win11_23H2_German_x64v2.iso"/>
<target dev="sda" bus="sata"/>
<readonly/>
<address type="drive" controller="0" bus="0" target="0" unit="0"/>
</disk>
<disk type="file" device="cdrom">
<driver name="qemu" type="raw"/>
<source file="/home/matthias/Downloads/virtio-win-0.1.240.iso"/>
<target dev="sdb" bus="sata"/>
<readonly/>
<address type="drive" controller="0" bus="0" target="0" unit="1"/>
</disk>
<controller type="usb" index="0" model="qemu-xhci" ports="15">
<address type="pci" domain="0x0000" bus="0x02" slot="0x00" function="0x0"/>
</controller>
<controller type="pci" index="0" model="pcie-root"/>
<controller type="pci" index="1" model="pcie-root-port">
<model name="pcie-root-port"/>
<target chassis="1" port="0x8"/>
<address type="pci" domain="0x0000" bus="0x00" slot="0x01" function="0x0" multifunction="on"/>
</controller>
<controller type="pci" index="2" model="pcie-root-port">
<model name="pcie-root-port"/>
<target chassis="2" port="0x9"/>
<address type="pci" domain="0x0000" bus="0x00" slot="0x01" function="0x1"/>
</controller>
<controller type="pci" index="3" model="pcie-root-port">
<model name="pcie-root-port"/>
<target chassis="3" port="0xa"/>
<address type="pci" domain="0x0000" bus="0x00" slot="0x01" function="0x2"/>
</controller>
<controller type="pci" index="4" model="pcie-root-port">
<model name="pcie-root-port"/>
<target chassis="4" port="0xb"/>
<address type="pci" domain="0x0000" bus="0x00" slot="0x01" function="0x3"/>
</controller>
<controller type="pci" index="5" model="pcie-root-port">
<model name="pcie-root-port"/>
<target chassis="5" port="0xc"/>
<address type="pci" domain="0x0000" bus="0x00" slot="0x01" function="0x4"/>
</controller>
<controller type="pci" index="6" model="pcie-root-port">
<model name="pcie-root-port"/>
<target chassis="6" port="0xd"/>
<address type="pci" domain="0x0000" bus="0x00" slot="0x01" function="0x5"/>
</controller>
<controller type="pci" index="7" model="pcie-root-port">
<model name="pcie-root-port"/>
<target chassis="7" port="0xe"/>
<address type="pci" domain="0x0000" bus="0x00" slot="0x01" function="0x6"/>
</controller>
<controller type="pci" index="8" model="pcie-root-port">
<model name="pcie-root-port"/>
<target chassis="8" port="0xf"/>
<address type="pci" domain="0x0000" bus="0x00" slot="0x01" function="0x7"/>
</controller>
<controller type="pci" index="9" model="pcie-root-port">
<model name="pcie-root-port"/>
<target chassis="9" port="0x10"/>
<address type="pci" domain="0x0000" bus="0x00" slot="0x02" function="0x0" multifunction="on"/>
</controller>
<controller type="pci" index="10" model="pcie-root-port">
<model name="pcie-root-port"/>
<target chassis="10" port="0x11"/>
<address type="pci" domain="0x0000" bus="0x00" slot="0x02" function="0x1"/>
</controller>
<controller type="pci" index="11" model="pcie-root-port">
<model name="pcie-root-port"/>
<target chassis="11" port="0x12"/>
<address type="pci" domain="0x0000" bus="0x00" slot="0x02" function="0x2"/>
</controller>
<controller type="pci" index="12" model="pcie-root-port">
<model name="pcie-root-port"/>
<target chassis="12" port="0x13"/>
<address type="pci" domain="0x0000" bus="0x00" slot="0x02" function="0x3"/>
</controller>
<controller type="pci" index="13" model="pcie-root-port">
<model name="pcie-root-port"/>
<target chassis="13" port="0x14"/>
<address type="pci" domain="0x0000" bus="0x00" slot="0x02" function="0x4"/>
</controller>
<controller type="pci" index="14" model="pcie-root-port">
<model name="pcie-root-port"/>
<target chassis="14" port="0x15"/>
<address type="pci" domain="0x0000" bus="0x00" slot="0x02" function="0x5"/>
</controller>
<controller type="sata" index="0">
<address type="pci" domain="0x0000" bus="0x00" slot="0x1f" function="0x2"/>
</controller>
<controller type="virtio-serial" index="0">
<address type="pci" domain="0x0000" bus="0x06" slot="0x00" function="0x0"/>
</controller>
<controller type="scsi" index="0" model="virtio-scsi">
<address type="pci" domain="0x0000" bus="0x05" slot="0x00" function="0x0"/>
</controller>
<interface type="network">
<mac address="52:54:00:27:88:9e"/>
<source network="default"/>
<model type="virtio"/>
<address type="pci" domain="0x0000" bus="0x01" slot="0x00" function="0x0"/>
</interface>
<serial type="pty">
<target type="isa-serial" port="0">
<model name="isa-serial"/>
</target>
</serial>
<console type="pty">
<target type="serial" port="0"/>
</console>
<input type="mouse" bus="ps2"/>
<input type="keyboard" bus="ps2"/>
<sound model="ich9">
<address type="pci" domain="0x0000" bus="0x00" slot="0x1b" function="0x0"/>
</sound>
<audio id="1" type="spice"/>
<hostdev mode="subsystem" type="pci" managed="yes">
<source>
<address domain="0x0000" bus="0x04" slot="0x00" function="0x0"/>
</source>
<boot order="1"/>
<address type="pci" domain="0x0000" bus="0x09" slot="0x00" function="0x0"/>
</hostdev>
<hostdev mode="subsystem" type="pci" managed="yes">
<source>
<address domain="0x0000" bus="0x03" slot="0x00" function="0x0"/>
</source>
<address type="pci" domain="0x0000" bus="0x03" slot="0x00" function="0x0" multifunction="on"/>
</hostdev>
<hostdev mode="subsystem" type="pci" managed="yes">
<source>
<address domain="0x0000" bus="0x03" slot="0x00" function="0x1"/>
</source>
<address type="pci" domain="0x0000" bus="0x03" slot="0x00" function="0x1"/>
</hostdev>
<redirdev bus="usb" type="spicevmc">
<address type="usb" bus="0" port="2"/>
</redirdev>
<redirdev bus="usb" type="spicevmc">
<address type="usb" bus="0" port="3"/>
</redirdev>
<memballoon model="virtio">
<address type="pci" domain="0x0000" bus="0x07" slot="0x00" function="0x0"/>
</memballoon>
</devices>
</domain>
and from the Logs I can give you the Qemu-command:
/usr/bin/qemu-system-x86_64 \
-name guest=ubuntu22.04,debug-threads=on \
-S \
-object '{"qom-type":"secret","id":"masterKey0","format":"raw","file":"/var/lib/libvirt/qemu/domain-6-ubuntu22.04/master-key.aes"}' \
-blockdev '{"driver":"file","filename":"/usr/share/OVMF/OVMF_CODE.fd","node-name":"libvirt-pflash0-storage","auto-read-only":true,"discard":"unmap"}' \
-blockdev '{"node-name":"libvirt-pflash0-format","read-only":true,"driver":"raw","file":"libvirt-pflash0-storage"}' \
-blockdev '{"driver":"file","filename":"/var/lib/libvirt/qemu/nvram/Ubuntu_VARS.fd","node-name":"libvirt-pflash1-storage","auto-read-only":true,"discard":"unmap">
-blockdev '{"node-name":"libvirt-pflash1-format","read-only":false,"driver":"raw","file":"libvirt-pflash1-storage"}' \
-machine pc-q35-6.2,usb=off,vmport=off,dump-guest-core=off,pflash0=libvirt-pflash0-format,pflash1=libvirt-pflash1-format,memory-backend=pc.ram \
-accel kvm \
-cpu host,migratable=on,kvm=off \
-m 16000 \
-object '{"qom-type":"memory-backend-ram","id":"pc.ram","size":16777216000}' \
-overcommit mem-lock=off \
-smp 16,sockets=16,cores=1,threads=1 \
-uuid 557c47b4-9f8a-40a0-b08b-44571d2d82ba \
-smbios 'type=0,vendor=INSYDE Corp.,version=03.03,date=03/27/2024,release=3.3' \
-smbios 'type=1,manufacturer=Framework,product=Laptop 16 (AMD Ryzen 7040 Series),version=A7,serial=DELETEDFORPRIVACY,sku=FRAGACCP07,family=16in Laptop' \
-smbios 'type=2,manufacturer=Framework,product=FRANMZCP07,version=A7,serial=DELETEDFORPRIVACY600XS,asset=*,location=*' \
-smbios type=3,manufacturer=Framework,version=A7,serial=DELETEDFORPRIVACY,asset=DELETEDFORPRIVACY,sku=FRAGACCP07 \
-smbios type=11,value=DELETEDFORPRIVACY,value=DELETEDFORPRIVACY,value=DELETEDFORPRIVACYP0,value=DELETEDFORPRIVACY,value=DELETEDFORPRIVACY,value=DELETEDFORPRIVACY>
-display none \
-no-user-config \
-nodefaults \
-chardev socket,id=charmonitor,fd=31,server=on,wait=off \
-mon chardev=charmonitor,id=monitor,mode=control \
-rtc base=utc,driftfix=slew \
-global kvm-pit.lost_tick_policy=delay \
-no-hpet \
-no-shutdown \
-global ICH9-LPC.disable_s3=1 \
-global ICH9-LPC.disable_s4=1 \
-boot strict=on \
-device pcie-root-port,port=8,chassis=1,id=pci.1,bus=pcie.0,multifunction=on,addr=0x1 \
-device pcie-root-port,port=9,chassis=2,id=pci.2,bus=pcie.0,addr=0x1.0x1 \
-device pcie-root-port,port=10,chassis=3,id=pci.3,bus=pcie.0,addr=0x1.0x2 \
-device pcie-root-port,port=11,chassis=4,id=pci.4,bus=pcie.0,addr=0x1.0x3 \
-device pcie-root-port,port=12,chassis=5,id=pci.5,bus=pcie.0,addr=0x1.0x4 \
-device pcie-root-port,port=13,chassis=6,id=pci.6,bus=pcie.0,addr=0x1.0x5 \
-device pcie-root-port,port=14,chassis=7,id=pci.7,bus=pcie.0,addr=0x1.0x6 \
-device pcie-root-port,port=15,chassis=8,id=pci.8,bus=pcie.0,addr=0x1.0x7 \
-device pcie-root-port,port=16,chassis=9,id=pci.9,bus=pcie.0,multifunction=on,addr=0x2 \
-device pcie-root-port,port=17,chassis=10,id=pci.10,bus=pcie.0,addr=0x2.0x1 \
-device pcie-root-port,port=18,chassis=11,id=pci.11,bus=pcie.0,addr=0x2.0x2 \
-device pcie-root-port,port=19,chassis=12,id=pci.12,bus=pcie.0,addr=0x2.0x3 \
-device pcie-root-port,port=20,chassis=13,id=pci.13,bus=pcie.0,addr=0x2.0x4 \
-device pcie-root-port,port=21,chassis=14,id=pci.14,bus=pcie.0,addr=0x2.0x5 \
-device qemu-xhci,p2=15,p3=15,id=usb,bus=pci.2,addr=0x0 \
-device virtio-scsi-pci,id=scsi0,bus=pci.5,addr=0x0 \
-device virtio-serial-pci,id=virtio-serial0,bus=pci.6,addr=0x0 \
-blockdev '{"driver":"file","filename":"/var/lib/libvirt/images/ubuntu22.04.qcow2","node-name":"libvirt-3-storage","auto-read-only":true,"discard":"unmap"}' \
-blockdev '{"node-name":"libvirt-3-format","read-only":false,"discard":"unmap","driver":"qcow2","file":"libvirt-3-storage","backing":null}' \
-device virtio-blk-pci,bus=pci.4,addr=0x0,drive=libvirt-3-format,id=virtio-disk0 \
-blockdev '{"driver":"file","filename":"/home/matthias/Schreibtisch/Win11_23H2_German_x64v2.iso","node-name":"libvirt-2-storage","auto-read-only":true,"discard":>
-blockdev '{"node-name":"libvirt-2-format","read-only":true,"driver":"raw","file":"libvirt-2-storage"}' \
-device ide-cd,bus=ide.0,drive=libvirt-2-format,id=sata0-0-0 \
-blockdev '{"driver":"file","filename":"/home/matthias/Downloads/virtio-win-0.1.240.iso","node-name":"libvirt-1-storage","auto-read-only":true,"discard":"unmap"}>
-blockdev '{"node-name":"libvirt-1-format","read-only":true,"driver":"raw","file":"libvirt-1-storage"}' \
-device ide-cd,bus=ide.1,drive=libvirt-1-format,id=sata0-0-1 \
-netdev tap,fd=33,id=hostnet0,vhost=on,vhostfd=35 \
-device virtio-net-pci,netdev=hostnet0,id=net0,mac=52:54:00:27:88:9e,bus=pci.1,addr=0x0 \
-chardev pty,id=charserial0 \
-device isa-serial,chardev=charserial0,id=serial0 \
-audiodev '{"id":"audio1","driver":"spice"}' \
-device ich9-intel-hda,id=sound0,bus=pcie.0,addr=0x1b \
-device hda-duplex,id=sound0-codec0,bus=sound0.0,cad=0,audiodev=audio1 \
-chardev spicevmc,id=charredir0,name=usbredir \
-device usb-redir,chardev=charredir0,id=redir0,bus=usb.0,port=2 \
-chardev spicevmc,id=charredir1,name=usbredir \
-device usb-redir,chardev=charredir1,id=redir1,bus=usb.0,port=3 \
-device vfio-pci,host=0000:04:00.0,id=hostdev0,bootindex=1,bus=pci.9,addr=0x0 \
-device vfio-pci,host=0000:03:00.0,id=hostdev1,bus=pci.3,multifunction=on,addr=0x0 \
-device vfio-pci,host=0000:03:00.1,id=hostdev2,bus=pci.3,addr=0x0.0x1 \
-device virtio-balloon-pci,id=balloon0,bus=pci.7,addr=0x0 \
-sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny \
-msg timestamp=on
Do you guys have any ideas on how to get it working?
Cheers,
Matthias