QEMU Windows 10 AMD RX480 Passthrough [Code 43]

woodcat · July 25, 2020, 8:07am

Host: Arch Linux (latest), Ryzen 3600, Host GPU: RX580, Guest GPU: RX480

I have an RX480 that I’m passing through to the guest, but after installing the AMD drivers on the guest the graphics card gives code 43. After a few hours of digging, I’m no closer to a solution. This kind of issue is pretty foreign to me so I’m not sure what other information would be useful to include here.

This is the current state of my xml for virt-manager:

<domain type="kvm">
  <name>win10-dev</name>
...
  <memory unit="KiB">8388608</memory>
  <currentMemory unit="KiB">8388608</currentMemory>
  <vcpu placement="static">4</vcpu>
  <os>
    <type arch="x86_64" machine="pc-q35-5.0">hvm</type>
    <loader readonly="yes" type="pflash">/usr/share/edk2-ovmf/x64/OVMF_CODE.fd</loader>
    <nvram>/var/lib/libvirt/qemu/nvram/win10-dev_VARS.fd</nvram>
    <bootmenu enable="no"/>
  </os>
  <features>
    <acpi/>
    <apic/>
    <hyperv>
      <relaxed state="on"/>
      <vapic state="on"/>
      <spinlocks state="on" retries="8191"/>
    </hyperv>
    <kvm>
      <hidden state="on"/>
    </kvm>
    <vmport state="off"/>
  </features>
  <cpu mode="custom" match="exact" check="none">
    <model fallback="forbid">EPYC</model>
    <topology sockets="1" dies="1" cores="2" threads="2"/>
  </cpu>
  <clock offset="localtime">
    <timer name="rtc" tickpolicy="catchup"/>
    <timer name="pit" tickpolicy="delay"/>
    <timer name="hpet" present="no"/>
    <timer name="hypervclock" present="yes"/>
  </clock>
  <on_poweroff>destroy</on_poweroff>
  <on_reboot>restart</on_reboot>
  <on_crash>destroy</on_crash>
  <pm>
    <suspend-to-mem enabled="no"/>
    <suspend-to-disk enabled="no"/>
   </pm>
   <devices>
    <emulator>/usr/bin/qemu-system-x86_64</emulator>
    <disk type="file" device="disk">
      <driver name="qemu" type="qcow2" cache="writeback"/>
      <source file="/media/virtual-machines/win10-dev.qcow2"/>
      <target dev="sda" bus="sata"/>
      <boot order="1"/>
      <address type="drive" controller="0" bus="0" target="0" unit="0"/>
    </disk>
    <controller type="usb" index="0" model="qemu-xhci" ports="15">
      <address type="pci" domain="0x0000" bus="0x02" slot="0x00" function="0x0"/>
    </controller>
    <controller type="sata" index="0">
      <address type="pci" domain="0x0000" bus="0x00" slot="0x1f" function="0x2"/>
    </controller>
    <controller type="pci" index="0" model="pcie-root"/>
    <controller type="pci" index="1" model="pcie-root-port">
      <model name="pcie-root-port"/>
      <target chassis="1" port="0x10"/>
      <address type="pci" domain="0x0000" bus="0x00" slot="0x02" function="0x0" multifunction="on"/>
    </controller>
    <controller type="pci" index="2" model="pcie-root-port">
      <model name="pcie-root-port"/>
      <target chassis="2" port="0x11"/>
      <address type="pci" domain="0x0000" bus="0x00" slot="0x02" function="0x1"/>
    </controller>
    <controller type="pci" index="3" model="pcie-root-port">
      <model name="pcie-root-port"/>
      <target chassis="3" port="0x12"/>
      <address type="pci" domain="0x0000" bus="0x00" slot="0x02" function="0x2"/>
    </controller>
    <controller type="pci" index="4" model="pcie-root-port">
      <model name="pcie-root-port"/>
      <target chassis="4" port="0x13"/>
      <address type="pci" domain="0x0000" bus="0x00" slot="0x02" function="0x3"/>
    </controller>
    <controller type="pci" index="5" model="pcie-root-port">
      <model name="pcie-root-port"/>
      <target chassis="5" port="0x14"/>
      <address type="pci" domain="0x0000" bus="0x00" slot="0x02" function="0x4"/>
    </controller>
    <controller type="pci" index="6" model="pcie-root-port">
      <model name="pcie-root-port"/>
      <target chassis="6" port="0x15"/>
      <address type="pci" domain="0x0000" bus="0x00" slot="0x02" function="0x5"/>
    </controller>
    <controller type="pci" index="7" model="pcie-root-port">
      <model name="pcie-root-port"/>
      <target chassis="7" port="0x16"/>
      <address type="pci" domain="0x0000" bus="0x00" slot="0x02" function="0x6"/>
    </controller>
    <controller type="pci" index="8" model="pcie-to-pci-bridge">
      <model name="pcie-pci-bridge"/>
      <address type="pci" domain="0x0000" bus="0x06" slot="0x00" function="0x0"/>
    </controller>
    <controller type="pci" index="9" model="pcie-root-port">
      <model name="pcie-root-port"/>
      <target chassis="9" port="0x17"/>
      <address type="pci" domain="0x0000" bus="0x00" slot="0x02" function="0x7"/>
    </controller>
    <controller type="virtio-serial" index="0">
      <address type="pci" domain="0x0000" bus="0x03" slot="0x00" function="0x0"/>
    </controller>
    <interface type="network">
      <mac address="52:54:00:ad:7e:eb"/>
      <source network="default"/>
      <model type="e1000e"/>
      <address type="pci" domain="0x0000" bus="0x01" slot="0x00" function="0x0"/>
    </interface>
    <serial type="pty">
      <target type="isa-serial" port="0">
        <model name="isa-serial"/>
      </target>
    </serial>
    <console type="pty">
      <target type="serial" port="0"/>
    </console>
    <channel type="spicevmc">
      <target type="virtio" name="com.redhat.spice.0"/>
      <address type="virtio-serial" controller="0" bus="0" port="1"/>
    </channel>
    <input type="tablet" bus="usb">
      <address type="usb" bus="0" port="1"/>
    </input>
    <input type="mouse" bus="ps2"/>
    <input type="keyboard" bus="ps2"/>
    <graphics type="spice" autoport="yes">
      <listen type="address"/>
      <image compression="off"/>
    </graphics>
    <sound model="ich9">
      <address type="pci" domain="0x0000" bus="0x00" slot="0x1b" function="0x0"/>
    </sound>
    <video>
      <model type="qxl" ram="65536" vram="65536" vgamem="16384" heads="1" primary="yes"/>
      <address type="pci" domain="0x0000" bus="0x00" slot="0x01" function="0x0"/>
    </video>
    <hostdev mode="subsystem" type="pci" managed="yes">
      <source>
        <address domain="0x0000" bus="0x09" slot="0x00" function="0x0"/>
      </source>
      <rom bar="off"/>
      <address type="pci" domain="0x0000" bus="0x05" slot="0x00" function="0x0"/>
    </hostdev>
    <hostdev mode="subsystem" type="pci" managed="yes">
      <source>
        <address domain="0x0000" bus="0x09" slot="0x00" function="0x1"/>
      </source>
      <rom bar="off"/>
      <address type="pci" domain="0x0000" bus="0x07" slot="0x00" function="0x0"/>
    </hostdev>
    <redirdev bus="usb" type="spicevmc">
      <address type="usb" bus="0" port="2"/>
    </redirdev>
    <redirdev bus="usb" type="spicevmc">
      <address type="usb" bus="0" port="3"/>
    </redirdev>
    <memballoon model="virtio">
      <address type="pci" domain="0x0000" bus="0x04" slot="0x00" function="0x0"/>
    </memballoon>
    <shmem name="looking-glass">
      <model type="ivshmem-plain"/>
      <size unit="M">32</size>
      <address type="pci" domain="0x0000" bus="0x08" slot="0x01" function="0x0"/>
    </shmem>
  </devices>
</domain>

Pollomoolokki · July 25, 2020, 10:08am

Hello and welcome to lvl1techs!

I found these two reddit posts after some google-fu. I used the search term:

vfio amd code 43

amd_rx480_code_43_error

amd_gpu_passtrough_code_43

The solution to both problems seem to be preventing the kernel from initializing the guest GPU on boot.

woodcat · July 25, 2020, 6:27pm

Thanks for the reply, I have seen both of these posts already and have tried adding video=efifb:off to the grub command line to no apparent effect. I had to remove the nomodeset that other users were using because it kind of goofed my whole setup.

From my /etc/default/grub:
GRUB_CMDLINE_LINUX_DEFAULT="loglevel=3 amd_iommu=on iommu=pt nofb video=efifb:off"

(yes, I remembered to sudo grub-mkconfig -o /boot/grub/grub.cfg)

If the solution is still through grub, I do not know it. The card is grabbed by vfio properly though, so I really have no idea what I’m missing here:
09:00.0 VGA compatible controller [0300]: Advanced Micro Devices, Inc. [AMD/ATI] Ellesmere [Radeon RX 470/480/570/570X/580/580X/590] [1002:67df] (rev c7) (prog-if 00 [VGA controller]) Subsystem: Gigabyte Technology Co., Ltd Device [1458:22d4] Flags: fast devsel, IRQ 15, IOMMU group 14 Memory at c0000000 (64-bit, prefetchable) [disabled] [size=256M] Memory at d0000000 (64-bit, prefetchable) [disabled] [size=2M] I/O ports at c000 [disabled] [size=256] Memory at fce00000 (32-bit, non-prefetchable) [disabled] [size=256K] Expansion ROM at fce40000 [disabled] [size=128K] Capabilities: <access denied> Kernel driver in use: vfio-pci Kernel modules: amdgpu

09:00.1 Audio device [0403]: Advanced Micro Devices, Inc. [AMD/ATI] Ellesmere HDMI Audio [Radeon RX 470/480 / 570/580/590] [1002:aaf0] Subsystem: Gigabyte Technology Co., Ltd Device [1458:aaf0] Flags: fast devsel, IRQ 5, IOMMU group 14 Memory at fce60000 (64-bit, non-prefetchable) [disabled] [size=16K] Capabilities: <access denied> Kernel driver in use: vfio-pci Kernel modules: snd_hda_intel

Pollomoolokki · July 25, 2020, 7:17pm

sudo grub-mkconfig -o /boot/grub/grub.cfg

I guess it’s just different on arch - but on ubuntu I’m used to running “update-grub” Probs just different command depending on the distro/boot loader. (On ubuntu the grub-file resides in different place. /etc/default/grub iirc)

I did some more googling.

Also try disabling “legacy boot” (sometimes called “CSM”) in your BIOS

pci-passthrough-error-code-43

Spice may give trouble when passing through a GPU as it presents a “virtual” PCI graphic card to the guest and some drivers have problems with that, even when both cards show up. It’s always worth a try to disable SPICE and check again if something fails.

Pci_passthrough

You could try to rebind the GPU and its audio via “pcie-root-port” or “ioh3420”

so_how_should_i_deal_with_code_43_but_for_amd

Error 43 can be caused by the GPU’s VBIOS without UEFI support.

UEFI_(OVMF)_compatibility_in_VBIOS

Iirc I “updated” the VBIOS via libvirt by adding the path to the .rom file in the vm’s .XML. It’s detailed in the link - You could try the same. WARNING Flashing your bios is dangerous!

You seem to have got all the steps right so far- Usually the last step is the hardest I hope you get it working!

woodcat · July 25, 2020, 7:20pm

Thanks for the tips! I’ll definitely be working through these to see what works. If I figure it out I’ll be sure to post my solution and maybe even update the arch wiki.

My gut feeling is that the issue is with the spice client, but it could just as easily be a rootport issue. I don’t suspect and bios/uefi problems, but I need to update that anyways for other reasons so maybe I’ll try that too.

woodcat · July 27, 2020, 1:30am

Update:

Gave up on virt-manager + libvirt for the time being since it was making troubleshooting a pain. Running qemu directly with the following script yielded some new information.

#!/bin/bash

exec qemu-system-x86_64 \
	-name "vplank",debug-threads=on \
	-M q35,accel=kvm,kernel-irqchip=on \
	-m size=4G \
	-cpu host,+topoext,hv_relaxed,hv_spinlocks=0x1fff,hv_vapic,hv_time,hv_synic,hv_stimer,hv_vpindex,hv_vendor_id=MicroStarInc \
	-smp cores=2,threads=2 \
	-drive if=pflash,format=raw,readonly,file=/usr/share/ovmf/x64/OVMF_CODE.fd \
	-drive if=pflash,format=raw,file=ovmf_vars.fd \
	-usb \
	-drive file=win10.qcow2,format=qcow2 \
	-device pcie-root-port,id=gpu,chassis=0,slot=0,bus=pcie.0 \
	-device vfio-pci,host=09:00.0,bus=gpu,addr=00.0,multifunction=on \
	-device vfio-pci,host=09:00.1,bus=gpu,addr=00.1 \
	-display none \
	-vga none \
	-device virtio-net,netdev=network0 \
	-netdev tap,id=network0,ifname=tap0,script=no,downscript=no \
	-monitor unix:monitor.sock,server,nowait \
	-qmp unix:qmp.sock,server,nowait \
	-boot menu=off "$@" \

I now get this when trying to boot:

qemu-system-x86_64: vfio: Unable to power on device, stuck in D3

I believe the issue to be related to motherboard’s bios/uefi and will update it soon and attempt again, not tonight though because I’m exhausted already. Motherboard for reference is an ASUS Prime X470-Pro.

woodcat · August 10, 2020, 9:17pm

Updated bios, still no video output from the card, but also no more errors I think? I now need to try going back and seeing if error 43 still exists. I have no idea how to get video out either way, but it’s a start at least.

woodcat · August 10, 2020, 10:51pm

Still gives Error 43 in windows.

Problem: 0x1F
Problem Status: 0xC01E0438

Does anyone have any idea how to get this working? I’ll start from scratch if necessary but this is kind of getting ridiculous. At the very least a rough prognosis of how much more suffering this will take would be appreciated.

woodcat · August 10, 2020, 11:44pm

I think it’s been pretty well narrowed down to just not being possible on this card without some sort of firmware update that likely does not exist. New plan is to sell the card and find an nvidia one to use instead