Error 43 with a Nvidia card on the first pcie slot

I’m trying to passthrough a Nvidia RTX2080 card to a Windows 10 guest (host PopOs 19.10). I can passthrough the card successfully, if I put it into the second pcie slot on my motherboard. But if I put the card into the first pcie slot, then I unfortunately encounter the error 43 on Windows (the error shows up in the Windows’ device manager). I would prefer to put the Nvidia card onto the first slot, because of a better placement/cooling.

Update:
I even tried to pass the GPU’s Bios to the VM (using this guide https://passthroughpo.st/explaining-csm-efifboff-setting-boot-gpu-manually/ ). I dumped the vBios with another computer and also tried to download the vBios from techpowerup. But in both cases, I still get the error 43. So I dont know how to solve the error 43 problem.

I have added these lines to the VM’s xml-file, so that the Nvidia card wouldnt find out that it is running on a VM:

<hyperv>
   ...
   <vendor_id state="on" value="abcdefgh"/>
   ...
</hyperv>
<kvm>      
     <hidden state='on'/>
</kvm>
<features>
   ...
   <ioapic driver="kvm"/>
</features>

But unfortunately this hasnt helped also.

Here’s the full xml file just in case

<domain type="kvm">
  <name>win10</name>
  <uuid>180397c6-e903-4618-9f94-7aa3e29cb345</uuid>
  <metadata>
    <libosinfo:libosinfo xmlns:libosinfo="http://libosinfo.org/xmlns/libvirt/domain/1.0">
      <libosinfo:os id="http://microsoft.com/win/10"/>
    </libosinfo:libosinfo>
  </metadata>
  <memory unit="KiB">33554432</memory>
  <currentMemory unit="KiB">33554432</currentMemory>
  <vcpu placement="static">24</vcpu>
  <iothreads>3</iothreads>
  <cputune>
    <vcpupin vcpu="0" cpuset="4"/>
    <vcpupin vcpu="1" cpuset="20"/>
    <vcpupin vcpu="2" cpuset="5"/>
    <vcpupin vcpu="3" cpuset="21"/>
    <vcpupin vcpu="4" cpuset="6"/>
    <vcpupin vcpu="5" cpuset="22"/>
    <vcpupin vcpu="6" cpuset="7"/>
    <vcpupin vcpu="7" cpuset="23"/>
    <vcpupin vcpu="8" cpuset="8"/>
    <vcpupin vcpu="9" cpuset="24"/>
    <vcpupin vcpu="10" cpuset="9"/>
    <vcpupin vcpu="11" cpuset="25"/>
    <vcpupin vcpu="12" cpuset="10"/>
    <vcpupin vcpu="13" cpuset="26"/>
    <vcpupin vcpu="14" cpuset="11"/>
    <vcpupin vcpu="15" cpuset="27"/>
    <vcpupin vcpu="16" cpuset="12"/>
    <vcpupin vcpu="17" cpuset="28"/>
    <vcpupin vcpu="18" cpuset="13"/>
    <vcpupin vcpu="19" cpuset="29"/>
    <vcpupin vcpu="20" cpuset="14"/>
    <vcpupin vcpu="21" cpuset="30"/>
    <vcpupin vcpu="22" cpuset="15"/>
    <vcpupin vcpu="23" cpuset="31"/>
    <emulatorpin cpuset="0-1"/>
    <iothreadpin iothread="1" cpuset="2-3"/>
    <iothreadpin iothread="2" cpuset="16-17"/>
    <iothreadpin iothread="3" cpuset="18-19"/>
  </cputune>
  <os>
    <type arch="x86_64" machine="pc-q35-4.0">hvm</type>
    <loader readonly="yes" type="pflash">/usr/share/OVMF/OVMF_CODE.fd</loader>
    <nvram>/var/lib/libvirt/qemu/nvram/win10_VARS.fd</nvram>
    <bootmenu enable="yes"/>
  </os>
  <features>
    <acpi/>
    <apic/>
    <hyperv>
      <relaxed state="on"/>
      <vapic state="on"/>
      <spinlocks state="on" retries="8191"/>
      <vpindex state="on"/>
      <synic state="on"/>
      <stimer state="on"/>
      <reset state="on"/>
      <vendor_id state="on" value="abcdefgh"/>
      <frequencies state="on"/>
    </hyperv>
    <kvm>
      <hidden state="on"/>
    </kvm>
    <vmport state="off"/>
    <ioapic driver="kvm"/>
  </features>
  <cpu mode="host-passthrough" check="none">
    <topology sockets="1" cores="12" threads="2"/>
    <cache mode="passthrough"/>
    <feature policy="require" name="topoext"/>
  </cpu>
  <clock offset="localtime">
    <timer name="rtc" tickpolicy="catchup"/>
    <timer name="pit" tickpolicy="delay"/>
    <timer name="hpet" present="no"/>
    <timer name="hypervclock" present="yes"/>
  </clock>
  <on_poweroff>destroy</on_poweroff>
  <on_reboot>restart</on_reboot>
  <on_crash>destroy</on_crash>
  <pm>
    <suspend-to-mem enabled="no"/>
    <suspend-to-disk enabled="no"/>
  </pm>
  <devices>
    <emulator>/usr/bin/qemu-system-x86_64</emulator>
    <disk type="file" device="cdrom">
      <driver name="qemu" type="raw"/>
      <source file="/home/username/VMs/Media/Win10_1909_Finnish_x64.iso"/>
      <target dev="sdb" bus="sata"/>
      <readonly/>
      <boot order="1"/>
      <address type="drive" controller="0" bus="0" target="0" unit="1"/>
    </disk>
    <disk type="file" device="cdrom">
      <driver name="qemu" type="raw"/>
      <source file="/home/username/VMs/Media/virtio-win-0.1.171.iso"/>
      <target dev="sdc" bus="sata"/>
      <readonly/>
      <boot order="2"/>
      <address type="drive" controller="0" bus="0" target="0" unit="2"/>
    </disk>
    <disk type="file" device="disk">
      <driver name="qemu" type="raw"/>
      <source file="/home/username/VMs/win10.img"/>
      <target dev="vda" bus="virtio"/>
      <boot order="3"/>
      <address type="pci" domain="0x0000" bus="0x0a" slot="0x00" function="0x0"/>
    </disk>
    <controller type="usb" index="0" model="qemu-xhci" ports="15">
      <address type="pci" domain="0x0000" bus="0x02" slot="0x00" function="0x0"/>
    </controller>
    <controller type="sata" index="0">
      <address type="pci" domain="0x0000" bus="0x00" slot="0x1f" function="0x2"/>
    </controller>
    <controller type="pci" index="0" model="pcie-root"/>
    <controller type="pci" index="1" model="pcie-root-port">
      <model name="pcie-root-port"/>
      <target chassis="1" port="0x10"/>
      <address type="pci" domain="0x0000" bus="0x00" slot="0x02" function="0x0" multifunction="on"/>
    </controller>
    <controller type="pci" index="2" model="pcie-root-port">
      <model name="pcie-root-port"/>
      <target chassis="2" port="0x11"/>
      <address type="pci" domain="0x0000" bus="0x00" slot="0x02" function="0x1"/>
    </controller>
    <controller type="pci" index="3" model="pcie-root-port">
      <model name="pcie-root-port"/>
      <target chassis="3" port="0x12"/>
      <address type="pci" domain="0x0000" bus="0x00" slot="0x02" function="0x2"/>
    </controller>
    <controller type="pci" index="4" model="pcie-root-port">
      <model name="pcie-root-port"/>
      <target chassis="4" port="0x13"/>
      <address type="pci" domain="0x0000" bus="0x00" slot="0x02" function="0x3"/>
    </controller>
    <controller type="pci" index="5" model="pcie-root-port">
      <model name="pcie-root-port"/>
      <target chassis="5" port="0x14"/>
      <address type="pci" domain="0x0000" bus="0x00" slot="0x02" function="0x4"/>
    </controller>
    <controller type="pci" index="6" model="pcie-root-port">
      <model name="pcie-root-port"/>
      <target chassis="6" port="0x15"/>
      <address type="pci" domain="0x0000" bus="0x00" slot="0x02" function="0x5"/>
    </controller>
    <controller type="pci" index="7" model="pcie-root-port">
      <model name="pcie-root-port"/>
      <target chassis="7" port="0x16"/>
      <address type="pci" domain="0x0000" bus="0x00" slot="0x02" function="0x6"/>
    </controller>
    <controller type="pci" index="8" model="pcie-root-port">
      <model name="pcie-root-port"/>
      <target chassis="8" port="0x17"/>
      <address type="pci" domain="0x0000" bus="0x00" slot="0x02" function="0x7"/>
    </controller>
    <controller type="pci" index="9" model="pcie-root-port">
      <model name="pcie-root-port"/>
      <target chassis="9" port="0x18"/>
      <address type="pci" domain="0x0000" bus="0x00" slot="0x03" function="0x0" multifunction="on"/>
    </controller>
    <controller type="pci" index="10" model="pcie-root-port">
      <model name="pcie-root-port"/>
      <target chassis="10" port="0x19"/>
      <address type="pci" domain="0x0000" bus="0x00" slot="0x03" function="0x1"/>
    </controller>
    <controller type="pci" index="11" model="pcie-root-port">
      <model name="pcie-root-port"/>
      <target chassis="11" port="0x1a"/>
      <address type="pci" domain="0x0000" bus="0x00" slot="0x03" function="0x2"/>
    </controller>
    <controller type="pci" index="12" model="pcie-to-pci-bridge">
      <model name="pcie-pci-bridge"/>
      <address type="pci" domain="0x0000" bus="0x01" slot="0x00" function="0x0"/>
    </controller>
    <controller type="pci" index="13" model="pcie-root-port">
      <model name="pcie-root-port"/>
      <target chassis="13" port="0x1b"/>
      <address type="pci" domain="0x0000" bus="0x00" slot="0x03" function="0x3"/>
    </controller>
    <controller type="virtio-serial" index="0">
      <address type="pci" domain="0x0000" bus="0x03" slot="0x00" function="0x0"/>
    </controller>
    <interface type="network">
      <mac address="52:54:00:5c:38:24"/>
      <source network="default"/>
      <model type="rtl8139"/>
      <address type="pci" domain="0x0000" bus="0x0c" slot="0x01" function="0x0"/>
    </interface>
    <channel type="spicevmc">
      <target type="virtio" name="com.redhat.spice.0"/>
      <address type="virtio-serial" controller="0" bus="0" port="1"/>
    </channel>
    <input type="mouse" bus="ps2"/>
    <input type="keyboard" bus="ps2"/>
    <graphics type="spice" autoport="yes">
      <listen type="address"/>
      <image compression="off"/>
    </graphics>
    <sound model="ich9">
      <address type="pci" domain="0x0000" bus="0x00" slot="0x1b" function="0x0"/>
    </sound>
    <video>
      <model type="qxl" ram="65536" vram="65536" vgamem="16384" heads="1" primary="yes"/>
      <address type="pci" domain="0x0000" bus="0x00" slot="0x01" function="0x0"/>
    </video>
    <hostdev mode="subsystem" type="pci" managed="yes">
      <source>
        <address domain="0x0000" bus="0x03" slot="0x00" function="0x0"/>
      </source>
      <address type="pci" domain="0x0000" bus="0x08" slot="0x00" function="0x0"/>
    </hostdev>
    <hostdev mode="subsystem" type="pci" managed="yes">
      <source>
        <address domain="0x0000" bus="0x32" slot="0x00" function="0x0"/>
      </source>
      <rom file="/home/username/MSI.RTX2080.8192.181018_1.rom"/>
      <address type="pci" domain="0x0000" bus="0x04" slot="0x00" function="0x0"/>
    </hostdev>
    <hostdev mode="subsystem" type="pci" managed="yes">
      <source>
        <address domain="0x0000" bus="0x32" slot="0x00" function="0x1"/>
      </source>
      <address type="pci" domain="0x0000" bus="0x05" slot="0x00" function="0x0"/>
    </hostdev>
    <hostdev mode="subsystem" type="pci" managed="yes">
      <source>
        <address domain="0x0000" bus="0x32" slot="0x00" function="0x2"/>
      </source>
      <address type="pci" domain="0x0000" bus="0x06" slot="0x00" function="0x0"/>
    </hostdev>
    <hostdev mode="subsystem" type="pci" managed="yes">
      <source>
        <address domain="0x0000" bus="0x32" slot="0x00" function="0x3"/>
      </source>
      <address type="pci" domain="0x0000" bus="0x07" slot="0x00" function="0x0"/>
    </hostdev>
    <redirdev bus="usb" type="spicevmc">
      <address type="usb" bus="0" port="2"/>
    </redirdev>
    <redirdev bus="usb" type="spicevmc">
      <address type="usb" bus="0" port="3"/>
    </redirdev>
    <memballoon model="virtio">
      <address type="pci" domain="0x0000" bus="0x09" slot="0x00" function="0x0"/>
    </memballoon>
  </devices>
</domain>

OLD MESSAGE (I got the romfile not found error solved already thanks to TheCakeIsNaOH, but I will leave the old message here, if you somebody else has similar problems. And for some reason I havent encountered anymore of the second error, so it is kinda “solved” for now):

I tried to fix the error 43 with the following guide ( https://passthroughpo.st/explaining-csm-efifboff-setting-boot-gpu-manually/ ). I downloaded bios that corresponded to my card model and also I tried to dump my gpu’s bios from another Windows computer with GPU-Z. But I have been getting these following two errors when I try to start up the VM. Here’s the most recent error that I encountered:

Error starting domain: internal error: qemu unexpectedly closed the monitor: 2020-05-04T14:29:29.897570Z qemu-system-x86_64: -device vfio-pci,host=31:00.0,id=hostdev1,bus=pci.4,addr=0x0,romfile=/home/username/MSI.RTX2080.8192.181018_1.rom: Failed to mmap 0000:31:00.0 BAR 3. Performance may be slow
2020-05-04T14:29:29.919654Z qemu-system-x86_64: -device vfio-pci,host=31:00.0,id=hostdev1,bus=pci.4,addr=0x0,romfile=/home/username/MSI.RTX2080.8192.181018_1.rom: failed to find romfile "/home/username/MSI.RTX2080.8192.181018_1.rom"

Traceback (most recent call last):
  File "/usr/share/virt-manager/virtManager/asyncjob.py", line 75, in cb_wrapper
    callback(asyncjob, *args, **kwargs)
  File "/usr/share/virt-manager/virtManager/asyncjob.py", line 111, in tmpcb
    callback(*args, **kwargs)
  File "/usr/share/virt-manager/virtManager/object/libvirtobject.py", line 66, in newfn
    ret = fn(self, *args, **kwargs)
  File "/usr/share/virt-manager/virtManager/object/domain.py", line 1279, in startup
    self._backend.create()
  File "/usr/lib/python3/dist-packages/libvirt.py", line 1080, in create
    if ret == -1: raise libvirtError ('virDomainCreate() failed', dom=self)
libvirt.libvirtError: internal error: qemu unexpectedly closed the monitor: 2020-05-04T14:29:29.897570Z qemu-system-x86_64: -device vfio-pci,host=31:00.0,id=hostdev1,bus=pci.4,addr=0x0,romfile=/home/username/MSI.RTX2080.8192.181018_1.rom: Failed to mmap 0000:31:00.0 BAR 3. Performance may be slow
2020-05-04T14:29:29.919654Z qemu-system-x86_64: -device vfio-pci,host=31:00.0,id=hostdev1,bus=pci.4,addr=0x0,romfile=/home/username/MSI.RTX2080.8192.181018_1.rom: failed to find romfile "/home/username/MSI.RTX2080.8192.181018_1.rom"

So I feel it has maybe something to do that the Qemu hasnt permission to access to files? I even tried to place the bios to /var/lib/libvirt/ . But I still got the same error.

This is the second error that I encountered (but I havent encountered it anymore, I dont know exactly why I havent got it anymore):

Error starting domain: internal error: connection closed due to keepalive timeout

Traceback (most recent call last):
  File "/usr/share/virt-manager/virtManager/asyncjob.py", line 75, in cb_wrapper
    callback(asyncjob, *args, **kwargs)
  File "/usr/share/virt-manager/virtManager/asyncjob.py", line 111, in tmpcb
    callback(*args, **kwargs)
  File "/usr/share/virt-manager/virtManager/object/libvirtobject.py", line 66, in newfn
    ret = fn(self, *args, **kwargs)
  File "/usr/share/virt-manager/virtManager/object/domain.py", line 1279, in startup
    self._backend.create()
  File "/usr/lib/python3/dist-packages/libvirt.py", line 1080, in create
    if ret == -1: raise libvirtError ('virDomainCreate() failed', dom=self)
libvirt.libvirtError: internal error: connection closed due to keepalive timeout

It has to be a bios from the exact model number of your card.

As a temporary test measure, try opening up the permissions to it(chmod 777 /path/to/romfile). If that works, then check what user qemu and libvirt are running as, then chown the file to that user, and chmod the permissions to something a bit more restrictive.

Aa sorry I meant the exact same model number.

Well hmm this is kinda weird. I put the romfile to " /var/lib/libvirt/", and “sudo chmod 777 /var/lib/libvirt/TU104.rom”, but still I get the same error posted in my first message that it doesnt find the romfile. It’s so weird, shouldnt the sudo chmod 777 allow access to everybody.

Normally it should. It could be an apparmor problem?

1 Like

Thanks for the quick reply! Yeah it seems to be a problem with the apparmor. I found a solution from here ( https://askubuntu.com/questions/985854/gpu-passthrough-problem-on-adding-dumped-rom ), so I moved the romfile under /usr/share/vgabios, and now the VM finds the romfile and the VM starts. Thanks for showing me to the right way!

But unfortunately, I still get the error 43 in Windows. I tried both the romfile that I dumped with another computer and then the romfile from techpowerup. But error 43 :frowning_face:

Edit: I also updated the first message on the topic that the romfile couldnt be found error has been now solved (but I left the original old message there, if anybody else encounters similar problems).

1 Like

Yay, I’ll keep /usr/share/vgabios in mind.

The startup logs are now clean with no warnings or errors?

1 Like

I think you are hitting the same error I did. You need to disable EFI frame buffer. It’s what keeping your GPU still bound to Linux host. I noticed with systemd that you can’t do video=vesa:off,efifb:off. Do just video=efifb:off. Remember, if you still see your mobo boot logo, you haven’t disabled framebuffer

1 Like

(Unfortunately have been bit busy, so a bit late reply)

I have the following command line arguments in “/etc/default/grub/”:
GRUB_CMDLINE_LINUX_DEFAULT=“quiet splash amd_iommu=on iommu=pt kvm_amd.npt=1 kvm_amd.avic=1 video=vesafb:off,efifb:off”

I tried without vesafb:off. But it unfortunately didn’t help.

Hmm. I can still see the motherboard boot logo on the Nvidia card (in first PCIE slot) even with or without vesafb:off. Or does your motherboard’s UEFI/BIOS allow you to set which PCIE slot / GPU to use by default? (Unfortunately I don’t have this option in Asrock X570 Creator)

Actually I get the following error in “/var/log/libvirt/qemu/win10.log”:

ite(0000:31:00.0:region3+0x1f1b15, 0x0,1) failed: Device or resource busy
2020-08-11T10:29:09.740958Z qemu-system-x86_64: vfio_region_write(0000:31:00.0:region3+0x1f1b16, 0x0,1) failed: Device or resource busy
2020-08-11T10:29:09.740964Z qemu-system-x86_64: vfio_region_write(0000:31:00.0:region3+0x1f1b17, 0x0,1) failed: Device or resource busy
2020-08-11T10:29:09.740972Z qemu-system-x86_64: vfio_region_write(0000:31:00.0:region3+0x1f1b18, 0x0,1) failed: Device or resource busy
2020-08-11T10:29:09.740980Z qemu-system-x86_64: vfio_region_write(0000:31:00.0:region3+0x1f1b19, 0x0,1) failed: Device or resource busy
2020-08-11T10:29:09.740987Z qemu-system-x86_64: vfio_region_write(0000:31:00.0:region3+0x1f1b1a, 0x0,1) failed: Device or resource busy
2020-08-11T10:29:09.740994Z qemu-system-x86_64: vfio_region_write(0000:31:00.0:region3+0x1f1b1b, 0x0,1) failed: Device or resource busy
2020-08-11T10:29:09.741000Z qemu-system-x86_64: vfio_region_write(0000:31:00.0:region3+0x1f1b1c, 0x0,1) failed: Device or resource busy
2020-08-11T10:29:09.741005Z qemu-system-x86_64: vfio_region_write(0000:31:00.0:region3+0x1f1b1d, 0x0,1) failed: Device or resource busy
2020-08-11T10:29:09.741011Z qemu-system-x86_64: vfio_region_write(0000:31:00.0:region3+0x1f1b1e, 0x0,1) failed: Device or resource busy
2020-08-11T10:29:09.741017Z qemu-system-x86_64: vfio_region_write(0000:31:00.0:region3+0x1f1b1f, 0x0,1) failed: Device or resource busy
2020-08-11T10:29:09.741022Z qemu-system-x86_64: vfio_region_write(0000:31:00.0:region3+0x1f1b20, 0x0,1) failed: Device or resource busy
2020-08-11T10:29:09.741028Z qemu-system-x86_64: vfio_region_write(0000:31:00.0:region3+0x1f1b21, 0x0,1) failed: Device or resource busy

So I’m not sure what the problem is. The error was the same if I provided a vBios or not (the error didn’t change). I read somewhere that you might need to alter the vBios before passing it to VM (remove some part of the vBios), but I haven’t yet found a clear guide for doing that. And I’m not sure would that fix this error (and the error 43 in the Windows guest OS).