This might be a little esoteric for most folks but appears to be some sort of looking-glass issue.
Recently I enabled virtio-fs to get better filesystem performance. It appears to work great - definitely WAY faster than samba fileshares. However, this requires setting up a <numa> node (I’m using virsh/libvirt VMM). My preferred configuration is for 32GB RAM allocated to this gpu-passthru guest.
This causes a problem that prevents the machine from starting, generating the following error:
Error starting domain: internal error: qemu unexpectedly closed the monitor: 2021-05-13T05:49:30.856608Z qemu-system-x86_64: -device vfio-pci,host=0000:01:00.0,id=hostdev0,bus=pci.6,addr=0x0: VFIO_MAP_DMA failed: Bad address 2021-05-13T05:49:31.147518Z qemu-system-x86_64: -device vfio-pci,host=0000:01:00.0,id=hostdev0,bus=pci.6,addr=0x0: vfio 0000:01:00.0: failed to setup container for group 1: memory listener initialization failed: Region ram-node0: vfio_dma_map(0x55c8f84f5480, 0x100000000, 0x780000000, 0x7fbf4bfff000) = -14 (Bad address)
However, when the VM is configured for 24GB RAM, it works.
The VM is started with the following command line (in this case it’s the 24GB config):
/usr/bin/qemu-system-x86_64 -name guest=gpu-template,debug-threads=on -S -object secret,id=masterKey0,format=raw,file=/var/lib/libvirt/qemu/domain-15-gpu-template/master-key.aes -blockdev {“driver”:“file”,“filename”:"/usr/share/OVMF/OVMF_CODE.fd",“node-name”:“libvirt-pflash0-storage”,“auto-read-only”:true,“discard”:“unmap”} -blockdev {“node-name”:“libvirt-pflash0-format”,“read-only”:true,“driver”:“raw”,“file”:“libvirt-pflash0-storage”} -blockdev {“driver”:“file”,“filename”:"/mnt/images/gpu-template/gpu-template_VARS.fd",“node-name”:“libvirt-pflash1-storage”,“auto-read-only”:true,“discard”:“unmap”} -blockdev {“node-name”:“libvirt-pflash1-format”,“read-only”:false,“driver”:“raw”,“file”:“libvirt-pflash1-storage”} -machine pc-q35-hirsute,accel=kvm,usb=off,vmport=off,dump-guest-core=off,pflash0=libvirt-pflash0-format,pflash1=libvirt-pflash1-format -cpu Skylake-Client-IBRS,ss=on,vmx=on,pdcm=on,hypervisor=on,tsc-adjust=on,clflushopt=on,umip=on,md-clear=on,stibp=on,arch-capabilities=on,ssbd=on,xsaves=on,pdpe1gb=on,ibpb=on,ibrs=on,amd-stibp=on,amd-ssbd=on,skip-l1dfl-vmentry=on,pschange-mc-no=on,hv-time,hv-relaxed,hv-vapic,hv-spinlocks=0x1fff,hv-vendor-id=1234567890ab,kvm=off -m 24576 -overcommit mem-lock=off -smp 8,sockets=1,dies=1,cores=4,threads=2 -object memory-backend-file,id=ram-node0,mem-path=/dev/shm/libvirt/qemu/15-gpu-template/ram-node0,share=yes,size=25769803776 -numa node,nodeid=0,cpus=0-7,memdev=ram-node0 -uuid 0-0-0-0-0 -no-user-config -nodefaults -chardev socket,id=charmonitor,fd=34,server,nowait -mon chardev=charmonitor,id=monitor,mode=control -rtc base=localtime,driftfix=slew -global kvm-pit.lost_tick_policy=delay -no-hpet -no-shutdown -global ICH9-LPC.disable_s3=1 -global ICH9-LPC.disable_s4=1 -boot strict=on -device pcie-root-port,port=0x10,chassis=1,id=pci.1,bus=pcie.0,multifunction=on,addr=0x2 -device pcie-root-port,port=0x11,chassis=2,id=pci.2,bus=pcie.0,addr=0x2.0x1 -device pcie-root-port,port=0x12,chassis=3,id=pci.3,bus=pcie.0,addr=0x2.0x2 -device pcie-root-port,port=0x13,chassis=4,id=pci.4,bus=pcie.0,addr=0x2.0x3 -device pcie-root-port,port=0x14,chassis=5,id=pci.5,bus=pcie.0,addr=0x2.0x4 -device pcie-root-port,port=0x15,chassis=6,id=pci.6,bus=pcie.0,addr=0x2.0x5 -device pcie-root-port,port=0x16,chassis=7,id=pci.7,bus=pcie.0,addr=0x2.0x6 -device pcie-root-port,port=0x17,chassis=8,id=pci.8,bus=pcie.0,addr=0x2.0x7 -device pcie-root-port,port=0x18,chassis=9,id=pci.9,bus=pcie.0,multifunction=on,addr=0x3 -device pcie-root-port,port=0x19,chassis=10,id=pci.10,bus=pcie.0,addr=0x3.0x1 -device pcie-pci-bridge,id=pci.11,bus=pci.10,addr=0x0 -device qemu-xhci,p2=15,p3=15,id=usb,bus=pci.2,addr=0x0 -device virtio-serial-pci,id=virtio-serial0,bus=pci.3,addr=0x0 -blockdev {“driver”:“file”,“filename”:"/mnt/images/gpu-template/gpu-template.img",“aio”:“threads”,“node-name”:“libvirt-1-storage”,“cache”:{“direct”:true,“no-flush”:false},“auto-read-only”:true,“discard”:“unmap”} -blockdev {“node-name”:“libvirt-1-format”,“read-only”:false,“cache”:{“direct”:true,“no-flush”:false},“driver”:“raw”,“file”:“libvirt-1-storage”} -device virtio-blk-pci,bus=pci.5,addr=0x0,drive=libvirt-1-format,id=virtio-disk0,bootindex=1,write-cache=on -chardev socket,id=chr-vu-fs0,path=/var/lib/libvirt/qemu/domain-15-gpu-template/fs0-fs.sock -device vhost-user-fs-pci,chardev=chr-vu-fs0,tag=Shared,bus=pci.8,addr=0x0 -netdev tap,fd=37,id=hostnet0,vhost=on,vhostfd=38 -device virtio-net-pci,netdev=hostnet0,id=net0,mac=52:54:00:00:00:03,bus=pci.1,addr=0x0 -chardev pty,id=charserial0 -device isa-serial,chardev=charserial0,id=serial0 -chardev spicevmc,id=charchannel0,name=vdagent -device virtserialport,bus=virtio-serial0.0,nr=1,chardev=charchannel0,id=channel0,name=com.redhat.spice.0 -device virtio-keyboard-pci,id=input2,bus=pci.9,addr=0x0 -spice port=5903,addr=127.0.0.1,disable-ticketing,image-compression=off,seamless-migration=on -device ich9-intel-hda,id=sound0,bus=pcie.0,addr=0x1b -device hda-duplex,id=sound0-codec0,bus=sound0.0,cad=0 -chardev spicevmc,id=charredir0,name=usbredir -device usb-redir,chardev=charredir0,id=redir0,bus=usb.0,port=1 -chardev spicevmc,id=charredir1,name=usbredir -device usb-redir,chardev=charredir1,id=redir1,bus=usb.0,port=2 -device vfio-pci,host=0000:01:00.0,id=hostdev0,bus=pci.6,addr=0x0 -device vfio-pci,host=0000:01:00.1,id=hostdev1,bus=pci.7,addr=0x0 -device usb-host,hostdevice=/dev/bus/usb/001/006,id=hostdev2,bus=usb.0,port=3 -device usb-host,hostdevice=/dev/bus/usb/001/005,id=hostdev3,bus=usb.0,port=4 -device usb-host,hostdevice=/dev/bus/usb/001/009,id=hostdev4,bus=usb.0,port=6 -device virtio-balloon-pci,id=balloon0,bus=pci.4,addr=0x0 -sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny -object memory-backend-file,id=shmmem-shmem0,mem-path=/dev/shm/looking-glass,size=67108864,share=yes -device ivshmem-plain,id=shmem0,memdev=shmmem-shmem0,bus=pci.11,addr=0x1 -msg timestamp=on
I see another mention of this issue here:
1830091 – VM does not start with PCI passthrough (comment 6)
Any suggestions for how to resolve or work around this issue?