Hi everyone,
I have a VM right now that is setup as my gaming machine with video pass-through, it works perfectly except when there's I/O on the host.
When there is I/O, the VM will have micro-freeze events where everything locks up for milliseconds up to a second at a time at frequent intervals, the host is on NVMe, CPUs are pinned with a dedicated core for emulator.
Whatever I put, virtio, virtio-scsi, cache=none, cache=writeback, threads or native, as soon as there is I/O, I get the same problem, to the point where it causes disconnections on the virtual USB bus.
VM is setup with hugepages, CPU pinning, I tried making qemu threads realtime with no success (even blue screens). I tried with dedicated cores (isolcpus), i440/q35, now I'm out of ideas. I do run a custom kernel to be able to use pcie_acs_override=downstream,multifunction in GRUB because I can't use passthrough otherwise.
This problem has been following me from my old setup to my new setup
Old setup:
ASUS B150 PLUS USB3
Core i7 6700K
16 GB DDR3
SATA raw LVM volume for Windows
New setup
ASRock Z390 Taichi
Core i7 9700K
48 GB DDR4
NVMe raw LVM volume for Windows
Any ideas would be appreciated.
Here is the libvirt XML definition (I excluded the buses):
<domain type='kvm'>
<name>win10-jerome</name>
<uuid>fc19f3c1-b8c6-4184-9b43-159157eff5b5</uuid>
<memory unit='KiB'>15360000</memory>
<currentMemory unit='KiB'>15360000</currentMemory>
<memoryBacking>
<hugepages/>
<nosharepages/>
<locked/>
</memoryBacking>
<vcpu placement='static'>4</vcpu>
<cputune>
<vcpupin vcpu='0' cpuset='0'/>
<vcpupin vcpu='1' cpuset='1'/>
<vcpupin vcpu='2' cpuset='2'/>
<vcpupin vcpu='3' cpuset='3'/>
<emulatorpin cpuset='7'/>
</cputune>
<resource>
<partition>/machine</partition>
</resource>
<os>
<type arch='x86_64' machine='pc-q35-4.0'>hvm</type>
<boot dev='hd'/>
<bootmenu enable='yes'/>
</os>
<features>
<acpi/>
<apic/>
<hyperv>
<relaxed state='on'/>
<vapic state='on'/>
<spinlocks state='on' retries='8191'/>
<vpindex state='on'/>
<runtime state='on'/>
<synic state='on'/>
<stimer state='on'/>
<reset state='on'/>
<frequencies state='on'/>
</hyperv>
<ioapic driver='kvm'/>
</features>
<cpu mode='host-passthrough' check='none'>
<topology sockets='1' cores='4' threads='1'/>
<feature policy='require' name='invtsc'/>
</cpu>
<clock offset='utc'>
<timer name='hypervclock' present='yes'/>
<timer name='hpet' present='yes'/>
</clock>
<devices>
<emulator>/usr/bin/kvm-spice</emulator>
<disk type='block' device='disk'>
<driver name='qemu' type='raw' cache='none' io='native'/>
<source dev='/dev/vgP4Root/win10jerome'/>
<backingStore/>
<target dev='sda' bus='sata'/>
<address type='drive' controller='0' bus='0' target='0' unit='0'/>
</disk>
<disk type='block' device='disk'>
<driver name='qemu' type='raw' cache='none' io='native' discard='ignore' detect_zeroes='off'/>
<source dev='/dev/vgP4Root/win10data'/>
<target dev='sdb' bus='scsi'/>
<address type='drive' controller='0' bus='0' target='0' unit='1'/>
</disk>
<disk type='block' device='disk'>
<driver name='qemu' type='raw' cache='none' io='native' discard='ignore' detect_zeroes='off'/>
<source dev='/dev/vgP4Root/win10swap'/>
<target dev='sdd' bus='scsi'/>
<address type='drive' controller='0' bus='0' target='0' unit='3'/>
</disk>
<interface type='direct'>
<mac address='52:54:00:31:1e:f9'/>
<source dev='eth0' mode='bridge'/>
<model type='virtio'/>
<address type='pci' domain='0x0000' bus='0x04' slot='0x00' function='0x0'/>
</interface>
<input type='mouse' bus='ps2'/>
<input type='keyboard' bus='ps2'/>
<graphics type='spice' autoport='yes' listen='127.0.0.1'>
<listen type='address' address='127.0.0.1'/>
</graphics>
<video>
<model type='qxl' ram='65536' vram='16384' vgamem='16384' heads='1' primary='yes'/>
<address type='pci' domain='0x00 <hostdev mode='subsystem' type='pci' managed='yes'>
</video>
<hostdev mode='subsystem' type='pci' managed='yes'>
<source>
<address domain='0x0000' bus='0x02' slot='0x00' function='0x0'/>
</source>
<rom bar='on'/>
<address type='pci' domain='0x0000' bus='0x02' slot='0x00' function='0x0' multifunction='on'/>
</hostdev>
<hostdev mode='subsystem' type='pci' managed='yes'>
<source>
<address domain='0x0000' bus='0x02' slot='0x00' function='0x1'/>
</source>
<rom bar='on'/>
<address type='pci' domain='0x0000' bus='0x02' slot='0x00' function='0x1'/>
</hostdev>
00' bus='0x00' slot='0x01' function='0x0'/>
<rng model='virtio'>
<backend model='random'>/dev/urandom</backend>
<address type='pci' domain='0x0000' bus='0x07' slot='0x00' function='0x0'/>
</rng>
</devices>
</domain>
Generated command line:
qemu-system-x86_64 -enable-kvm -name guest=win10-jerome,debug-threads=on -S -object secret,id=masterKey0,format=raw,file=/var/lib/libvirt/qemu/domain-22-win10-jerome/master-key.aes -machine pc-q35-4.0,accel=kvm,usb=off,dump-guest-core=off,mem-merge=off,kernel_irqchip=on -cpu host,invtsc=on,hv_time,hv_relaxed,hv_vapic,hv_spinlocks=0x1fff,hv_vpindex,hv_runtime,hv_synic,hv_stimer,hv_reset,hv_frequencies -m 15000 -mem-prealloc -mem-path /dev/hugepages/libvirt/qemu/22-win10-jerome -overcommit mem-lock=on -smp 4,sockets=1,cores=4,threads=1 -uuid fc19f3c1-b8c6-4184-9b43-159157eff5b5 -no-user-config -nodefaults -chardev socket,id=charmonitor,fd=25,server,nowait -mon chardev=charmonitor,id=monitor,mode=control -rtc base=utc -no-shutdown -global ICH9-LPC.disable_s3=0 -global ICH9-LPC.disable_s4=0 -boot menu=on,strict=on -device pcie-root-port,port=0x10,chassis=1,id=pci.1,bus=pcie.0,multifunction=on,addr=0x2 -device pcie-root-port,port=0x11,chassis=2,id=pci.2,bus=pcie.0,addr=0x2.0x1 -device pcie-root-port,port=0x12,chassis=3,id=pci.3,bus=pcie.0,addr=0x2.0x2 -device pcie-root-port,port=0x13,chassis=4,id=pci.4,bus=pcie.0,addr=0x2.0x3 -device pcie-root-port,port=0x14,chassis=5,id=pci.5,bus=pcie.0,addr=0x2.0x4 -device pcie-root-port,port=0x15,chassis=6,id=pci.6,bus=pcie.0,addr=0x2.0x5 -device pcie-root-port,port=0x16,chassis=7,id=pci.7,bus=pcie.0,addr=0x2.0x6 -device pcie-root-port,port=0x17,chassis=8,id=pci.8,bus=pcie.0,addr=0x2.0x7 -device i82801b11-bridge,id=pci.9,bus=pcie.0,addr=0x1e -device pci-bridge,chassis_nr=10,id=pci.10,bus=pci.9,addr=0x0 -device ich9-usb-ehci1,id=usb,bus=pcie.0,addr=0x1d.0x7 -device ich9-usb-uhci1,masterbus=usb.0,firstport=0,bus=pcie.0,multifunction=on,addr=0x1d -device ich9-usb-uhci2,masterbus=usb.0,firstport=2,bus=pcie.0,addr=0x1d.0x1 -device ich9-usb-uhci3,masterbus=usb.0,firstport=4,bus=pcie.0,addr=0x1d.0x2 -device virtio-scsi-pci,id=scsi0,bus=pci.5,addr=0x0 -device virtio-serial-pci,id=virtio-serial0,bus=pci.1,addr=0x0 -drive file=/dev/vgP4Root/win10jerome,format=raw,if=none,id=drive-sata0-0-0,cache=none,aio=native -device ide-hd,bus=ide.0,drive=drive-sata0-0-0,id=sata0-0-0,bootindex=1,write-cache=on -drive file=/dev/vgP4Root/win10data,format=raw,if=none,id=drive-scsi0-0-0-1,cache=none,discard=ignore,detect-zeroes=off,aio=native -device scsi-hd,bus=scsi0.0,channel=0,scsi-id=0,lun=1,device_id=drive-scsi0-0-0-1,drive=drive-scsi0-0-0-1,id=scsi0-0-0-1,write-cache=on -drive file=/dev/vgP4Root/win10swap,format=raw,if=none,id=drive-scsi0-0-0-3,cache=none,discard=ignore,detect-zeroes=off,aio=native -device scsi-hd,bus=scsi0.0,channel=0,scsi-id=0,lun=3,device_id=drive-scsi0-0-0-3,drive=drive-scsi0-0-0-3,id=scsi0-0-0-3,write-cache=on -netdev tap,fd=27,id=hostnet0,vhost=on,vhostfd=28 -device virtio-net-pci,netdev=hostnet0,id=net0,mac=52:54:00:31:1e:f9,bus=pci.4,addr=0x0 -spice port=5901,addr=127.0.0.1,disable-ticketing,seamless-migration=on -device qxl-vga,id=video0,ram_size=67108864,vram_size=16777216,vram64_size_mb=0,vgamem_mb=16,max_outputs=1,bus=pcie.0,addr=0x1 -chardev spicevmc,id=charredir0,name=usbredir -device usb-redir,chardev=charredir0,id=redir0,bus=usb.0,port=2 -chardev spicevmc,id=charredir1,name=usbredir -device usb-redir,chardev=charredir1,id=redir1,bus=usb.0,port=3 -device vfio-pci,host=02:00.0,id=hostdev0,bus=pci.2,multifunction=on,addr=0x0,rombar=1 -device vfio-pci,host=02:00.1,id=hostdev1,bus=pci.2,addr=0x0.0x1,rombar=1 -device usb-host,hostbus=1,hostaddr=7,id=hostdev2,bus=usb.0,port=5 -device usb-host,hostbus=1,hostaddr=56,id=hostdev3,bus=usb.0,port=1 -device usb-host,hostbus=1,hostaddr=25,id=hostdev4,bus=usb.0,port=4 -device virtio-balloon-pci,id=balloon0,bus=pci.3,addr=0x0 -object rng-random,id=objrng0,filename=/dev/urandom -device virtio-rng-pci,rng=objrng0,id=rng0,bus=pci.7,addr=0x0 -sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny -msg timestamp=on