Re: Windows 10 passthrough lag/micro-freeze issues during disk I/O

[Th3R3al]

Hello Jérôme,

just want to give a +1. But have the exact issue you described with
network i/o. If i try to transfer a big file from a samba share (at
around 40MB/s~) the VM is freezing. Nearly identical configuration to yours.

Greetings

Th3R3al

Am 20.03.20 um 20:46 schrieb Jérôme Poulin:
> Hi everyone,
>
> I have a VM right now that is setup as my gaming machine with video
> pass-through, it works perfectly except when there's I/O on the host.
>
> When there is I/O, the VM will have micro-freeze events where
> everything locks up for milliseconds up to a second at a time at
> frequent intervals, the host is on NVMe, CPUs are pinned with a
> dedicated core for emulator.
>
> Whatever I put, virtio, virtio-scsi, cache=none, cache=writeback,
> threads or native, as soon as there is I/O, I get the same problem, to
> the point where it causes disconnections on the virtual USB bus.
>
> VM is setup with hugepages, CPU pinning, I tried making qemu threads
> realtime with no success (even blue screens). I tried with dedicated
> cores (isolcpus), i440/q35, now I'm out of ideas. I do run a custom
> kernel to be able to use pcie_acs_override=downstream,multifunction in
> GRUB because I can't use passthrough otherwise.
>
> This problem has been following me from my old setup to my new setup
> Old setup:
> ASUS B150 PLUS USB3
> Core i7 6700K
> 16 GB DDR3
> SATA raw LVM volume for Windows
>
> New setup
> ASRock Z390 Taichi
> Core i7 9700K
> 48 GB DDR4
> NVMe raw LVM volume for Windows
>
> Any ideas would be appreciated.
>
> Here is the libvirt XML definition (I excluded the buses):
> <domain type='kvm'>
>   <name>win10-jerome</name>
>   <uuid>fc19f3c1-b8c6-4184-9b43-159157eff5b5</uuid>
>   <memory unit='KiB'>15360000</memory>
>   <currentMemory unit='KiB'>15360000</currentMemory>
>   <memoryBacking>
>     <hugepages/>
>     <nosharepages/>
>     <locked/>
>   </memoryBacking>
>   <vcpu placement='static'>4</vcpu>
>   <cputune>
>     <vcpupin vcpu='0' cpuset='0'/>
>     <vcpupin vcpu='1' cpuset='1'/>
>     <vcpupin vcpu='2' cpuset='2'/>
>     <vcpupin vcpu='3' cpuset='3'/>
>     <emulatorpin cpuset='7'/>
>   </cputune>
>   <resource>
>     <partition>/machine</partition>
>   </resource>
>   <os>
>     <type arch='x86_64' machine='pc-q35-4.0'>hvm</type>
>     <boot dev='hd'/>
>     <bootmenu enable='yes'/>
>   </os>
>   <features>
>     <acpi/>
>     <apic/>
>     <hyperv>
>       <relaxed state='on'/>
>       <vapic state='on'/>
>       <spinlocks state='on' retries='8191'/>
>       <vpindex state='on'/>
>       <runtime state='on'/>
>       <synic state='on'/>
>       <stimer state='on'/>
>       <reset state='on'/>
>       <frequencies state='on'/>
>     </hyperv>
>     <ioapic driver='kvm'/>
>   </features>
>   <cpu mode='host-passthrough' check='none'>
>     <topology sockets='1' cores='4' threads='1'/>
>     <feature policy='require' name='invtsc'/>
>   </cpu>
>   <clock offset='utc'>
>     <timer name='hypervclock' present='yes'/>
>     <timer name='hpet' present='yes'/>
>   </clock>
>   <devices>
>     <emulator>/usr/bin/kvm-spice</emulator>
>     <disk type='block' device='disk'>
>       <driver name='qemu' type='raw' cache='none' io='native'/>
>       <source dev='/dev/vgP4Root/win10jerome'/>
>       <backingStore/>
>       <target dev='sda' bus='sata'/>
>       <address type='drive' controller='0' bus='0' target='0' unit='0'/>
>     </disk>
>     <disk type='block' device='disk'>
>       <driver name='qemu' type='raw' cache='none' io='native'
> discard='ignore' detect_zeroes='off'/>
>       <source dev='/dev/vgP4Root/win10data'/>
>       <target dev='sdb' bus='scsi'/>
>       <address type='drive' controller='0' bus='0' target='0' unit='1'/>
>     </disk>
>     <disk type='block' device='disk'>
>       <driver name='qemu' type='raw' cache='none' io='native'
> discard='ignore' detect_zeroes='off'/>
>       <source dev='/dev/vgP4Root/win10swap'/>
>       <target dev='sdd' bus='scsi'/>
>       <address type='drive' controller='0' bus='0' target='0' unit='3'/>
>     </disk>
>     <interface type='direct'>
>       <mac address='52:54:00:31:1e:f9'/>
>       <source dev='eth0' mode='bridge'/>
>       <model type='virtio'/>
>       <address type='pci' domain='0x0000' bus='0x04' slot='0x00'
> function='0x0'/>
>     </interface>
>     <input type='mouse' bus='ps2'/>
>     <input type='keyboard' bus='ps2'/>
>     <graphics type='spice' autoport='yes' listen='127.0.0.1'>
>       <listen type='address' address='127.0.0.1'/>
>     </graphics>
>     <video>
>       <model type='qxl' ram='65536' vram='16384' vgamem='16384'
> heads='1' primary='yes'/>
>       <address type='pci' domain='0x00    <hostdev mode='subsystem'
> type='pci' managed='yes'>
>     </video>
>     <hostdev mode='subsystem' type='pci' managed='yes'>
>       <source>
>         <address domain='0x0000' bus='0x02' slot='0x00' function='0x0'/>
>       </source>
>       <rom bar='on'/>
>       <address type='pci' domain='0x0000' bus='0x02' slot='0x00'
> function='0x0' multifunction='on'/>
>     </hostdev>
>     <hostdev mode='subsystem' type='pci' managed='yes'>
>       <source>
>         <address domain='0x0000' bus='0x02' slot='0x00' function='0x1'/>
>       </source>
>       <rom bar='on'/>
>       <address type='pci' domain='0x0000' bus='0x02' slot='0x00'
> function='0x1'/>
>     </hostdev>
> 00' bus='0x00' slot='0x01' function='0x0'/>
>     <rng model='virtio'>
>       <backend model='random'>/dev/urandom</backend>
>       <address type='pci' domain='0x0000' bus='0x07' slot='0x00'
> function='0x0'/>
>     </rng>
>   </devices>
> </domain>
>
> Generated command line:
> qemu-system-x86_64 -enable-kvm -name
> guest=win10-jerome,debug-threads=on -S -object
> secret,id=masterKey0,format=raw,file=/var/lib/libvirt/qemu/domain-22-win10-jerome/master-key.aes
> -machine
> pc-q35-4.0,accel=kvm,usb=off,dump-guest-core=off,mem-merge=off,kernel_irqchip=on
> -cpu
> host,invtsc=on,hv_time,hv_relaxed,hv_vapic,hv_spinlocks=0x1fff,hv_vpindex,hv_runtime,hv_synic,hv_stimer,hv_reset,hv_frequencies
> -m 15000 -mem-prealloc -mem-path
> /dev/hugepages/libvirt/qemu/22-win10-jerome -overcommit mem-lock=on
> -smp 4,sockets=1,cores=4,threads=1 -uuid
> fc19f3c1-b8c6-4184-9b43-159157eff5b5 -no-user-config -nodefaults
> -chardev socket,id=charmonitor,fd=25,server,nowait -mon
> chardev=charmonitor,id=monitor,mode=control -rtc base=utc -no-shutdown
> -global ICH9-LPC.disable_s3=0 -global ICH9-LPC.disable_s4=0 -boot
> menu=on,strict=on -device
> pcie-root-port,port=0x10,chassis=1,id=pci.1,bus=pcie.0,multifunction=on,addr=0x2
> -device
> pcie-root-port,port=0x11,chassis=2,id=pci.2,bus=pcie.0,addr=0x2.0x1
> -device
> pcie-root-port,port=0x12,chassis=3,id=pci.3,bus=pcie.0,addr=0x2.0x2
> -device
> pcie-root-port,port=0x13,chassis=4,id=pci.4,bus=pcie.0,addr=0x2.0x3
> -device
> pcie-root-port,port=0x14,chassis=5,id=pci.5,bus=pcie.0,addr=0x2.0x4
> -device
> pcie-root-port,port=0x15,chassis=6,id=pci.6,bus=pcie.0,addr=0x2.0x5
> -device
> pcie-root-port,port=0x16,chassis=7,id=pci.7,bus=pcie.0,addr=0x2.0x6
> -device
> pcie-root-port,port=0x17,chassis=8,id=pci.8,bus=pcie.0,addr=0x2.0x7
> -device i82801b11-bridge,id=pci.9,bus=pcie.0,addr=0x1e -device
> pci-bridge,chassis_nr=10,id=pci.10,bus=pci.9,addr=0x0 -device
> ich9-usb-ehci1,id=usb,bus=pcie.0,addr=0x1d.0x7 -device
> ich9-usb-uhci1,masterbus=usb.0,firstport=0,bus=pcie.0,multifunction=on,addr=0x1d
> -device
> ich9-usb-uhci2,masterbus=usb.0,firstport=2,bus=pcie.0,addr=0x1d.0x1
> -device
> ich9-usb-uhci3,masterbus=usb.0,firstport=4,bus=pcie.0,addr=0x1d.0x2
> -device virtio-scsi-pci,id=scsi0,bus=pci.5,addr=0x0 -device
> virtio-serial-pci,id=virtio-serial0,bus=pci.1,addr=0x0 -drive
> file=/dev/vgP4Root/win10jerome,format=raw,if=none,id=drive-sata0-0-0,cache=none,aio=native
> -device
> ide-hd,bus=ide.0,drive=drive-sata0-0-0,id=sata0-0-0,bootindex=1,write-cache=on
> -drive
> file=/dev/vgP4Root/win10data,format=raw,if=none,id=drive-scsi0-0-0-1,cache=none,discard=ignore,detect-zeroes=off,aio=native
> -device
> scsi-hd,bus=scsi0.0,channel=0,scsi-id=0,lun=1,device_id=drive-scsi0-0-0-1,drive=drive-scsi0-0-0-1,id=scsi0-0-0-1,write-cache=on
> -drive
> file=/dev/vgP4Root/win10swap,format=raw,if=none,id=drive-scsi0-0-0-3,cache=none,discard=ignore,detect-zeroes=off,aio=native
> -device
> scsi-hd,bus=scsi0.0,channel=0,scsi-id=0,lun=3,device_id=drive-scsi0-0-0-3,drive=drive-scsi0-0-0-3,id=scsi0-0-0-3,write-cache=on
> -netdev tap,fd=27,id=hostnet0,vhost=on,vhostfd=28 -device
> virtio-net-pci,netdev=hostnet0,id=net0,mac=52:54:00:31:1e:f9,bus=pci.4,addr=0x0
> -spice
> port=5901,addr=127.0.0.1,disable-ticketing,seamless-migration=on
> -device
> qxl-vga,id=video0,ram_size=67108864,vram_size=16777216,vram64_size_mb=0,vgamem_mb=16,max_outputs=1,bus=pcie.0,addr=0x1
> -chardev spicevmc,id=charredir0,name=usbredir -device
> usb-redir,chardev=charredir0,id=redir0,bus=usb.0,port=2 -chardev
> spicevmc,id=charredir1,name=usbredir -device
> usb-redir,chardev=charredir1,id=redir1,bus=usb.0,port=3 -device
> vfio-pci,host=02:00.0,id=hostdev0,bus=pci.2,multifunction=on,addr=0x0,rombar=1
> -device
> vfio-pci,host=02:00.1,id=hostdev1,bus=pci.2,addr=0x0.0x1,rombar=1
> -device usb-host,hostbus=1,hostaddr=7,id=hostdev2,bus=usb.0,port=5
> -device usb-host,hostbus=1,hostaddr=56,id=hostdev3,bus=usb.0,port=1
> -device usb-host,hostbus=1,hostaddr=25,id=hostdev4,bus=usb.0,port=4
> -device virtio-balloon-pci,id=balloon0,bus=pci.3,addr=0x0 -object
> rng-random,id=objrng0,filename=/dev/urandom -device
> virtio-rng-pci,rng=objrng0,id=rng0,bus=pci.7,addr=0x0 -sandbox
> on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny
> -msg timestamp=on