rendered paste bodyGoal: I want to pass the PCI-E device 04:00.0 to KVM, and keep the device 03:00.0 working on the host (it's got the host OS).
proxmox:~# dmesg | grep -e DMAR -e IOMMU
ACPI: DMAR 00000000bf79e0c0 00118 (v01 AMI OEMDMAR 00000001 MSFT 00000097)
Intel-IOMMU: enabled
DMAR: Host address width 39
DMAR: DRHD base: 0x000000fbffe000 flags: 0x1
IOMMU fbffe000: ver 1:0 cap c90780106f0462 ecap f020f6
DMAR: RMRR base: 0x000000000ec000 end: 0x000000000effff
DMAR: RMRR base: 0x000000bf7ec000 end: 0x000000bf7fffff
DMAR: ATSR flags: 0x0
IOMMU 0xfbffe000: using Queued invalidation
IOMMU: Setting RMRR:
IOMMU: Setting identity map for device 0000:00:1d.0 [0xbf7ec000 - 0xbf800000]
IOMMU: Setting identity map for device 0000:00:1d.1 [0xbf7ec000 - 0xbf800000]
IOMMU: Setting identity map for device 0000:00:1d.2 [0xbf7ec000 - 0xbf800000]
IOMMU: Setting identity map for device 0000:00:1d.7 [0xbf7ec000 - 0xbf800000]
IOMMU: Setting identity map for device 0000:00:1a.0 [0xbf7ec000 - 0xbf800000]
IOMMU: Setting identity map for device 0000:00:1a.1 [0xbf7ec000 - 0xbf800000]
IOMMU: Setting identity map for device 0000:00:1a.2 [0xbf7ec000 - 0xbf800000]
IOMMU: Setting identity map for device 0000:00:1a.7 [0xbf7ec000 - 0xbf800000]
IOMMU: Setting identity map for device 0000:00:1d.0 [0xec000 - 0xf0000]
IOMMU: Setting identity map for device 0000:00:1d.1 [0xec000 - 0xf0000]
IOMMU: Setting identity map for device 0000:00:1d.2 [0xec000 - 0xf0000]
IOMMU: Setting identity map for device 0000:00:1d.7 [0xec000 - 0xf0000]
IOMMU: Setting identity map for device 0000:00:1a.0 [0xec000 - 0xf0000]
IOMMU: Setting identity map for device 0000:00:1a.1 [0xec000 - 0xf0000]
IOMMU: Setting identity map for device 0000:00:1a.2 [0xec000 - 0xf0000]
IOMMU: Setting identity map for device 0000:00:1a.7 [0xec000 - 0xf0000]
IOMMU: Prepare 0-16MiB unity mapping for LPC
IOMMU: Setting identity map for device 0000:00:1f.0 [0x0 - 0x1000000]
proxmox:~# grep iommu /boot/grub/menu.lst
kernel /vmlinuz-2.6.32-6-pve root=/dev/mapper/pve-root ro intel_iommu=on
proxmox:~# lspci | grep -i 3ware
03:00.0 RAID bus controller: 3ware Inc 9750 SAS2/SATA-II RAID PCIe (rev 05)
04:00.0 RAID bus controller: 3ware Inc 9750 SAS2/SATA-II RAID PCIe (rev 05)
proxmox:~# lspci -n | tail -n 6
02:00.0 0300: 10de:0659 (rev a1)
03:00.0 0104: 13c1:1010 (rev 05)
04:00.0 0104: 13c1:1010 (rev 05)
05:00.0 0200: 8086:10d3
06:00.0 0200: 8086:10d3
07:08.0 0c00: 104c:8023
proxmox:~# cat /sys/bus/pci/devices/0000:03:00.0/irq
16
proxmox:~# cat /sys/bus/pci/devices/0000:04:00.0/irq
16
proxmox:~# echo "13c1 1010" > /sys/bus/pci/drivers/pci-stub/new_id
proxmox:~# echo "0000:04:00.0" > /sys/bus/pci/drivers/3w-sas/unbind
proxmox:~# echo "0000:04:00.0" > /sys/bus/pci/drivers/pci-stub/bind
proxmox:~# echo "13c1 1010" > /sys/bus/pci/drivers/pci-stub/remove_id
proxmox:~# /usr/bin/kvm -name FreeNAS_dgroup -smp sockets=2,cores=1 -nodefaults -boot menu=on -vga cirrus -tdf -k en-us -drive file=/var/lib/vz/images/102/vm-102-disk-1.raw,if=ide,index=0,cache=none,boot=on -m 12288 -netdev type=tap,id=vlan1d0,ifname=tap102i1d0,script=/var/lib/qemu-server/bridge-vlan -device rtl8139,romfile=,mac=5E:6A:83:F4:31:D1,netdev=vlan1d0 -id 102 -cpuunits 1000 -device pci-assign,host=04:00.0
Failed to assign device "(null)" : Operation not permitted
kvm: -device pci-assign,host=04:00.0: Device 'pci-assign' could not be initialized
proxmox:~# strace /usr/bin/kvm -name FreeNAS_dgroup -smp sockets=2,cores=1 -nodefaults -boot menu=on -vga cirrus -tdf -k en-us -drive file=/var/lib/vz/images/102/vm-102-disk-1.raw,if=ide,index=0,cache=none,boot=on -m 12288 -netdev type=tap,id=vlan1d0,ifname=tap102i1d0,script=/var/lib/qemu-server/bridge-vlan -device rtl8139,romfile=,mac=5E:6A:83:F4:31:D1,netdev=vlan1d0 -id 102 -cpuunits 1000 -device pci-assign,host=04:00.0
<snip>
brk(0x42c7000) = 0x42c7000
brk(0x42eb000) = 0x42eb000
access("/usr/share/kvm/bios.bin", R_OK) = 0
open("/usr/share/kvm/bios.bin", O_RDONLY) = 13
lseek(13, 0, SEEK_END) = 131072
close(13) = 0
mmap(NULL, 139264, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ff2835f0000
madvise(0x7ff2835f1000, 131072, 0xc /* MADV_??? */) = 0
ioctl(3, 0xae03, 0x10) = 1
access("/usr/share/kvm/bios.bin", R_OK) = 0
open("/usr/share/kvm/bios.bin", O_RDONLY) = 13
lseek(13, 0, SEEK_END) = 131072
mmap(NULL, 135168, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ff2835cf000
lseek(13, 0, SEEK_SET) = 0
read(13, "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 131072) = 131072
close(13) = 0
ioctl(4, 0x4020ae46, 0x7fff1918f7c0) = 0
mmap(NULL, 139264, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ff2835ad000
madvise(0x7ff2835ae000, 131072, 0xc /* MADV_??? */) = 0
ioctl(3, 0xae03, 0x10) = 1
ioctl(4, 0x4020ae46, 0x7fff1918f7c0) = 0
ioctl(4, 0x4020ae46, 0x7fff1918f7c0) = 0
ioctl(3, 0xae03, 0x19) = 1024
access("/usr/share/kvm/extboot.bin", R_OK) = 0
open("/usr/share/kvm/extboot.bin", O_RDONLY) = 13
lseek(13, 0, SEEK_END) = 1536
lseek(13, 0, SEEK_SET) = 0
read(13, "U\252\3fP\0361\300\216\330\376\6u\4f\241d\0f\243\0\2\307\6d\0$\0\214\16f\0\37"..., 1536) = 1536
close(13) = 0
access("/usr/share/kvm/vapic.bin", R_OK) = 0
open("/usr/share/kvm/vapic.bin", O_RDONLY) = 13
lseek(13, 0, SEEK_END) = 9216
lseek(13, 0, SEEK_SET) = 0
read(13, "U\252\22\6\16\0071\300\271\0\20\215>\0\3\374\363\253\7\270 \0\347~\313\17\37\200\0\0\0\0k"..., 9216) = 9216
close(13) = 0
brk(0x431d000) = 0x431d000
mmap(NULL, 16785408, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fef7e1a2000
madvise(0x7fef7e1a3000, 16777216, 0xc /* MADV_??? */) = 0
mremap(0x7fef7f1a4000, 3149824, 3153920, MREMAP_MAYMOVE) = 0x7fef7dea0000
ioctl(3, 0xae03, 0x10) = 1
ioctl(4, 0x4010ae67, 0x7fff1918f7d0) = 0
mmap(NULL, 1232896, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fef7f378000
access("/usr/share/kvm/vgabios-cirrus.bin", R_OK) = 0
open("/usr/share/kvm/vgabios-cirrus.bin", O_RDONLY) = 13
lseek(13, 0, SEEK_END) = 35840
close(13) = 0
madvise(0x42fe000, 65536, 0xc /* MADV_??? */) = 0
ioctl(3, 0xae03, 0x10) = 1
open("/usr/share/kvm/vgabios-cirrus.bin", O_RDONLY) = 13
lseek(13, 0, SEEK_END) = 35840
lseek(13, 0, SEEK_SET) = 0
read(13, "U\252F\351!\1\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\17\1\0\0\0\0IBM"..., 35840) = 35840
close(13) = 0
brk(0x4340000) = 0x4340000
open("/etc/localtime", O_RDONLY) = 13
fstat(13, {st_mode=S_IFREG|0644, st_size=327, ...}) = 0
fstat(13, {st_mode=S_IFREG|0644, st_size=327, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ff2835ac000
read(13, "TZif2\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\3\0\0\0\3\0\0\0\0\0"..., 4096) = 327
lseek(13, -197, SEEK_CUR) = 130
read(13, "TZif2\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\4\0\0\0\4\0\0\0\0\0"..., 4096) = 197
close(13) = 0
munmap(0x7ff2835ac000, 4096) = 0
mmap(NULL, 135168, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ff28358c000
mmap(NULL, 135168, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ff28356b000
mmap(NULL, 135168, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fef7f357000
mmap(NULL, 135168, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fef7f336000
pread(9, "\3741\300\216\300\216\330\216\320\274\0|\211\346\277\0\6\271\0\1\363\245\211\375\261\10\363\253\376E\362\351\0"..., 512, 0) = 512
ioctl(11, 0x4090ae82, 0x7fff1918f7a0) = 0
ioctl(11, 0x41a0ae8d, 0x7fff1918f680) = 0
ioctl(11, 0x4138ae84, 0x7fff1918f860) = 0
ioctl(11, 0x4008ae89, 0x7fff1918f1f0) = 53
ioctl(11, 0x4004ae99, 0x7fff1918f850) = 0
ioctl(11, 0x4400ae8f, 0x7fff1918f430) = 0
ioctl(11, 0x4040aea0, 0x7fff1918f860) = 0
ioctl(12, 0x4090ae82, 0x7fff1918f7a0) = 0
ioctl(12, 0x41a0ae8d, 0x7fff1918f680) = 0
ioctl(12, 0x4138ae84, 0x7fff1918f860) = 0
ioctl(12, 0x4008ae89, 0x7fff1918f1f0) = 53
ioctl(12, 0x4004ae99, 0x7fff1918f850) = 0
ioctl(12, 0x4400ae8f, 0x7fff1918f430) = 0
ioctl(12, 0x4040aea0, 0x7fff1918f860) = 0
open("/sys/bus/pci/devices/0000:04:00.0/config", O_RDWR) = 13
read(13, "\301\23\20\20\3\0\20\0\5\0\4\1@\0\0\0\1\310\0\0\4\300\271\373\0\0\0\0\4\0\274\373\0"..., 256) = 256
open("/sys/bus/pci/devices/0000:04:00.0/resource", O_RDONLY) = 14
fstat(14, {st_mode=S_IFREG|0444, st_size=4096, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fef7f335000
read(14, "0x000000000000c800 0x000000000000"..., 4096) = 741
open("/sys/bus/pci/devices/0000:04:00.0/resource0", O_RDWR) = 15
open("/sys/bus/pci/devices/0000:04:00.0/resource1", O_RDWR) = 16
open("/sys/bus/pci/devices/0000:04:00.0/resource3", O_RDWR) = 17
close(14) = 0
munmap(0x7fef7f335000, 4096) = 0
open("/sys/bus/pci/devices/0000:04:00.0/vendor", O_RDONLY) = 14
fstat(14, {st_mode=S_IFREG|0444, st_size=4096, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fef7f335000
read(14, "0x13c1\n"..., 4096) = 7
read(14, ""..., 4096) = 0
close(14) = 0
munmap(0x7fef7f335000, 4096) = 0
open("/sys/bus/pci/devices/0000:04:00.0/device", O_RDONLY) = 14
fstat(14, {st_mode=S_IFREG|0444, st_size=4096, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fef7f335000
read(14, "0x1010\n"..., 4096) = 7
read(14, ""..., 4096) = 0
close(14) = 0
munmap(0x7fef7f335000, 4096) = 0
stat("/sys/bus/pci/devices/0000:04:00.0/physfn/", 0x7fff1918f4c0) = -1 ENOENT (No such file or directory)
pread(15, 0x7fff1918f6ec, 3, 0) = -1 EINVAL (Invalid argument)
mmap(NULL, 16384, PROT_READ|PROT_WRITE, MAP_SHARED, 16, 0) = 0x7fef7f332000
ioctl(3, 0xae03, 0x10) = 1
mmap(NULL, 262144, PROT_READ|PROT_WRITE, MAP_SHARED, 17, 0) = 0x7fef7f2f2000
ioctl(3, 0xae03, 0x10) = 1
pread(13, "\20"..., 1, 6) = 1
pread(13, "P"..., 1, 52) = 1
pread(13, "\1"..., 1, 80) = 1
pread(13, "h"..., 1, 81) = 1
pread(13, "\20"..., 1, 104) = 1
pread(13, "\320"..., 1, 105) = 1
pread(13, "\3"..., 1, 208) = 1
pread(13, "\250"..., 1, 209) = 1
pread(13, "\5"..., 1, 168) = 1
pread(13, "\20"..., 1, 6) = 1
pread(13, "P"..., 1, 52) = 1
pread(13, "\1"..., 1, 80) = 1
pread(13, "h"..., 1, 81) = 1
pread(13, "\20"..., 1, 104) = 1
pread(13, "\320"..., 1, 105) = 1
pread(13, "\3"..., 1, 208) = 1
pread(13, "\250"..., 1, 209) = 1
pread(13, "\5"..., 1, 168) = 1
pread(13, "\0"..., 1, 169) = 1
pread(13, "\20"..., 1, 6) = 1
pread(13, "P"..., 1, 52) = 1
pread(13, "\1"..., 1, 80) = 1
pread(13, "\20"..., 1, 6) = 1
pread(13, "P"..., 1, 52) = 1
pread(13, "\1"..., 1, 80) = 1
pread(13, "h"..., 1, 81) = 1
pread(13, "\20"..., 1, 104) = 1
pread(13, "\20"..., 1, 6) = 1
pread(13, "P"..., 1, 52) = 1
pread(13, "\1"..., 1, 80) = 1
pread(13, "h"..., 1, 81) = 1
pread(13, "\20"..., 1, 104) = 1
pread(13, "\320"..., 1, 105) = 1
pread(13, "\3"..., 1, 208) = 1
pread(13, "\250"..., 1, 209) = 1
pread(13, "\5"..., 1, 168) = 1
pread(13, "\0"..., 1, 169) = 1
pread(13, "\20"..., 1, 6) = 1
pread(13, "P"..., 1, 52) = 1
pread(13, "\1"..., 1, 80) = 1
pread(13, "h"..., 1, 81) = 1
pread(13, "\20"..., 1, 104) = 1
pread(13, "\320"..., 1, 105) = 1
pread(13, "\3"..., 1, 208) = 1
pread(13, "\20"..., 1, 6) = 1
pread(13, "P"..., 1, 52) = 1
pread(13, "\1"..., 1, 80) = 1
pread(13, "h"..., 1, 81) = 1
pread(13, "\20"..., 1, 104) = 1
pread(13, "\320"..., 1, 105) = 1
pread(13, "\3"..., 1, 208) = 1
pread(13, "\250"..., 1, 209) = 1
pread(13, "\5"..., 1, 168) = 1
pread(13, "\0"..., 1, 169) = 1
ioctl(3, 0xae03, 0x2f) = 0
ioctl(3, 0xae03, 0x12) = 1
ioctl(4, 0xffffffff8040ae69, 0x7fff1918f6f0) = -1 EPERM (Operation not permitted)
write(2, "Failed to assign device \"(null)\" "..., 59Failed to assign device "(null)" : Operation not permitted
) = 59
close(15) = 0
munmap(0x7fef7f332000, 16384) = 0
close(16) = 0
munmap(0x7fef7f2f2000, 262144) = 0
close(17) = 0
close(13) = 0
write(2, "kvm:"..., 4kvm:) = 4
write(2, " -device"..., 8 -device) = 8
write(2, " pci-assign,host=04:00.0"..., 24 pci-assign,host=04:00.0) = 24
write(2, ": "..., 2: ) = 2
write(2, "Device 'pci-assign' could not be "..., 44Device 'pci-assign' could not be initialized) = 44
write(2, "\n"..., 1
) = 1
exit_group(1) = ?
proxmox:~#