rendered paste bodyGoal: To pass the PCI-E device 04:00.0 to KVM, and keep the device 03:00.0 working on the host (it's got the host OS).
proxmox:~# uname -a
Linux proxmox 2.6.32-6-pve #1 SMP Mon Sep 26 06:32:53 CEST 2011 x86_64 GNU/Linux
proxmox:~# cat /proc/cpuinfo
processor : 0
vendor_id : GenuineIntel
cpu family : 6
model : 26
model name : Intel(R) Core(TM) i7 CPU 960 @ 3.20GHz
stepping : 5
cpu MHz : 3200.165
cache size : 8192 KB
physical id : 0
siblings : 8
core id : 0
cpu cores : 4
apicid : 0
initial apicid : 0
fpu : yes
fpu_exception : yes
cpuid level : 11
wp : yes
flags : fpu vme de pse tsc msr pae mce cx8 apic mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx rdtscp lm constant_tsc arch_perfmon pebs bts rep_good xtopology nonstop_tsc aperfmperf pni dtes64 monitor ds_cpl vmx est tm2 ssse3 cx16 xtpr pdcm sse4_1 sse4_2 popcnt lahf_lm ida tpr_shadow vnmi flexpriority ept vpid
bogomips : 6400.33
clflush size : 64
cache_alignment : 64
address sizes : 36 bits physical, 48 bits virtual
power management:
<snip other 7 cores>
proxmox:~# dmesg | grep -e DMAR -e IOMMU
ACPI: DMAR 00000000bf79e0c0 00118 (v01 AMI OEMDMAR 00000001 MSFT 00000097)
Intel-IOMMU: enabled
DMAR: Host address width 39
DMAR: DRHD base: 0x000000fbffe000 flags: 0x1
IOMMU fbffe000: ver 1:0 cap c90780106f0462 ecap f020f6
DMAR: RMRR base: 0x000000000ec000 end: 0x000000000effff
DMAR: RMRR base: 0x000000bf7ec000 end: 0x000000bf7fffff
DMAR: ATSR flags: 0x0
IOMMU 0xfbffe000: using Queued invalidation
IOMMU: Setting RMRR:
IOMMU: Setting identity map for device 0000:00:1d.0 [0xbf7ec000 - 0xbf800000]
IOMMU: Setting identity map for device 0000:00:1d.1 [0xbf7ec000 - 0xbf800000]
IOMMU: Setting identity map for device 0000:00:1d.2 [0xbf7ec000 - 0xbf800000]
IOMMU: Setting identity map for device 0000:00:1d.7 [0xbf7ec000 - 0xbf800000]
IOMMU: Setting identity map for device 0000:00:1a.0 [0xbf7ec000 - 0xbf800000]
IOMMU: Setting identity map for device 0000:00:1a.1 [0xbf7ec000 - 0xbf800000]
IOMMU: Setting identity map for device 0000:00:1a.2 [0xbf7ec000 - 0xbf800000]
IOMMU: Setting identity map for device 0000:00:1a.7 [0xbf7ec000 - 0xbf800000]
IOMMU: Setting identity map for device 0000:00:1d.0 [0xec000 - 0xf0000]
IOMMU: Setting identity map for device 0000:00:1d.1 [0xec000 - 0xf0000]
IOMMU: Setting identity map for device 0000:00:1d.2 [0xec000 - 0xf0000]
IOMMU: Setting identity map for device 0000:00:1d.7 [0xec000 - 0xf0000]
IOMMU: Setting identity map for device 0000:00:1a.0 [0xec000 - 0xf0000]
IOMMU: Setting identity map for device 0000:00:1a.1 [0xec000 - 0xf0000]
IOMMU: Setting identity map for device 0000:00:1a.2 [0xec000 - 0xf0000]
IOMMU: Setting identity map for device 0000:00:1a.7 [0xec000 - 0xf0000]
IOMMU: Prepare 0-16MiB unity mapping for LPC
IOMMU: Setting identity map for device 0000:00:1f.0 [0x0 - 0x1000000]
proxmox:~# grep iommu /boot/grub/menu.lst
kernel /vmlinuz-2.6.32-6-pve root=/dev/mapper/pve-root ro intel_iommu=on
proxmox:~# lspci | grep -i 3ware
03:00.0 RAID bus controller: 3ware Inc 9750 SAS2/SATA-II RAID PCIe (rev 05)
04:00.0 RAID bus controller: 3ware Inc 9750 SAS2/SATA-II RAID PCIe (rev 05)
proxmox:~# lspci -n | tail -n 6
02:00.0 0300: 10de:0659 (rev a1)
03:00.0 0104: 13c1:1010 (rev 05)
04:00.0 0104: 13c1:1010 (rev 05)
05:00.0 0200: 8086:10d3
06:00.0 0200: 8086:10d3
07:08.0 0c00: 104c:8023
proxmox:~# cat /sys/bus/pci/devices/0000:03:00.0/irq
16
proxmox:~# cat /sys/bus/pci/devices/0000:04:00.0/irq
16
proxmox:~# echo "13c1 1010" > /sys/bus/pci/drivers/pci-stub/new_id
proxmox:~# echo "0000:04:00.0" > /sys/bus/pci/drivers/3w-sas/unbind
proxmox:~# echo "0000:04:00.0" > /sys/bus/pci/drivers/pci-stub/bind
proxmox:~# echo "13c1 1010" > /sys/bus/pci/drivers/pci-stub/remove_id
proxmox:~# /usr/bin/kvm -name FreeNAS_dgroup -smp sockets=2,cores=1 -nodefaults -boot menu=on -vga cirrus -tdf -k en-us -drive file=/var/lib/vz/images/102/vm-102-disk-1.raw,if=ide,index=0,cache=none,boot=on -m 12288 -netdev type=tap,id=vlan1d0,ifname=tap102i1d0,script=/var/lib/qemu-server/bridge-vlan -device rtl8139,romfile=,mac=5E:6A:83:F4:31:D1,netdev=vlan1d0 -id 102 -cpuunits 1000 -device pci-assign,host=04:00.0
Failed to assign device "(null)" : Operation not permitted
kvm: -device pci-assign,host=04:00.0: Device 'pci-assign' could not be initialized
proxmox:~# strace /usr/bin/kvm -name FreeNAS_dgroup -smp sockets=2,cores=1 -nodefaults -boot menu=on -vga cirrus -tdf -k en-us -drive file=/var/lib/vz/images/102/vm-102-disk-1.raw,if=ide,index=0,cache=none,boot=on -m 12288 -netdev type=tap,id=vlan1d0,ifname=tap102i1d0,script=/var/lib/qemu-server/bridge-vlan -device rtl8139,romfile=,mac=5E:6A:83:F4:31:D1,netdev=vlan1d0 -id 102 -cpuunits 1000 -device pci-assign,host=04:00.0
<snip>
brk(0x42c7000) = 0x42c7000
brk(0x42eb000) = 0x42eb000
access("/usr/share/kvm/bios.bin", R_OK) = 0
open("/usr/share/kvm/bios.bin", O_RDONLY) = 13
lseek(13, 0, SEEK_END) = 131072
close(13) = 0
mmap(NULL, 139264, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ff2835f0000
madvise(0x7ff2835f1000, 131072, 0xc /* MADV_??? */) = 0
ioctl(3, 0xae03, 0x10) = 1
access("/usr/share/kvm/bios.bin", R_OK) = 0
open("/usr/share/kvm/bios.bin", O_RDONLY) = 13
lseek(13, 0, SEEK_END) = 131072
mmap(NULL, 135168, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ff2835cf000
lseek(13, 0, SEEK_SET) = 0
read(13, "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 131072) = 131072
close(13) = 0
ioctl(4, 0x4020ae46, 0x7fff1918f7c0) = 0
mmap(NULL, 139264, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ff2835ad000
madvise(0x7ff2835ae000, 131072, 0xc /* MADV_??? */) = 0
ioctl(3, 0xae03, 0x10) = 1
ioctl(4, 0x4020ae46, 0x7fff1918f7c0) = 0
ioctl(4, 0x4020ae46, 0x7fff1918f7c0) = 0
ioctl(3, 0xae03, 0x19) = 1024
access("/usr/share/kvm/extboot.bin", R_OK) = 0
open("/usr/share/kvm/extboot.bin", O_RDONLY) = 13
lseek(13, 0, SEEK_END) = 1536
lseek(13, 0, SEEK_SET) = 0
read(13, "U\252\3fP\0361\300\216\330\376\6u\4f\241d\0f\243\0\2\307\6d\0$\0\214\16f\0\37"..., 1536) = 1536
close(13) = 0
access("/usr/share/kvm/vapic.bin", R_OK) = 0
open("/usr/share/kvm/vapic.bin", O_RDONLY) = 13
lseek(13, 0, SEEK_END) = 9216
lseek(13, 0, SEEK_SET) = 0
read(13, "U\252\22\6\16\0071\300\271\0\20\215>\0\3\374\363\253\7\270 \0\347~\313\17\37\200\0\0\0\0k"..., 9216) = 9216
close(13) = 0
brk(0x431d000) = 0x431d000
mmap(NULL, 16785408, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fef7e1a2000
madvise(0x7fef7e1a3000, 16777216, 0xc /* MADV_??? */) = 0
mremap(0x7fef7f1a4000, 3149824, 3153920, MREMAP_MAYMOVE) = 0x7fef7dea0000
ioctl(3, 0xae03, 0x10) = 1
ioctl(4, 0x4010ae67, 0x7fff1918f7d0) = 0
mmap(NULL, 1232896, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fef7f378000
access("/usr/share/kvm/vgabios-cirrus.bin", R_OK) = 0
open("/usr/share/kvm/vgabios-cirrus.bin", O_RDONLY) = 13
lseek(13, 0, SEEK_END) = 35840
close(13) = 0
madvise(0x42fe000, 65536, 0xc /* MADV_??? */) = 0
ioctl(3, 0xae03, 0x10) = 1
open("/usr/share/kvm/vgabios-cirrus.bin", O_RDONLY) = 13
lseek(13, 0, SEEK_END) = 35840
lseek(13, 0, SEEK_SET) = 0
read(13, "U\252F\351!\1\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\17\1\0\0\0\0IBM"..., 35840) = 35840
close(13) = 0
brk(0x4340000) = 0x4340000
open("/etc/localtime", O_RDONLY) = 13
fstat(13, {st_mode=S_IFREG|0644, st_size=327, ...}) = 0
fstat(13, {st_mode=S_IFREG|0644, st_size=327, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ff2835ac000
read(13, "TZif2\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\3\0\0\0\3\0\0\0\0\0"..., 4096) = 327
lseek(13, -197, SEEK_CUR) = 130
read(13, "TZif2\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\4\0\0\0\4\0\0\0\0\0"..., 4096) = 197
close(13) = 0
munmap(0x7ff2835ac000, 4096) = 0
mmap(NULL, 135168, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ff28358c000
mmap(NULL, 135168, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ff28356b000
mmap(NULL, 135168, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fef7f357000
mmap(NULL, 135168, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fef7f336000
pread(9, "\3741\300\216\300\216\330\216\320\274\0|\211\346\277\0\6\271\0\1\363\245\211\375\261\10\363\253\376E\362\351\0"..., 512, 0) = 512
ioctl(11, 0x4090ae82, 0x7fff1918f7a0) = 0
ioctl(11, 0x41a0ae8d, 0x7fff1918f680) = 0
ioctl(11, 0x4138ae84, 0x7fff1918f860) = 0
ioctl(11, 0x4008ae89, 0x7fff1918f1f0) = 53
ioctl(11, 0x4004ae99, 0x7fff1918f850) = 0
ioctl(11, 0x4400ae8f, 0x7fff1918f430) = 0
ioctl(11, 0x4040aea0, 0x7fff1918f860) = 0
ioctl(12, 0x4090ae82, 0x7fff1918f7a0) = 0
ioctl(12, 0x41a0ae8d, 0x7fff1918f680) = 0
ioctl(12, 0x4138ae84, 0x7fff1918f860) = 0
ioctl(12, 0x4008ae89, 0x7fff1918f1f0) = 53
ioctl(12, 0x4004ae99, 0x7fff1918f850) = 0
ioctl(12, 0x4400ae8f, 0x7fff1918f430) = 0
ioctl(12, 0x4040aea0, 0x7fff1918f860) = 0
open("/sys/bus/pci/devices/0000:04:00.0/config", O_RDWR) = 13
read(13, "\301\23\20\20\3\0\20\0\5\0\4\1@\0\0\0\1\310\0\0\4\300\271\373\0\0\0\0\4\0\274\373\0"..., 256) = 256
open("/sys/bus/pci/devices/0000:04:00.0/resource", O_RDONLY) = 14
fstat(14, {st_mode=S_IFREG|0444, st_size=4096, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fef7f335000
read(14, "0x000000000000c800 0x000000000000"..., 4096) = 741
open("/sys/bus/pci/devices/0000:04:00.0/resource0", O_RDWR) = 15
open("/sys/bus/pci/devices/0000:04:00.0/resource1", O_RDWR) = 16
open("/sys/bus/pci/devices/0000:04:00.0/resource3", O_RDWR) = 17
close(14) = 0
munmap(0x7fef7f335000, 4096) = 0
open("/sys/bus/pci/devices/0000:04:00.0/vendor", O_RDONLY) = 14
fstat(14, {st_mode=S_IFREG|0444, st_size=4096, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fef7f335000
read(14, "0x13c1\n"..., 4096) = 7
read(14, ""..., 4096) = 0
close(14) = 0
munmap(0x7fef7f335000, 4096) = 0
open("/sys/bus/pci/devices/0000:04:00.0/device", O_RDONLY) = 14
fstat(14, {st_mode=S_IFREG|0444, st_size=4096, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fef7f335000
read(14, "0x1010\n"..., 4096) = 7
read(14, ""..., 4096) = 0
close(14) = 0
munmap(0x7fef7f335000, 4096) = 0
stat("/sys/bus/pci/devices/0000:04:00.0/physfn/", 0x7fff1918f4c0) = -1 ENOENT (No such file or directory)
pread(15, 0x7fff1918f6ec, 3, 0) = -1 EINVAL (Invalid argument)
mmap(NULL, 16384, PROT_READ|PROT_WRITE, MAP_SHARED, 16, 0) = 0x7fef7f332000
ioctl(3, 0xae03, 0x10) = 1
mmap(NULL, 262144, PROT_READ|PROT_WRITE, MAP_SHARED, 17, 0) = 0x7fef7f2f2000
ioctl(3, 0xae03, 0x10) = 1
pread(13, "\20"..., 1, 6) = 1
pread(13, "P"..., 1, 52) = 1
pread(13, "\1"..., 1, 80) = 1
pread(13, "h"..., 1, 81) = 1
pread(13, "\20"..., 1, 104) = 1
pread(13, "\320"..., 1, 105) = 1
pread(13, "\3"..., 1, 208) = 1
pread(13, "\250"..., 1, 209) = 1
pread(13, "\5"..., 1, 168) = 1
pread(13, "\20"..., 1, 6) = 1
pread(13, "P"..., 1, 52) = 1
pread(13, "\1"..., 1, 80) = 1
pread(13, "h"..., 1, 81) = 1
pread(13, "\20"..., 1, 104) = 1
pread(13, "\320"..., 1, 105) = 1
pread(13, "\3"..., 1, 208) = 1
pread(13, "\250"..., 1, 209) = 1
pread(13, "\5"..., 1, 168) = 1
pread(13, "\0"..., 1, 169) = 1
pread(13, "\20"..., 1, 6) = 1
pread(13, "P"..., 1, 52) = 1
pread(13, "\1"..., 1, 80) = 1
pread(13, "\20"..., 1, 6) = 1
pread(13, "P"..., 1, 52) = 1
pread(13, "\1"..., 1, 80) = 1
pread(13, "h"..., 1, 81) = 1
pread(13, "\20"..., 1, 104) = 1
pread(13, "\20"..., 1, 6) = 1
pread(13, "P"..., 1, 52) = 1
pread(13, "\1"..., 1, 80) = 1
pread(13, "h"..., 1, 81) = 1
pread(13, "\20"..., 1, 104) = 1
pread(13, "\320"..., 1, 105) = 1
pread(13, "\3"..., 1, 208) = 1
pread(13, "\250"..., 1, 209) = 1
pread(13, "\5"..., 1, 168) = 1
pread(13, "\0"..., 1, 169) = 1
pread(13, "\20"..., 1, 6) = 1
pread(13, "P"..., 1, 52) = 1
pread(13, "\1"..., 1, 80) = 1
pread(13, "h"..., 1, 81) = 1
pread(13, "\20"..., 1, 104) = 1
pread(13, "\320"..., 1, 105) = 1
pread(13, "\3"..., 1, 208) = 1
pread(13, "\20"..., 1, 6) = 1
pread(13, "P"..., 1, 52) = 1
pread(13, "\1"..., 1, 80) = 1
pread(13, "h"..., 1, 81) = 1
pread(13, "\20"..., 1, 104) = 1
pread(13, "\320"..., 1, 105) = 1
pread(13, "\3"..., 1, 208) = 1
pread(13, "\250"..., 1, 209) = 1
pread(13, "\5"..., 1, 168) = 1
pread(13, "\0"..., 1, 169) = 1
ioctl(3, 0xae03, 0x2f) = 0
ioctl(3, 0xae03, 0x12) = 1
ioctl(4, 0xffffffff8040ae69, 0x7fff1918f6f0) = -1 EPERM (Operation not permitted)
write(2, "Failed to assign device \"(null)\" "..., 59Failed to assign device "(null)" : Operation not permitted
) = 59
close(15) = 0
munmap(0x7fef7f332000, 16384) = 0
close(16) = 0
munmap(0x7fef7f2f2000, 262144) = 0
close(17) = 0
close(13) = 0
write(2, "kvm:"..., 4kvm:) = 4
write(2, " -device"..., 8 -device) = 8
write(2, " pci-assign,host=04:00.0"..., 24 pci-assign,host=04:00.0) = 24
write(2, ": "..., 2: ) = 2
write(2, "Device 'pci-assign' could not be "..., 44Device 'pci-assign' could not be initialized) = 44
write(2, "\n"..., 1
) = 1
exit_group(1) = ?
proxmox:~#