目的
测试vpp ipsec转发性能，想当作openstack VPNaas的一种driver，提高ipsec网关的转发能力。
环境
测试资源很有限，没有测试仪，物理机之外的交换机不受控制。
只有两台物理机，各有一张测试网卡，网卡有两个10G口，可能确定的是两台物理机一个10G口之间联到同一个TOR交换机可能互通，另一对10G口之间测试不通。
想法就是vpp两个口，一个口当作网关，另一个口用来做tunnel。
vpp版本是master commit 6e39ff03a6fa28a2655d767454394413252a269d，早期版本ipsec有bug.
vpp用到了dpdk，需要先创建hugepage和网卡绑定igb_uio，不再特殊强调。
第一种方法
创建一对veth pair，一个放在namespace中，一个放在vpp上当host interface。
机器1上操作：
代码语言：javascript复制#服务器1配置
ip netns add ns0
ip link add vpp0 type veth peer name host_vpp0
ip link set vpp0 up
ip link set host_vpp0 netns ns0
ip netns exec ns0 ip link set lo up
ip netns exec ns0 ip link set host_vpp0 up
ip netns exec ns0 ip addr add 192.168.1.1/24 dev host_vpp0
ip netns exec ns0 ip route add default via 192.168.1.2 dev host_vpp0
#启动vpp
/home/huiwei/vpp/build-root/build-vpp-native/vpp/bin/vpp -c /home/huiwei/vsap/configs/startup.conf
#vpp1的配置
set interface state TenGigabitEthernet3/0/1 up
set interface ip address TenGigabitEthernet3/0/1 192.168.3.1/24
set interface promiscuous on TenGigabitEthernet3/0/1
create host-interface name vpp0 hw-addr fa:16:19:19:19:19
set interface state host-vpp0 up
set interface ip address host-vpp0 192.168.1.2/24
create ipip tunnel src 192.168.3.1 dst 192.168.3.2
ipsec sa add 10 spi 1000 esp crypto-key 4339314b55523947594d6d3547666b45 crypto-alg aes-cbc-128 integ-key 4339314b55523947594d6d3547666b45 integ-alg sha1-96
ipsec tunnel protect ipip0 sa-in 10 sa-out 10 192.168.3.2
ip route add 192.168.2.0/24 via 192.168.3.2 ipip0
set interface unnumbered ipip0 use TenGigabitEthernet3/0/1
set interface state ipip0 up机器2上操作：
代码语言：javascript复制#服务器2配置
ip netns add ns0
ip link add vpp0 type veth peer name host_vpp0
ip link set vpp0 up
ip link set host_vpp0 netns ns0
ip netns exec ns0 ip link set lo up
ip netns exec ns0 ip link set host_vpp0 up
ip netns exec ns0 ip addr add 192.168.2.1/24 dev host_vpp0
ip netns exec ns0 ip route add default via 192.168.2.2 dev host_vpp0
#启动vpp
/home/huiwei/vpp/build-root/build-vpp-native/vpp/bin/vpp -c /home/huiwei/vsap/configs/startup.conf
#vpp2的配置
set interface state TenGigabitEthernet3/0/1 up
set interface ip address TenGigabitEthernet3/0/1 192.168.3.2/24
set interface promiscuous on TenGigabitEthernet3/0/1
create host-interface name vpp0 hw-addr fa:16:29:29:29:29
set interface state host-vpp0 up
set interface ip address host-vpp0 192.168.2.2/24
create ipip tunnel src 192.168.3.2 dst 192.168.3.1
ipsec sa add 10 spi 1000 esp crypto-key 4339314b55523947594d6d3547666b45 crypto-alg aes-cbc-128 integ-key 4339314b55523947594d6d3547666b45 integ-alg sha1-96
ipsec tunnel protect ipip0 sa-in 10 sa-out 10 192.168.3.1
ip route add 192.168.1.0/24 via 192.168.3.1 ipip0
set interface unnumbered ipip0 use TenGigabitEthernet3/0/1
set interface state ipip0 up测试结果比较差，分析原因第一是vpp host interface收包时大量丢包，第二就是iperf3测试工具不好用。
代码语言：javascript复制ip netns exec ns0 iperf3 -c 192.168.1.1 -u -l 64 -P 128 -t 5
#no ipsec
[SUM] 0.00-5.00 sec 64.5 MBytes 108 Mbits/sec 42.081 ms 554038/1056128 (52%)
#aesni_mb
[SUM] 0.00-5.00 sec 67.6 MBytes 113 Mbits/sec 26.476 ms 775537/1105258 (70%)
#openssl
[SUM] 0.00-5.00 sec 69.2 MBytes 116 Mbits/sec 11.179 ms 886194/1132434 (78%)第二种方法
既然veth pair不好用，那就用物理网卡，但卡又不够用，外面交换机又不受控制，突然想到了sriov，多虚出来几个物理网卡。iperf3用dpdk-ptkgen或者test-pmd替代。dpdk用一个vf作为host，vpp用另一个vf作为网关，这两个vf之间互通用网卡内嵌的switch。
PS：sriov有个问题就是网卡内嵌的switch只转发它自己的pf和vf的mac，不学习mac，所以openstack中sriov虚机和同host上virtio虚机或者dvr网关是通不了的。
vpp另一个网卡本来想直接用整个物理网卡，但是结果用着用着就莫名其妙NO-CARRIER了，提示没有接网线，原因不明，reboot物理机就好了，试着用vf就没再碰到这个问题
dpdk test-pmd只显示发包和收包个数，不能显示实时速率，采用low一点的手法，用手机秒表计时300s计算pps，不是非常精确。
代码语言：javascript复制#纯IP转发vpp1
set interface state VirtualFunctionEthernet3/10/1 up
set interface state VirtualFunctionEthernet3/10/0 up
set interface ip address VirtualFunctionEthernet3/10/1 192.168.3.1/24
set interface ip address VirtualFunctionEthernet3/10/0 192.168.1.2/24
ip route add 192.168.2.0/24 via 192.168.3.2 VirtualFunctionEthernet3/10/1
#纯IP转发vpp2
set interface state VirtualFunctionEthernet3/10/1 up
set interface state VirtualFunctionEthernet3/10/0 up
set interface ip address VirtualFunctionEthernet3/10/1 192.168.3.2/24
set interface ip address VirtualFunctionEthernet3/10/0 192.168.2.2/24
ip route add 192.168.1.0/24 via 192.168.3.1 VirtualFunctionEthernet3/10/1
#ipsec转发vpp1
set interface state VirtualFunctionEthernet3/10/1 up
set interface ip address VirtualFunctionEthernet3/10/1 192.168.3.1/24
set interface state VirtualFunctionEthernet3/10/0 up
set interface ip address VirtualFunctionEthernet3/10/0 192.168.1.2/24
create ipip tunnel src 192.168.3.1 dst 192.168.3.2
ipsec sa add 10 spi 1000 esp crypto-key 4339314b55523947594d6d3547666b45 crypto-alg aes-cbc-128 integ-key 4339314b55523947594d6d3547666b45 integ-alg sha1-96
ipsec tunnel protect ipip0 sa-in 10 sa-out 10 192.168.3.2
ip route add 192.168.2.0/24 via 192.168.3.2 ipip0
set interface unnumbered ipip0 use VirtualFunctionEthernet3/10/1
set interface state ipip0 up
ip neigh VirtualFunctionEthernet3/10/0 192.168.1.1 22:FA:5E:56:7C:5C static
#ipsec转发vpp2
set interface state VirtualFunctionEthernet3/10/1 up
set interface ip address VirtualFunctionEthernet3/10/1 192.168.3.2/24
set interface state VirtualFunctionEthernet3/10/0 up
set interface ip address VirtualFunctionEthernet3/10/0 192.168.2.2/24
create ipip tunnel src 192.168.3.2 dst 192.168.3.1
ipsec sa add 10 spi 1000 esp crypto-key 4339314b55523947594d6d3547666b45 crypto-alg aes-cbc-128 integ-key 4339314b55523947594d6d3547666b45 integ-alg sha1-96
ipsec tunnel protect ipip0 sa-in 10 sa-out 10 192.168.3.1
ip route add 192.168.1.0/24 via 192.168.3.1 ipip0
set interface unnumbered ipip0 use VirtualFunctionEthernet3/10/1
set interface state ipip0 up两边test-pmd参数
代码语言：javascript复制./testpmd -l 1-2 -n 4 -b 0000:03:10.0 -b 0000:03:10.1 -- -i --nb-cores=1 --nb-ports=1 --burst=512 --max-pkt-len=64 --txq=4 --eth-peer=0,3a:54:11:3a:72:1f --tx-ip=192.168.2.1,192.168.1.1 --forward-mode=txonly
./testpmd -l 1-2 -n 4 -b 0000:03:10.0 -b 0000:03:10.1 -- -i --nb-cores=1 --nb-ports=1 --burst=512 --rxq=4 --forward-mode=rxonly试着收方向用没用test-pmd，tcpdump抓包看了一上没问题
代码语言：javascript复制[root@slb1 huiwei]# tcpdump -nn -e -i eth6 -c 10
tcpdump: verbose output suppressed, use -v or -vv for full protocol decode
listening on eth6, link-type EN10MB (Ethernet), capture size 65535 bytes
17:25:01.265838 02:09:c0:8b:6c:68 > 22:fa:5e:56:7c:5c, ethertype IPv4 (0x0800), length 64: 192.168.2.1.9 > 192.168.1.1.9: UDP, length 22
17:25:01.265842 02:09:c0:8b:6c:68 > 22:fa:5e:56:7c:5c, ethertype IPv4 (0x0800), length 64: 192.168.2.1.9 > 192.168.1.1.9: UDP, length 22
17:25:01.265844 02:09:c0:8b:6c:68 > 22:fa:5e:56:7c:5c, ethertype IPv4 (0x0800), length 64: 192.168.2.1.9 > 192.168.1.1.9: UDP, length 22测试结果300s的数据，测试了vpp三层转发，用openssl的ipsec和用dpdk mb crypto的ipsec，分别是5Mpps,1.4Mpps和1.2Mpps。
代码语言：javascript复制[root@slb2 huiwei]#
======================================================================================
#dpdk sw pmd ipsec三层转发300s
testpmd> stop
Telling cores to stop...
Waiting for lcores to finish...
---------------------- Forward statistics for port 0 ----------------------
RX-packets: 13 RX-dropped: 0 RX-total: 13
TX-packets: 1943715560 TX-dropped: 1 TX-total: 1943715561
----------------------------------------------------------------------------
Accumulated forward statistics for all ports
RX-packets: 13 RX-dropped: 0 RX-total: 13
TX-packets: 1943715560 TX-dropped: 1 TX-total: 1943715561
Done.
vpp# show int
Name Idx State MTU (L3/IP4/IP6/MPLS) Counter Count
VirtualFunctionEthernet3/10/0 1 up 9000/0/0/0 rx packets 460111460
rx bytes 29447134830
drops 5
ip4 460111460
VirtualFunctionEthernet3/10/1 2 up 9000/0/0/0 rx packets 140
rx bytes 13194
tx packets 460111455
tx bytes 61654934970
drops 140
ip4 20
ipip0 3 up 9000/0/0/0 tx packets 920222910
tx bytes 87421176450
local0 0 down 0/0/0/0
vpp# show error
Count Node Reason
460111455 dpdk-esp4-encrypt-tun ESP pkts received
460111455 dpdk-crypto-input Crypto ops dequeued
5 dpdk-input no error
17 dpdk-input no error
120 arp-reply IP4 source address not local to subnet
3 ip4-local ip4 source lookup miss
vpp#
vpp# show int
Name Idx State MTU (L3/IP4/IP6/MPLS) Counter Count
VirtualFunctionEthernet3/10/0 1 up 9000/0/0/0 rx packets 141
rx bytes 8460
tx packets 364223104
tx bytes 23310278656
drops 141
VirtualFunctionEthernet3/10/1 2 up 9000/0/0/0 rx packets 364223212
rx bytes 48805906364
drops 108
ip4 364223120
ipip0 3 up 9000/0/0/0 rx packets 728446208
rx bytes 54633465600
ip4 728446208
local0 0 down 0/0/0/0
vpp# show error
Count Node Reason
141 arp-reply IP4 source address not local to subnet
364223104 dpdk-esp4-decrypt-post ESP post pkts
364223104 dpdk-esp4-decrypt ESP pkts received
364223104 dpdk-crypto-input Crypto ops dequeued
14 dpdk-input no error
93 arp-reply IP4 source address not local to subnet
364223104 ipip4-input packets decapsulated
364223104 ipsec4-tun-input good packets received
2 ip4-local ip4 source lookup miss
======================================================================================
#openssl ipsec三层转发300s
testpmd> stop
Telling cores to stop...
Waiting for lcores to finish...
---------------------- Forward statistics for port 0 ----------------------
RX-packets: 321 RX-dropped: 0 RX-total: 321
TX-packets: 1652694742 TX-dropped: 0 TX-total: 1652694742
----------------------------------------------------------------------------
Accumulated forward statistics for all ports
RX-packets: 321 RX-dropped: 0 RX-total: 321
TX-packets: 1652694742 TX-dropped: 0 TX-total: 1652694742
Done.
testpmd>
vpp# show int
Name Idx State MTU (L3/IP4/IP6/MPLS) Counter Count
VirtualFunctionEthernet3/10/0 1 up 9000/0/0/0 rx packets 460111460
rx bytes 29447134830
drops 5
ip4 460111460
VirtualFunctionEthernet3/10/1 2 up 9000/0/0/0 rx packets 140
rx bytes 13194
tx packets 460111455
tx bytes 61654934970
drops 140
ip4 20
ipip0 3 up 9000/0/0/0 tx packets 920222910
tx bytes 87421176450
local0 0 down 0/0/0/0
vpp# show error
Count Node Reason
460111455 dpdk-esp4-encrypt-tun ESP pkts received
460111455 dpdk-crypto-input Crypto ops dequeued
5 dpdk-input no error
17 dpdk-input no error
120 arp-reply IP4 source address not local to subnet
3 ip4-local ip4 source lookup miss
vpp#
testpmd> stop
Telling cores to stop...
Waiting for lcores to finish...
---------------------- Forward statistics for port 0 ----------------------
RX-packets: 364223212 RX-dropped: 0 RX-total: 364223212
TX-packets: 0 TX-dropped: 0 TX-total: 0
----------------------------------------------------------------------------
Accumulated forward statistics for all ports
RX-packets: 364223212 RX-dropped: 0 RX-total: 364223212
TX-packets: 0 TX-dropped: 0 TX-total: 0
Done.
===================================================================================================
#线三层转发300s
testpmd> stop
Telling cores to stop...
Waiting for lcores to finish...
---------------------- Forward statistics for port 0 ----------------------
RX-packets: 316 RX-dropped: 0 RX-total: 316
TX-packets: 1648023286 TX-dropped: 0 TX-total: 1648023286
----------------------------------------------------------------------------
Accumulated forward statistics for all ports
RX-packets: 316 RX-dropped: 0 RX-total: 316
TX-packets: 1648023286 TX-dropped: 0 TX-total: 1648023286
Done.
testpmd>
vpp#
vpp# show int
Name Idx State MTU (L3/IP4/IP6/MPLS) Counter Count
VirtualFunctionEthernet3/10/0 1 up 9000/0/0/0 rx packets 1644736479
rx bytes 105263139660
tx packets 298
tx bytes 12516
drops 18
ip4 1644736479
VirtualFunctionEthernet3/10/1 2 up 9000/0/0/0 rx packets 535
rx bytes 46936
tx packets 1644736462
tx bytes 105263133546
drops 535
ip4 328
local0 0 down 0/0/0/0
vpp#
vpp# show int
Name Idx State MTU (L3/IP4/IP6/MPLS) Counter Count
VirtualFunctionEthernet3/10/0 1 up 9000/0/0/0 rx packets 527
rx bytes 40858
tx packets 1544198741
tx bytes 98828718874
drops 201
ip4 310
VirtualFunctionEthernet3/10/1 2 up 9000/0/0/0 rx packets 1544219410
rx bytes 98830046460
tx packets 307
tx bytes 28162
drops 20693
ip4 1544219219
local0 0 down 0/0/0/0最后附vpp的配置文件示例
代码语言：javascript复制#startup.conf
unix {
interactive
log /var/log/vpp/vpp.log
full-coredump
cli-listen /run/vpp/cli.sock
}
api-trace {
on
}
cpu {
main-core 3
corelist-workers 4-5
}
dpdk {
socket-mem 20480,0
dev 0000:03:10.1 {
}
dev 0000:03:10.0 {
}
#enable-cryptodev
vdev cryptodev_aesni_mb_pmd,max_nb_queue_pairs=2,max_nb_sessions=1024,socket_id=0
dev default {
num-rx-queues 4
num-tx-queues 4
num-rx-desc 512
num-tx-desc 512
}
}
