目的

测试vpp ipsec转发性能，想当作openstack VPNaas的一种driver，提高ipsec网关的转发能力。

环境

测试资源很有限，没有测试仪，物理机之外的交换机不受控制。

只有两台物理机，各有一张测试网卡，网卡有两个10G口，可能确定的是两台物理机一个10G口之间联到同一个TOR交换机可能互通，另一对10G口之间测试不通。

想法就是vpp两个口，一个口当作网关，另一个口用来做tunnel。

vpp版本是master commit 6e39ff03a6fa28a2655d767454394413252a269d，早期版本ipsec有bug.

vpp用到了dpdk，需要先创建hugepage和网卡绑定igb_uio，不再特殊强调。

第一种方法

创建一对veth pair，一个放在namespace中，一个放在vpp上当host interface。

机器1上操作：

#服务器1配置
ip netns add ns0
ip link add vpp0 type veth peer name host_vpp0
ip link set vpp0 upip link set host_vpp0 netns ns0
ip netns exec ns0 ip link set lo up
ip netns exec ns0 ip link set host_vpp0 up
ip netns exec ns0 ip addr add 192.168.1.1/24 dev host_vpp0
ip netns exec ns0 ip route add default via 192.168.1.2 dev host_vpp0#启动vpp
/home/huiwei/vpp/build-root/build-vpp-native/vpp/bin/vpp -c /home/huiwei/vsap/configs/startup.conf#vpp1的配置
set interface state TenGigabitEthernet3/0/1 up
set interface ip address TenGigabitEthernet3/0/1 192.168.3.1/24
set interface promiscuous on TenGigabitEthernet3/0/1create host-interface name vpp0  hw-addr fa:16:19:19:19:19
set interface state host-vpp0 up
set interface ip address host-vpp0  192.168.1.2/24create ipip tunnel src 192.168.3.1 dst 192.168.3.2
ipsec sa add 10 spi 1000 esp crypto-key 4339314b55523947594d6d3547666b45 crypto-alg aes-cbc-128 integ-key 4339314b55523947594d6d3547666b45 integ-alg sha1-96
ipsec tunnel protect ipip0 sa-in 10 sa-out 10 192.168.3.2ip route add 192.168.2.0/24 via 192.168.3.2  ipip0
set interface unnumbered ipip0 use TenGigabitEthernet3/0/1
set interface state ipip0 up

复制

机器2上操作：

#服务器2配置
ip netns add ns0
ip link add vpp0 type veth peer name host_vpp0
ip link set vpp0 upip link set host_vpp0 netns ns0
ip netns exec ns0 ip link set lo up
ip netns exec ns0 ip link set host_vpp0 up
ip netns exec ns0 ip addr add 192.168.2.1/24 dev host_vpp0
ip netns exec ns0 ip route add default via 192.168.2.2 dev host_vpp0#启动vpp
/home/huiwei/vpp/build-root/build-vpp-native/vpp/bin/vpp -c /home/huiwei/vsap/configs/startup.conf#vpp2的配置
set interface state TenGigabitEthernet3/0/1 up
set interface ip address TenGigabitEthernet3/0/1 192.168.3.2/24
set interface promiscuous on TenGigabitEthernet3/0/1create host-interface name vpp0  hw-addr fa:16:29:29:29:29
set interface state host-vpp0 up
set interface ip address host-vpp0  192.168.2.2/24create ipip tunnel src 192.168.3.2 dst 192.168.3.1
ipsec sa add 10 spi 1000 esp crypto-key 4339314b55523947594d6d3547666b45 crypto-alg aes-cbc-128 integ-key 4339314b55523947594d6d3547666b45 integ-alg sha1-96
ipsec tunnel protect ipip0 sa-in 10 sa-out 10 192.168.3.1ip route add 192.168.1.0/24 via 192.168.3.1 ipip0
set interface unnumbered ipip0 use TenGigabitEthernet3/0/1
set interface state ipip0 up

复制

测试结果比较差，分析原因第一是vpp host interface收包时大量丢包，第二就是iperf3测试工具不好用。

ip netns exec ns0 iperf3 -c 192.168.1.1 -u -l 64 -P 128 -t 5
#no ipsec
[SUM]   0.00-5.00   sec  64.5 MBytes   108 Mbits/sec  42.081 ms  554038/1056128 (52%)
#aesni_mb
[SUM]   0.00-5.00   sec  67.6 MBytes   113 Mbits/sec  26.476 ms  775537/1105258 (70%)
#openssl
[SUM]   0.00-5.00   sec  69.2 MBytes   116 Mbits/sec  11.179 ms  886194/1132434 (78%)

复制

第二种方法

既然veth pair不好用，那就用物理网卡，但卡又不够用，外面交换机又不受控制，突然想到了sriov，多虚出来几个物理网卡。iperf3用dpdk-ptkgen或者test-pmd替代。dpdk用一个vf作为host，vpp用另一个vf作为网关，这两个vf之间互通用网卡内嵌的switch。

PS：sriov有个问题就是网卡内嵌的switch只转发它自己的pf和vf的mac，不学习mac，所以openstack中sriov虚机和同host上virtio虚机或者dvr网关是通不了的。

vpp另一个网卡本来想直接用整个物理网卡，但是结果用着用着就莫名其妙NO-CARRIER了，提示没有接网线，原因不明，reboot物理机就好了，试着用vf就没再碰到这个问题

dpdk test-pmd只显示发包和收包个数，不能显示实时速率，采用low一点的手法，用手机秒表计时300s计算pps，不是非常精确。

#纯IP转发vpp1
set interface state VirtualFunctionEthernet3/10/1 up
set interface state VirtualFunctionEthernet3/10/0 up
set interface ip address VirtualFunctionEthernet3/10/1 192.168.3.1/24
set interface ip address VirtualFunctionEthernet3/10/0 192.168.1.2/24
ip route add 192.168.2.0/24 via 192.168.3.2 VirtualFunctionEthernet3/10/1
#纯IP转发vpp2
set interface state VirtualFunctionEthernet3/10/1 up
set interface state VirtualFunctionEthernet3/10/0 up
set interface ip address VirtualFunctionEthernet3/10/1 192.168.3.2/24
set interface ip address VirtualFunctionEthernet3/10/0 192.168.2.2/24
ip route add 192.168.1.0/24 via 192.168.3.1 VirtualFunctionEthernet3/10/1#ipsec转发vpp1
set interface state VirtualFunctionEthernet3/10/1 up
set interface ip address VirtualFunctionEthernet3/10/1 192.168.3.1/24
set interface state VirtualFunctionEthernet3/10/0 up
set interface ip address VirtualFunctionEthernet3/10/0 192.168.1.2/24create ipip tunnel src 192.168.3.1 dst 192.168.3.2
ipsec sa add 10 spi 1000 esp crypto-key 4339314b55523947594d6d3547666b45 crypto-alg aes-cbc-128 integ-key 4339314b55523947594d6d3547666b45 integ-alg sha1-96
ipsec tunnel protect ipip0 sa-in 10 sa-out 10 192.168.3.2ip route add 192.168.2.0/24 via 192.168.3.2  ipip0
set interface unnumbered ipip0 use VirtualFunctionEthernet3/10/1
set interface state ipip0 upip neigh VirtualFunctionEthernet3/10/0 192.168.1.1 22:FA:5E:56:7C:5C static#ipsec转发vpp2
set interface state VirtualFunctionEthernet3/10/1 up
set interface ip address VirtualFunctionEthernet3/10/1 192.168.3.2/24
set interface state VirtualFunctionEthernet3/10/0 up
set interface ip address VirtualFunctionEthernet3/10/0 192.168.2.2/24create ipip tunnel src 192.168.3.2 dst 192.168.3.1
ipsec sa add 10 spi 1000 esp crypto-key 4339314b55523947594d6d3547666b45 crypto-alg aes-cbc-128 integ-key 4339314b55523947594d6d3547666b45 integ-alg sha1-96
ipsec tunnel protect ipip0 sa-in 10 sa-out 10 192.168.3.1ip route add 192.168.1.0/24 via 192.168.3.1  ipip0
set interface unnumbered ipip0 use VirtualFunctionEthernet3/10/1
set interface state ipip0 up

（免费订阅,永久学习）学习地址: Dpdk/网络协议栈/vpp/OvS/DDos/NFV/虚拟化/高性能专家-学习视频教程-腾讯课堂

更多DPDK相关学习资料有需要的可以自行报名学习,免费订阅,永久学习,或点击这里加qun免费
领取,关注我持续更新哦! !   

复制

两边test-pmd参数

./testpmd -l 1-2 -n 4 -b 0000:03:10.0 -b 0000:03:10.1 -- -i --nb-cores=1 --nb-ports=1 --burst=512 --max-pkt-len=64 --txq=4 --eth-peer=0,3a:54:11:3a:72:1f --tx-ip=192.168.2.1,192.168.1.1 --forward-mode=txonly./testpmd -l 1-2 -n 4 -b 0000:03:10.0 -b 0000:03:10.1 -- -i --nb-cores=1 --nb-ports=1 --burst=512 --rxq=4 --forward-mode=rxonly

复制

试着收方向用没用test-pmd，tcpdump抓包看了一上没问题

[root@slb1 huiwei]# tcpdump -nn -e -i eth6 -c 10
tcpdump: verbose output suppressed, use -v or -vv for full protocol decode
listening on eth6, link-type EN10MB (Ethernet), capture size 65535 bytes
17:25:01.265838 02:09:c0:8b:6c:68 > 22:fa:5e:56:7c:5c, ethertype IPv4 (0x0800), length 64: 192.168.2.1.9 > 192.168.1.1.9: UDP, length 22
17:25:01.265842 02:09:c0:8b:6c:68 > 22:fa:5e:56:7c:5c, ethertype IPv4 (0x0800), length 64: 192.168.2.1.9 > 192.168.1.1.9: UDP, length 22
17:25:01.265844 02:09:c0:8b:6c:68 > 22:fa:5e:56:7c:5c, ethertype IPv4 (0x0800), length 64: 192.168.2.1.9 > 192.168.1.1.9: UDP, length 22

复制

测试结果300s的数据，测试了vpp三层转发，用openssl的ipsec和用dpdk mb crypto的ipsec，分别是5Mpps,1.4Mpps和1.2Mpps。

[root@slb2 huiwei]#======================================================================================
#dpdk sw pmd ipsec三层转发300s
testpmd> stop
Telling cores to stop...
Waiting for lcores to finish...---------------------- Forward statistics for port 0  ----------------------RX-packets: 13             RX-dropped: 0             RX-total: 13TX-packets: 1943715560     TX-dropped: 1             TX-total: 1943715561----------------------------------------------------------------------------+++++++++++++++ Accumulated forward statistics for all ports+++++++++++++++RX-packets: 13             RX-dropped: 0             RX-total: 13TX-packets: 1943715560     TX-dropped: 1             TX-total: 1943715561++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++Done.
vpp# show intName               Idx    State  MTU (L3/IP4/IP6/MPLS)     Counter          Count
VirtualFunctionEthernet3/10/0     1      up          9000/0/0/0     rx packets             460111460rx bytes             29447134830drops                          5ip4                    460111460
VirtualFunctionEthernet3/10/1     2      up          9000/0/0/0     rx packets                   140rx bytes                   13194tx packets             460111455tx bytes             61654934970drops                        140ip4                           20
ipip0                             3      up          9000/0/0/0     tx packets             920222910tx bytes             87421176450
local0                            0     down          0/0/0/0
vpp# show errorCount                    Node                  Reason460111455          dpdk-esp4-encrypt-tun         ESP pkts received460111455            dpdk-crypto-input           Crypto ops dequeued5               dpdk-input               no error17               dpdk-input               no error120                arp-reply               IP4 source address not local to subnet3                ip4-local               ip4 source lookup miss
vpp#vpp# show intName               Idx    State  MTU (L3/IP4/IP6/MPLS)     Counter          Count
VirtualFunctionEthernet3/10/0     1      up          9000/0/0/0     rx packets                   141rx bytes                    8460tx packets             364223104tx bytes             23310278656drops                        141
VirtualFunctionEthernet3/10/1     2      up          9000/0/0/0     rx packets             364223212rx bytes             48805906364drops                        108ip4                    364223120
ipip0                             3      up          9000/0/0/0     rx packets             728446208rx bytes             54633465600ip4                    728446208
local0                            0     down          0/0/0/0vpp# show errorCount                    Node                  Reason141                arp-reply               IP4 source address not local to subnet364223104         dpdk-esp4-decrypt-post         ESP post pkts364223104            dpdk-esp4-decrypt           ESP pkts received364223104            dpdk-crypto-input           Crypto ops dequeued14               dpdk-input               no error93                arp-reply               IP4 source address not local to subnet364223104               ipip4-input              packets decapsulated364223104            ipsec4-tun-input            good packets received2                ip4-local               ip4 source lookup miss======================================================================================
#openssl ipsec三层转发300s
testpmd> stop
Telling cores to stop...
Waiting for lcores to finish...---------------------- Forward statistics for port 0  ----------------------RX-packets: 321            RX-dropped: 0             RX-total: 321TX-packets: 1652694742     TX-dropped: 0             TX-total: 1652694742----------------------------------------------------------------------------+++++++++++++++ Accumulated forward statistics for all ports+++++++++++++++RX-packets: 321            RX-dropped: 0             RX-total: 321TX-packets: 1652694742     TX-dropped: 0             TX-total: 1652694742++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++Done.
testpmd>vpp# show intName               Idx    State  MTU (L3/IP4/IP6/MPLS)     Counter          Count
VirtualFunctionEthernet3/10/0     1      up          9000/0/0/0     rx packets             460111460rx bytes             29447134830drops                          5ip4                    460111460
VirtualFunctionEthernet3/10/1     2      up          9000/0/0/0     rx packets                   140rx bytes                   13194tx packets             460111455tx bytes             61654934970drops                        140ip4                           20
ipip0                             3      up          9000/0/0/0     tx packets             920222910tx bytes             87421176450
local0                            0     down          0/0/0/0
vpp# show errorCount                    Node                  Reason460111455          dpdk-esp4-encrypt-tun         ESP pkts received460111455            dpdk-crypto-input           Crypto ops dequeued5               dpdk-input               no error17               dpdk-input               no error120                arp-reply               IP4 source address not local to subnet3                ip4-local               ip4 source lookup miss
vpp#
testpmd> stop
Telling cores to stop...
Waiting for lcores to finish...---------------------- Forward statistics for port 0  ----------------------RX-packets: 364223212      RX-dropped: 0             RX-total: 364223212TX-packets: 0              TX-dropped: 0             TX-total: 0----------------------------------------------------------------------------+++++++++++++++ Accumulated forward statistics for all ports+++++++++++++++RX-packets: 364223212      RX-dropped: 0             RX-total: 364223212TX-packets: 0              TX-dropped: 0             TX-total: 0++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++Done.===================================================================================================
#线三层转发300s
testpmd> stop
Telling cores to stop...
Waiting for lcores to finish...---------------------- Forward statistics for port 0  ----------------------RX-packets: 316            RX-dropped: 0             RX-total: 316TX-packets: 1648023286     TX-dropped: 0             TX-total: 1648023286----------------------------------------------------------------------------+++++++++++++++ Accumulated forward statistics for all ports+++++++++++++++RX-packets: 316            RX-dropped: 0             RX-total: 316TX-packets: 1648023286     TX-dropped: 0             TX-total: 1648023286++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++Done.
testpmd>vpp#
vpp# show intName               Idx    State  MTU (L3/IP4/IP6/MPLS)     Counter          Count
VirtualFunctionEthernet3/10/0     1      up          9000/0/0/0     rx packets            1644736479rx bytes            105263139660tx packets                   298tx bytes                   12516drops                         18ip4                   1644736479
VirtualFunctionEthernet3/10/1     2      up          9000/0/0/0     rx packets                   535rx bytes                   46936tx packets            1644736462tx bytes            105263133546drops                        535ip4                          328
local0                            0     down          0/0/0/0
vpp#vpp# show intName               Idx    State  MTU (L3/IP4/IP6/MPLS)     Counter          Count
VirtualFunctionEthernet3/10/0     1      up          9000/0/0/0     rx packets                   527rx bytes                   40858tx packets            1544198741tx bytes             98828718874drops                        201ip4                          310
VirtualFunctionEthernet3/10/1     2      up          9000/0/0/0     rx packets            1544219410rx bytes             98830046460tx packets                   307tx bytes                   28162drops                      20693ip4                   1544219219
local0                            0     down          0/0/0/0

复制

最后附vpp的配置文件示例

#startup.conf
unix {interactivelog /var/log/vpp/vpp.logfull-coredumpcli-listen /run/vpp/cli.sock
}api-trace {on
}cpu {main-core 3corelist-workers 4-5
}dpdk {socket-mem 20480,0dev 0000:03:10.1 {}dev 0000:03:10.0 {}#enable-cryptodevvdev cryptodev_aesni_mb_pmd,max_nb_queue_pairs=2,max_nb_sessions=1024,socket_id=0dev default {num-rx-queues 4num-tx-queues 4num-rx-desc 512num-tx-desc 512}
}

原文链接：https://cloud.tencent.com/developer/article/1792341