惠伟：networking-sfc deep dive 2zhuanlan.zhihu.com
上篇介绍MPLS封装流量的networking-sfc，这次介绍一下NSH，毕竟这才是主流，看看NSH的流表和报文格式，体验一下。
环境
修改代码
openstack stein版本的networking-sfc代码有bug。
- encap(nsh)后外层默认的ethernet头src mac和dst mac都为空，需要设置一下，否则后续无法match或者认为是非法报文。
http://man7.org/linux/man-pages/man7/ovs-actions.7.html
The encap(ethernet) variant encapsulate a bare L3 packet in an
Ethernet frame. The Ethernet type is initialized to the L3 packet’s
type, e.g. 0x0800 if the L3 packet is IPv4. The Ethernet source and
destination are initially zeroed.
#不修改openvswitch内核模块报错
2019-12-26T09:39:54.560Z|05881|dpif(handler162)|WARN|system@ovs-system: failed to put[create] (Invalid argument) ufid:4f1aa182-ef16-4c52-b24e-52f6fd29cfd9 recirc_id(0x1f),dp_hash(0xc772ac0e/0xf),skb_priority(0/0),in_port(28),skb_mark(0/0),ct_state(0/0),ct_zone(0/0),ct_mark(0/0),ct_label(0/0),eth(src=fa:16:3e:43:c7:d0,dst=fa:16:e3:43:f3:43),eth_type(0x0800),ipv4(src=1.0.0.143/0.0.0.0,dst=2.0.0.4/0.0.0.0,proto=1/0,tos=0/0,ttl=64/0,frag=no),icmp(type=8/0,code=0/0), actions:set(eth(dst=fa:16:3e:94:4a:b1)),push_nsh(flags=0,ttl=63,mdtype=1,np=3,spi=0x2,si=255,c1=0x0,c2=0x0,c3=0x0,c4=0x0),push_eth(src=fa:16:3e:43:c7:d0,dst=fa:16:3e:94:4a:b1),pop_eth,pop_nsh(),set(eth(src=fa:16:3e:43:c7:d0,dst=fa:16:3e:94:4a:b1)),recirc(0x6ad)- 另一个问题是decap nsh时设置了dst mac，但ovs报错了，decap时不需要设置，encap时内层dst mac已经修改过了。
#nsh table 10报错
openvswitch-agent.log:5718:2019-12-24 15:24:17.148 3652358 ERROR neutron.agent.common.ovs_lib [req-62f90557-d9f9-4a94-a081-db4cc021738c - - - - -] Unable to execute ['ovs-ofctl', 'add-flows', '-O', 'OpenFlow13', 'br-int', '-']. Exception: Exit code: 1; Stdin: hard_timeout=0,idle_timeout=0,priority=1,cookie=2103133625977426012,nsh_spi=2,nsh_mdtype=1,eth_type=35151,table=10,dl_dst=fa:16:3e:28:ca:68,dl_vlan=18,nsh_si=255,actions=strip_vlan,move:NXM_OF_ETH_DST->OXM_OF_PKT_REG0[0..47],decap(),decap(),move:OXM_OF_PKT_REG0[0..47]->NXM_OF_ETH_DST,output:226;
Stdout: ; Stderr: 2019-12-24T07:24:17Z|00001|meta_flow|WARN|destination field eth_dst lacks correct prerequisites我的代码修改如下：
Gerrit Code Reviewreview.opendev.org
代码语言：javascript复制huiwei@huiwei-l1 MINGW64 /d/src/openstack_community/networking-sfc ((8bee6b4...))
$ git diff
diff --git a/networking_sfc/services/sfc/agent/extensions/openvswitch/sfc_driver.py b/networking_sfc/services/sfc/agent/extensions/openvswitch/sfc_driver.py
index 563b3f5..7052612 100644
--- a/networking_sfc/services/sfc/agent/extensions/openvswitch/sfc_driver.py
b/networking_sfc/services/sfc/agent/extensions/openvswitch/sfc_driver.py
@@ -415,10 415,40 @@ class SfcOVSAgentDriver(sfc.SfcAgentDriver):
subnet_actions_list.append(across_flow)
if item['local_endpoint'] == self.local_ip:
- subnet_actions = 'resubmit(,%d)' % INGRESS_TABLE
if flowrule['fwd_path']:
subnet_actions = (
'set_field:%s->eth_src,'
'set_field:%s->eth_dst,'
'resubmit(,%d)' %
(self.br_int.get_vif_port_by_id(flowrule['egress']).vif_mac,
item['in_mac_address'],
INGRESS_TABLE))
else:
subnet_actions = (
'set_field:%s->eth_src,'
'set_field:%s->eth_dst,'
'resubmit(,%d)' %
(self.br_int.get_vif_port_by_id(flowrule['egress']).vif_mac,
item['mac_address'],
INGRESS_TABLE))
else:
# same subnet with next hop
- subnet_actions = 'output:%s' % self.patch_tun_ofport
if flowrule['fwd_path']:
subnet_actions = (
'set_field:%s->eth_src,'
'set_field:%s->eth_dst,'
'output:%s' %
(self.br_int.get_vif_port_by_id(flowrule['egress']).vif_mac,
item['in_mac_address'],
self.patch_tun_ofport))
else:
subnet_actions = (
'set_field:%s->eth_src,'
'set_field:%s->eth_dst,'
'output:%s' %
(self.br_int.get_vif_port_by_id(flowrule['egress']).vif_mac,
item['mac_address'],
self.patch_tun_ofport))
subnet_actions_list.append(subnet_actions)
eth_type = constants.ETH_TYPE_IP
@@ -615,9 645,9 @@ class SfcOVSAgentDriver(sfc.SfcAgentDriver):
def _build_proxy_sfc_nsh(self, flowrule, vif_port, vlan):
match_field = self._build_ingress_match_field_sfc_nsh(
flowrule, vif_port, vlan)
- actions = ("strip_vlan,move:NXM_OF_ETH_DST->OXM_OF_PKT_REG0[0..47],"
actions = ("strip_vlan,"
"decap(),decap(),"
- "move:OXM_OF_PKT_REG0[0..47]->NXM_OF_ETH_DST,output:%s"
"output:%s"
"" % vif_port.ofport)
match_field['actions'] = actions
return match_field我用的openvswitch版本号是2.11.0，四个VM都在同一台处理机上报错了，还没太搞明白，参考了一下网上，简单修改了一下代码能运行了。
Re: [ovs-discuss] [SFC NSH] Update "eth dst" for original packet after decap nsh encapsulation packet.www.mail-archive.com
代码语言：javascript复制huiwei@huiwei-l1 MINGW64 /d/src/ovs ((v2.11.0))
$ git diff
diff --git a/datapath/flow_netlink.c b/datapath/flow_netlink.c
index e5e469a84..956bb4f13 100644
--- a/datapath/flow_netlink.c
b/datapath/flow_netlink.c
@@ -2670,7 2670,13 @@ static int validate_set(const struct nlattr *a,
if (key_type > OVS_KEY_ATTR_MAX ||
!check_attr_len(key_len, ovs_key_lens[key_type].len))
return -EINVAL;
-
/************************************************************/
if (key_len == 28) {
return 0;
} else if (key_type == OVS_KEY_ATTR_ETHERNET) {
mac_proto = MAC_PROTO_ETHERNET;
}
/************************************************************/
if (masked && !validate_masked(nla_data(ovs_key), key_len))
return -EINVAL;
@@ -3065,11 3071,11 @@ static int __ovs_nla_copy_actions(struct net *net, const struct nlattr *attr,
case OVS_ACTION_ATTR_POP_NSH: {
__be16 inner_proto;
- if (eth_type != htons(ETH_P_NSH))
- return -EINVAL;
- inner_proto = tun_p_to_eth_p(key->nsh.base.np);
- if (!inner_proto)
- return -EINVAL;
//if (eth_type != htons(ETH_P_NSH))
// return -EINVAL;
//inner_proto = tun_p_to_eth_p(key->nsh.base.np);
//if (!inner_proto)
// return -EINVAL;
if (key->nsh.base.np == TUN_P_ETHERNET)
mac_proto = MAC_PROTO_ETHERNET;
else
diff --git a/datapath/linux/compat/include/linux/static_key.h b/datapath/linux/compat/include/linux/static_key.h
index 01c6a93f0..142c72945 100644
--- a/datapath/linux/compat/include/linux/static_key.h
b/datapath/linux/compat/include/linux/static_key.h
@@ -1,6 1,6 @@
#ifndef _STATIC_KEY_WRAPPER_H
#define _STATIC_KEY_WRAPPER_H
-
#include <asm/atomic.h>
#include_next <linux/static_key.h>
#ifndef HAVE_UPSTREAM_STATIC_KEY
/*Re: [ovs-discuss] [SFC NSH] Update "eth dst" for original packet after decap nsh encapsulation packet.huiwei@huiwei-l1 MINGW64 /d/src/ovs ((v2.11.0))
$ git diff
diff --git a/datapath/flow_netlink.c b/datapath/flow_netlink.c
index e5e469a84..956bb4f13 100644
--- a/datapath/flow_netlink.c
b/datapath/flow_netlink.c
@@ -2670,7 2670,13 @@ static int validate_set(const struct nlattr *a,
if (key_type > OVS_KEY_ATTR_MAX ||
!check_attr_len(key_len, ovs_key_lens[key_type].len))
return -EINVAL;
-
/************************************************************/
if (key_len == 28) {
return 0;
} else if (key_type == OVS_KEY_ATTR_ETHERNET) {
mac_proto = MAC_PROTO_ETHERNET;
}
/************************************************************/
if (masked && !validate_masked(nla_data(ovs_key), key_len))
return -EINVAL;
@@ -3065,11 3071,11 @@ static int __ovs_nla_copy_actions(struct net *net, const struct nlattr *attr,
case OVS_ACTION_ATTR_POP_NSH: {
__be16 inner_proto;
- if (eth_type != htons(ETH_P_NSH))
- return -EINVAL;
- inner_proto = tun_p_to_eth_p(key->nsh.base.np);
- if (!inner_proto)
- return -EINVAL;
//if (eth_type != htons(ETH_P_NSH))
// return -EINVAL;
//inner_proto = tun_p_to_eth_p(key->nsh.base.np);
//if (!inner_proto)
// return -EINVAL;
if (key->nsh.base.np == TUN_P_ETHERNET)
mac_proto = MAC_PROTO_ETHERNET;
else
diff --git a/datapath/linux/compat/include/linux/static_key.h b/datapath/linux/compat/include/linux/static_key.h
index 01c6a93f0..142c72945 100644
--- a/datapath/linux/compat/include/linux/static_key.h
b/datapath/linux/compat/include/linux/static_key.h
@@ -1,6 1,6 @@
#ifndef _STATIC_KEY_WRAPPER_H
#define _STATIC_KEY_WRAPPER_H
-
#include <asm/atomic.h>
#include_next <linux/static_key.h>
#ifndef HAVE_UPSTREAM_STATIC_KEY
/*修改后要编译和重新加载openvswitch.ko
代码语言：javascript复制make
make modules_install
modprobe tunnel6
ovs-dpctl del-dp ovs-system
rmmod vport_vxlan
rmmod openvswitch
modprobe openvswitch
systemctl restart openvswitch
systemctl restart neutron-openvswitch-agent命令
代码语言：javascript复制openstack subnet create --network sfc-net-test1 --subnet-range 1.0.0.0/24 sfc-subnet-test1
openstack subnet create --network sfc-net-test2 --subnet-range 2.0.0.0/24 sfc-subnet-test2
openstack port create --network sfc-net-test1 --enable --no-security-group --disable-port-security sfc-port-p1
openstack port create --network sfc-net-test1 --enable --no-security-group --disable-port-security sfc-port-p2
openstack port create --network sfc-net-test1 --enable --no-security-group --disable-port-security sfc-port-p4
openstack port create --network sfc-net-test2 --enable --no-security-group --disable-port-security sfc-port-p3
openstack port create --network sfc-net-test2 --enable --no-security-group --disable-port-security sfc-port-p5
openstack port create --network sfc-net-test2 --enable --no-security-group --disable-port-security sfc-port-p6
openstack server create --image centos7-hw --flavor centos7-flavor --port sfc-port-p1 sfc-src
openstack server create --image centos7-hw --flavor centos7-flavor --port sfc-port-p2 --port sfc-port-p3 sfc-sf1
openstack server create --image centos7-hw --flavor centos7-flavor --port sfc-port-p4 --port sfc-port-p5 sfc-sf2
openstack server create --image centos7-hw --flavor centos7-flavor --port sfc-port-p6 sfc-dst
openstack sfc port pair create --ingress sfc-port-p2 --egress sfc-port-p3 --service-function-parameters correlation=None,weight=1 port-pair-1
openstack sfc port pair create --ingress sfc-port-p4 --egress sfc-port-p5 --service-function-parameters correlation=None,weight=2 port-pair-2
openstack sfc port pair group create --port-pair port-pair-1 --port-pair port-pair-2 --port-pair-group-parameters lb-fields=ip_dst port-pair-group-1
openstack sfc flow classifier create --source-ip-prefix 1.0.0.0/24 --destination-ip-prefix 2.0.0.0/24 --logical-source-port sfc-port-p1 --logical-destination-port sfc-port-p6 sfc-flow-classifier-1
openstack sfc port chain create --chain-parameters correlation=nsh,symmetric=true --flow-classifier sfc-flow-classifier-1 --port-pair-group port-pair-group-1 sfc-port-chain-1流表
代码语言：javascript复制#nsh整体流表
[root@test25g05 /home/huiwei]# ovs-ofctl dump-flows -O OpenFlow13 br-int
cookie=0x12870a3164aefc04, duration=303.984s, table=0, n_packets=0, n_bytes=0, priority=65535,vlan_tci=0x0fff/0x1fff actions=drop
cookie=0x3a35346be9d7bfe8, duration=291.764s, table=0, n_packets=268, n_bytes=26264, priority=30,ip,in_port="qvo327f1f20-19",nw_src=1.0.0.0/24,nw_dst=2.0.0.0/24 actions=group:3
cookie=0x3a35346be9d7bfe8, duration=289.832s, table=0, n_packets=141, n_bytes=13818, priority=30,ip,in_port="qvo44589e56-db",nw_src=2.0.0.0/24,nw_dst=1.0.0.0/24 actions=group:7003
cookie=0x3a35346be9d7bfe8, duration=281.836s, table=0, n_packets=0, n_bytes=0, priority=30,ip,in_port="qvo5a5a289f-68",nw_src=2.0.0.0/24,nw_dst=1.0.0.0/24 actions=NORMAL
cookie=0x3a35346be9d7bfe8, duration=281.735s, table=0, n_packets=559, n_bytes=54782, priority=30,ip,in_port="qvof309fd48-14",nw_src=1.0.0.0/24,nw_dst=2.0.0.0/24 actions=NORMAL
cookie=0x3a35346be9d7bfe8, duration=278.092s, table=0, n_packets=16, n_bytes=1568, priority=30,ip,in_port="qvo82826950-34",nw_src=2.0.0.0/24,nw_dst=1.0.0.0/24 actions=NORMAL
cookie=0x3a35346be9d7bfe8, duration=277.987s, table=0, n_packets=0, n_bytes=0, priority=30,ip,in_port="qvoa72eb67c-36",nw_src=1.0.0.0/24,nw_dst=2.0.0.0/24 actions=NORMAL
cookie=0x3a35346be9d7bfe8, duration=302.622s, table=0, n_packets=0, n_bytes=0, priority=20,mpls actions=goto_table:10
cookie=0x3a35346be9d7bfe8, duration=302.619s, table=0, n_packets=0, n_bytes=0, priority=20,dl_type=0x894f actions=goto_table:10
#NSH encap
cookie=0x3a35346be9d7bfe8, duration=291.908s, table=5, n_packets=772, n_bytes=75656, priority=0,ip,dl_dst=fa:16:3e:94:4a:b1 actions=encap(nsh),set_field:0x2->nsh_spi,set_field:255->nsh_si,encap(ethernet),push_vlan:0x8100,set_field:4112->vlan_vid,set_field:fa:16:3e:43:c7:d0->eth_src,set_field:fa:16:3e:94:4a:b1->eth_dst,resubmit(,10)
cookie=0x3a35346be9d7bfe8, duration=291.870s, table=5, n_packets=0, n_bytes=0, priority=0,ip,dl_dst=fa:16:3e:4b:5c:85 actions=encap(nsh),set_field:0x2->nsh_spi,set_field:255->nsh_si,encap(ethernet),push_vlan:0x8100,set_field:4112->vlan_vid,set_field:fa:16:3e:43:c7:d0->eth_src,set_field:fa:16:3e:4b:5c:85->eth_dst,resubmit(,10)
cookie=0x3a35346be9d7bfe8, duration=289.975s, table=5, n_packets=559, n_bytes=54782, priority=0,ip,dl_dst=fa:16:3e:28:ca:68 actions=encap(nsh),set_field:0x2->nsh_spi,set_field:255->nsh_si,encap(ethernet),push_vlan:0x8100,set_field:4114->vlan_vid,set_field:fa:16:3e:f8:63:19->eth_src,set_field:fa:16:3e:28:ca:68->eth_dst,resubmit(,10)
cookie=0x3a35346be9d7bfe8, duration=289.938s, table=5, n_packets=0, n_bytes=0, priority=0,ip,dl_dst=fa:16:3e:6b:0d:53 actions=encap(nsh),set_field:0x2->nsh_spi,set_field:255->nsh_si,encap(ethernet),push_vlan:0x8100,set_field:4114->vlan_vid,set_field:fa:16:3e:f8:63:19->eth_src,set_field:fa:16:3e:6b:0d:53->eth_dst,resubmit(,10)
#NSH decap
cookie=0x3a35346be9d7bfe8, duration=281.789s, table=10, n_packets=0, n_bytes=0, priority=1,dl_vlan=18,dl_dst=fa:16:3e:6b:0d:53 actions=pop_vlan,decap(),decap(),output:"qvof309fd48-14"
cookie=0x3a35346be9d7bfe8, duration=281.688s, table=10, n_packets=1315, n_bytes=128870, priority=1,dl_vlan=16,dl_dst=fa:16:3e:94:4a:b1 actions=pop_vlan,decap(),decap(),output:"qvo5a5a289f-68"
cookie=0x3a35346be9d7bfe8, duration=278.043s, table=10, n_packets=141, n_bytes=13818, priority=1,dl_vlan=18,dl_dst=fa:16:3e:28:ca:68 actions=pop_vlan,decap(),decap(),output:"qvoa72eb67c-36"
cookie=0x3a35346be9d7bfe8, duration=277.942s, table=10, n_packets=0, n_bytes=0, priority=1,dl_vlan=16,dl_dst=fa:16:3e:4b:5c:85 actions=pop_vlan,decap(),decap(),output:"qvo82826950-34"
cookie=0x3a35346be9d7bfe8, duration=302.617s, table=10, n_packets=741, n_bytes=72618, priority=0 actions=drop
#做SF1和SF2做LB
[root@test25g05 /home/huiwei]# ovs-ofctl dump-groups -O OpenFlow13 br-int
OFPST_GROUP_DESC reply (OF1.3) (xid=0x2):
group_id=3,type=select,bucket=actions=set_field:fa:16:3e:94:4a:b1->eth_dst,resubmit(,5),bucket=actions=set_field:fa:16:3e:4b:5c:85->eth_dst,resubmit(,5)
group_id=7003,type=select,bucket=actions=set_field:fa:16:3e:28:ca:68->eth_dst,resubmit(,5),bucket=actions=set_field:fa:16:3e:6b:0d:53->eth_dst,resubmit(,5)问题
- restart openvswitch报错
重启openvswitch报错，update_port_chain和update_network先后问题
1979:2019-12-24 17:03:39.703 3746024 ERROR neutron.agent.common.ovs_lib [req-6d0f9605-0ece-4d07-bc04-e6d9daa3bf1b - - - - -] Unable to execute ['ovs-ofctl', 'add-flows', '-O', 'OpenFlow13', 'br-int', '-']. Exception: Exit code: 1; Stdin: hard_timeout=0,idle_timeout=0,priority=1,cookie=17912164061049090790,nsh_spi=2,nsh_mdtype=1,eth_type=35151,table=10,dl_dst=fa:16:3e:28:ca:68,dl_vlan=None,nsh_si=255,actions=strip_vlan,move:NXM_OF_ETH_DST->OXM_OF_PKT_REG0[0..47],decap(),decap(),move:OXM_OF_PKT_REG0[0..47]->NXM_OF_ETH_DST,output:226; Stdout: ; Stderr: ovs-ofctl: -:1: None: bad syntax for dl_vlan value- live migration后流表不对
- 看流表要指定openflow版本号
[root@test25g05 /home/huiwei]# ovs-ofctl dump-flows br-int
2019-12-02T09:07:48Z|00001|ofp_actions|WARN|unknown OpenFlow10 action for vendor 0x2320 and type 46
2019-12-02T09:07:48Z|00002|ofp_actions|WARN|bad action at offset 0 (OFPBAC_BAD_VENDOR_TYPE):
00000000 ff ff 00 10 00 00 23 20-00 2e 00 00 00 01 89 4f
00000010 ff ff 00 18 00 00 23 20-00 21 ff ff 08 08 00 5a
00000020 d6 50 00 00 00 02 00 00-ff ff 00 18 00 00 23 20
00000030 00 21 ff ff 0a 05 00 5a-d6 50 ff 00 00 00 00 00
00000040 ff ff 00 10 00 00 23 20-00 2e 00 00 00 00 00 00
00000050 ff ff 00 10 00 00 23 20-00 28 00 00 00 00 00 03
2019-12-02T09:07:48Z|00003|ofp_flow|WARN|OFPST_FLOW reply bad instructions
2019-12-02T09:07:48Z|00004|vconn|WARN|parse error in reply (OFPBAC_BAD_VENDOR_TYPE)
ovs-ofctl: dump flows (Protocol error)
解决办法：
ovs-ofctl -O OpenFlow13 dump-flows br-int总结
正常情况下通用，异常情况问题比较多，回退考虑不周全，需要社区投入人力修改，欢迎大家参与。
