ovs patch端口实现原理

101 阅读 0 评论 67 点赞

我是靠谱客的博主潇洒石头，这篇文章主要介绍ovs patch端口实现原理，现在分享给大家，希望可以做个参考。

ovs的patch端口，用于连接两个网桥，命令如下

复制代码

1
2
ovs-vsctl add-port  br10 patch3  -- set interface patch3 type=patch options:peer=patch4 -- add-port br12 patch4 -- set interface patch4 type=patch options:peer=patch3

添加patch端口流程

添加端口时，会先后调用 port_construct 和 port_add，下面看一下这两个函数对于patch端口的特殊处理
a. port_construct

复制代码

static int
port_construct(struct ofport *port_)
    if (netdev_vport_is_patch(netdev)) {
        /* By bailing out here, we don't submit the port to the sFlow module
         * to be considered for counter polling export.  This is correct
         * because the patch port represents an interface that sFlow considers
         * to be "internal" to the switch as a whole, and therefore not a
         * candidate for counter polling. */
        port->odp_port = ODPP_NONE;
        ofport_update_peer(port);
        return 0;
    }

更新peer设备，比如上面的命令，给br10添加patch3，给br12添加patch4，
会调用两次ofport_update_peer，第一次调用给br10添加patch3时，遍历所有bridge，寻找peer patch4时会失败，因为patch4还没有添加
到bridge；第二次调用给br12添加patch4时，遍历所有bridge，寻找peer patch3会成功，此时会同时设置patch3和patch4的peer设备。
static void
ofport_update_peer(struct ofport_dpif *ofport)
{
    const struct ofproto_dpif *ofproto;
    struct dpif_backer *backer;
    char *peer_name;

if (!netdev_vport_is_patch(ofport->up.netdev)) {
        return;
    }

backer = ofproto_dpif_cast(ofport->up.ofproto)->backer;
    backer->need_revalidate = REV_RECONFIGURE;

if (ofport->peer) {
        ofport->peer->peer = NULL;
        ofport->peer = NULL;
    }

peer_name = netdev_vport_patch_peer(ofport->up.netdev);
    if (!peer_name) {
        return;
    }

//遍历所有bridge，寻找peer
    HMAP_FOR_EACH (ofproto, all_ofproto_dpifs_node, &all_ofproto_dpifs) {
        struct ofport *peer_ofport;
        struct ofport_dpif *peer;
        char *peer_peer;

//patch peer必须在同一个datapath中
        if (ofproto->backer != backer) {
            continue;
        }

peer_ofport = shash_find_data(&ofproto->up.port_by_name, peer_name);
        if (!peer_ofport) {
            continue;
        }

//找到peer设备后，设置本设备的peer为peer设备，同时也要设置peer设备的peer为本设备
        peer = ofport_dpif_cast(peer_ofport);
        peer_peer = netdev_vport_patch_peer(peer->up.netdev);
        if (peer_peer && !strcmp(netdev_get_name(ofport->up.netdev),
                                 peer_peer)) {
            ofport->peer = peer;
            ofport->peer->peer = ofport;
        }
        free(peer_peer);

break;
    }
    free(peer_name);
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
static int
port_construct(struct ofport *port_)
    if (netdev_vport_is_patch(netdev)) {
        /* By bailing out here, we don't submit the port to the sFlow module
         * to be considered for counter polling export.  This is correct
         * because the patch port represents an interface that sFlow considers
         * to be "internal" to the switch as a whole, and therefore not a
         * candidate for counter polling. */
        port->odp_port = ODPP_NONE;
        ofport_update_peer(port);
        return 0;
    }

更新peer设备，比如上面的命令，给br10添加patch3，给br12添加patch4，
会调用两次ofport_update_peer，第一次调用给br10添加patch3时，遍历所有bridge，寻找peer patch4时会失败，因为patch4还没有添加
到bridge；第二次调用给br12添加patch4时，遍历所有bridge，寻找peer patch3会成功，此时会同时设置patch3和patch4的peer设备。
static void
ofport_update_peer(struct ofport_dpif *ofport)
{
    const struct ofproto_dpif *ofproto;
    struct dpif_backer *backer;
    char *peer_name;

    if (!netdev_vport_is_patch(ofport->up.netdev)) {
        return;
    }

    backer = ofproto_dpif_cast(ofport->up.ofproto)->backer;
    backer->need_revalidate = REV_RECONFIGURE;

    if (ofport->peer) {
        ofport->peer->peer = NULL;
        ofport->peer = NULL;
    }

    peer_name = netdev_vport_patch_peer(ofport->up.netdev);
    if (!peer_name) {
        return;
    }

    //遍历所有bridge，寻找peer
    HMAP_FOR_EACH (ofproto, all_ofproto_dpifs_node, &all_ofproto_dpifs) {
        struct ofport *peer_ofport;
        struct ofport_dpif *peer;
        char *peer_peer;

        //patch peer必须在同一个datapath中
        if (ofproto->backer != backer) {
            continue;
        }

        peer_ofport = shash_find_data(&ofproto->up.port_by_name, peer_name);
        if (!peer_ofport) {
            continue;
        }

        //找到peer设备后，设置本设备的peer为peer设备，同时也要设置peer设备的peer为本设备
        peer = ofport_dpif_cast(peer_ofport);
        peer_peer = netdev_vport_patch_peer(peer->up.netdev);
        if (peer_peer && !strcmp(netdev_get_name(ofport->up.netdev),
                                 peer_peer)) {
            ofport->peer = peer;
            ofport->peer->peer = ofport;
        }
        free(peer_peer);

        break;
    }
    free(peer_name);
}

b. port_add
如果是patch类型端口，不会将其添加到datapath中，所以通过 ovs-appctl dpctl/show 是看不到patch端口的

复制代码

static int
port_add(struct ofproto *ofproto_, struct netdev *netdev)
{
    struct ofproto_dpif *ofproto = ofproto_dpif_cast(ofproto_);
    const char *devname = netdev_get_name(netdev);
    char namebuf[NETDEV_VPORT_NAME_BUFSIZE];
    const char *dp_port_name;

//如果是patch类型端口，则返回。不会将其添加到datapath中
    if (netdev_vport_is_patch(netdev)) {
        sset_add(&ofproto->ghost_ports, netdev_get_name(netdev));
        return 0;
    }
    
    dp_port_name = netdev_vport_get_dpif_port(netdev, namebuf, sizeof namebuf);
    if (!dpif_port_exists(ofproto->backer->dpif, dp_port_name)) {
        odp_port_t port_no = ODPP_NONE;
        int error;

将端口添加到datapath中
        error = dpif_port_add(ofproto->backer->dpif, netdev, &port_no);
        if (error) {
            return error;
        }
        if (netdev_get_tunnel_config(netdev)) {
            simap_put(&ofproto->backer->tnl_backers,
                      dp_port_name, odp_to_u32(port_no));
        }
    }

if (netdev_get_tunnel_config(netdev)) {
        sset_add(&ofproto->ghost_ports, devname);
    } else {
        sset_add(&ofproto->ports, devname);
    }
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
static int
port_add(struct ofproto *ofproto_, struct netdev *netdev)
{
    struct ofproto_dpif *ofproto = ofproto_dpif_cast(ofproto_);
    const char *devname = netdev_get_name(netdev);
    char namebuf[NETDEV_VPORT_NAME_BUFSIZE];
    const char *dp_port_name;

    //如果是patch类型端口，则返回。不会将其添加到datapath中
    if (netdev_vport_is_patch(netdev)) {
        sset_add(&ofproto->ghost_ports, netdev_get_name(netdev));
        return 0;
    }
    
    dp_port_name = netdev_vport_get_dpif_port(netdev, namebuf, sizeof namebuf);
    if (!dpif_port_exists(ofproto->backer->dpif, dp_port_name)) {
        odp_port_t port_no = ODPP_NONE;
        int error;

        将端口添加到datapath中
        error = dpif_port_add(ofproto->backer->dpif, netdev, &port_no);
        if (error) {
            return error;
        }
        if (netdev_get_tunnel_config(netdev)) {
            simap_put(&ofproto->backer->tnl_backers,
                      dp_port_name, odp_to_u32(port_no));
        }
    }

    if (netdev_get_tunnel_config(netdev)) {
        sset_add(&ofproto->ghost_ports, devname);
    } else {
        sset_add(&ofproto->ports, devname);
    }
}

br10和br12必须在同一个datapath，否则寻找peer就会失败，ofport->peer就会为空，后面数据转发时也不会从peer设备发出。

patch端口数据转发的处理

假如出端口为patch port，流程如下：

复制代码

static void
compose_output_action__(struct xlate_ctx *ctx, ofp_port_t ofp_port,
                        const struct xlate_bond_recirc *xr, bool check_stp)
    //如果patch端口有peer，则发送给peer
    if (xport->peer) {
       apply_nested_clone_actions(ctx, xport, xport->peer);
       return;
    }
    
    //如果patch端口没有peer
    if (xport->is_tunnel) {
        ...
    } else {
        //因为xport没有peer，所以xport->odp_port 为ODPP_NONE。
        //比如通过patch端口连接两个不同类型datapath的网桥，patch端口的peer就是空的，所以也不能互相转发。
        odp_port = xport->odp_port;
        out_port = odp_port;
    }

if (out_port != ODPP_NONE) {
    }
    
static void
apply_nested_clone_actions(struct xlate_ctx *ctx, const struct xport *in_dev,
              struct xport *out_dev)
    //in_port 修改成出端口的id
    flow->in_port.ofp_port = out_dev->ofp_port;
    //临时设置成peer端口所在的桥
    ctx->xbridge = out_dev->xbridge;
    //查找peer端口所在的桥的openflow table，相当于peer端口收到报文的处理
    xlate_table_action(ctx, flow->in_port.ofp_port, 0, true, true,false);
        //查找 openflow table
        rule = rule_dpif_lookup_from_table();
        xlate_recursively(ctx, rule, table_id <= old_table_id);
            actions = rule_get_actions(&rule->up);
            //执行action
            do_xlate_actions(actions->ofpacts, actions->ofpacts_len, ctx);
    //恢复出端口所在桥
    ctx->xbridge = in_dev->xbridge;

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
static void
compose_output_action__(struct xlate_ctx *ctx, ofp_port_t ofp_port,
                        const struct xlate_bond_recirc *xr, bool check_stp)
    //如果patch端口有peer，则发送给peer
    if (xport->peer) {
       apply_nested_clone_actions(ctx, xport, xport->peer);
       return;
    }
    
    //如果patch端口没有peer
    if (xport->is_tunnel) {
        ...
    } else {
        //因为xport没有peer，所以xport->odp_port 为ODPP_NONE。
        //比如通过patch端口连接两个不同类型datapath的网桥，patch端口的peer就是空的，所以也不能互相转发。
        odp_port = xport->odp_port;
        out_port = odp_port;
    }

    if (out_port != ODPP_NONE) {
    }
    
static void
apply_nested_clone_actions(struct xlate_ctx *ctx, const struct xport *in_dev,
              struct xport *out_dev)
    //in_port 修改成出端口的id
    flow->in_port.ofp_port = out_dev->ofp_port;
    //临时设置成peer端口所在的桥
    ctx->xbridge = out_dev->xbridge;
    //查找peer端口所在的桥的openflow table，相当于peer端口收到报文的处理
    xlate_table_action(ctx, flow->in_port.ofp_port, 0, true, true,false);
        //查找 openflow table
        rule = rule_dpif_lookup_from_table();
        xlate_recursively(ctx, rule, table_id <= old_table_id);
            actions = rule_get_actions(&rule->up);
            //执行action
            do_xlate_actions(actions->ofpacts, actions->ofpacts_len, ctx);
    //恢复出端口所在桥
    ctx->xbridge = in_dev->xbridge;

最后会将流表和action安装到datapath中，指导后续报文转发。

实验

下面做一个小实验，拓扑图如下:

image.png

创建三个网桥br10,br11和br12，其中br10和br12为netdev类型，br11为system类型。
br10和br11通过patch端口patch1/patch2相连接，br10和br12通过patch端口patch3/patch4相连接。
br10上还添加了一个物理网卡enp129s0f0，其直连的网卡enp129s0f1配置ip地址2.2.2.2/24。

复制代码

root@ubuntu:~# ovs-vsctl add-br br10 -- set bridge br10 datapath_type=netdev
root@ubuntu:~# ovs-vsctl add-br br12 -- set bridge br12 datapath_type=netdev
root@ubuntu:~# ovs-vsctl add-port  br10 patch3  -- set interface patch3 type=patch options:peer=patch4 -- add-port br12 patch4 -- set interface patch4 type=patch options:peer=patch3
root@ubuntu:~# ovs-vsctl add-port  br10 enp129s0f0
root@ubuntu:~# ovs-vsctl add-br br11 -- set bridge br11 datapath_type=system
root@ubuntu:~# ovs-vsctl add-port  br10 patch1  -- set interface patch1 type=patch options:peer=patch2 -- add-port br11 patch2 -- set interface patch2 type=patch options:peer=patch1

root@ubuntu:~# ovs-vsctl show
e436075d-bffe-4cce-8cab-91cf99f0a4b2
    Bridge "br10"
        Port "enp129s0f0"
            Interface "enp129s0f0"
        Port "patch3"
            Interface "patch3"
                type: patch
                options: {peer="patch4"}
        Port "patch1"
            Interface "patch1"
                type: patch
                options: {peer="patch2"}
        Port "br10"
            Interface "br10"
                type: internal
    Bridge "br11"
        Port "br11"
            Interface "br11"
                type: internal
        Port "patch2"
            Interface "patch2"
                type: patch
                options: {peer="patch1"}
    Bridge "br12"
        Port "patch4"
            Interface "patch4"
                type: patch
                options: {peer="patch3"}
        Port "br12"
            Interface "br12"
                type: internal

//查看datapath信息
//可看到有两个datapath：netdev和system，并且datapath中不包含任何patch端口
root@ubuntu:~# ovs-appctl dpctl/show
netdev@ovs-netdev:
        lookups: hit:2022051 missed:25 lost:0
        flows: 1
        port 0: ovs-netdev (tap)
        port 1: br10 (tap)
        port 2: enp129s0f0
        port 3: br12 (tap)
system@ovs-system:
        lookups: hit:0 missed:0 lost:0
        flows: 0
        masks: hit:0 total:1 hit/pkt:0.00
        port 0: ovs-system (internal)
        port 1: br11 (internal)

//给enp129s0f1 配置ip，并ping任意同网段的ip，目的是发送arp广播报文
root@ubuntu:~# ip link set dev enp129s0f1 2.2.2.2/24
root@ubuntu:~# ping 2.2.2.7
PING 2.2.2.7 (2.2.2.7) 56(84) bytes of data.
From 2.2.2.2 icmp_seq=1 Destination Host Unreachable
...

//只有netdev datapath有流表，system datapath没有
root@ubuntu:~# ovs-appctl dpctl/dump-flows netdev@ovs-netdev
flow-dump from non-dpdk interfaces:
recirc_id(0),in_port(2),packet_type(ns=0,id=0),eth(src=3c:fd:fe:a2:1f:a7,dst=ff:ff:ff:ff:ff:ff),eth_type(0x0806),arp(sip=2.2.2.2,tip=2.2.2.7,op=1/0xff), packets:1522, bytes:91320, used:0.030s, actions:1,3
root@ubuntu:~#
root@ubuntu:~# ovs-appctl dpctl/dump-flows system@ovs-system
root@ubuntu:~#

//查看网桥br10上端口统计信息，enp129s0f0收到arp广播报文后，flood到br10上所有端口(patch1没有，因为它连接的是不同类型的网桥)
root@ubuntu:~# ovs-ofctl dump-ports br10
OFPST_PORT reply (xid=0x2): 4 ports
  port  patch1: rx pkts=0, bytes=0, drop=?, errs=?, frame=?, over=?, crc=?
           tx pkts=0, bytes=0, drop=?, errs=?, coll=?
  port  patch3: rx pkts=0, bytes=0, drop=?, errs=?, frame=?, over=?, crc=?
           tx pkts=17260, bytes=1045224, drop=?, errs=?, coll=?
  port LOCAL: rx pkts=34, bytes=2780, drop=0, errs=0, frame=0, over=0, crc=0
           tx pkts=2025556, bytes=147632755, drop=0, errs=0, coll=0
  port  enp129s0f0: rx pkts=1035304, bytes=64562809, drop=0, errs=0, frame=0, over=0, crc=0
           tx pkts=0, bytes=0, drop=0, errs=0, coll=0
//查看网桥br11上端口统计信息，因为patch2没收到任何报文，所有br11上端口统计信息全0
root@ubuntu:~# ovs-ofctl dump-ports br11
OFPST_PORT reply (xid=0x2): 2 ports
  port LOCAL: rx pkts=0, bytes=0, drop=0, errs=0, frame=0, over=0, crc=0
           tx pkts=0, bytes=0, drop=0, errs=0, coll=0
  port  patch2: rx pkts=0, bytes=0, drop=?, errs=?, frame=?, over=?, crc=?
           tx pkts=0, bytes=0, drop=?, errs=?, coll=?
//查看网桥br12上端口统计信息，patch4从patch3收到报文，flood到其他端口
root@ubuntu:~# ovs-ofctl dump-ports br12
OFPST_PORT reply (xid=0x2): 2 ports
  port LOCAL: rx pkts=8, bytes=648, drop=0, errs=0, frame=0, over=0, crc=0
           tx pkts=19764, bytes=1197319, drop=0, errs=0, coll=0
  port  patch4: rx pkts=17430, bytes=1055750, drop=?, errs=?, frame=?, over=?, crc=?
           tx pkts=0, bytes=0, drop=?, errs=?, coll=?

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
root@ubuntu:~# ovs-vsctl add-br br10 -- set bridge br10 datapath_type=netdev
root@ubuntu:~# ovs-vsctl add-br br12 -- set bridge br12 datapath_type=netdev
root@ubuntu:~# ovs-vsctl add-port  br10 patch3  -- set interface patch3 type=patch options:peer=patch4 -- add-port br12 patch4 -- set interface patch4 type=patch options:peer=patch3
root@ubuntu:~# ovs-vsctl add-port  br10 enp129s0f0
root@ubuntu:~# ovs-vsctl add-br br11 -- set bridge br11 datapath_type=system
root@ubuntu:~# ovs-vsctl add-port  br10 patch1  -- set interface patch1 type=patch options:peer=patch2 -- add-port br11 patch2 -- set interface patch2 type=patch options:peer=patch1

root@ubuntu:~# ovs-vsctl show
e436075d-bffe-4cce-8cab-91cf99f0a4b2
    Bridge "br10"
        Port "enp129s0f0"
            Interface "enp129s0f0"
        Port "patch3"
            Interface "patch3"
                type: patch
                options: {peer="patch4"}
        Port "patch1"
            Interface "patch1"
                type: patch
                options: {peer="patch2"}
        Port "br10"
            Interface "br10"
                type: internal
    Bridge "br11"
        Port "br11"
            Interface "br11"
                type: internal
        Port "patch2"
            Interface "patch2"
                type: patch
                options: {peer="patch1"}
    Bridge "br12"
        Port "patch4"
            Interface "patch4"
                type: patch
                options: {peer="patch3"}
        Port "br12"
            Interface "br12"
                type: internal

//查看datapath信息
//可看到有两个datapath：netdev和system，并且datapath中不包含任何patch端口
root@ubuntu:~# ovs-appctl dpctl/show
netdev@ovs-netdev:
        lookups: hit:2022051 missed:25 lost:0
        flows: 1
        port 0: ovs-netdev (tap)
        port 1: br10 (tap)
        port 2: enp129s0f0
        port 3: br12 (tap)
system@ovs-system:
        lookups: hit:0 missed:0 lost:0
        flows: 0
        masks: hit:0 total:1 hit/pkt:0.00
        port 0: ovs-system (internal)
        port 1: br11 (internal)

//给enp129s0f1 配置ip，并ping任意同网段的ip，目的是发送arp广播报文
root@ubuntu:~# ip link set dev enp129s0f1 2.2.2.2/24
root@ubuntu:~# ping 2.2.2.7
PING 2.2.2.7 (2.2.2.7) 56(84) bytes of data.
From 2.2.2.2 icmp_seq=1 Destination Host Unreachable
...

//只有netdev datapath有流表，system datapath没有
root@ubuntu:~# ovs-appctl dpctl/dump-flows netdev@ovs-netdev
flow-dump from non-dpdk interfaces:
recirc_id(0),in_port(2),packet_type(ns=0,id=0),eth(src=3c:fd:fe:a2:1f:a7,dst=ff:ff:ff:ff:ff:ff),eth_type(0x0806),arp(sip=2.2.2.2,tip=2.2.2.7,op=1/0xff), packets:1522, bytes:91320, used:0.030s, actions:1,3
root@ubuntu:~#
root@ubuntu:~# ovs-appctl dpctl/dump-flows system@ovs-system
root@ubuntu:~#

//查看网桥br10上端口统计信息，enp129s0f0收到arp广播报文后，flood到br10上所有端口(patch1没有，因为它连接的是不同类型的网桥)
root@ubuntu:~# ovs-ofctl dump-ports br10
OFPST_PORT reply (xid=0x2): 4 ports
  port  patch1: rx pkts=0, bytes=0, drop=?, errs=?, frame=?, over=?, crc=?
           tx pkts=0, bytes=0, drop=?, errs=?, coll=?
  port  patch3: rx pkts=0, bytes=0, drop=?, errs=?, frame=?, over=?, crc=?
           tx pkts=17260, bytes=1045224, drop=?, errs=?, coll=?
  port LOCAL: rx pkts=34, bytes=2780, drop=0, errs=0, frame=0, over=0, crc=0
           tx pkts=2025556, bytes=147632755, drop=0, errs=0, coll=0
  port  enp129s0f0: rx pkts=1035304, bytes=64562809, drop=0, errs=0, frame=0, over=0, crc=0
           tx pkts=0, bytes=0, drop=0, errs=0, coll=0
//查看网桥br11上端口统计信息，因为patch2没收到任何报文，所有br11上端口统计信息全0
root@ubuntu:~# ovs-ofctl dump-ports br11
OFPST_PORT reply (xid=0x2): 2 ports
  port LOCAL: rx pkts=0, bytes=0, drop=0, errs=0, frame=0, over=0, crc=0
           tx pkts=0, bytes=0, drop=0, errs=0, coll=0
  port  patch2: rx pkts=0, bytes=0, drop=?, errs=?, frame=?, over=?, crc=?
           tx pkts=0, bytes=0, drop=?, errs=?, coll=?
//查看网桥br12上端口统计信息，patch4从patch3收到报文，flood到其他端口
root@ubuntu:~# ovs-ofctl dump-ports br12
OFPST_PORT reply (xid=0x2): 2 ports
  port LOCAL: rx pkts=8, bytes=648, drop=0, errs=0, frame=0, over=0, crc=0
           tx pkts=19764, bytes=1197319, drop=0, errs=0, coll=0
  port  patch4: rx pkts=17430, bytes=1055750, drop=?, errs=?, frame=?, over=?, crc=?
           tx pkts=0, bytes=0, drop=?, errs=?, coll=?