第20周：L3 转发器实现

字数统计: 2.2k字   |   阅读时长≈ 12分

第20周：L3 转发器实现

目标：基于 DPDK 实现 IPv4 三层转发，包括路由查找、MAC 重写、ARP 处理。

1. IPv4 转发原理

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
转发流程：

收包（RX）
  │
  ▼
验证以太网类型（ETHERTYPE_IP）
  │
  ▼
解析 IP 头（rte_ipv4_hdr）
  │
  ├── TTL -= 1
  ├── 校验和重新计算
  │
  ▼
路由查找（路由表）
  │
  ├── 找到出接口
  ├── 下一跳 IP
  │
  ▼
ARP 解析（下一跳 MAC）
  │
  ├── 缓存命中 → 直接使用
  ├── 缓存未命中 → 发送 ARP 请求，等待回复
  │
  ▼
重写 MAC 地址
  │
  ├── 源 MAC = 本机出接口 MAC
  ├── 目的 MAC = 下一跳 MAC
  │
  ▼
发送（TX）

2. 路由表

2.1 路由条目

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
// 路由条目
struct route_entry {
    uint32_t ip;        // 目的 IP
    uint32_t mask;      // 掩码
    uint8_t  prefix_len; // 前缀长度
    uint16_t out_port;   // 出端口
    uint32_t next_hop;   // 下一跳 IP
};

// 路由表（线性搜索，生产环境用 trie）
struct route_table {
    struct route_entry entries[MAX_ROUTES];
    int count;
};

// 最长前缀匹配（LPM）
int route_lookup(struct route_table *rt, uint32_t dst_ip,
                 struct route_entry *out) {
    int best_match = -1;
    int best_prefix = -1;

    for (int i = 0; i < rt->count; i++) {
        if ((dst_ip & rt->entries[i].mask) == rt->entries[i].ip) {
            if (rt->entries[i].prefix_len > best_prefix) {
                best_prefix = rt->entries[i].prefix_len;
                best_match = i;
            }
        }
    }

    if (best_match >= 0) {
        *out = rt->entries[best_match];
        return 0;
    }
    return -1;  // 无路由
}

2.2 初始化路由表

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
void init_route_table(struct route_table *rt) {
    rt->count = 0;

    // 添加直连路由
    // 192.168.1.0/24 → port 0（直接可达）
    rt->entries[rt->count++] = (struct route_entry){
        .ip = rte_cpu_to_be_32(0xC0A80100),  // 192.168.1.0
        .mask = rte_cpu_to_be_32(0xFFFFFF00),
        .prefix_len = 24,
        .out_port = 0,
        .next_hop = rte_cpu_to_be_32(0xC0A80101),  // 192.168.1.1
    };

    // 10.0.0.0/8 → port 1（通过网关）
    rt->entries[rt->count++] = (struct route_entry){
        .ip = rte_cpu_to_be_32(0x0A000000),  // 10.0.0.0
        .mask = rte_cpu_to_be_32(0xFF000000),
        .prefix_len = 8,
        .out_port = 1,
        .next_hop = rte_cpu_to_be_32(0xC0A80102),  // 192.168.1.2
    };

    // 默认路由 → port 1
    rt->entries[rt->count++] = (struct route_entry){
        .ip = 0,
        .mask = 0,
        .prefix_len = 0,
        .out_port = 1,
        .next_hop = rte_cpu_to_be_32(0xC0A80102),  // 网关
    };
}

3. ARP 缓存与处理

3.1 ARP 缓存

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
#include <rte_hash.h>
#include <rte_jhash.h>

#define ARP_CACHE_SIZE  1024

struct arp_entry {
    uint32_t ip;                        // IP 地址
    struct rte_ether_addr mac;           // MAC 地址
    uint64_t timestamp;                 // 最后更新时间
    uint8_t state;                      // 状态（0=未完成 1=完成）
};

struct arp_cache {
    struct arp_entry entries[ARP_CACHE_SIZE];
    struct rte_hash *hash_table;
};

// 初始化 ARP 缓存
struct arp_cache *arp_init(void) {
    struct arp_cache *cache = rte_zmalloc(NULL, sizeof(*cache), 0);

    // 创建 hash 表（ip → arp_entry*）
    struct rte_hash_parameters params = {
        .name = "ARP_HASH",
        .entries = ARP_CACHE_SIZE,
        .key_len = sizeof(uint32_t),
        .hash_func = rte_jhash,
        .hash_func_init_val = 0,
        .socket_id = SOCKET_ID_ANY,
    };
    cache->hash_table = rte_hash_create(&params);

    return cache;
}

// 查找
struct arp_entry *arp_lookup(struct arp_cache *cache, uint32_t ip) {
    struct arp_entry **entry;
    int ret = rte_hash_lookup_data(cache->hash_table, &ip,
                                   (void **)&entry);
    if (ret >= 0) return *entry;
    return NULL;
}

// 添加/更新
int arp_update(struct arp_cache *cache, uint32_t ip,
               struct rte_ether_addr *mac) {
    struct arp_entry *entry = arp_lookup(cache, ip);
    if (entry) {
        // 更新
        rte_memcpy(&entry->mac, mac, sizeof(*mac));
        entry->timestamp = rte_rdtsc();
        entry->state = 1;
        return 0;
    }

    // 新建
    entry = &cache->entries[cache->count++];
    entry->ip = ip;
    rte_memcpy(&entry->mac, mac, sizeof(*mac));
    entry->timestamp = rte_rdtsc();
    entry->state = 1;

    // 添加到 hash
    rte_hash_add_key_data(cache->hash_table, &ip, &entry);
    return 0;
}

3.2 构建 ARP 请求

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
// 构建 ARP 请求包（发给下一跳）
struct rte_mbuf *build_arp_request(struct rte_mbuf *orig_mbuf,
                                   struct rte_ether_addr *src_mac,
                                   uint32_t src_ip,
                                   uint32_t target_ip) {
    struct rte_mbuf *arp_mbuf = rte_pktmbuf_alloc(mbuf_pool);
    if (!arp_mbuf) return NULL;

    // 预留以太网头 + ARP 头的空间
    uint32_t pkt_len = sizeof(struct rte_ether_hdr) + sizeof(struct rte_arp_hdr);
    char *data = (char *)rte_pktmbuf_append(arp_mbuf, pkt_len);

    // 以太网头
    struct rte_ether_hdr *eth = (struct rte_ether_hdr *)data;
    // 目的 MAC = 广播
    rte_eth_broadcast_addr(&eth->dst_addr);
    // 源 MAC = 本机
    rte_memcpy(&eth->src_addr, src_mac, sizeof(*src_mac));
    eth->ether_type = rte_cpu_to_be_16(RTE_ETHER_TYPE_ARP);

    // ARP 头
    struct rte_arp_hdr *arp = (struct rte_arp_hdr *)(eth + 1);
    arp->arp_hardware = rte_cpu_to_be_16(RTE_ARP_HRD_ETHER);
    arp->arp_protocol = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPv4);
    arp->arp_hlen = RTE_ETHER_ADDR_LEN;
    arp->arp_plen = 4;
    arp->arp_opcode = rte_cpu_to_be_16(RTE_ARP_OP_REQUEST);
    rte_memcpy(&arp->arp_data_arp_sip, &src_ip, sizeof(src_ip));
    rte_memcpy(&arp->arp_data_arp_tha, eth->dst_addr, sizeof(struct rte_ether_addr));
    rte_memcpy(&arp->arp_data_arp_sha, src_mac, sizeof(struct rte_ether_addr));
    rte_memcpy(&arp->arp_data_arp_tip, &target_ip, sizeof(target_ip));

    arp_mbuf->pkt_len = pkt_len;
    arp_mbuf->data_len = pkt_len;

    return arp_mbuf;
}

3.3 构建 ARP 应答

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
struct rte_mbuf *build_arp_reply(struct rte_mbuf *req,
                                 struct rte_ether_addr *src_mac,
                                 uint32_t src_ip) {
    struct rte_mbuf *reply = rte_pktmbuf_alloc(mbuf_pool);
    if (!reply) return NULL;

    struct rte_ether_hdr *req_eth = rte_pktmbuf_mtod(req, struct rte_ether_hdr *);
    struct rte_arp_hdr *req_arp = (struct rte_arp_hdr *)(req_eth + 1);

    uint32_t pkt_len = sizeof(struct rte_ether_hdr) + sizeof(struct rte_arp_hdr);
    char *data = (char *)rte_pktmbuf_append(reply, pkt_len);

    struct rte_ether_hdr *eth = (struct rte_ether_hdr *)data;
    rte_memcpy(&eth->dst_addr, &req_arp->arp_data_arp_sha, sizeof(struct rte_ether_addr));
    rte_memcpy(&eth->src_addr, src_mac, sizeof(struct rte_ether_addr));
    eth->ether_type = rte_cpu_to_be_16(RTE_ETHER_TYPE_ARP);

    struct rte_arp_hdr *arp = (struct rte_arp_hdr *)(eth + 1);
    arp->arp_hardware = rte_cpu_to_be_16(RTE_ARP_HRD_ETHER);
    arp->arp_protocol = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPv4);
    arp->arp_hlen = RTE_ETHER_ADDR_LEN;
    arp->arp_plen = 4;
    arp->arp_opcode = rte_cpu_to_be_16(RTE_ARP_OP_REPLY);
    rte_memcpy(&arp->arp_data_arp_sip, &src_ip, sizeof(src_ip));
    rte_memcpy(&arp->arp_data_arp_sha, src_mac, sizeof(struct rte_ether_addr));
    rte_memcpy(&arp->arp_data_arp_tip, &req_arp->arp_data_arp_sip, sizeof(uint32_t));
    rte_memcpy(&arp->arp_data_arp_tha, &req_arp->arp_data_arp_sha, sizeof(struct rte_ether_addr));

    reply->pkt_len = pkt_len;
    reply->data_len = pkt_len;

    return reply;
}

4. L3 转发器实现

4.1 完整转发逻辑

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
// l3fwd.c — DPDK L3 转发器
#include <rte_eal.h>
#include <rte_ethdev.h>
#include <rte_mbuf.h>
#include <rte_ip.h>
#include <rte_ether.h>
#include <rte_arp.h>
#include <rte_hash.h>
#include <rte_jhash.h>

#define NB_MBUF    65536
#define RX_RINGS   1
#define TX_RINGS   1
#define RX_DESC    1024
#define TX_DESC    1024
#define MAX_ROUTES 256
#define ARP_CACHE_SIZE 1024

// 路由表
struct route_entry {
    uint32_t ip;
    uint32_t mask;
    uint8_t  prefix_len;
    uint16_t out_port;
    uint32_t next_hop;
};
static struct route_table {
    struct route_entry entries[MAX_ROUTES];
    int count;
} g_rt;

// ARP 缓存
struct arp_entry {
    uint32_t ip;
    struct rte_ether_addr mac;
    uint64_t timestamp;
    uint8_t state;
};
static struct {
    struct arp_entry entries[ARP_CACHE_SIZE];
    struct rte_hash *hash;
    int count;
} g_arp;

static struct rte_mempool *mbuf_pool;

// === 路由查找 ===
static int route_lookup(uint32_t dst_ip, struct route_entry *out) {
    int best = -1, best_pref = -1;
    for (int i = 0; i < g_rt.count; i++) {
        if ((dst_ip & g_rt.entries[i].mask) == g_rt.entries[i].ip) {
            if (g_rt.entries[i].prefix_len > best_pref) {
                best_pref = g_rt.entries[i].prefix_len;
                best = i;
            }
        }
    }
    if (best >= 0) {
        *out = g_rt.entries[best];
        return 0;
    }
    return -1;
}

// === ARP 查找 ===
static struct arp_entry *arp_find(uint32_t ip) {
    struct arp_entry **entry;
    int ret = rte_hash_lookup_data(g_arp.hash, &ip, (void **)&entry);
    if (ret >= 0) return *entry;
    return NULL;
}

// === 构建 ARP 请求 ===
static struct rte_mbuf *make_arp_request(uint16_t out_port,
                                          uint32_t src_ip,
                                          uint32_t target_ip) {
    struct rte_mbuf *m = rte_pktmbuf_alloc(mbuf_pool);
    if (!m) return NULL;

    uint32_t total = sizeof(struct rte_ether_hdr) + sizeof(struct rte_arp_hdr);
    char *data = (char *)rte_pktmbuf_append(m, total);

    struct rte_ether_hdr *eth = (struct rte_ether_hdr *)data;
    rte_eth_broadcast_addr(&eth->dst_addr);
    rte_eth_macaddr_get(out_port, &eth->src_addr);
    eth->ether_type = rte_cpu_to_be_16(RTE_ETHER_TYPE_ARP);

    struct rte_arp_hdr *arp = (struct rte_arp_hdr *)(eth + 1);
    arp->arp_hardware = rte_cpu_to_be_16(1);   // Ethernet
    arp->arp_protocol = rte_cpu_to_be_16(0x0800); // IPv4
    arp->arp_hlen = 6;
    arp->arp_plen = 4;
    arp->arp_opcode = rte_cpu_to_be_16(RTE_ARP_OP_REQUEST);
    rte_memcpy(&arp->arp_data_arp_sip, &src_ip, 4);
    rte_eth_macaddr_get(out_port, &arp->arp_data_arp_sha);
    memset(&arp->arp_data_arp_tha, 0, sizeof(struct rte_ether_addr));
    rte_memcpy(&arp->arp_data_arp_tip, &target_ip, 4);

    m->pkt_len = total;
    m->data_len = total;
    return m;
}

// === 转发处理 ===
static int forward_packet(struct rte_mbuf *m, uint16_t in_port) {
    struct rte_ether_hdr *eth = rte_pktmbuf_mtod(m, struct rte_ether_hdr *);
    uint16_t eth_type = rte_be_to_cpu_16(eth->ether_type);

    // === ARP 包 ===
    if (eth_type == RTE_ETHER_TYPE_ARP) {
        struct rte_arp_hdr *arp = (struct rte_arp_hdr *)(eth + 1);
        uint16_t op = rte_be_to_cpu_16(arp->arp_opcode);

        if (op == RTE_ARP_OP_REQUEST) {
            // ARP Request：应答
            uint32_t target_ip;
            rte_memcpy(&target_ip, &arp->arp_data_arp_tip, 4);

            struct rte_ether_addr src_mac;
            rte_eth_macaddr_get(in_port, &src_mac);

            struct rte_mbuf *reply = make_arp_reply(m, &src_mac, target_ip);
            if (reply) {
                rte_eth_tx_burst(in_port, 0, &reply, 1);
                if (reply) rte_pktmbuf_free(reply);
            }
        } else if (op == RTE_ARP_OP_REPLY) {
            // ARP Reply：更新缓存
            uint32_t sip;
            rte_memcpy(&sip, &arp->arp_data_arp_sip, 4);
            arp_update(sip, &arp->arp_data_arp_sha);
        }
        return 0;
    }

    // === IP 包 ===
    if (eth_type != RTE_ETHER_TYPE_IPv4) return -1;

    struct rte_ipv4_hdr *ip = (struct rte_ipv4_hdr *)(eth + 1);

    // 路由查找
    struct route_entry rt;
    if (route_lookup(ip->dst_addr, &rt) < 0) {
        // 无路由，丢弃
        return -1;
    }

    // TTL 减一
    ip->time_to_live--;

    // 检查 TTL
    if (ip->time_to_live == 0) {
        // 发送 ICMP Time Exceeded（简化：直接丢弃）
        return -1;
    }

    // 重新计算 IP 校验和
    ip->hdr_checksum = 0;
    ip->hdr_checksum = rte_ipv4_cksum(ip);

    // ARP 查找下一跳 MAC
    struct arp_entry *ae = arp_find(rt.next_hop);
    if (!ae) {
        // 缓存未命中，发送 ARP 请求
        uint32_t src_ip = ip->src_addr;  // 简化：用包内源 IP
        struct rte_ether_addr src_mac;
        rte_eth_macaddr_get(in_port, &src_mac);

        struct rte_mbuf *arp_req = make_arp_request(rt.out_port,
                                                      src_ip,
                                                      rt.next_hop);
        if (arp_req) {
            rte_eth_tx_burst(rt.out_port, 0, &arp_req, 1);
            rte_pktmbuf_free(arp_req);
        }
        return 0;  // 包暂存或丢弃（简化处理）
    }

    // 重写 MAC 地址
    struct rte_ether_addr out_mac;
    rte_eth_macaddr_get(rt.out_port, &out_mac);
    rte_memcpy(&eth->src_addr, &out_mac, sizeof(struct rte_ether_addr));
    rte_memcpy(&eth->dst_addr, &ae->mac, sizeof(struct rte_ether_addr));

    // 发送到出接口
    rte_eth_tx_burst(rt.out_port, 0, &m, 1);
    return 0;
}

// === 主循环 ===
void l3fwd_loop(void) {
    struct rte_mbuf *pkts[32];
    uint16_t nb_rx;

    printf("L3FWD running on lcore %u\n", rte_lcore_id());

    while (1) {
        nb_rx = rte_eth_rx_burst(0, 0, pkts, 32);
        if (nb_rx == 0) continue;

        for (int i = 0; i < nb_rx; i++) {
            if (forward_packet(pkts[i], 0) < 0)
                rte_pktmbuf_free(pkts[i]);
        }
    }
}

5. 校验和计算

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
// IP 校验和（DPDK 内置）
#include <rte_ip.h>

// IPv4 校验和
uint16_t rte_ipv4_cksum(struct rte_ipv4_hdr *ip);

// TCP/UDP 校验和（包含 pseudo-header）
uint16_t rte_ipv4_udptcp_cksum(struct rte_ipv4_hdr *ip, void *l4_hdr);

// 手动计算（理解原理）
uint16_t calc_checksum(uint16_t *data, int len) {
    uint32_t sum = 0;
    for (int i = 0; i < len / 2; i++) {
        sum += data[i];
        if (sum > 0xFFFF)
            sum = (sum & 0xFFFF) + 1;
    }
    return ~sum;
}

// IP 校验和更新（TTL 减一后必须重新计算）
void update_ip_checksum(struct rte_ipv4_hdr *ip) {
    // 方法 1：全量重新计算（简单）
    ip->hdr_checksum = 0;
    ip->hdr_checksum = rte_ipv4_cksum(ip);

    // 方法 2：增量更新（只修改 TTL 时更高效）
    //   旧校验和 = old_cs
    //   新校验和 = ~(~old_cs + (old_ttl << 8) + new_ttl)
    uint32_t old_cs = ntohs(ip->hdr_checksum);
    uint32_t old_ttl = ip->time_to_life + 1;
    uint32_t new_ttl = ip->time_to_life;
    uint32_t sum = ~old_cs + ((old_ttl << 8) + new_ttl);
    while (sum >> 16) sum = (sum & 0xFFFF) + (sum >> 16);
    ip->hdr_checksum = htons(~sum);
}

6. IPv4 头结构

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
// <rte_ip.h>
struct rte_ipv4_hdr {
    uint8_t  version_ihl;   // ver(4) | ihl(4)
    uint8_t  type_of_service;
    uint16_t total_length;   // 总长度（头+数据）
    uint16_t packet_id;
    uint16_t fragment_offset;
    uint8_t  time_to_live;
    uint8_t  next_proto_id;  // 上层协议（TCP=6, UDP=17）
    uint16_t hdr_checksum;
    uint32_t src_addr;
    uint32_t dst_addr;
    // options 可选
} __rte_aligned(4);

// 访问辅助函数
RTE_IPV4_HDR_IHL_LEN(iphdr)   // IHL * 4 = 头长度
rte_be_to_cpu_16(iphdr->total_length)  // 网络序→主机序
rte_ipv4_hdr_len(iphdr)        // 返回头长度

7. 以太网头结构

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
// <rte_ether.h>
struct rte_ether_hdr {
    struct rte_ether_addr dst_addr;  // 目的 MAC（6 bytes）
    struct rte_ether_addr src_addr;  // 源 MAC（6 bytes）
    uint16_t ether_type;             // 类型（0x0800=IPv4, 0x0806=ARP）
} __rte_aligned(2);

struct rte_ether_addr {
    uint8_t addr_bytes[RTE_ETHER_ADDR_LEN];  // 6 bytes
};

// 辅助宏
RTE_ETHER_TYPE_IPv4    = 0x0800
RTE_ETHER_TYPE_ARP     = 0x0806
RTE_ETHER_TYPE_IPv6    = 0x86DD
RTE_ETHER_MAX_LEN      = 1518

// 常用操作
rte_eth_macaddr_get(port, &mac)      // 获取 MAC
rte_eth_broadcast_addr(&addr)        // 设为广播地址
rte_is_broadcast_addr(&addr)         // 是否为广播
rte_is_unicast_addr(&addr)           // 是否为单播
rte_ether_addr_cmp(&a, &b)           // MAC 比较
赏

谢谢你请我吃糖果

支付宝
微信

扫一扫，分享到微信

tag:

    缺失模块。
    1、请确保node版本大于6.2
    2、在博客根目录（注意不是yilia-plus根目录）执行以下命令：
    npm i hexo-generator-json-content --save

    3、在根目录_config.yml里添加配置： 

      jsonContent:
    meta: false
    pages: false
    posts:
      title: true
      date: true
      path: true
      text: false
      raw: false
      content: false
      slug: false
      updated: false
      comments: false
      link: false
      permalink: false
      excerpt: false
      categories: false
      tags: true

主要涉及技术：
Java后端开发、Linux开发、
公众号开发、开源爱好者、Linux

联系QQ:562054870