Linux 源码分析之 udp 分析二 sendto

udp sendto过程

udp发送过程前面几个系统调用与Raw socket类似。

1、 sendto系统调用

SYSCALL_DEFINE6(sendto, int, fd, void __user *, buff, size_t, len,unsigned int, flags, struct sockaddr __user *, addr,int, addr_len)
{return __sys_sendto(fd, buff, len, flags, addr, addr_len);
}

int __sys_sendto(int fd, void __user *buff, size_t len, unsigned int flags,struct sockaddr __user *addr,  int addr_len)
{struct socket *sock;struct msghdr msg;//将用户空间的buff挂到msg.msg_iter上err = import_single_range(WRITE, buff, len, &iov, &msg.msg_iter);//根据fd查找socketsock = sockfd_lookup_light(fd, &err, &fput_needed);msg.msg_control = NULL;msg.msg_controllen = 0;if (addr) {//封装用户层的addr结构体，打包成msghdr，并设置flagerr = move_addr_to_kernel(addr, addr_len, &address);if (err < 0)goto out_put;msg.msg_name = (struct sockaddr *)&address;msg.msg_namelen = addr_len;}if (sock->file->f_flags & O_NONBLOCK)flags |= MSG_DONTWAIT;msg.msg_flags = flags;//会继续调用sock->ops->sendmsgerr = sock_sendmsg(sock, &msg);
}

对于udp来说，sock->ops = inet_dgram_ops,如下：

const struct proto_ops inet_stream_ops = {.family        = PF_INET,.listen        = inet_listen,.bind          = inet_bind,.sendmsg       = inet_sendmsg,.recvmsg       = inet_recvmsg,
…
}

调用的是inet_sendmsg，继而会调用udp_sendmsg,这个函数是分析的重点。

2、 udp_sendmsg

int udp_sendmsg(struct sock *sk, struct msghdr *msg, size_t len)
{//获取inetstruct inet_sock *inet = inet_sk(sk);struct udp_sock *up = udp_sk(sk);//usin获取用户层sendto传入的sockaddr_in，包含dst addrDECLARE_SOCKADDR(struct sockaddr_in *, usin, msg->msg_name);struct flowi4 fl4_stack;struct flowi4 *fl4;int ulen = len;struct ipcm_cookie ipc;struct rtable *rt = NULL;int free = 0;int connected = 0;__be32 daddr, faddr, saddr;__be16 dport;u8  tos;int err, is_udplite = IS_UDPLITE(sk);//flag | MSG_MORE，如果设置MSG_MORE，会选择不同的发送分支int corkreq = READ_ONCE(up->corkflag) || msg->msg_flags&MSG_MORE;int (*getfrag)(void *, char *, int, int, int, struct sk_buff *);struct sk_buff *skb;struct ip_options_data opt_copy;//长度大于65535，报错if (len > 0xFFFF)return -EMSGSIZE;//+udp报文的头ulen += sizeof(struct udphdr);//获取目的ipdaddr = usin->sin_addr.s_addr;//获取目的端口dport = usin->sin_port;//初始话ipcipcm_init_sk(&ipc, inet);//这里由于udp未绑定src,因此saddr为空saddr = ipc.addr;//赋值为dst addripc.addr = faddr = daddr;//如果已connect,获取路由表rtable,connect过程会建立路由表if (connected)rt = (struct rtable *)sk_dst_check(sk, 0);//获取路由表if (!rt) {struct net *net = sock_net(sk);__u8 flow_flags = inet_sk_flowi_flags(sk);fl4 = &fl4_stack;//初始化fl4结构flowi4_init_output(fl4, ipc.oif, ipc.sockc.mark, tos,RT_SCOPE_UNIVERSE, sk->sk_protocol,flow_flags,faddr, saddr, dport, inet->inet_sport,sk->sk_uid);security_sk_classify_flow(sk, flowi4_to_flowi(fl4));//路由查找rt = ip_route_output_flow(net, fl4, sk);if (IS_ERR(rt)) {err = PTR_ERR(rt);rt = NULL;if (err == -ENETUNREACH)IP_INC_STATS(net, IPSTATS_MIB_OUTNOROUTES);goto out;}err = -EACCES;if ((rt->rt_flags & RTCF_BROADCAST) &&!sock_flag(sk, SOCK_BROADCAST))goto out;if (connected)sk_dst_set(sk, dst_clone(&rt->dst));
}//如果没设置MSG_MORE,直接发送if (!corkreq) {struct inet_cork cork;skb = ip_make_skb(sk, fl4, getfrag, msg, ulen,sizeof(struct udphdr), &ipc, &rt,&cork, msg->msg_flags);err = PTR_ERR(skb);if (!IS_ERR_OR_NULL(skb))err = udp_send_skb(skb, fl4, &cork);goto out;}//设置了MSG_MORE   fl4 = &inet->cork.fl.u.ip4;fl4->daddr = daddr;fl4->saddr = saddr;fl4->fl4_dport = dport;fl4->fl4_sport = inet->inet_sport;up->pending = AF_INET;do_append_data:up->len += ulen;//构建ip的头err = ip_append_data(sk, fl4, getfrag, msg, ulen,sizeof(struct udphdr), &ipc, &rt,corkreq ? msg->msg_flags|MSG_MORE : msg->msg_flags);if (err)udp_flush_pending_frames(sk);else if (!corkreq)err = udp_push_pending_frames(sk);return err;}

分析一下路由查找过程，对于udp来讲，默认为经过connect，报文第一次发送时需要查找出口dev，构建路由表rtable。
1.fl4初始化：

    fl4->flowi4_oif = oif;//out dev的索引值，这里还没有fl4->flowi4_iif = LOOPBACK_IFINDEX;//入口默认lookback设备fl4->flowi4_proto = proto;fl4->daddr = daddr;//dst addr是用户层传入fl4->saddr = saddr;//src addr为空fl4->fl4_dport = dport;fl4->fl4_sport = sport;

2.路由查找
rt = ip_route_output_flow(net, fl4, sk)会最终调用：

struct rtable *ip_route_output_key_hash_rcu(struct net *net, struct flowi4 *fl4, struct fib_result *res, const struct sk_buff *skb)
{//出口设备struct net_device *dev_out = NULL;//出口设备indexint orig_oif = fl4->flowi4_oif;unsigned int flags = 0;struct rtable *rth;//src addr为空，这里不执行if (fl4->saddr) {if (fl4->flowi4_oif == 0 &&(ipv4_is_multicast(fl4->daddr) ||ipv4_is_lbcast(fl4->daddr))) {//通过saddr查找out_dev ,这里查询的是RT_TABLE_LOCALdev_out = __ip_dev_find(net, fl4->saddr, false);if (!dev_out)goto out;…}//如果设置了fl4的out dev的索引值if (fl4->flowi4_oif){//通过索引查找out devdev_out = dev_get_by_index_rcu(net, fl4->flowi4_oif);//如果没设置src addr，则获取out_dev的addrif (!fl4->saddr) {if (ipv4_is_multicast(fl4->daddr))fl4->saddr = inet_select_addr(dev_out, 0,fl4->flowi4_scope);else if (!fl4->daddr)fl4->saddr = inet_select_addr(dev_out, 0,RT_SCOPE_HOST);}}//如果没设置dst addrif (!fl4->daddr) {//将dst addr设置为src addrfl4->daddr = fl4->saddr;//如果src addr也没有设置,直接设置out dev为lookbackif (!fl4->daddr)fl4->daddr = fl4->saddr = htonl(INADDR_LOOPBACK);dev_out = net->loopback_dev;fl4->flowi4_oif = LOOPBACK_IFINDEX;res->type = RTN_LOCAL;flags |= RTCF_LOCAL;goto make_route;}//进行路由查找,这里查询RT_TABLE_MAIN,返回值在reserr = fib_lookup(net, fl4, res, 0);if (res->type == RTN_LOCAL) {//如果时LOCAL模式，即本机地址，则out dev设置为loopback_devdev_out = l3mdev_master_dev_rcu(FIB_RES_DEV(*res)) ? :net->loopback_dev;       }//若saddr未设置，这里会设置fib_select_path(net, res, fl4, skb);//正常情况下，从res获取dev_outdev_out = FIB_RES_DEV(*res);
make_route://创建路由表rtable并返回rth = __mkroute_output(res, fl4, orig_oif, dev_out, flags);return rth;
}

路由下一跳查找的关键函数为：

int fib_lookup(struct net *net, const struct flowi4 *flp,struct fib_result *res, unsigned int flags)
{struct fib_table *tb;int err = -ENETUNREACH;rcu_read_lock();//从MAIN TABLE中查找tb = fib_get_table(net, RT_TABLE_MAIN);if (tb)err = fib_table_lookup(tb, flp, res, flags | FIB_LOOKUP_NOREF);if (err == -EAGAIN)err = -ENETUNREACH;rcu_read_unlock();return err;
}

int fib_table_lookup(struct fib_table *tb, const struct flowi4 *flp, struct fib_result *res, int fib_flags)
{// dst addr转换成key值         const t_key key = ntohl(flp->daddr);struct key_vector *n, *pn;//根据key查找key_vectorfor (;;) {index = get_cindex(key, n);n = get_child_rcu(n, index);}......
//遍历key_vector->leaf链表hlist_for_each_entry_rcu(fa, &n->leaf, fa_list) {struct fib_info *fi = fa->fa_info;struct fib_nh_common *nhc;//获取下一跳nhc = nexthop_get_nhc_lookup(fi->nh, fib_flags, flp,&nhsel);
}
set_result:res->prefix = htonl(n->key);res->prefixlen = KEYLENGTH - fa->fa_slen;res->nh_sel = nhsel;res->nhc = nhc;res->type = fa->fa_type;res->scope = fi->fib_scope;res->fi = fi;res->table = tb;res->fa_head = &n->leaf;
}

获取下一跳nexthop_get_nhc_lookup最终会执行如下：

bool fib_lookup_good_nhc(const struct fib_nh_common *nhc, int fib_flags, const struct flowi4 *flp)
{if (nhc->nhc_flags & RTNH_F_DEAD)return false;//如果链路link down,失败if (ip_ignore_linkdown(nhc->nhc_dev) &&nhc->nhc_flags & RTNH_F_LINKDOWN &&!(fib_flags & FIB_LOOKUP_IGNORE_LINKSTATE))return false;//如果flp->flowi4_oif != nhc->nhc_oif 失败if (!(flp->flowi4_flags & FLOWI_FLAG_SKIP_NH_OIF)) {if (flp->flowi4_oif &&flp->flowi4_oif != nhc->nhc_oif)return false;}return true;
}

因此，本质是根据daddr映射的key查找一个没有link down，并且oif与设置值一致的下一跳(如果oif设置过)，最终找到一个fib_nh_common结构体，其 *nhc_dev成员即出口设备。具体结构体关系如下：

最后，利用返回的res创建rtable，并缓存起来：

rth = __mkroute_output(res, fl4, orig_oif, dev_out, flags);rth = rt_dst_alloc(dev_out, flags, type,IN_DEV_CONF_GET(in_dev, NOPOLICY),IN_DEV_CONF_GET(in_dev, NOXFRM));

初始化rtable过程：

struct rtable *rt_dst_alloc(struct net_device *dev,unsigned int flags, u16 type,bool nopolicy, bool noxfrm)
{struct rtable *rt;rt = dst_alloc(&ipv4_dst_ops, dev, 1, DST_OBSOLETE_FORCE_CHK,(nopolicy ? DST_NOPOLICY : 0) |(noxfrm ? DST_NOXFRM : 0));if (rt) {rt->rt_genid = rt_genid_ipv4(dev_net(dev));rt->rt_flags = flags;rt->rt_type = type;…rt->dst.output = ip_output;if (flags & RTCF_LOCAL)rt->dst.input = ip_local_deliver;}return rt;
}

会设置如下属性：

 dst->dev = dev;//out devdst->ops = ops;//ipv4_dst_opsdst->input = dst_discard;dst->output = dst_discard_out;

查找到路由之后，进入发送模式。
1.未设置MSG_MORE属性，不会将多个数据包合并发送，来一个发一个。
调用ip_make_skb构建skb

skb = ip_make_skb(sk, fl4, getfrag, msg, ulen,sizeof(struct udphdr), &ipc, &rt,&cork, msg->msg_flags);

@getfrage()函数用于将L4的指定的数据拷贝到一个个的skb中
@from: 待拷贝数据的用户态起始地址
@length：待拷贝数据长度
@transhdrlen：传输层报文长度，对于UDP就是sizeof(struct udphdr)
@ipc：临时的IP控制信息
@rtp：路由信息，调用者必须已经查询过路由
@cork：待填充字段
@flags：控制标记，这里我们关心的只有MSG_MOREstruct sk_buff *ip_make_skb(struct sock *sk,struct flowi4 *fl4,int getfrag(void *from, char *to, int offset,int len, int odd, struct sk_buff *skb),void *from, int length, int transhdrlen,struct ipcm_cookie *ipc, struct rtable **rtp,struct inet_cork *cork, unsigned int flags)
{struct sk_buff_head queue;int err;if (flags & MSG_PROBE)return NULL;//初始化queue,prev、next指向自身__skb_queue_head_init(&queue);cork->flags = 0;cork->addr = 0;cork->opt = NULL;//填充corkerr = ip_setup_cork(sk, cork, ipc, rtp);if (err)return ERR_PTR(err);//创建一个skberr = __ip_append_data(sk, fl4, &queue, cork,&current->task_frag, getfrag,from, length, transhdrlen, flags);if (err) {__ip_flush_pending_frames(sk, &queue, cork);return ERR_PTR(err);}//给创建好的skb添加ip部首return __ip_make_skb(sk, fl4, &queue, cork);
}

__ip_append_data用于构建skb，具体分析一下：
该函数分两个分支：
1.首次创建skb，即发送缓冲区queue为空，对于未设置MSG_MORE的udp来说，走的是这条分支。

fragheaderlen = sizeof(struct iphdr) + (opt ? opt->optlen : 0);
//最大未分片数据长度，最大0xFFFF
maxnonfragsize = ip_sk_ignore_df(sk) ? IP_MAX_MTU : mtu;
//判断数据长度是否大于maxnonfragsize，大于则报错
if (cork->length + length > maxnonfragsize - fragheaderlen) {ip_local_error(sk, EMSGSIZE, fl4->daddr, inet->inet_dport,mtu - (opt ? opt->optlen : 0));return -EMSGSIZE;
}while (length > 0) {char *data;struct sk_buff *skb_prev;
alloc_new_skb:if (transhdrlen) {skb = sock_alloc_send_skb(sk, alloclen,(flags & MSG_DONTWAIT), &err);}skb->ip_summed = csummode;skb->csum = 0;//skb->tail += len;//skb->len  += len;skb_reserve(skb, hh_len);data = skb_put(skb, fraglen + exthdrlen - pagedlen);data += fragheaderlen + exthdrlen;copy = datalen - transhdrlen - fraggap - pagedlen;//拷贝用户数据至skb->data+ transhdrlen,空出了传输层头长度if (copy > 0 && getfrag(from, data + transhdrlen, offset, copy, fraggap, skb) < 0) {err = -EFAULT;kfree_skb(skb);goto error;}offset += copy;length -= copy + transhdrlen;//skb挂到发送缓冲区尾部__skb_queue_tail(queue, skb);continue;} // end of if (copy <= 0)

skb构建完成，会添加ip部首：

__ip_make_skb(sk, fl4, &queue, cork);
struct sk_buff *__ip_make_skb(struct sock *sk,struct flowi4 *fl4,struct sk_buff_head *queue,struct inet_cork *cork)
{struct sk_buff *skb, *tmp_skb;struct sk_buff **tail_skb;struct inet_sock *inet = inet_sk(sk);struct net *net = sock_net(sk);struct ip_options *opt = NULL;struct rtable *rt = (struct rtable *)cork->dst;struct iphdr *iph;__be16 df = 0;
__u8 ttl;
//将__ip_append_data构建的skb取出skb = __skb_dequeue(queue);if (!skb)goto out;// 获取第一个skb共享结构中的frag_list指针tail_skb = &(skb_shinfo(skb)->frag_list);// 将发送队列中的后续skb全部接到第一个skb的frag_list列表中while ((tmp_skb = __skb_dequeue(&sk->sk_write_queue)) != NULL) {__skb_pull(tmp_skb, skb_network_header_len(skb));*tail_skb = tmp_skb;tail_skb = &(tmp_skb->next);skb->len += tmp_skb->len;skb->data_len += tmp_skb->len;skb->truesize += tmp_skb->truesize;__sock_put(tmp_skb->sk);tmp_skb->destructor = NULL;tmp_skb->sk = NULL;}//添加ip的头iph = ip_hdr(skb);iph->version = 4;iph->ihl = 5;iph->tos = (cork->tos != -1) ? cork->tos : inet->tos;iph->frag_off = df;iph->ttl = ttl;iph->protocol = sk->sk_protocol;//设置dst addrip_copy_addrs(iph, fl4);ip_select_ident(net, skb, sk);//设置skb的rtable缓存skb_dst_set(skb, &rt->dst);return skb;
}

至此skb构建，及添加ip头完成，下一步发送：

udp_send_skb(skb, fl4, &cork);
static int udp_send_skb(struct sk_buff *skb, struct flowi4 *fl4,struct inet_cork *cork)
{struct sock *sk = skb->sk;struct inet_sock *inet = inet_sk(sk);struct udphdr *uh;int err = 0;int is_udplite = IS_UDPLITE(sk);int offset = skb_transport_offset(skb);int len = skb->len - offset;int datalen = len - sizeof(*uh);__wsum csum = 0;//构建UDP的头uh = udp_hdr(skb);uh->source = inet->inet_sport;uh->dest = fl4->fl4_dport;uh->len = htons(len);uh->check = 0;
…
send://调用ip层发送err = ip_send_skb(sock_net(sk), skb);
}

3、 ip_send_skb

ip_send_skb是ip层的发送接口，至此，udp数据包进入网络层。该接口最终调用如下：

int ip_output(struct net *net, struct sock *sk, struct sk_buff *skb)
{//获取rtable里的出口dev，前面路由阶段已获取到struct net_device *dev = skb_dst(skb)->dev, *indev = skb->dev;IP_UPD_PO_STATS(net, IPSTATS_MIB_OUT, skb->len);//设置skb出口devskb->dev = dev;//设置协议skb->protocol = htons(ETH_P_IP);//调用ip_finish_outputreturn NF_HOOK_COND(NFPROTO_IPV4, NF_INET_POST_ROUTING,net, sk, skb, indev, dev,ip_finish_output,!(IPCB(skb)->flags & IPSKB_REROUTED));
}

ip_finish_output往下走：

static int __ip_finish_output(struct net *net, struct sock *sk, struct sk_buff *skb)
{unsigned int mtu;
mtu = ip_skb_dst_mtu(sk, skb);
// gso用来delay 大包的分片,dev_hard_start_xmit函数会用到if (skb_is_gso(skb))return ip_finish_output_gso(net, sk, skb, mtu);//如果大于MTU，则分片if (skb->len > mtu || IPCB(skb)->frag_max_size)return ip_fragment(net, sk, skb, mtu, ip_finish_output2);return ip_finish_output2(net, sk, skb);
}

最终走到 ip_finish_output2 ：

int ip_finish_output2(struct net *net, struct sock *sk, struct sk_buff *skb)
{struct dst_entry *dst = skb_dst(skb);//获取路由表struct rtable *rt = (struct rtable *)dst;//获取出口设备struct net_device *dev = dst->dev;unsigned int hh_len = LL_RESERVED_SPACE(dev);struct neighbour *neigh;bool is_v6gw = false;…
rcu_read_lock_bh();
//邻居子系统查询目的MACneigh = ip_neigh_for_gw(rt, skb, &is_v6gw);if (!IS_ERR(neigh)) {int res;sock_confirm_neigh(skb, neigh);/* if crossing protocols, can not use the cached header */res = neigh_output(neigh, skb, is_v6gw);rcu_read_unlock_bh();return res;}rcu_read_unlock_bh();net_dbg_ratelimited("%s: No header cache and no neighbour!\n",__func__);kfree_skb(skb);return -EINVAL;
}

邻居子系统主要用于获取目的MAC，可能会发出arp报文：

static inline struct neighbour *ip_neigh_gw4(struct net_device *dev,__be32 daddr)
{struct neighbour *neigh;//从缓存中获取邻居项neigh = __ipv4_neigh_lookup_noref(dev, (__force u32)daddr);if (unlikely(!neigh))//缓存没有，则创建neigh = __neigh_create(&arp_tbl, &daddr, dev, false);return neigh;
}

struct neighbour *
___neigh_create(struct neigh_table *tbl, const void *pkey,struct net_device *dev, u8 flags,bool exempt_from_gc, bool want_ref)
{u32 hash_val, key_len = tbl->key_len;struct neighbour *n1, *rc, *n;struct neigh_hash_table *nht;//分配neighbour空间n = neigh_alloc(tbl, dev, flags, exempt_from_gc);trace_neigh_create(tbl, dev, pkey, n, exempt_from_gc);//拷贝daddr参数memcpy(n->primary_key, pkey, key_len);//设置出口设备n->dev = dev;//执行arp_tbl->constructor, 即arp_constructorif (tbl->constructor && (error = tbl->constructor(n)) < 0) {rc = ERR_PTR(error);goto out_neigh_release;}//放入哈希表缓存rcu_assign_pointer(nht->hash_buckets[hash_val], n);rc = n;return rc;
}

}
看一下arp_constructor执行了什么：

static int arp_constructor(struct neighbour *neigh)
{//关键步骤neigh->ops = &arp_generic_ops;neigh->output = neigh->ops->output;
}

}
arp_generic_ops如下：

static const struct neigh_ops arp_generic_ops = {.family =       AF_INET,.solicit =      arp_solicit,.error_report =     arp_error_report,.output =       neigh_resolve_output,.connected_output = neigh_connected_output,
};

因此neigh->output = neigh_resolve_output.
然后调用neigh_output:

static inline int neigh_output(struct neighbour *n, struct sk_buff *skb,bool skip_cache)
{const struct hh_cache *hh = &n->hh;return n->output(n, skb);
}

调用的即neigh_resolve_output：

int neigh_resolve_output(struct neighbour *neigh, struct sk_buff *skb)
{int rc = 0;//发送arp报文，获取目的MAC，存入neigh->haif (!neigh_event_send(neigh, skb)) {int err;struct net_device *dev = neigh->dev;unsigned int seq;if (dev->header_ops->cache && !READ_ONCE(neigh->hh.hh_len))neigh_hh_init(neigh);do {__skb_pull(skb, skb_network_offset(skb));seq = read_seqbegin(&neigh->ha_lock);//目的MAC存入skberr = dev_hard_header(skb, dev, ntohs(skb->protocol),neigh->ha, NULL, skb->len);} while (read_seqretry(&neigh->ha_lock, seq));if (err >= 0)//链路层发送rc = dev_queue_xmit(skb);elsegoto out_kfree_skb;}
out:return rc;
out_kfree_skb:rc = -EINVAL;kfree_skb(skb);goto out;
}

arp报文发出：

int __neigh_event_send(struct neighbour *neigh, struct sk_buff *skb)
{neigh->nud_state     = NUD_INCOMPLETE;neigh->updated = now;next = now + max(NEIGH_VAR(neigh->parms, RETRANS_TIME),HZ/100);//设置超时时间neigh_add_timer(neigh, next);immediate_probe = true;out_unlock_bh:if (immediate_probe)//arp发送neigh_probe(neigh);elsewrite_unlock(&neigh->lock);local_bh_enable();trace_neigh_event_send_done(neigh, rc);
return rc;
}

neigh_probe具体执行arp发出的功能：

void neigh_probe(struct neighbour *neigh)__releases(neigh->lock)
{struct sk_buff *skb = skb_peek_tail(&neigh->arp_queue);/* keep skb alive even if arp_queue overflows */if (skb)skb = skb_clone(skb, GFP_ATOMIC);write_unlock(&neigh->lock);
if (neigh->ops->solicit)//调用arp_solicit发出arp报文neigh->ops->solicit(neigh, skb);atomic_inc(&neigh->probes);consume_skb(skb);
}

发出arp报文，待arp结果返回后，将目的MAC存入neigh->ha(这一步应该是内核线程做的)，然后调用dev_hard_header设置skb目的MAC。
最后调用dev_queue_xmit(skb)进入链路层发送。
创建的邻居子系统neighbor结构体如下：

总结：
udp发送报文会调用udp_prot的udp_sendmsg方法。udp_sendmsg会判断报文长度、标志位，然后根据目的IP addr查找路由下一跳的出口设备。进入邻居子系统查找邻居项填充目的MAC,最终进入链路层发送。