recvfrom(2) receives UDP broadcast twice, but tcpdump(8) receives it only once

Summary: I want to receive packets from a single interface, but setsockopt(sock, SOL_SOCKET, SO_BINDTODEVICE, iface, 1 + strlen(iface)) doesn't play well with recvfrom and shows all packets on all interfaces. tcpdump works well, however.

I have a strong feeling that there's something wrong with the receiver program, but I haven't been able to figure it out.

---

I'm working with a Netronome Agilio CX SmartNIC. The two ports on the NIC are connected together with one cable, and the port on the motherboard are connected to the wall (so I can SSH into it). The board-loaded NIC is eth0 in the OS, while the SmartNIC presents two interfaces as enp1s0np0 and enp1s0np1.

Because the two interfaces on the SmartNIC has no associated IP addresses, I have to send broadcast to one port so it arrives at the other port. For now, I send to enp1s0np0 and expect it from enp1s0np1.

I have also deployed a XDP offload program that modifies part of the packet so I can know whether the packet arrives at enp1s0np1. The program changes the string at position 28~35 to another string (of the form !......!).

The problem I am having is, I wrote a receiver program myself, and it receives two packets for every packet I send - the first is the original, while the second is the XDP-modified packet. However, tcpdump only receives the modified packet (expected behavior).

I am unsure why my program is getting it twice - I don't think it should be able to see the unmodified packet.

This is the packet sender program. It reads 32 double-precision floating point numbers and packs them into a 256-byte block, and prepends the block with 16 bytes of "magic numbers".

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <sys/socket.h>
#include <sys/types.h>
#include <arpa/inet.h>
#include <netinet/in.h>

#include "config.h"
#include "util.h"

typedef unsigned char byte;

void sanity_check(void);

int main(int argc, char** argv) {
    sanity_check();

    int sock = socket(AF_INET, SOCK_DGRAM, 0);
    if (sock < 0)
        errorexit("socket");

    char iface[16] = "enp1s0np0";
    if (setsockopt(sock, SOL_SOCKET, SO_BINDTODEVICE, iface, 1 + strlen(iface)))
        errorexit("setsockopt");
    int optval = -1;
    if (setsockopt(sock, SOL_SOCKET, SO_BROADCAST | SO_REUSEADDR, &optval, sizeof(int)))
        errorexit("setsockopt");

    struct sockaddr_in target_addr = {
        .sin_family = AF_INET,
        .sin_addr.s_addr = 0xFFFFFFFF,
        .sin_port = htons(6666)
    };

    byte buf[272];
    // Prepare data
    {
        unsigned long magic = MAGIC;
        memcpy(buf + 0, &magic, sizeof magic);
        unsigned long zero = 0UL;
        memcpy(buf + 8, &zero, sizeof zero);
        double data;
        for (int i = 0; i < 32; i++) {
            scanf(" %lf", &data);
            memcpy(buf + 16 + 8 * i, &data, sizeof data);
        }
    }

    int sent = sendto(sock, buf, sizeof(buf), 0, (struct sockaddr*)&target_addr, sizeof(struct sockaddr));
    if (sent != 0)
        errorexit("send");
    printf("%d bytes sent.\n", sent);

    // if (shutdown(sock, SHUT_RDWR))
    if (close(sock))
        errorexit("close");

    return 0;
}

void sanity_check(void) {
    if (getuid() || geteuid()) {
        fprintf(stderr, "Need root to proceed\n");
        exit(1);
    }
}

This is the receiver program. In fact, it's receiving every single packet that comes into the machine, with most of them being SSH data. I had to add checks for the magic number or it just spams the terminal. I guess it just failed to listen to the specific interface. (Check is if (buf[28] != '!' || buf[35] != '!') continue;)

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <sys/socket.h>
#include <sys/types.h>
#include <arpa/inet.h>
#include <netinet/in.h>

#include "config.h"
#include "util.h"

typedef unsigned char byte;

void sanity_check(void);

int main(int argc, char** argv) {
    sanity_check();

    int sock = socket(AF_PACKET, SOCK_DGRAM, htons(3));
    if (sock < 0)
        errorexit("socket");

    char iface[16] = "enp1s0np1";
    if (setsockopt(sock, SOL_SOCKET, SO_BINDTODEVICE, iface, 1 + strlen(iface)))
        errorexit("setsockopt");

    struct sockaddr_in target_addr = {
        .sin_family = AF_INET,
        .sin_addr.s_addr = htonl(INADDR_ANY),
        .sin_port = htons(UDP_PORT)
    };

    size_t bufsize = 8192;
    byte *buf = malloc(bufsize);
    unsigned long magic = MAGIC;
    int saddr_size = sizeof(struct sockaddr);
    printf("Preparing to receive... ");
    fflush(stdout);
    while (1) {
        int received = recvfrom(sock, buf, bufsize, 0,(struct sockaddr *)&target_addr , (socklen_t*)&saddr_size);
        if (received < 0)
            errorexit("receive");
        if (received == 0)
            break;
        else if (buf[28] != '!' || buf[35] != '!') // Magic check
            continue;
        printf("%d bytes received.\n", received);
        hexdump(buf, received);
    }

    if (close(sock))
        errorexit("close");

    free(buf);
    return 0;
}

void sanity_check(void) {
    if (getuid() || geteuid()) {
        fprintf(stderr, "Need root to proceed\n");
        exit(1);
    }
}

The file util.c (link to Gist) contains two utility functions (errorexit which is just a wrapper of perror and exit, and a badly hand-crafted hexdump function for displaying) and is irrelevant here.

The constant MAGIC is defined as

#define MAGIC 0x216C7174786A7A21UL // string "!zjxtql!"

Here's the console output of my program (recv.c compiled into recv) and the tcpdump command, with irrelevant data truncated. Before both programs are killed, only one packet is sent from the sender program. The special thing to note is the data at position 28 (was originally !zjxtql!, should be modified to !wjfskb! by the XDP offload program).

$ sudo ./recv
Preparing to receive...
300 bytes received.
00000000  45 00 01 2C 4C 1A 40 00  40 11 B4 7F 72 D6 C6 51  |E..,L.@.@...r..Q|
00000010  FF FF FF FF B0 9F 1A 0A  01 18 3A 51 21 7A 6A 78  |..........:Q!zjx|
00000020  74 71 6C 21 00 00 00 00  00 00 00 00 29 5C 8F C2  |tql!........)\..|
00000120  14 AE F7 3F AE 47 E1 7A  14 AE F7 3F              |...?.G.z...?|
0000012C
300 bytes received.
00000000  45 00 01 2C 4C 1A 40 00  40 11 B4 7F 72 D6 C6 51  |E..,L.@.@...r..Q|
00000010  FF FF FF FF B0 9F 1A 0A  01 18 C0 9B 21 77 6A 66  |............!wjf|
00000020  73 6B 62 21 00 00 00 00  00 00 00 00 29 5C 8F C2  |skb!........)\..|
00000120  14 AE F7 3F AE 47 E1 7A  14 AE F7 3F              |...?.G.z...?|
0000012C
^C
$ sudo tcpdump -vv -X -i enp1s0np1 port 6666
tcpdump: listening on enp1s0np1, link-type EN10MB (Ethernet), capture size 262144 bytes
04:53:52.819657 IP (tos 0x0, ttl 64, id 8595, offset 0, flags [DF], proto UDP (17), length 300)
    agilio1415.47585 > 255.255.255.255.ircu-2: [bad udp cksum 0xb759 -> 0xc274!] UDP, length 272
        0x0000:  4500 012c 2193 4000 4011 df06 72d6 c651  E..,!.@.@...r..Q
        0x0010:  ffff ffff b9e1 1a0a 0118 b759 2177 6a66  ...........Y!wjf
        0x0020:  736b 6221 0000 0000 0000 0000 295c 8fc2  skb!........)\..
        0x0120:  14ae f73f ae47 e17a 14ae f73f            ...?.G.z...?
^C

I have tried straceing tcpdump and trying its job with setsockopt:

sudo strace -e setsockopt tcpdump -vv -X -i enp1s0np1 port 6666

which gives

setsockopt(3, SOL_PACKET, PACKET_ADD_MEMBERSHIP, {mr_ifindex=if_nametoindex("enp1s0np1"), mr_type=PACKET_MR_PROMISC, mr_alen=0, mr_address=}, 16) = 0
setsockopt(3, SOL_PACKET, PACKET_AUXDATA, [1], 4) = 0
setsockopt(3, SOL_PACKET, PACKET_VERSION, [1], 4) = 0
setsockopt(3, SOL_PACKET, PACKET_RESERVE, [4], 4) = 0
setsockopt(3, SOL_PACKET, PACKET_RX_RING, 0x7ffe9d8d1510, 28) = 0
setsockopt(7, SOL_SOCKET, SO_RCVBUF, [8388608], 4) = 0
setsockopt(7, SOL_SOCKET, SO_SNDBUF, [8388608], 4) = 0
setsockopt(3, SOL_SOCKET, SO_ATTACH_FILTER, {len=1, filter=0x7fa5289de000}, 16) = 0
setsockopt(3, SOL_SOCKET, SO_ATTACH_FILTER, {len=24, filter=0x56025631f280}, 16) = 0
tcpdump: listening on enp1s0np1, link-type EN10MB (Ethernet), capture size 262144 bytes

because I don't understand the others, I mimicked only the first call to setsockopt of tcpdump:

struct packet_mreq mreq = {
    .mr_ifindex = if_nametoindex(iface),
    .mr_type = PACKET_MR_PROMISC,
    .mr_alen = 0
};
if (setsockopt(sock, SOL_PACKET, PACKET_ADD_MEMBERSHIP, &mreq, sizeof(mreq)))
    errorexit("setsockopt");

The above code is a replacement for the setsockopt(SO_BINDTODEVICE) call in the receiver program, but I haven't observed any difference (still all packets from all interfaces are caught).

It looks like all that I'm missing is a bind(2). Unfortunately, SO_BINDTOINTERFACE doesn't work with AF_PACKET, so bind(2) is the only solution.

The code isn't any complex:

struct sockaddr_ll sll = {
    .sll_family = AF_PACKET,
    .sll_ifindex = if_nametoindex(iface),
    .sll_protocol = htons(3) // 3 = ETH_P_ALL
};
if (bind(sock, (struct sockaddr*)&sll, sizeof sll))
    errorexit("sock");

From the same socket.7 page:

SO_BINDTOSOCKET

... Note that this works only for some socket types, particularly AF_INET sockets. It is not supported for packet sockets (use normal bind(2) there).

Hmmm, guess I should've read the manual more thoroughly.