From d3c104598cc3de2d3f1433db749426cde141053d Mon Sep 17 00:00:00 2001 From: Eelco Chaudron Date: Tue, 3 Sep 2019 07:19:29 -0400 Subject: [PATCH 1/4] vagrant: add vagrant support to build latest bpf-next in VM This adds support for a Vagrant configuration that will build a VM running fedora29, with the latest bpf-next kernel. In addition, it will install and build the xdp-tutorials. Signed-off-by: Eelco Chaudron --- .gitignore | 3 ++ vagrant/README.org | 77 ++++++++++++++++++++++++++++++++ vagrant/Vagrantfile | 22 +++++++++ vagrant/bootstrap.sh | 56 +++++++++++++++++++++++ vagrant/bootstrap_post_kernel.sh | 9 ++++ 5 files changed, 167 insertions(+) create mode 100644 vagrant/README.org create mode 100644 vagrant/Vagrantfile create mode 100644 vagrant/bootstrap.sh create mode 100644 vagrant/bootstrap_post_kernel.sh diff --git a/.gitignore b/.gitignore index c659ebb0..af651670 100644 --- a/.gitignore +++ b/.gitignore @@ -66,3 +66,6 @@ trace_load_and_stats trace_load_and_stats trace_read xdp_sample_pkts_user + +# Vagrant directory +.vagrant/ diff --git a/vagrant/README.org b/vagrant/README.org new file mode 100644 index 00000000..bc5aa7a3 --- /dev/null +++ b/vagrant/README.org @@ -0,0 +1,77 @@ +# -*- fill-column: 76; -*- +#+TITLE: Vagrant image for xdp-tutorial with latest bpf-next kernel +#+OPTIONS: ^:nil + +This Vagrant configuration can be used to test the xdp-tutorial using a +virtual machine based on Fedora29 with the latest bpf-next kernel. + +* Configure Vagrant +This example uses the libvirt back-end, which needs to be installed: + +#+begin_example sh +vagrant plugin install vagrant-libvirt +#+end_example + +In addition, we also need the /reload/ plugin to reboot the machine: + +#+begin_example sh +vagrant plugin install vagrant-reload +#+end_example + + +* Configure the virtual machine +The more cores and memory you will assign to the VM the faster it will build +the kernel. Please modify the Valgrantfile to change the default 8 cores and +4G of memory: + +#+begin_example ruby + config.vm.provider :libvirt do |libvirt| + libvirt.cpus = 8 + libvirt.memory = 4096 + libvirt.storage :file, :size => '50G', :bus => 'scsi' + end +#+end_example + + +* Start the Vagrant vm +To start the virtual machine you need to execute the below from the vagrant +directory: + +#+begin_example sh +$ vagrant up +Bringing machine 'default' up with 'libvirt' provider... +==> default: Checking if box 'generic/fedora29' version '1.9.24' is up to date... +==> default: Creating image (snapshot of base box volume). +==> default: Creating domain with the following settings... +==> default: -- Name: vagrant_default +==> default: -- Domain type: kvm +==> default: -- Cpus: 8 +==> default: -- Feature: acpi +... +... + default: -D __BPF_TRACING__ \ + default: -I../libbpf/src//root/usr/include/ -g -I/usr/include/x86_64-linux-gnu -I../headers/ \ + default: -Wall \ + default: -Wno-unused-value \ + default: -Wno-pointer-sign \ + default: -Wno-compare-distinct-pointer-types \ + default: -Werror \ + default: -O2 -emit-llvm -c -g -o af_xdp_kern.ll af_xdp_kern.c + default: llc -march=bpf -filetype=obj -o af_xdp_kern.o af_xdp_kern.ll + default: make[1]: Leaving directory '/home/vagrant/xdp-tutorial/advanced03-AF_XDP' +#+end_example + +NOTE: The above will take some time as it will build the kernel from scratch. + +If the installation completed successfully you can access the VM as follows: + +#+begin_example sh +$ vagrant ssh default +[vagrant@xdp-tutorial ~]$ ls +dp-tutorial +[vagrant@xdp-tutorial ~]$ cd xdp-tutorial/advanced03-AF_XDP/ +[vagrant@xdp-tutorial advanced03-AF_XDP]$ +[vagrant@xdp-tutorial advanced03-AF_XDP]$ ls +af_xdp_kern.c af_xdp_kern.o af_xdp_user.c README.org +af_xdp_kern.ll af_xdp_user Makefile +#+end_example diff --git a/vagrant/Vagrantfile b/vagrant/Vagrantfile new file mode 100644 index 00000000..30a59ad1 --- /dev/null +++ b/vagrant/Vagrantfile @@ -0,0 +1,22 @@ +# +# NOTE: This script requeres the reload pluging, install it as follows: +# vagrant plugin install vagrant-reload +# + +Vagrant.configure("2") do |config| + + config.vm.provider :libvirt do |libvirt| + libvirt.cpus = 8 + libvirt.memory = 4096 + libvirt.storage :file, :size => '50G', :bus => 'scsi' + # libvirt.storage_pool_path = '/home/user/.local/share/libvirt/images' + end + + config.vm.box = "generic/fedora29" + config.vm.hostname = "xdp-tutorial" + + config.vm.provision :shell, path: "bootstrap.sh" + config.vm.provision :reload + config.vm.provision :shell, privileged: false, path: "bootstrap_post_kernel.sh" + +end diff --git a/vagrant/bootstrap.sh b/vagrant/bootstrap.sh new file mode 100644 index 00000000..5adff78f --- /dev/null +++ b/vagrant/bootstrap.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash + +# +# Install required packages +# +dnf install -y \ + bc \ + bpftool \ + clang \ + elfutils-libelf-devel \ + emacs-nox \ + fedora-packager \ + fedpkg \ + flex bison \ + kernel-headers \ + libpcap-devel \ + llvm \ + ncurses-devel \ + openssl-devel \ + perf \ + pesign \ + rpmdevtools + +# +# Initialize the extra disk needed to build the kernel +# +if ! grep -q $(blkid /dev/sdb1 -o value -s UUID) /etc/fstab +then + parted /dev/sdb mklabel gpt + parted /dev/sdb mkpart primary 0% 100% + mkfs.xfs /dev/sdb1 + mkdir /data + echo "UUID=$(blkid /dev/sdb1 -o value -s UUID) /data xfs defaults 0 0" >> /etc/fstab + mount /data +fi + +# +# Build the latest bpf-next kernel +# +cd /data +git clone https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git linux_kernel +# This was successfully tested with commit bdb15a29cc28f8155e20f7fb58b60ffc452f2d1b +cd linux_kernel +cp /boot/config-$(uname -r) .config +make olddefconfig +make -j $(nproc) bzImage +make -j $(nproc) modules +make modules_install +make headers_install +make install + +# +# Boot the new kernel, and make sure IPv6 is enabled +# +grubby --set-default-index=1 +sed -i 's/net.ipv6.conf.all.disable_ipv6 = 1/net.ipv6.conf.all.disable_ipv6 = 0/g' /etc/sysctl.conf diff --git a/vagrant/bootstrap_post_kernel.sh b/vagrant/bootstrap_post_kernel.sh new file mode 100644 index 00000000..7bd64561 --- /dev/null +++ b/vagrant/bootstrap_post_kernel.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +# +# Clone and build the tutorial +# +git clone --recurse-submodules https://github.com/xdp-project/xdp-tutorial.git +# This was successfully tested with commit 94471eed572a733a71b096975b3cb72509113e6f +cd xdp-tutorial +make From eaf1e96e4dd84658b329b317b39ace5bb34872db Mon Sep 17 00:00:00 2001 From: Eelco Chaudron Date: Tue, 3 Sep 2019 07:27:01 -0400 Subject: [PATCH 2/4] advanced03: add support for the XDP_USE_NEED_WAKEUP API Adding this will give improved performance. In my setup, it goes from 6,428kpps to 6,661kpps for 128 bytes packets with the IPv6 ping packets replier (assignment 3). Signed-off-by: Eelco Chaudron --- advanced03-AF_XDP/af_xdp_user.c | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/advanced03-AF_XDP/af_xdp_user.c b/advanced03-AF_XDP/af_xdp_user.c index e259407c..dc48e0a7 100644 --- a/advanced03-AF_XDP/af_xdp_user.c +++ b/advanced03-AF_XDP/af_xdp_user.c @@ -30,6 +30,9 @@ #include "../common/common_user_bpf_xdp.h" #include "../common/common_libbpf.h" +#ifdef XDP_USE_NEED_WAKEUP +#define HAS_XDP_NEED_WAKEUP 1 +#endif #define NUM_FRAMES 4096 #define FRAME_SIZE XSK_UMEM__DEFAULT_FRAME_SIZE @@ -186,6 +189,9 @@ static struct xsk_socket_info *xsk_configure_socket(struct config *cfg, xsk_cfg.libbpf_flags = 0; xsk_cfg.xdp_flags = cfg->xdp_flags; xsk_cfg.bind_flags = cfg->xsk_bind_flags; +#ifdef HAS_XDP_NEED_WAKEUP + xsk_cfg.bind_flags |= XDP_USE_NEED_WAKEUP; +#endif ret = xsk_socket__create(&xsk_info->xsk, cfg->ifname, cfg->xsk_if_queue, umem->umem, &xsk_info->rx, &xsk_info->tx, &xsk_cfg); @@ -234,7 +240,11 @@ static void complete_tx(struct xsk_socket_info *xsk) if (!xsk->outstanding_tx) return; - sendto(xsk_socket__fd(xsk->xsk), NULL, 0, MSG_DONTWAIT, NULL, 0); +#ifdef HAVE_XDP_NEED_WAKEUP + if (xsk_ring_prod__needs_wakeup(&xsk->tx)) +#endif + sendto(xsk_socket__fd(xsk->xsk), NULL, 0, MSG_DONTWAIT, + NULL, 0); /* Collect/free completed TX buffers */ @@ -343,8 +353,19 @@ static void handle_receive_packets(struct xsk_socket_info *xsk) int ret; rcvd = xsk_ring_cons__peek(&xsk->rx, RX_BATCH_SIZE, &idx_rx); - if (!rcvd) + if (!rcvd) { +#ifdef HAS_XDP_NEED_WAKEUP + if (xsk_ring_prod__needs_wakeup(&xsk->umem->fq)) { + struct pollfd fds[2]; + + memset(fds, 0, sizeof(fds)); + fds[0].fd = xsk_socket__fd(xsk->xsk); + fds[0].events = POLLIN; + poll(fds, 1, 0); + } +#endif return; + } /* Stuff the ring with as much frames as possible */ stock_frames = xsk_prod_nb_free(&xsk->umem->fq, From ce93ed3b8f8d5d7e45f979645adde70655605ff7 Mon Sep 17 00:00:00 2001 From: Eelco Chaudron Date: Wed, 25 Sep 2019 09:25:34 -0400 Subject: [PATCH 3/4] advanced03: add some basic kernel version checking for AF_XDP support Add some basic version checking support to warn if a kernel version is not supporting AF_XDP. Signed-off-by: Eelco Chaudron --- advanced03-AF_XDP/af_xdp_user.c | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/advanced03-AF_XDP/af_xdp_user.c b/advanced03-AF_XDP/af_xdp_user.c index dc48e0a7..15572e5e 100644 --- a/advanced03-AF_XDP/af_xdp_user.c +++ b/advanced03-AF_XDP/af_xdp_user.c @@ -14,6 +14,7 @@ #include #include +#include #include #include @@ -534,6 +535,29 @@ int main(int argc, char **argv) struct xsk_socket_info *xsk_socket; struct bpf_object *bpf_obj = NULL; pthread_t stats_poll_thread; + struct utsname uname_info; + unsigned int major, minor, revision; + + /* Some basic version checking for AF_XDP */ + if (uname(&uname_info) == -1) { + fprintf(stderr, "ERROR: failed calling uname(): %s\n", + strerror(errno)); + exit(EXIT_FAILURE); + } + if (sscanf(uname_info.release, "%u.%u.%u-", + &major, &minor, &revision) != 3) { + fprintf(stderr, "ERROR: failed to extract Kernel version\n"); + exit(EXIT_FAILURE); + } + if (major < 5 || (major == 5 && minor < 2)) { + printf("WARNING: For AF_XDP you need at least an upstream " + "kernel of 5.2!\n"); + } + if (major == 5 && minor == 2) { + printf("WARNING: Although AF_XDP is supported in upstream " + "kernel 5.2, due to known libbpf issues it's " + "recommended to use at least version 5.3!\n"); + } /* Global shutdown handler */ signal(SIGINT, exit_application); @@ -602,7 +626,8 @@ int main(int argc, char **argv) /* Open and configure the AF_XDP (xsk) socket */ xsk_socket = xsk_configure_socket(&cfg, umem); if (xsk_socket == NULL) { - fprintf(stderr, "ERROR: Can't setup AF_XDP socket \"%s\"\n", + fprintf(stderr, "ERROR: Can't setup AF_XDP socket \"%s\"\n" + "Make sure Kernel is build with CONFIG_XDP_SOCKETS=y\n", strerror(errno)); exit(EXIT_FAILURE); } From 3f89b0e90f67eabd132bcecda9253d2f08144c17 Mon Sep 17 00:00:00 2001 From: Eelco Chaudron Date: Wed, 25 Sep 2019 10:49:20 -0400 Subject: [PATCH 4/4] advanced03: fix compilation of custom xdp program on older kernels This fixes execution failure when trying to load the custom XDP program with the tutorial and kernels < 5.3. Fixes https://github.com/xdp-project/xdp-tutorial/issues/65 Signed-off-by: Eelco Chaudron --- advanced03-AF_XDP/af_xdp_kern.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/advanced03-AF_XDP/af_xdp_kern.c b/advanced03-AF_XDP/af_xdp_kern.c index 2ec5df60..a3951687 100644 --- a/advanced03-AF_XDP/af_xdp_kern.c +++ b/advanced03-AF_XDP/af_xdp_kern.c @@ -1,6 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 */ #include +#include #include "bpf_helpers.h" @@ -18,6 +19,17 @@ struct bpf_map_def SEC("maps") xdp_stats_map = { .max_entries = 64, }; +#if LINUX_VERSION_CODE < KERNEL_VERSION(5,3,0) + +/* Kernel version before 5.3 needed an additional map */ +struct bpf_map_def SEC("maps") qidconf_map = { + .type = BPF_MAP_TYPE_ARRAY, + .key_size = sizeof(int), + .value_size = sizeof(int), + .max_entries = 64, +}; +#endif + SEC("xdp_sock") int xdp_sock_prog(struct xdp_md *ctx) {