lkl: integrate with zpoline for alternate hijack backend

This commit introduces an integration with zpoline (*1), which is a
mechanism to rewrite binary upon loading.  zpoline allows us to replace
symbols of system call to different functions, which current LKL hijack
library does the same thing in a different way, by using symbol
replacement via LD_PRELOAD.

The benefit of zpoline is that the replacement of syscall is at the
instruction of `syscall` or `sysenter`, which userspace program can
catch all syscalls,  while the approach based on LD_PRELOAD cannot when
the symbols of interet are hidden within libc (e.g., __socket).

For more detail about the internal of zpoline, take a look at *1.

*1: https://github.com/yasukata/zpoline

Signed-off-by: Hajime Tazaki <[email protected]>

.github/workflows/ci.yml

@@ -36,6 +36,12 @@ jobs:
             runs_on: ubuntu-22.04
             shell: bash
             build_options: "LKL_FUZZING=1 fuzzers"
+          - displayTargetName: zpoline
+          # maybe integrate with default Linux build once the function becomes stable
+            os: unix
+            runs_on: ubuntu-22.04
+            shell: bash
+            build_options: "zpoline=./zpoline"
     timeout-minutes: 100
     env:
       CCACHE_DIR: ${{ github.workspace }}/.ccache
@@ -128,6 +134,23 @@ jobs:
           which gcc
           ccache -z
 
+      - name: install zpoline
+        if: matrix.displayTargetName == 'zpoline'
+        run: |
+          sudo apt install -y binutils-dev
+          git clone https://github.com/yasukata/zpoline
+          cd zpoline
+          # FIXME: this is needed for the moment to silence debug messages
+          # from zpoline. without that, our tests to verify console outputs won't pass.
+          git fetch origin pull/3/head:pr-3
+          git checkout pr-3
+          make
+          cd ..
+          # This is the whole point of zpoline
+          echo "== setting mmap_min_addr =="
+          sudo sh -c "echo 0 > /proc/sys/vm/mmap_min_addr"
+          echo "setting env variable (debug)"
+          echo "ZPOLINE_DEBUG=0" >> "$GITHUB_ENV"
       - name: Build
         run: |
           make -j4 -C tools/lkl ${{ matrix.build_options }}

Documentation/lkl.txt

@@ -460,6 +460,71 @@ The following are the list of keys to describe a JSON file.
      "nameserver":"8.8.8.8"
 ```
 
+LKL hijack library with zpoline
+-------------------------------
+
+[zpoline](https://github.com/yasukata/zpoline) is an alternative to
+syscall hijack based on LD_PRELOAD, which is still default on LKL.
+The zpoline library works with binary rewrites to the loaded programs
+upon instantiation, then load hook function for the original syscalls.
+The LKL hijack library works together with zpoline by loading LKL.
+
+zpoline currently only works on x86_64 machines.
+
+To use the zpoline-enabled hijack library, please follow the
+instruction below.
+
+- Build
+```
+make -C tools/lkl -j8 zpoine=../zpoline
+```
+
+Suppose `zpoline` is downloaded at `../zpoline` and already build
+before LKL build.
+
+- Execution
+
+zpoline rewrites the memory address 0x0 to hook syscalls, but non-root
+users don't have a privilege to operate that address.  The following
+configuration allows us to use zpoline without root privilege.
+
+```
+sudo sh -c "echo 0 > /proc/sys/vm/mmap_min_addr"
+```
+
+then, execute command with the environment variable `LKL_HIJACK_ZPOLINE=1`.
+
+```
+LKL_HIJACK_ZPOLINE=1 LKL_HIJACK_CONFIG_FILE=lkl-tap.json \
+  ./tools/lkl/bin/lkl-hijack.sh ping www.google.com
+```
+
+The file `lkl-tap.json` can be prepared like this.
+
+```
+{
+    "gateway": "172.17.0.1",
+    "nameserver": "8.8.8.8",
+    "interfaces": [
+        {
+            "ip": "172.17.0.39",
+            "masklen": "16",
+            "mac": "00:0d:0b:94:4e:97",
+            "param": "tap0",
+            "type": "tap"
+        }
+    ],
+}
+```
+
+With the preload hijack library, which is the default one, it uses the
+host name resolver and if the host uses a nameserver, defined at
+`/etc/resolv.conf`, like 127.0.0.53, is not accepting DNS requests, in
+a view of the LKL instance.
+
+But with zpoline, it can successfully replace all syscalls for name
+resolution so can `ping` with a name.
+
 FAQ
 ===
 

tools/lkl/Makefile.autoconf

@@ -62,6 +62,11 @@ define virtio_net_vde
   LDLIBS += $(shell pkg-config --libs vdeplug)
 endef
 
+define zpoline_conf
+  $(eval zpoline_dir=$(abspath $(srctree)/$(1)))
+  $(if $(strip $(foreach f, $(zpoline_dir), $(wildcard $(f)/libzpoline.so))),$(call set_autoconf_var,ZPOLINE_DIR,$(zpoline_dir)))
+endef
+
 define posix_host
   $(call set_autoconf_var,POSIX,y)
   $(call set_autoconf_var,VIRTIO_NET,y)
@@ -82,6 +87,7 @@ define posix_host
   $(if $(filter $(1),elf64-x86-64-freebsd),$(call set_autoconf_var,NEEDS_LARGP,y))
   $(if $(filter $(1),elf32-i386),$(call set_autoconf_var,I386,y))
   $(if $(strip $(call find_include,jsmn.h)),$(call set_autoconf_var,JSMN,y))
+  $(if $(filter %,$(zpoline)),$(call zpoline_conf,$(zpoline)))
 endef
 
 define nt64_host

tools/lkl/Targets

@@ -2,13 +2,17 @@ libs-y += lib/liblkl
 
 ifneq ($(LKL_HOST_CONFIG_BSD),y)
 libs-$(LKL_HOST_CONFIG_POSIX) += lib/hijack/liblkl-hijack
+libs-$(LKL_HOST_CONFIG_POSIX) += lib/hijack/liblkl-zpoline
 endif
 LDFLAGS_lib/hijack/liblkl-hijack-y += -shared -nodefaultlibs
 LDLIBS_lib/hijack/liblkl-hijack-y += -ldl
 LDLIBS_lib/hijack/liblkl-hijack-$(LKL_HOST_CONFIG_ARM) += -lgcc -lc
 LDLIBS_lib/hijack/liblkl-hijack-$(LKL_HOST_CONFIG_AARCH64) += -lgcc -lc
 LDLIBS_lib/hijack/liblkl-hijack-$(LKL_HOST_CONFIG_I386) += -lc_nonshared
 
+LDFLAGS_lib/hijack/liblkl-zpoline-$(LKL_HOST_CONFIG_POSIX) += -shared -nodefaultlibs
+LDLIBS_lib/hijack/liblkl-zpoline-$(LKL_HOST_CONFIG_POSIX) += -ldl -lc
+
 progs-$(LKL_HOST_CONFIG_FUSE) += lklfuse
 LDLIBS_lklfuse-y := -lfuse
 

tools/lkl/bin/lkl-hijack.sh

@@ -13,10 +13,19 @@
 ##
 
 script_dir=$(cd $(dirname ${BASH_SOURCE:-$0}); pwd)
+. ${script_dir}/../tests/autoconf.sh
 
 export LD_LIBRARY_PATH=${script_dir}/../lib/hijack
 if [ -n ${LKL_HIJACK_DEBUG+x}  ]
 then
   trap '' TSTP
 fi
-LD_PRELOAD=liblkl-hijack.so $*
+
+
+if [ -n "${LKL_HIJACK_ZPOLINE}" ]
+then
+    export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${LKL_HOST_CONFIG_ZPOLINE_DIR}
+    LD_PRELOAD=libzpoline.so LIBZPHOOK=liblkl-zpoline.so $*
+else
+    LD_PRELOAD=liblkl-hijack.so $*
+fi

tools/lkl/lib/hijack/Build

@@ -1,4 +1,12 @@
+liblkl-hijack-y += preload.o
 liblkl-hijack-y += hijack.o
 liblkl-hijack-y += init.o
 liblkl-hijack-y += xlate.o
 
+liblkl-zpoline-y += zpoline.o
+liblkl-zpoline-y += hijack.o
+liblkl-zpoline-y += init.o
+liblkl-zpoline-y += xlate.o
+
+CFLAGS_preload.o = -I$(srctree)/include
+CFLAGS_zpoline.o = -I$(srctree)/tools/include

tools/lkl/lib/hijack/hijack.c

@@ -31,8 +31,9 @@
 
 #include "xlate.h"
 #include "init.h"
+#include "hijack.h"
 
-static int is_lklfd(int fd)
+int is_lklfd(int fd)
 {
 	if (fd < LKL_FD_OFFSET)
 		return 0;
@@ -167,8 +168,8 @@ HOST_CALL(write)
 HOST_CALL(pipe2)
 
 HOST_CALL(setsockopt);
-int setsockopt(int fd, int level, int optname, const void *optval,
-	       socklen_t optlen)
+int hijack_setsockopt(int fd, int level, int optname, const void *optval,
+		      socklen_t optlen)
 {
 	CHECK_HOST_CALL(setsockopt);
 	if (!is_lklfd(fd))
@@ -178,7 +179,7 @@ int setsockopt(int fd, int level, int optname, const void *optval,
 }
 
 HOST_CALL(getsockopt);
-int getsockopt(int fd, int level, int optname, void *optval, socklen_t *optlen)
+int hijack_getsockopt(int fd, int level, int optname, void *optval, socklen_t *optlen)
 {
 	CHECK_HOST_CALL(getsockopt);
 	if (!is_lklfd(fd))
@@ -240,7 +241,7 @@ int fcntl(int fd, int cmd, ...)
 }
 
 HOST_CALL(poll);
-int poll(struct pollfd *fds, nfds_t nfds, int timeout)
+int hijack_poll(struct pollfd *fds, nfds_t nfds, int timeout)
 {
 	unsigned int i, lklfds = 0, hostfds = 0;
 
@@ -264,10 +265,8 @@ int poll(struct pollfd *fds, nfds_t nfds, int timeout)
 	return lkl_sys_poll((struct lkl_pollfd *)fds, nfds, timeout);
 }
 
-int __poll(struct pollfd *, nfds_t, int) __attribute__((alias("poll")));
-
 HOST_CALL(select);
-int select(int nfds, fd_set *r, fd_set *w, fd_set *e, struct timeval *t)
+int hijack_select(int nfds, fd_set *r, fd_set *w, fd_set *e, struct timeval *t)
 {
 	int fd, hostfds = 0, lklfds = 0;
 
@@ -324,7 +323,7 @@ int close(int fd)
 }
 
 HOST_CALL(epoll_create);
-int epoll_create(int size)
+int hijack_epoll_create(int size)
 {
 	int host_fd;
 
@@ -346,7 +345,7 @@ int epoll_create(int size)
 }
 
 HOST_CALL(epoll_create1);
-int epoll_create1(int flags)
+int hijack_epoll_create1(int flags)
 {
 	int host_fd;
 
@@ -369,7 +368,7 @@ int epoll_create1(int flags)
 
 
 HOST_CALL(epoll_ctl);
-int epoll_ctl(int epollfd, int op, int fd, struct epoll_event *event)
+int hijack_epoll_ctl(int epollfd, int op, int fd, struct epoll_event *event)
 {
 	CHECK_HOST_CALL(epoll_ctl);
 
@@ -404,7 +403,7 @@ static void *host_epollwait(void *arg)
 	return (void *)(intptr_t)ret;
 }
 
-int epoll_wait(int epfd, struct epoll_event *events,
+int hijack_epoll_wait(int epfd, struct epoll_event *events,
 	       int maxevents, int timeout)
 {
 	CHECK_HOST_CALL(epoll_wait);
@@ -541,7 +540,7 @@ int epoll_wait(int epfd, struct epoll_event *events,
 	return ret;
 }
 
-int eventfd(unsigned int count, int flags)
+int hijack_eventfd(unsigned int count, int flags)
 {
 	if (!lkl_running) {
 		int (*f)(unsigned int, int) = resolve_sym("eventfd");
@@ -553,7 +552,7 @@ int eventfd(unsigned int count, int flags)
 }
 
 HOST_CALL(eventfd_read);
-int eventfd_read(int fd, uint64_t *value)
+int hijack_eventfd_read(int fd, uint64_t *value)
 {
 	CHECK_HOST_CALL(eventfd_read);
 
@@ -565,7 +564,7 @@ int eventfd_read(int fd, uint64_t *value)
 }
 
 HOST_CALL(eventfd_write);
-int eventfd_write(int fd, uint64_t value)
+int hijack_eventfd_write(int fd, uint64_t value)
 {
 	CHECK_HOST_CALL(eventfd_write);
 

tools/lkl/lib/hijack/hijack.h

@@ -0,0 +1,21 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#include <sys/socket.h>
+#include <sys/epoll.h>
+#include <poll.h>
+
+
+int is_lklfd(int fd);
+int hijack_setsockopt(int fd, int level, int optname, const void *optval,
+		      socklen_t optlen);
+int hijack_getsockopt(int fd, int level, int optname, void *optval, socklen_t *optlen);
+int hijack_poll(struct pollfd *fds, nfds_t nfds, int timeout);
+int hijack_select(int nfds, fd_set *r, fd_set *w, fd_set *e, struct timeval *t);
+int hijack_eventfd(unsigned int count, int flags);
+int hijack_epoll_create(int size);
+int hijack_epoll_create1(int flags);
+int hijack_epoll_ctl(int epollfd, int op, int fd, struct epoll_event *event);
+int hijack_epoll_wait(int epfd, struct epoll_event *events,
+	      int maxevents, int timeout);
+int hijack_eventfd_read(int fd, uint64_t *value);
+int hijack_eventfd_write(int fd, uint64_t value);

tools/lkl/lib/hijack/init.c

@@ -112,8 +112,8 @@ static int config_load(void)
 	return ret;
 }
 
-void __attribute__((constructor))
-hijack_init(void)
+void
+__hijack_init(void)
 {
 	int ret, i, dev_null;
 	int single_cpu_mode = 0;
@@ -225,8 +225,17 @@ hijack_init(void)
 	lkl_load_config_post(cfg);
 }
 
-void __attribute__((destructor))
-hijack_fini(void)
+void __attribute__((constructor))
+hijack_init(void)
+{
+	if (getenv("LKL_HIJACK_ZPOLINE"))
+		return;
+
+	return __hijack_init();
+}
+
+void
+__hijack_fini(void)
 {
 	int i;
 	int err;
@@ -257,3 +266,12 @@ hijack_fini(void)
 
 	lkl_cleanup();
 }
+
+void __attribute__((destructor))
+hijack_fini(void)
+{
+	if (getenv("LKL_HIJACK_ZPOLINE"))
+		return;
+
+	return __hijack_fini();
+}

tools/lkl/lib/hijack/init.h

@@ -4,4 +4,7 @@
 extern int lkl_running;
 extern int dual_fds[];
 
+void __hijack_init(void);
+void __hijack_fini(void);
+
 #endif /*_LKL_HIJACK_INIT_H */