Libvirt fuzzing technique

Fuzzing is done by socket calls to the monolithic libvirtd daemon using the libvirt RPC protocol and preloading the desock library to create dummy sockets, as afl++ natively does not support network interface fuzzing.

Prepare environment

Prepare workdir

Optionally, run podman container or continue in alt p10/sisyphus:

$ podman run -ti alt:p10

Create /fuzz directory:

$ mkdir /fuzz
$ cd /fuzz

Prepare patches and config

Put following files to /fuzz directory:

deferred_init.patch

diff --git a/src/remote/remote_daemon.c b/src/remote/remote_daemon.c
index d880711c91..d77ffbbbe4 100644
--- a/src/remote/remote_daemon.c
+++ b/src/remote/remote_daemon.c
@@ -1023,7 +1023,9 @@ int main(int argc, char **argv) {
         ret = VIR_DAEMON_ERR_DRIVER;
         goto cleanup;
     }
-
+#ifdef __AFL_HAVE_MANUAL_CONTROL
+  __AFL_INIT();
+#endif
     if (!(srv = virNetServerNew(DAEMON_NAME, 1,
                                 config->min_workers,
                                 config->max_workers,

virnetsocket.patch

diff --git a/src/rpc/virnetsocket.c b/src/rpc/virnetsocket.c
index 910fb8dd67..2b88f58e7f 100644
--- a/src/rpc/virnetsocket.c
+++ b/src/rpc/virnetsocket.c
@@ -266,18 +266,6 @@ virNetSocketNew(virSocketAddr *localAddr,
     sock->isClient = isClient;
     sock->unlinkUNIX = unlinkUNIX;
 
-    /* Disable nagle for TCP sockets */
-    if (sock->localAddr.data.sa.sa_family == AF_INET ||
-        sock->localAddr.data.sa.sa_family == AF_INET6) {
-        if (setsockopt(fd, IPPROTO_TCP, TCP_NODELAY,
-                       &no_slow_start,
-                       sizeof(no_slow_start)) < 0) {
-            virReportSystemError(errno, "%s",
-                                 _("Unable to disable nagle algorithm"));
-            goto error;
-        }
-    }
-
 
     if (localAddr &&
         !(sock->localAddrStrSASL = virSocketAddrFormatFull(localAddr, true, ";")))

fix-sigsev-when-desock-and-capng.patch

From 1ca99bdecd4ba7d28f56d8f38f01c59eba725ed9 Mon Sep 17 00:00:00 2001
From: Alexandr Shashkin <dutyrok@altlinux.org>
Date: Fri, 25 Aug 2023 15:17:37 +0300
Subject: [PATCH 1/1] desock.c: move initialization of pointer variables of
 original functions to their overrides

If we use __attribute__((constructor)) void preeny_desock_orig() to save
original function pointer it will lead to segmentation fault at init_lib()
(src/cap-ng.c:230 at 0.8.2-alt1), when "/proc/sys/kernel/cap_last_cap" fd is closed.
That remains to be examined why constructor from cap-ng called earlier than one
from preloaded library.
Moving initialization to workaround this issue.
---
 src/desock.c | 35 ++++++++++++++++++++++++-----------
 1 file changed, 24 insertions(+), 11 deletions(-)

diff --git a/src/desock.c b/src/desock.c
index 19be82d..affe238 100644
--- a/src/desock.c
+++ b/src/desock.c
@@ -159,20 +159,12 @@ int (*original_connect)(int sockfd, const struct sockaddr *addr, socklen_t addrl
 int (*original_close)(int fd);
 int (*original_shutdown)(int sockfd, int how);
 int (*original_getsockname)(int sockfd, struct sockaddr *addr, socklen_t *addrlen);
-__attribute__((constructor)) void preeny_desock_orig()
-{
-	original_socket = dlsym(RTLD_NEXT, "socket");
-	original_listen = dlsym(RTLD_NEXT, "listen");
-	original_accept = dlsym(RTLD_NEXT, "accept");
-	original_bind = dlsym(RTLD_NEXT, "bind");
-	original_connect = dlsym(RTLD_NEXT, "connect");
-	original_close = dlsym(RTLD_NEXT, "close");
-	original_shutdown = dlsym(RTLD_NEXT, "shutdown");
-	original_getsockname = dlsym(RTLD_NEXT, "getsockname");
-}
 
 int socket(int domain, int type, int protocol)
 {
+	if (original_socket == NULL)
+		original_socket = dlsym(RTLD_NEXT, "socket");
+
 	int fds[2];
 	int front_socket;
 	int back_socket;
@@ -222,6 +214,9 @@ int socket(int domain, int type, int protocol)
 
 int accept(int sockfd, struct sockaddr *addr, socklen_t *addrlen)
 {
+	if (original_accept == NULL)
+		original_accept = dlsym(RTLD_NEXT, "accept");
+
 	if (preeny_desock_accepted_sock >= 0)
 	{
                 errno = ECONNRESET;
@@ -257,6 +252,9 @@ int accept4(int sockfd, struct sockaddr *addr, socklen_t *addrlen, int flags)
 
 int bind(int sockfd, const struct sockaddr *addr, socklen_t addrlen)
 {
+	if (original_bind == NULL)
+		original_bind = dlsym(RTLD_NEXT, "bind");
+
 	if (preeny_socket_threads_to_front[sockfd])
 	{
 		preeny_info("Emulating bind on port %d\n", ntohs(((struct sockaddr_in*)addr)->sin_port));
@@ -270,17 +268,26 @@ int bind(int sockfd, const struct sockaddr *addr, socklen_t addrlen)
 
 int listen(int sockfd, int backlog)
 {
+	if (original_listen == NULL)
+		original_listen = dlsym(RTLD_NEXT, "listen");
+
 	if (preeny_socket_threads_to_front[sockfd]) return 0;
 	else return original_listen(sockfd, backlog);
 }
 
 int connect(int sockfd, const struct sockaddr *addr, socklen_t addrlen)
 {
+	if (original_connect == NULL)
+		original_connect = dlsym(RTLD_NEXT, "connect");
+
 	if (preeny_socket_threads_to_front[sockfd]) return 0;
 	else return original_connect(sockfd, addr, addrlen);
 }
 
 int close(int fd) {
+	if (original_close == NULL)
+		original_close = dlsym(RTLD_NEXT, "close");
+
 	if (preeny_desock_accepted_sock != -1 && preeny_desock_accepted_sock == fd)
 		exit(0);
 
@@ -288,6 +295,9 @@ int close(int fd) {
 }
 
 int shutdown(int sockfd, int how) {
+	if (original_shutdown == NULL)
+		original_shutdown = dlsym(RTLD_NEXT, "shutdown");
+
 	if (preeny_desock_accepted_sock != -1 && preeny_desock_accepted_sock == sockfd)
 		exit(0);
 
@@ -295,6 +305,9 @@ int shutdown(int sockfd, int how) {
 }
 
 int getsockname(int sockfd, struct sockaddr *addr, socklen_t *addrlen) {
+	if (original_getsockname == NULL)
+		original_getsockname = dlsym(RTLD_NEXT, "getsockname");
+
 	struct sockaddr_in target;
 	socklen_t copylen = sizeof(target);
 
-- 
2.33.8

libvirtd.conf

listen_tls = 0
listen_tcp = 1
tcp_port = "16509"
listen_addr = "127.0.0.1"
unix_sock_group = "vmusers"
auth_tcp = "none"
min_workers = 1
max_workers = 1
prio_workers = 1
admin_min_workers = 1
admin_max_workers = 1

Install dependencies, build libvirt & preeny

Update system and install deps:

$ apt-get update
$ apt-get dist-upgrade -y
$ apt-get install -y git make lcov

# install libvirt deps
$ apt-get install -y xsltproc xml-utils libgnutls-devel libnl-devel \
    libdevmapper-devel libpciaccess-devel udev libudev-devel \
    libpciaccess-devel libyajl-devel libpcap-devel libselinux-devel dnsmasq \
    iptables radvd ebtables libsasl2-devel util-linux lvm2 libparted-devel \
    parted dmsetup libdevmapper-devel open-iscsi libiscsi-devel libnuma-devel \
    libcap-ng-devel libcurl-gnutls-compat libcurl-devel libaudit-devel \
    pm-utils bridge-utils libblkid-devel libgcrypt-devel libp11-kit-devel \
    libreadline-devel libtasn1-devel attr libxml2-devel xsltproc python3 \
    python3-dev iproute2 dmidecode libtirpc-devel kmod autoconf-archive \
    python3-module-flake8 gnutls-utils python3-module-docutils meson rpcgen \
    libgio-devel

# install preeny deps
$ apt-get install -y libini_config-devel libseccomp-devel

Install AFLplusplus and it's deps:

$ apt-get install -y AFLplusplus llvm15.0

Clone repos:

$ cd fuzz
$ git clone git://git.altlinux.org/gears/l/libvirt.git
$ git clone https://github.com/zardus/preeny.git

Build libvirt with afl++ instrumentation:

$ cd libvirt
$ git checkout 9.3.0-alt1

$ git apply /fuzz/virnetsocket.patch
$ git apply /fuzz/deferred_init.patch

$ export CC=afl-clang-lto \
       CXX=afl-clang-lto++ \
       CFLAGS="-O0 -Wframe-larger-than=30000" \
       CXXFLAGS="-O0 -Wframe-larger-than=30000" \
       AR=llvm-ar \
       RANLIB=llvm-ranlib \
       AS=llvm-as \
       GIT_SSL_NO_VERIFY=1

$ meson build -Dsystem=true -Dgit_werror=disabled
$ cd build
$ ninja

Build preeny:

$ cd /fuzz/preeny
$ git apply /fuzz/fix-sigsev-when-desock-and-capng.patch

$ export CC="gcc" \
       CFLAGS="-Werror=frame-larger-than=70000"
       
$ make -C src desock.so

Run fuzzing

Run AFLplusplus:

$ mkdir input
$ echo "testinput" > input/1

$ export INPUT=/fuzz/input \
       OUTPUT=/fuzz/out \
       LD_LIBRARY_PATH=/fuzz/libvirt/build/src/ \
       TARGET="/fuzz/libvirt/build/src/libvirtd -l -f /fuzz/libvirtd.conf" \
       AFL_PRELOAD="$(ls /fuzz/libvirt/build/src/*.so | tr -s '[:space:]' ' ') /fuzz/preeny/src/desock.so"

$ afl-fuzz -i $INPUT -o $OUTPUT -- $TARGET

Gather coverage

Rebuild libvirt:

$ git clone git://git.altlinux.org/gears/l/libvirt.git /fuzz/libvirt-cov
$ cd /fuzz/libvirt-cov
$ git fetch origin
$ git checkout 9.3.0-alt1

$ git apply /fuzz/virnetsocket.patch
$ export CC=gcc \
       CXX=g++ \
       CFLAGS="-Wno-error -O0 -Werror=frame-larger-than=30000" \
       CFLAGS="$CFLAGS -fprofile-arcs -ftest-coverage" \
       CXXFLAGS="-Wno-eqrror -O0 -Werror=frame-larger-than=30000" \
       CXXFLAGS="$CXXFLAGS -fprofile-arcs -ftest-coverage" \
       GIT_SSL_NO_VERIFY=1


$ meson build -Dsystem=true -Dgit_werror=disabled
$ cd build
$ ninja

Install afl-cov:

$ git clone https://github.com/vanhauser-thc/afl-cov.git /fuzz/afl-cov
$ cd /fuzz/afl-cov
$ make install

Gather coverage:

$ export OUT=/fuzz/out/ \
       TARGET="/fuzz/libvirt-cov/build/src/libvirtd -l -f /fuzz/libvirtd.conf" \
       SOURCE_CODE=/fuzz/libvirt-cov/build/src \
       LD_LIBRARY_PATH=/fuzz/libvirt-cov/build/src/ \
       LD_PRELOAD="$(ls /fuzz/libvirt-cov/build/src/*.so | tr -s '[:space:]' ' ') /fuzz/preeny/src/desock.so"

$ afl-cov --overwrite -v -d $OUT --coverage-cmd "$TARGET" \
    --code-dir $SOURCE_CODE