Hi everyone! I’m really happy to tell you about my experimenting adventure today. I decided to experiment with KCOV and see how I can hook it into libfuzzer and boot the kernel without spending too much on building a root file system.

First of all why not just using syzkaller? because why not? this may cover more State space.

after some googling I found a very interesting blog post by cloudflare

they have had answered my second question on how to boot newly built linux kernel with current root file system with virtme so basicall Virtme is a set of simple tools to run a virtualized Linux kernel that uses the host Linux distribution or a simple rootfs instead of a whole disk image. Virtme is tiny, easy to use, and makes testing kernel changes quite simple. I also borrowd some script from them.

So let’s get started: clone virtme and linux kernel

git clone --depth 1 https://github.com/torvalds/linux.git
git clone --depth 1 https://github.com/amluto/virtme.git
cd linux

you have to enable kcov for all targets with KCOV_INSTRUMENT_ALL or specific makefile. Enable KCOV in all “fs” subdirectory:

find "fs" -name Makefile \
    | xargs -L1 -I {} bash -c 'echo "KCOV_INSTRUMENT := y" >> {}'

then build linux kernel with KCOV and KASAN and some other flags needed by virtme

../virtme/virtme-configkernel  --defconfig
 
 ./scripts/config \
    -e KCOV \
    -d KCOV_INSTRUMENT_ALL \
    -e KCOV_ENABLE_COMPARISONS
   
   
./scripts/config \
    -e DEBUG_FS -e DEBUG_INFO \
    -e KALLSYMS -e KALLSYMS_ALL \
    -e NAMESPACES -e UTS_NS -e IPC_NS -e PID_NS -e NET_NS -e USER_NS \
    -e CGROUP_PIDS -e MEMCG -e CONFIGFS_FS -e SECURITYFS \
    -e KASAN -e KASAN_INLINE -e WARNING \
    -e FAULT_INJECTION -e FAULT_INJECTION_DEBUG_FS \
    -e FAILSLAB -e FAIL_PAGE_ALLOC \
    -e FAIL_MAKE_REQUEST -e FAIL_IO_TIMEOUT -e FAIL_FUTEX \
    -e LOCKDEP -e PROVE_LOCKING \
    -e DEBUG_ATOMIC_SLEEP \
    -e PROVE_RCU -e DEBUG_VM \
    -e REFCOUNT_FULL -e FORTIFY_SOURCE \
    -e HARDENED_USERCOPY -e LOCKUP_DETECTOR \
    -e SOFTLOCKUP_DETECTOR -e HARDLOCKUP_DETECTOR \
    -e BOOTPARAM_HARDLOCKUP_PANIC \
    -e DETECT_HUNG_TASK -e WQ_WATCHDOG \
    --set-val DEFAULT_HUNG_TASK_TIMEOUT 140 \
    --set-val RCU_CPU_STALL_TIMEOUT 100 \
    -e UBSAN \
    -d RANDOMIZE_BASE

in order to provied kenrnel code coverage to libfuzzer we can use __libfuzzer_extra_counters, you can see a good example in syzkaller and its documentation in kernel website

almost every kernel attack vector is Stateful APIs. you can’t just feed raw buffer to it. we have to use Structure-Aware Fuzzing, I deciede to use libprotobuf-mutator, which has show is very powerfull tool.

there are tons of resource out there about using libprotobuf-mutator. I can’t explain better then original google fuzzing doc

Protocol Buffers As Intermediate Format Protobufs provide a convenient way to serialize structured data, and LPM provides an easy way to mutate protobufs for structure-aware fuzzing. Thus, it is tempting to use libFuzzer+LPM for APIs that consume structured data other than protobufs.

but simply clone the repo and replace following code with this file

git clone https://github.com/google/libprotobuf-mutator.git

you can comment out other files in CMakeLists.txt because we want to modify .proto file.

#include <cmath>
#include <iostream>

#include "examples/libfuzzer/libfuzzer_example.pb.h"
#include "port/protobuf.h"
#include "src/libfuzzer/libfuzzer_macro.h"
#include <errno.h>
#include <fcntl.h>
#include <linux/bpf.h>
#include <memory.h>
#include <stdarg.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <sys/ioctl.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <sys/syscall.h>
#include <sys/types.h>
#include <unistd.h>
void fail(const char* msg, ...);
void kcov_start();
void kcov_stop();
#define KCOV_COVER_SIZE (256 << 10)
#define KCOV_TRACE_PC 0
#define KCOV_INIT_TRACE64 _IOR('c', 1, uint64_t)
#define KCOV_ENABLE _IO('c', 100)

__attribute__((section("__libfuzzer_extra_counters"))) unsigned char libfuzzer_coverage[32 << 10];
uint64_t* kcov_data;
int kcov;
extern "C" int LLVMFuzzerInitialize(int *argc, char ***argv) {
	
	kcov = open("/sys/kernel/debug/kcov", O_RDWR);
	if (kcov `-1)
		fail("open of /sys/kernel/debug/kcov failed");
	if (ioctl(kcov, KCOV_INIT_TRACE64, KCOV_COVER_SIZE))
		fail("init trace write failed");
	kcov_data = (uint64_t*)mmap(NULL, KCOV_COVER_SIZE * sizeof(kcov_data[0]),
				    PROT_READ | PROT_WRITE, MAP_SHARED, kcov, 0);
	if (kcov_data `MAP_FAILED)
		fail("mmap failed");
	if (ioctl(kcov, KCOV_ENABLE, KCOV_TRACE_PC))
		fail("enable write trace failed");
	//close(kcov); // where to close this?

 	return 0;
}
void kcov_start()
{
	__atomic_store_n(&kcov_data[0], 0, __ATOMIC_RELAXED);
 	 if (ioctl(kcov, KCOV_ENABLE, KCOV_TRACE_PC))
             perror("ioctl"), exit(1);
}
void kcov_stop()
{
	uint64_t ncov = __atomic_load_n(&kcov_data[0], __ATOMIC_RELAXED);
	if (ncov >= KCOV_COVER_SIZE)
		fail("too much cover: %llu", ncov);
	for (uint64_t i = 0; i < ncov; i++) {
		uint64_t pc = __atomic_load_n(&kcov_data[i + 1], __ATOMIC_RELAXED);
		libfuzzer_coverage[pc % sizeof(libfuzzer_coverage)]++;
	}
 	if (ioctl(kcov, KCOV_DISABLE, 0))
            perror("ioctl"), exit(1);
}

void fail(const char* msg, ...)
{
	int e = errno;
	va_list args;
	va_start(args, msg);
	vfprintf(stderr, msg, args);
	va_end(args);
	fprintf(stderr, " (errno %d)\n", e);
	_exit(1);
}

DEFINE_PROTO_FUZZER(const libfuzzer_example::Msg& message) {
protobuf_mutator::protobuf::FileDescriptorProto file;

        kcov_start();
        // your logic should be here:

        // std::cerr << message.DebugString() << "\n";	
        // Emulate a bug.
        //int fd = syscall(SYS_open, "example.txt", 4, message.sample_int());
        int fd = syscall(SYS_open, "example.txt", 4, 0x4141);
        syscall(SYS_close,fd);

        kcov_stop();
}

for test you can use SockFuzzer to fuzz network stack.

The intriguing phase starts. If a kasan panics, libfuzzer lacks awareness and will dispose of the sample. Therefore, to preserve the triggering sample of the crash, we must inform libfuzzer about the kernel panic.

Initially, I explored alternative methods to notify the fuzzer about the panic. However, I opted to simulate SIGSEGV and dispatch a signal to libfuzzer whenever a kasan panic occurs in the kernel. Upon receiving this signal, libfuzzer will preserve the sample and terminate. so we have to modify the linux kernel and build it once more. add send_sigsegv_to_process function to print_error_description in /mm/kasan/report.c.

make sure

  • kernel.panic_on_warn=0
  • kernel.panic_on_oops=0
#include <linux/sched/signal.h>
#include <linux/sched.h>
#include <asm/siginfo.h>

void send_sigsegv_to_process(void*  access_addr );
void send_sigsegv_to_process(void*  access_addr ) {

        kernel_siginfo_t info;
        memset(&info, 0, sizeof(kernel_siginfo_t));
        info.si_signo = SIGSEGV;  // Signal type
        info.si_pid = current->pid;  // Process ID to send the signal to
        info.si_code = SEGV_MAPERR;   // Signal code for a memory access error
        info.si_addr = access_addr;          // Address that caused the fault
        send_sig_info(SIGSEGV, &info, current);
}

static void print_error_description(struct kasan_report_info *info)
{

        send_sigsegv_to_process((void*)info->access_addr);

        pr_err("BUG: KASAN: %s in %pS\n", info->bug_type, (void *)info->ip);

        if (info->type != KASAN_REPORT_ACCESS) {
                pr_err("Free of addr %px by task %s/%d\n",
                        info->access_addr, current->comm, task_pid_nr(current));
                return;
        }

        if (info->access_size)
                pr_err("%s of size %zu at addr %px by task %s/%d\n",
                        info->is_write ? "Write" : "Read", info->access_size,
                        info->access_addr, current->comm, task_pid_nr(current));
        else
                pr_err("%s at addr %px by task %s/%d\n",
                        info->is_write ? "Write" : "Read",
                        info->access_addr, current->comm, task_pid_nr(current));
}

copy libprotobuf example binary to testfuzz. now you can boot the new kernel and run the fuzzer with

cd linux
 ../virtme/virtme-run --kimg arch/x86/boot/bzImage --rwdir ../testfuzz/ --qemu-opts  -m 2G -smp 2 -enable-kvm

The next step involves incorporating APIs and system calls into the proto file and ensuring they are included in the DEFINE_PROTO_FUZZER. This will enable the ability to fuzz additional subsystems within the Linux kernel.