linux - 编译时找不到外部头文件

Question

我正在尝试在树外编译 bpf 样本。这是我的文件夹:

.
├── bpf_load.c
├── bpf_load.h
├── bpf_load.o
├── libbpf.h
├── Makefile
├── xdp1
├── xdp1_kern.c
├── xdp1_kern.o
├── xdp1_user.c
├── xdp2_kern.c
└── xdp2_user.c

这是生成文件:

#
# Makefile for out-of-tree building eBPF programs
#  similar to kernel/samples/bpf/
#
# Still depend on a kernel source tree.
#
TARGETS = xdp1

TOOLS_PATH = /usr/src/kernels/$(shell uname -r)/tools

TARGETS_ALL = $(TARGETS)

# Generate file name-scheme based on TARGETS
KERN_SOURCES = ${TARGETS_ALL:=_kern.c}
USER_SOURCES = ${TARGETS_ALL:=_user.c}
KERN_OBJECTS = ${KERN_SOURCES:.c=.o}
USER_OBJECTS = ${USER_SOURCES:.c=.o}

# Notice: the kbuilddir can be redefined on make cmdline
kbuilddir ?= /lib/modules/`uname -r`/build/
KERNEL=$(kbuilddir)

CFLAGS := -g -O2 -Wall

# Local copy of include/linux/bpf.h kept under ./kernel-usr-include
#
CFLAGS += /usr/include/linux/bpf.h
#
# Interacting with libbpf
CFLAGS += -I$(TOOLS_PATH)/lib
CFLAGS += -I$(TOOLS_PATH)/testing/selftests/bpf

LDFLAGS= -lelf

# Objects that xxx_user program is linked with:
OBJECT_LOADBPF = bpf_load.o
OBJECTS = $(OBJECT_LOADBPF)
#
# The static libbpf library
LIBBPF = $(TOOLS_PATH)/lib/bpf/libbpf.a

# Allows pointing LLC/CLANG to another LLVM backend, redefine on cmdline:
#  make LLC=~/git/llvm/build/bin/llc CLANG=~/git/llvm/build/bin/clang
LLC ?= llc
CLANG ?= clang

CC = gcc

NOSTDINC_FLAGS := -nostdinc -isystem $(shell $(CC) -print-file-name=include)

# TODO: can we remove(?) copy of uapi/linux/bpf.h stored here: ./tools/include/
# LINUXINCLUDE := -I./tools/include/
#
# bpf_helper.h need newer version of uapi/linux/bpf.h
# (as this git-repo use new devel kernel features)
KERNEL_PATH = /usr/src/kernels/$(shell uname -r)
LINUXINCLUDE := -I$(KERNEL_PATH)/include
#
LINUXINCLUDE += -I$(KERNEL_PATH)/arch/x86/include
LINUXINCLUDE += -I$(KERNEL_PATH)/arch/x86/include/generated/uapi
LINUXINCLUDE += -I$(KERNEL_PATH)/arch/x86/include/generated
LINUXINCLUDE += -I$(KERNEL_PATH)/include
LINUXINCLUDE += -I$(KERNEL_PATH)/arch/x86/include/uapi
LINUXINCLUDE += -I$(KERNEL_PATH)/include/uapi
LINUXINCLUDE += -I$(KERNEL_PATH)/include/generated/uapi
LINUXINCLUDE += -include $(KERNEL_PATH)/include/linux/kconfig.h
#LINUXINCLUDE += -I$(KERNEL)/tools/lib
EXTRA_CFLAGS=-Werror

all: dependencies $(TARGETS_ALL) $(KERN_OBJECTS)

.PHONY: dependencies clean verify_cmds verify_llvm_target_bpf $(CLANG) $(LLC)

# Most xxx_user program still depend on old bpf_load.c
$(OBJECT_LOADBPF): bpf_load.c bpf_load.h
    $(CC) $(CFLAGS) -o $@ -c $<

LIBBPF_SOURCES  = $(TOOLS_PATH)/lib/bpf/*.c

# New ELF-loaded avail in libbpf (in bpf/libbpf.c)
$(LIBBPF): $(LIBBPF_SOURCES) $(TOOLS_PATH)/lib/bpf/Makefile
    make -C $(TOOLS_PATH)/lib/bpf/ all

# Compiling of eBPF restricted-C code with LLVM
#  clang option -S generated output file with suffix .ll
#   which is the non-binary LLVM assembly language format
#   (normally LLVM bitcode format .bc is generated)
#
# Use -Wno-address-of-packed-member as eBPF verifier enforces
# unaligned access checks where necessary
#
$(KERN_OBJECTS): %.o: %.c Makefile
    $(CLANG) -S $(NOSTDINC_FLAGS) $(LINUXINCLUDE) $(EXTRA_CFLAGS) \
        -D__KERNEL__ -D__ASM_SYSREG_H \
        -D__BPF_TRACING__ \
        -Wall \
        -Wno-unused-value -Wno-pointer-sign \
        -D__TARGET_ARCH_$(ARCH) \
        -Wno-compare-distinct-pointer-types \
        -Wno-gnu-variable-sized-type-not-at-end \
        -Wno-tautological-compare \
        -Wno-unknown-warning-option \
        -Wno-address-of-packed-member \
        -O2 -emit-llvm -c $< -o ${@:.o=.ll}
    $(LLC) -march=bpf -filetype=obj -o $@ ${@:.o=.ll}

$(TARGETS): %: xdp1_user.c $(OBJECTS) $(LIBBPF) Makefile
    $(CC) $(CFLAGS) $(OBJECTS) $(LDFLAGS) -o $@ $<  $(LIBBPF)

我几乎只是复制了我在这里找到的 makfile: https://github.com/netoptimizer/prototype-kernel/blob/master/kernel/samples/bpf/Makefile并删除了一堆我不需要的东西，并将其更改为更加动态并使用“uname -r”计算不同的路径。问题在于原始 Makefile 假定 bfp_helpers.h 与文件位于同一目录中。但是我的 xdp1_kern.c 使用它，我不能将它放在同一目录中。我添加了 -I(bpf_helpers.h 的路径)，但是当我运行它时它仍然会抛出错误。

 make LLC=<path to llc> CLANG=<path to clang>
gcc -g -O2 -Wall /usr/include/linux/bpf.h -I/usr/src/kernels/4.18.0-mlnx/tools/lib -I/usr/src/kernels/4.18.0-mlnx/tools/testing/selftests/bpf bpf_load.o -lelf -o xdp1 xdp1_user.c  /usr/src/kernels/4.18.0-mlnx/tools/lib/bpf/libbpf.a
/.autodirect/net_linux_verification/tools/clang+llvm-3.8.0-linux-x86_64-centos6/bin/clang -S -nostdinc -isystem /usr/lib/gcc/x86_64-redhat-linux/7/include -I/usr/src/kernels/4.18.0-mlnx/include -I/usr/src/kernels/4.18.0-mlnx/arch/x86/include -I/usr/src/kernels/4.18.0-mlnx/arch/x86/include/generated/uapi -I/usr/src/kernels/4.18.0-mlnx/arch/x86/include/generated -I/usr/src/kernels/4.18.0-mlnx/include -I/usr/src/kernels/4.18.0-mlnx/arch/x86/include/uapi -I/usr/src/kernels/4.18.0-mlnx/include/uapi -I/usr/src/kernels/4.18.0-mlnx/include/generated/uapi -include /usr/src/kernels/4.18.0-mlnx/include/linux/kconfig.h -Werror \
    -D__KERNEL__ -D__ASM_SYSREG_H \
    -D__BPF_TRACING__ \
    -Wall \
    -Wno-unused-value -Wno-pointer-sign \
    -D__TARGET_ARCH_ \
    -Wno-compare-distinct-pointer-types \
    -Wno-gnu-variable-sized-type-not-at-end \
    -Wno-tautological-compare \
    -Wno-unknown-warning-option \
    -Wno-address-of-packed-member \
    -O2 -emit-llvm -c xdp1_kern.c -o xdp1_kern.ll
xdp1_kern.c:15:10: fatal error: 'bpf_helpers.h' file not found
#include "bpf_helpers.h"
         ^
1 error generated.
make: *** [Makefile:93: xdp1_kern.o] Error 1

我确定添加 -I/usr/src/kernels/4.18.0-mlnx/tools/testing/selftests/bpf 会解决它，因为 bpf_helpers.h 在我的机器上。但它没有，它只解决了 bpf_utils.h 的包含问题。

Answer 1

在尝试编译 BPF 程序时，不要使用修改过的 CFLAGS。

在编译用户空间程序时确实使用了CFLAGS:

# Most xxx_user program still depend on old bpf_load.c
$(OBJECT_LOADBPF): bpf_load.c bpf_load.h
    $(CC) $(CFLAGS) -o $@ -c $<

和

$(TARGETS): %: xdp1_user.c $(OBJECTS) $(LIBBPF) Makefile
    $(CC) $(CFLAGS) $(OBJECTS) $(LDFLAGS) -o $@ $<  $(LIBBPF)

后者为您提供了前两行输出，其中确实出现了您要搜索的目录:

 make LLC=<path to llc> CLANG=<path to clang>
gcc -g -O2 -Wall [...] -I/usr/src/kernels/4.18.0-mlnx/tools/testing/selftests/bpf [...]

然而，从下一行开始，这是一个不同的命令，调用 clang 来编译你的 BPF 程序。它来自 Makefile 代码:

$(KERN_OBJECTS): %.o: %.c Makefile
    $(CLANG) -S $(NOSTDINC_FLAGS) $(LINUXINCLUDE) $(EXTRA_CFLAGS) \
        [...]

... 不使用 $(CFLAGS)。只需将它添加到 Makefile 中的 clang 命令，这应该可以解决您的问题。