Allow custom search path for libnl; try pkg-config when needed

This is helpful when libnl packages are installed in a location outside
the default search paths of the libraries.
Gentoo and NixOS need this feature to build.

Co-authored-by: Franz Pletz <[email protected]>
Signed-off-by: Kang-Che Sung <[email protected]>

Author: Explorer09

configure.ac

@@ -1033,6 +1033,29 @@ case "$enable_capabilities" in
 esac
 
 
+# $1: libnl-3 search path
+htop_try_link_libnl3 () {
+   htop_save_LDFLAGS=$LDFLAGS
+   htop_save_LIBS=$LIBS
+
+   LIBS="-lnl-3 $LIBS"
+   if test "x$1" != x; then
+      # New library path searched after what user has specified
+      LDFLAGS="$LDFLAGS -L$1"
+   fi
+   AC_LINK_IFELSE([AC_LANG_PROGRAM([[
+/* struct nl_sock* nl_socket_alloc(void); */
+void* nl_socket_alloc(void);
+      ]], [[
+void* sock = nl_socket_alloc();
+      ]])],
+      [htop_libnl3_link_succeed=yes],
+      [htop_libnl3_link_succeed=no])
+
+   LDFLAGS=$htop_save_LDFLAGS
+   LIBS=$htop_save_LIBS
+} # htop_try_link_libnl3
+
 AC_ARG_ENABLE([delayacct],
               [AS_HELP_STRING([--enable-delayacct],
                               [enable Linux delay accounting support; requires libnl-3 and libnl-genl-3 @<:@default=check@:>@])],
@@ -1061,19 +1084,41 @@ case "$enable_delayacct" in
          LIBNL3_CFLAGS="-I/usr/include/libnl3"
       ])
 
-      old_CFLAGS="$CFLAGS"
+      htop_save_CFLAGS=$CFLAGS
       # New include path searched after what user has specified
       CFLAGS="$CFLAGS $LIBNL3_CFLAGS"
       AC_CHECK_HEADERS([netlink/attr.h netlink/handlers.h netlink/msg.h],
          [],
          [if test "$enable_delayacct" = yes; then
             AC_MSG_ERROR([can not find required header files netlink/attr.h, netlink/handlers.h, netlink/msg.h])
          fi
-         enable_delayacct=no
-         break])
-      CFLAGS="$old_CFLAGS"
+         enable_delayacct=no])
+      CFLAGS=$htop_save_CFLAGS
 
       if test "$enable_delayacct" != no; then
+         AC_MSG_CHECKING([the search path of libnl-3])
+
+         htop_libnl3_link_succeed=no
+         htop_try_link_libnl3 "$LIBNL3_LIBDIR"
+         if test "$htop_libnl3_link_succeed$LIBNL3_LIBDIR" = no && test "x$PKG_CONFIG" != x; then
+            LIBNL3_LIBDIR=`$PKG_CONFIG --variable=libdir libnl-3.0 2>/dev/null`
+            if test "x$LIBNL3_LIBDIR" != x; then
+               htop_try_link_libnl3 "$LIBNL3_LIBDIR"
+            fi
+         fi
+
+         if test "x$LIBNL3_LIBDIR" = x; then
+            AC_MSG_RESULT([(default)])
+         else
+            # The path must end with a slash
+            LIBNL3_LIBDIR=`echo "x$LIBNL3_LIBDIR" | sed 's/^x//; s|/*$|/|'`
+            AC_MSG_RESULT([$LIBNL3_LIBDIR])
+         fi
+         AC_DEFINE_UNQUOTED([LIBNL3_LIBDIR], ["$LIBNL3_LIBDIR"], [libnl-3 search path; use the default search paths if empty])
+         if test "$htop_libnl3_link_succeed" = no; then
+            AC_MSG_WARN([libnl-3 binary currently not present; will be needed in htop runtime for delay accounting support])
+         fi
+
          enable_delayacct=yes
       fi
       ;;

linux/LibNl.c

@@ -78,17 +78,17 @@ static int load_libnl(void) {
    if (libnlHandle && libnlGenlHandle)
       return 0;
 
-   libnlHandle = dlopen("libnl-3.so", RTLD_LAZY);
+   libnlHandle = dlopen(LIBNL3_LIBDIR "libnl-3.so", RTLD_LAZY);
    if (!libnlHandle) {
-      libnlHandle = dlopen("libnl-3.so.200", RTLD_LAZY);
+      libnlHandle = dlopen(LIBNL3_LIBDIR "libnl-3.so.200", RTLD_LAZY);
       if (!libnlHandle) {
          goto dlfailure;
       }
    }
 
-   libnlGenlHandle = dlopen("libnl-genl-3.so", RTLD_LAZY);
+   libnlGenlHandle = dlopen(LIBNL3_LIBDIR "libnl-genl-3.so", RTLD_LAZY);
    if (!libnlGenlHandle) {
-      libnlGenlHandle = dlopen("libnl-genl-3.so.200", RTLD_LAZY);
+      libnlGenlHandle = dlopen(LIBNL3_LIBDIR "libnl-genl-3.so.200", RTLD_LAZY);
       if (!libnlGenlHandle) {
          goto dlfailure;
       }