Support Clang -shared-libsan

...instead of using the default -static-libsan, where dynamic libraries must be
built without -z defs and where all executables must link against the static
libsan.  Dropping the latter requirement means that we no longer need the static
sal hack in Library_jpipe.

But instead, at runtime, when a process loads any library that links against the
dynamic libsan, then that dynamic libsan must be loaded first into the process.
For executables that are themselves linked against the dynamic libsan, that is
already the case.  For scenarios where we invoke a Java executable, we now use
LD_PRELOAD to fulfil that requirement.  (And for invocation of Python
executables, this commit assumes a --enable-python=fully-internal configuration,
so that executable will be itself linked against the dynamic libsan.)

Dropping the requirement to build without -z defs revealed that using
-fsanitize=vptr often requires RTTI symbols from libraries that are not usually
linked against.  Such cases are now handled by conditionally including those
libraries in gb_*_use_libraries calls.  (This could be restricted even further
to only be done under -shared-libsan, but it arguably doesn't hurt to also do it
under the default -static-libsan.)

The expectation is that -shared-libsan will explicitly be passed into the
configuration along with any other -fsanitize... arguments via
--with-extra-cc-flags and --with-extra-cxx-flags.  This commit does not yet
remove the code that is only needed for the (default) -static-libsan caes, but
that code could eventually be cleaned up, once all (Clang) builds using
sanitizers use -shared-libsan.  (GCC already defaults to a shared model.
Instead of a combined -shared-libsan, it has individual -shared-libasan etc.,
though.  This commit does not intend to make any changes to any potential GCC
sanitizer builds; my assumption is that we do not have any such, and that they
wouldn't work out of the box, neither before nor after this change.)

Change-Id: I932e4b75c299c158211e122adf33d7988892a87e
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/200935
Tested-by: Jenkins
Reviewed-by: Stephan Bergmann <stephan.bergmann@collabora.com>

bin/verapdf.sh.in

@@ -1,2 +1,2 @@
 #!/usr/bin/env bash
-java -jar @TARFILE_LOCATION_NATIVE@/@VERAPDF_JAR@ --nonpdfext "$@"
+@JAVAINTERPRETER@ -jar @TARFILE_LOCATION_NATIVE@/@VERAPDF_JAR@ --nonpdfext "$@"

configure.ac

@@ -8837,8 +8837,8 @@ _ACEOF
         if ! $CXX $CXXFLAGS $CPPFLAGS $LINKFLAGSSHL -fPIC -fvisibility-inlines-hidden conftestlib1.cc -o libconftest1$DLLPOST >/dev/null 2>&5; then
             gccvisinlineshiddenok=no
         else
-            dnl At least Clang -fsanitize=address and -fsanitize=undefined are
-            dnl known to not work with -z defs (unsetting which makes the test
+            dnl At least Clang -fsanitize=address and -fsanitize=undefined (and no -shared-libsan)
+            dnl are known to not work with -z defs (unsetting which makes the test
             dnl moot, though):
             my_linkflagsnoundefs=$LINKFLAGSNOUNDEFS
             if test "$COM_IS_CLANG" = TRUE; then
@@ -8846,6 +8846,14 @@ _ACEOF
                     case $i in
                     -fsanitize=*)
                         my_linkflagsnoundefs=
+                        for i in $CXX $CXXFLAGS; do
+                            case $i in
+                            -shared-libsan)
+                                my_linkflagsnoundefs=$LINKFLAGSNOUNDEFS
+                                break
+                                ;;
+                            esac
+                        done
                         break
                         ;;
                     esac
@@ -9567,6 +9575,17 @@ if test "$ENABLE_JAVA" != "" -a "$cross_compiling" != "yes"; then
     JDK_SECURITYMANAGER_DISALLOWED_FOR_BUILD=$JDK_SECURITYMANAGER_DISALLOWED
 fi
 
+if test -n "$JAVAINTERPRETER"; then
+    for i in $CXX $CXXFLAGS; do
+        case $i in
+        -shared-libsan)
+            JAVAINTERPRETER="env LD_PRELOAD=libclang_rt.asan.so $JAVAINTERPRETER"
+            break
+            ;;
+        esac
+    done
+fi
+
 AC_SUBST(JAVACFLAGS)
 AC_SUBST(JAVACOMPILER)
 AC_SUBST(JAVAINTERPRETER)

desktop/CppunitTest_desktop_lib.mk

@@ -33,6 +33,7 @@ $(eval $(call gb_CppunitTest_use_libraries,desktop_lib, \
 	utl \
 	tl \
 	vcl \
+	$(if $(filter -fsanitize=vptr,$(gb_CXX)),svl) \
 ))
 
 $(eval $(call gb_CppunitTest_use_externals,desktop_lib, \

external/firebird/UnpackedTarball_firebird.mk

@@ -69,6 +69,7 @@ ifneq ($(filter -fsanitize=%,$(CC)),)
 $(eval $(call gb_UnpackedTarball_add_patches,firebird, \
     external/firebird/sanitizer.patch \
     external/firebird/sanitizer-rtti.patch \
+    $(if $(filter -shared-libsan,$(CC) $(LDFLAGS)),,external/firebird/sanitizer-undefs.patch) \
 ))
 endif
 

external/firebird/sanitizer-undefs.patch

@@ -0,0 +1,11 @@
+--- builds/posix/make.defaults
++++ builds/posix/make.defaults
+@@ -263,7 +263,7 @@
+ # LINKER OPTIONS
+ #
+ 
+-UNDEF_PLATFORM = -Wl,--no-undefined
++UNDEF_PLATFORM =
+ ifeq ($(TARGET),Debug)
+   UNDEF_FLAGS = $(UNDEF_PLATFORM)
+ endif

external/firebird/sanitizer.patch

@@ -21,15 +21,6 @@
 +_ZTI*
 --- builds/posix/make.defaults
 +++ builds/posix/make.defaults
-@@ -263,7 +263,7 @@
- # LINKER OPTIONS
- #
- 
--UNDEF_PLATFORM = -Wl,--no-undefined
-+UNDEF_PLATFORM =
- ifeq ($(TARGET),Debug)
-   UNDEF_FLAGS = $(UNDEF_PLATFORM)
- endif
 @@ -315,7 +315,7 @@
  LIB_LINK_MAPFILE= -Wl,--version-script,$(1)
  FIREBIRD_LIBRARY_LINK= -L$(LIB) -lfbclient $(MATHLIB)

external/nss/UnpackedTarball_nss.mk

@@ -45,10 +45,12 @@ $(eval $(call gb_UnpackedTarball_add_patches,nss,\
 
 ifeq ($(COM_IS_CLANG),TRUE)
 ifneq ($(filter -fsanitize=%,$(CC)),)
+ifeq ($(filter -shared-libsan,$(CC) $(LDFLAGS)),)
 $(eval $(call gb_UnpackedTarball_add_patches,nss,\
     external/nss/asan.patch.1 \
 ))
 endif
 endif
+endif
 
 # vim: set noet sw=4 ts=4:

jurt/Library_jpipe.mk

@@ -21,9 +21,11 @@ else
 
 ifeq ($(COM_IS_CLANG),TRUE)
 ifneq ($(filter -fsanitize=%,$(gb_CC)),)
+ifeq ($(filter -shared-libsan,$(CC) $(LDFLAGS)),)
 Library_jpipe__staticsalhack = TRUE
 endif
 endif
+endif
 
 ifeq ($(Library_jpipe__staticsalhack),)
 

sax/CppunitTest_sax_xmlimport.mk

@@ -25,6 +25,7 @@ $(eval $(call gb_CppunitTest_use_libraries,sax_xmlimport, \
     unotest \
     utl \
     salhelper \
+    $(if $(filter -fsanitize=vptr,$(gb_CXX)),tl) \
 ))
 
 $(eval $(call gb_CppunitTest_use_api,sax_xmlimport,\

sc/CppunitTest_sc_copypaste.mk

@@ -37,6 +37,7 @@ $(eval $(call gb_CppunitTest_use_libraries,sc_copypaste, \
     unotest \
     utl \
     vcl \
+    $(if $(filter -fsanitize=vptr,$(gb_CXX)),salhelper) \
 ))
 
 $(eval $(call gb_CppunitTest_set_include,sc_copypaste,\

sc/CppunitTest_sc_tiledrendering.mk

@@ -40,6 +40,7 @@ $(eval $(call gb_CppunitTest_use_libraries,sc_tiledrendering, \
     vcl \
     tl \
     utl \
+    $(if $(filter -fsanitize=vptr,$(gb_CXX)),salhelper) \
 ))
 
 $(eval $(call gb_CppunitTest_use_externals,sc_tiledrendering,\

sc/CppunitTest_sc_uicalc2.mk

@@ -40,6 +40,7 @@ $(eval $(call gb_CppunitTest_use_libraries,sc_uicalc2, \
     unotest \
     utl \
     vcl \
+    $(if $(filter -fsanitize=vptr,$(gb_CXX)),salhelper) \
 ))
 
 $(eval $(call gb_CppunitTest_set_include,sc_uicalc2,\

sc/Library_scqahelper.mk

@@ -64,6 +64,7 @@ $(eval $(call gb_Library_use_libraries,scqahelper,\
 	ucbhelper \
 	unotest \
 	vcl \
+	$(if $(filter -fsanitize=vptr,$(gb_CXX)),salhelper) \
 ))
 
 $(eval $(call gb_Library_add_exception_objects,scqahelper,\

sc/Library_scui.mk

@@ -68,6 +68,7 @@ $(eval $(call gb_Library_use_libraries,scui,\
 	tl \
 	utl \
 	vcl \
+	$(if $(filter -fsanitize=vptr,$(gb_CXX)),salhelper) \
 ))
 
 $(eval $(call gb_Library_add_exception_objects,scui,\

sc/Library_vbaobj.mk

@@ -54,6 +54,7 @@ $(eval $(call gb_Library_use_libraries,vbaobj,\
 	utl \
 	vbahelper \
 	vcl \
+	$(if $(filter -fsanitize=vptr,$(gb_CXX)),salhelper) \
 ))
 
 $(eval $(call gb_Library_add_exception_objects,vbaobj,\

shell/Library_kf5be.mk

@@ -20,6 +20,7 @@ $(eval $(call gb_Library_use_libraries,kf5be1,\
 	cppuhelper \
 	sal \
 	vcl \
+	$(if $(filter -fsanitize=vptr,$(gb_CXX)),comphelper) \
 ))
 
 $(eval $(call gb_Library_set_componentfile,kf5be1,shell/source/backends/kf5be/kf5be1,services))

solenv/gbuild/platform/linux.mk

@@ -10,9 +10,11 @@
 gb__LinkTarget_LDFLAGS_zdefs := -Wl,-z,defs
 ifeq ($(COM_IS_CLANG),TRUE)
 ifneq ($(filter -fsanitize=%,$(CC) $(LDFLAGS)),)
+ifeq ($(filter -shared-libsan,$(CC) $(LDFLAGS)),)
 gb__LinkTarget_LDFLAGS_zdefs :=
 endif
 endif
+endif
 gb_LinkTarget_LDFLAGS += $(gb__LinkTarget_LDFLAGS_zdefs)
 
 ifneq ($(findstring lld,$(USE_LD)),)

sw/CppunitTest_sw_core_text.mk

@@ -43,6 +43,7 @@ $(eval $(call gb_CppunitTest_use_libraries,sw_core_text, \
     unotest \
     utl \
     vcl \
+    $(if $(filter -fsanitize=vptr,$(gb_CXX)),svxcore) \
 ))
 
 $(eval $(call gb_CppunitTest_use_externals,sw_core_text,\

sw/CppunitTest_sw_rtfimport2.mk

@@ -33,6 +33,7 @@ $(eval $(call gb_CppunitTest_use_libraries,sw_rtfimport2, \
     vcl \
     tl \
     utl \
+    $(if $(filter -fsanitize=vptr,$(gb_CXX)),editeng) \
 ))
 
 $(eval $(call gb_CppunitTest_use_externals,sw_rtfimport2,\

sw/CppunitTest_sw_uibase_fldui.mk

@@ -34,6 +34,7 @@ $(eval $(call gb_CppunitTest_use_libraries,sw_uibase_fldui, \
     svt \
     tl \
     svl \
+    $(if $(filter -fsanitize=vptr,$(gb_CXX)),salhelper) \
 ))
 
 $(eval $(call gb_CppunitTest_use_externals,sw_uibase_fldui,\

sw/Library_msword.mk

@@ -68,6 +68,8 @@ $(eval $(call gb_Library_use_libraries,msword,\
     ucbhelper \
     utl \
     vcl \
+    $(if $(filter -fsanitize=vptr,$(gb_CXX)),i18npool) \
+    $(if $(filter -fsanitize=vptr,$(gb_CXX)),salhelper) \
 ))
 
 $(eval $(call gb_Library_use_externals,msword,\

unotest/Library_embindtest.mk

@@ -22,6 +22,7 @@ $(eval $(call gb_Library_use_libraries,embindtest, \
     salhelper \
     tl \
     vcl \
+    $(if $(filter -fsanitize=vptr,$(gb_CXX)),comphelper) \
 ))
 
 $(eval $(call gb_Library_use_sdk_api,embindtest))