Re: [Tarantool-patches] [PATCH] cmake: add LTO support for building luajit

[Olga Arkhangelskaia <arkholga@tarantool.org>]

Hello Sergey!

Thanks for review and testing.

However,  -DENABLE_LTO=ON is not set by default, yu have to set it.

I followed your way:

  cmake . -DCMAKE_LINKER=/usr/bin/ld.gold -DENABLE_LTO=ON
-- ld version string: GNU ld version 2.30-58.el8_1.2
-- Found ld.bfd version: 2.30
CMake Error at cmake/lto.cmake:79 (message):
   ld.bfd >= 2.31 is needed for LTO
Call Stack (most recent call first):
   CMakeLists.txt:68 (include)

As far as I understand there gold/bfd is the frontend to binutils, so it 
is impossible om msc.

I did the same at the debian docker container:

Generating third_party/luajit/src/libluajit.a
cd /tarantool/third_party/luajit && make BUILDMODE=static 
HOST_CC="/usr/bin/cc" TARGET_CC="/usr/bin/cc" 
TARGET_CFLAGS="-fexceptions -funwind-tables -fno-omit-frame-pointer 
-fno-stack-protector -fno-common -fopenmp -msse2 
-Wno-parentheses-equality -Wno-tautological-compare 
-Wno-misleading-indentation -Wno-varargs -Wno-implicit-fallthrough -flto 
-fuse-linker-plugin -fno-fat-lto-objects" TARGET_LD="/usr/bin/ld" 
TARGET_LDFLAGS=" -Wno-lto-type-mismatch" TARGET_AR="gcc-ar rcus" 
TARGET_STRIP="/usr/bin/strip" TARGET_SYS="Linux" CCOPT="-O2" CCDEBUG="" 
XCFLAGS="-D_GNU_SOURCE -D_FILE_OFFSET_BITS=64 -DNDEBUG -DNVALGRIND 
-DNVALGRIND=1 -DLUAJIT_ENABLE_PAIRSMM=1 -DLUAJIT_SMART_STRINGS=1" Q='' 
MACOSX_DEPLOYMENT_TARGET="" clean

So the -flto is passet to luajit.

I will check formatting once again.

25.05.2020 15:58, Sergey Bronnikov пишет:
> Olya,
>
> I have reviewed patch in a branch and below my comments.
>
> formatting issues still exist:
>
> +        if (CMAKE_COMPILER_IS_CLANG)
> +            if (${CMAKE_CXX_COMPILER_VERSION} VERSION_LESS 3.9)
> +                set (luajit_cflags ${luajit_cflags} -flto=full)
> +            else()
> +                # ThinLTO that is both scalable and incremental
> +                # due to parallel IPO, available since 3.9 and above.
> +                # See http://blog.llvm.org/2016/06/thinlto-scalable-and-incremental-lto.html
> +                set (luajit_cflags ${luajit_cflags} -flto=thin)
> +            endif()
> +                # XCode linker dynamically load libLTO.dylib to perform
> +                # link-time optimization.
> +                # http://lists.llvm.org/pipermail/llvm-dev/2009-November/027103.html
> ^^^^^^^^^^^^^^^^^^^^^^^^
>
> +            if (NOT TARGET_OS_DARWIN)
> +                # llvm-ar is just a wrapper over ar to pass --plugin= option,
> +                # so that ar can understand LTO objects.
> +                # See https://lists.llvm.org/pipermail/llvm-dev/2018-March/122018.html
> +                set (CMAKE_AR llvm-ar)
> +            endif()
> +        else()
> +        # GNU opts to support lto
> ^^^^^^^^^^^^^^^^^^^^^^
>
> +            # Due to some problems (bugs, slow work, etc) we support LTO
> +            # only for 5.0+. The same is for binutils prior to 2.27
> +            # See comments in scripts/Makefile.lto in scope of
>
>
> +            # the patch: https://patchwork.kernel.org/patch/10078207/
> +            if (NOT ${CMAKE_CXX_COMPILER_VERSION} VERSION_LESS 5.0
> +                AND NOT ${linker_version} VERSION_LESS 2.27)
> +                set (luajit_cflags ${luajit_cflags} -flto -fuse-linker-plugin -fno-fat-lto-objects)
> +             endif()
> ^^^^^^^^^^^^^^^^^^^^^^
> +            # gcc-ar is just a wrapper over ar to pass --plugin option
>
> Unfortunatlely I have no minimal required version of binutils on my system
> and I tested patch with ld gold (cmake -DCMAKE_LINKER=/usr/bin/ld.gold ..),
> as Timur Safin advised.
>
> How I tested:
> $ mkdir build; cd build
> $ make build_bundled_libs VERBOSE=1 2>&1 | tee log.txt
>
> with commands above output in log.txt doesn't contain LTO related options
> in compilation lines for luajit.
>
> gcc 8.3.1, clang 8.0.1, CentOS 8.1.1911
>
> Sergey
>
> On 17:33 Tue 14 Apr , Olga Arkhangelskaia wrote:
>> Fixed tabs:
>>
>>   +            # Depending of compiler we need to set appropriate llvm/gcc-ar
>> lib
>>   +            # llvm/gcc -ar is just a wrapper over ar to pass --plugin=
>> option,
>>   +            # so that ar can understand LTO objects.
>>
>> 14.04.2020 12:18, Igor Munkin пишет:
>>> Olya,
>>>
>>> Thanks for the patch! I left several comments below, please consider
>>> them.
>>>
>>> On 12.03.20, Olga Arkhangelskaia wrote:
>>>> Tarantool has LTO support, however while building luajit this opt. was
>>>> omitted. Patch adds necessary flag to turn it on.
>>>>
>>>> Minimum compiler/linker versions: clang 3.4, gcc 5.0+, ld 2.27+.
>>>>
>>>> Closes #3743
>>>> ---
>>>> Branch: OKriw/gh-3743-LuaJIT-does-not-use-LTO-with-DENABLE_LTO=ON-full-ci
>>>>    cmake/lto.cmake    |  3 ++-
>>>>    cmake/luajit.cmake | 26 ++++++++++++++++++++++++++
>>>>    2 files changed, 28 insertions(+), 1 deletion(-)
>>>>
>>>> diff --git a/cmake/lto.cmake b/cmake/lto.cmake
>>>> index 95ade75f4..9f29f3324 100644
>>>> --- a/cmake/lto.cmake
>>>> +++ b/cmake/lto.cmake
>>>> @@ -78,7 +78,8 @@ if (NOT TARGET_OS_DARWIN)
>>>>            if (linker_version VERSION_LESS "2.31")
>>>>                message(FATAL_ERROR "ld.bfd >= 2.31 is needed for LTO")
>>>>            endif()
>>>> -    elseif(matched_gold)
>>>> +
>>>> +	elseif(matched_gold)
>>> Typo: there is a mess with whitespace above.
>>>
>>> However this chunk provides no changes except whitespace and newline.
>>> Please drop it.
>>>
>>>>            set(linker_version ${CMAKE_MATCH_1})
>>>>            message(STATUS "Found ld.gold version: ${linker_version}")
>>>> diff --git a/cmake/luajit.cmake b/cmake/luajit.cmake
>>>> index 072db8269..da1b4926d 100644
>>>> --- a/cmake/luajit.cmake
>>>> +++ b/cmake/luajit.cmake
>>>> @@ -225,6 +225,32 @@ macro(luajit_build)
>>>>            set(luajit_xcflags ${luajit_xcflags} -D${def})
>>>>        endforeach()
>>>> +    # Add LTO option to luajit
>>>> +    if (CMAKE_INTERPROCEDURAL_OPTIMIZATION)
>>>> +	    message("Setting LTO flags for building luajit")
>>>> +	# Clang opt to support LTO
>>>> +	if (CMAKE_COMPILER_IS_CLANG AND
>>>> +	    NOT ${CMAKE_CXX_COMPILER_VERSION} VERSION_LESS 3.4)
>>> So, if clang toolchain is used but version is less than 3.4, then gcc
>>> toolchain settings are used. It's either totally wrong or definitely
>>> need to be clarified with a corresponding comment.
>>>
>>> Also, AFAICS LTO support for toolchain is checked via CheckIPOSupported
>>> CMake module. So additional check here looks to be excess or also need
>>> to be clarified.
>>>
>>>> +	    if (${CMAKE_CXX_COMPILER_VERSION} VERSION_LESS 3.9)
>>>> +		set(luajit_cflags ${luajit_cflags} -flto=full)
>>>> +	    else()
>>>> +	        set(luajit_cflags ${luajit_cflags} -flto=full)
>>>> +	    endif()
>>> I see no difference in the branches above.
>>>
>>>> +	if (NOT TARGET_OS_DARWIN)
>>>> +	   # Depending of compiler we need to set appropriate llvm/gcc-ar lib
>>>> +	   set (CMAKE_AR llvm-ar)
>>> What ar is used on systems different from Darwin based ones when clang
>>> toolchain is used?
>>>
>>>> +	endif()
>>>> +	# GNU opts to support lto
>>>> +	else()
>>>> +	   #Due to some problems (bugs, slow work, etc) we support LTO only for 5.0+
>>>> +	   #The same is for binutils prior 2.27
>>>> +	   if (NOT ${CMAKE_CXX_COMPILER_VERSION} VERSION_LESS 5.0
>>>> +	       AND NOT ${linker_version} VERSION_LESS 2.27)
>>>> +	       set(luajit_cflags ${luajit_cflags} -flto -fuse-linker-plugin -fno-fat-lto-objects)
>>>> +	   endif()
>>>> +	   set (CMAKE_AR gcc-ar)
>>> Could you please clarify why ar is set explicitly here?
>>>
>>>> +        endif()
>>>> +    endif()
>>> Typo: there is a mess with indent above. Unfortunately I failed to find
>>> our CMake style guide, so please adjust your changes considering the
>>> code nearby.
>>>
>>>>        # Pass the same toolchain that is used for building of
>>>>        # tarantool itself, because tools from different toolchains
>>>>        # can be incompatible. A compiler and a linker are already set
>>>> -- 
>>>> 2.20.1 (Apple Git-117)
>>>>