Building an LLVM-based tool Lessons Learned EuroLLVM 2019, Brussels - - PowerPoint PPT Presentation

▶
building an llvm based tool
SMART_READER_LITE
LIVE PREVIEW

Building an LLVM-based tool Lessons Learned EuroLLVM 2019, Brussels - - PowerPoint PPT Presentation

Feb 29, 2024 276 likes •874 views

Building an LLVM-based tool Lessons Learned EuroLLVM 2019, Brussels whoami Alex Denisov Software Engineer at PTScientists GmbH LLVM Hacker https://lowlevelbits.org https://twitter.com/1101_debian Agenda Build System

slide-1
SLIDE 1

Building an LLVM-based tool

Lessons Learned EuroLLVM 2019, Brussels

slide-2
SLIDE 2

whoami

  • Alex Denisov
  • Software Engineer at PTScientists GmbH
  • LLVM Hacker
  • https://lowlevelbits.org
  • https://twitter.com/1101_debian
slide-3
SLIDE 3

Agenda

  • Build System
  • Memory Management
  • Parallelization
  • Multi Version Support
  • Multi OS Support
  • Bitcode Extraction
  • And more
slide-4
SLIDE 4

Mull

  • https://github.com/mull-project/mull
  • Mutation Testing: Leaving the Stone Age. FOSDEM 2017


https://www.youtube.com/watch?v=YEgiyiICkpQ

  • Mull it over: mutation testing based on LLVM


https://ieeexplore.ieee.org/document/8411727

  • Works on Linux, macOS, FreeBSD
  • Works with LLVM 3.9 - 8.0
slide-5
SLIDE 5

Disclaimer

I am an Expert => Blindly Follow My Advice!

slide-6
SLIDE 6

Disclaimer

I am an Expert => Blindly Follow My Advice!

YMMV

slide-7
SLIDE 7

An LLVM-based tool

  • Works with LLVM Bitcode
  • Load
  • Analyze
  • Transform
  • Process and report results
slide-8
SLIDE 8

llvm-config

> clang -c `llvm-config --cxxflags` \ foo.cpp -o foo.o > clang -c `llvm-config --cxxflags` \ bar.cpp -o bar.o > clang `llvm-config --ldflags` \ `llvm-config --libs core support` \ bar.o foo.o -o foobar.bin

slide-9
SLIDE 9

llvm-config

> llvm-config --cxxflags

  • I/opt/llvm/6.0.0/include

...

  • Werror=unguarded-availability-new
  • O3 -DNDEBUG

...

slide-10
SLIDE 10

llvm-config

> llvm-config --cxxflags

  • I/opt/llvm/6.0.0/include

...

  • Werror=unguarded-availability-new
  • O3 -DNDEBUG

...

slide-11
SLIDE 11

llvm-config

> llvm-config --libs core

  • lLLVMCore
  • lLLVMBinaryFormat
  • lLLVMSupport
  • lLLVMDemangle
slide-12
SLIDE 12

llvm-config

/usr/lib/llvm-4.0/lib/libLLVM.dylib /usr/lib/llvm-6.0/lib/libLLVM.dylib

slide-13
SLIDE 13

llvm-config

/usr/lib/llvm-4.0/lib/libLLVM.dylib /usr/lib/llvm-6.0/lib/libLLVM.dylib > clang foo.o bar.o -lLLVMSupport -o foobar.bin > ./foobar.bin LLVM ERROR: inconsistency in registered CommandLine

  • ptions
slide-14
SLIDE 14

llvm-config

/usr/lib/llvm-4.0/lib/libLLVM.dylib /usr/lib/llvm-6.0/lib/libLLVM.dylib > clang foo.o bar.o -lLLVM -o foobar.bin > ./foobar.bin All good.

slide-15
SLIDE 15

llvm-config

if macOS LDFLAGS=-lLLVM -lclangEdit else LDFLAGS=-Wl,--start-group -lLLVM -lclangEdit -Wl,--end-group endif clang foo.o bar.o $LDFLAGS -o foobar.bin

slide-16
SLIDE 16

CMake

add_executable(foo foo.cpp bar.cpp ) target_include_directories(foo /usr/include /usr/local/include ) target_link_libraries(foo m sqlite3 ncurses )

https://llvm.org/docs/CMakePrimer.html

slide-17
SLIDE 17

CMake

set (search_paths ${PATH_TO_LLVM} ${PATH_TO_LLVM}/lib/cmake ${PATH_TO_LLVM}/lib/cmake/llvm ${PATH_TO_LLVM}/lib/cmake/clang ${PATH_TO_LLVM}/share/clang/cmake/ ${PATH_TO_LLVM}/share/llvm/cmake/ ) find_package(LLVM REQUIRED CONFIG PATHS ${search_paths} NO_DEFAULT_PATH)

slide-18
SLIDE 18

CMake

find_package(LLVM REQUIRED CONFIG PATHS ${search_paths} NO_DEFAULT_PATH) find_package(Clang REQUIRED CONFIG PATHS ${search_paths} NO_DEFAULT_PATH)

slide-19
SLIDE 19

CMake

target_include_directories(mull PUBLIC ${LLVM_INCLUDE_DIRS}) target_link_libraries(mull LLVMSupport clangTooling)

slide-20
SLIDE 20

CMake

target_include_directories(mull PUBLIC ${LLVM_INCLUDE_DIRS}) target_link_libraries(mull LLVMSupport clangTooling) LLVM ERROR: inconsistency in registered CommandLine options

slide-21
SLIDE 21

CMake

target_include_directories(mull PUBLIC ${LLVM_INCLUDE_DIRS}) if (LLVM IN_LIST LLVM_AVAILABLE_LIBS) target_link_libraries(mull LLVM clangTooling) else() target_link_libraries(mull LLVMSupport clangTooling) endif()

slide-22
SLIDE 22

Multiple LLVM Versions

#if LLVM_VERSION_CODE >= LLVM_VERSION(4, 0) #include <llvm/Bitcode/BitcodeReader.h> #else #include <llvm/Bitcode/ReaderWriter.h> #endif #if LLVM_VERSION_CODE >= LLVM_VERSION(5, 0) assert(ii->getNumOperands() == 3 && "wrong number of arguments"); #else assert(ii->getNumOperands() == 2 && "wrong number of arguments"); #endif

http://github.com/klee/klee

slide-23
SLIDE 23

Multiple LLVM Versions

LLVMCompatibility/ 3.9.x CMakeLists.txt LLVMCompatibility.cpp LLVMCompatibility.h 4.x.x CMakeLists.txt LLVMCompatibility.cpp LLVMCompatibility.h 5.x.x 6.x.x 7.x.x 8.x.x

slide-24
SLIDE 24

Multiple LLVM Versions

if (EXISTS LLVMCompatibility/${LLVM_VERSION}) add_subdirectory(LLVMCompatibility/${LLVM_VERSION}) else() message(FATAL_ERROR "LLVM-${LLVM_VERSION} is not supported”) endif()

slide-25
SLIDE 25

Multiple LLVM Versions

#include <llvm/ExecutionEngine/Orc/CompileUtils.h> #include <llvm/ExecutionEngine/Orc/ExecutionUtils.h> #include <llvm/ExecutionEngine/RuntimeDyld.h> namespace llvm_compat { using namespace llvm; typedef LegacyJITSymbolResolver SymbolResolver; typedef JITSymbol JITSymbolInfo; typedef JITSymbol JITSymbol;

  • bject::OwningBinary<object::ObjectFile>

compileModule(orc::SimpleCompiler &compiler, Module &module); std::unique_ptr<Module> parseBitcode(MemoryBufferRef bufferRef, LLVMContext &context); } #include <llvm/ExecutionEngine/Orc/CompileUtils.h> #include <llvm/ExecutionEngine/Orc/ExecutionUtils.h> #include <llvm/ExecutionEngine/Orc/JITSymbol.h> #include <llvm/ExecutionEngine/RuntimeDyld.h> namespace llvm_compat { using namespace llvm; typedef RuntimeDyld::SymbolResolver SymbolResolver; typedef RuntimeDyld::SymbolInfo JITSymbolInfo; typedef orc::JITSymbol JITSymbol;

  • bject::OwningBinary<object::ObjectFile>

compileModule(orc::SimpleCompiler &compiler, Module &module); std::unique_ptr<Module> parseBitcode(MemoryBufferRef bufferRef, LLVMContext &context); }

LLVM 3.9 LLVM 8.0

slide-26
SLIDE 26

Multiple LLVM Versions

#include <llvm/ExecutionEngine/Orc/CompileUtils.h> #include <llvm/ExecutionEngine/Orc/ExecutionUtils.h> #include <llvm/ExecutionEngine/RuntimeDyld.h> namespace llvm_compat { using namespace llvm; typedef LegacyJITSymbolResolver SymbolResolver; typedef JITSymbol JITSymbolInfo; typedef JITSymbol JITSymbol;

  • bject::OwningBinary<object::ObjectFile>

compileModule(orc::SimpleCompiler &compiler, Module &module); std::unique_ptr<Module> parseBitcode(MemoryBufferRef bufferRef, LLVMContext &context); } #include <llvm/ExecutionEngine/Orc/CompileUtils.h> #include <llvm/ExecutionEngine/Orc/ExecutionUtils.h> #include <llvm/ExecutionEngine/Orc/JITSymbol.h> #include <llvm/ExecutionEngine/RuntimeDyld.h> namespace llvm_compat { using namespace llvm; typedef RuntimeDyld::SymbolResolver SymbolResolver; typedef RuntimeDyld::SymbolInfo JITSymbolInfo; typedef orc::JITSymbol JITSymbol;

  • bject::OwningBinary<object::ObjectFile>

compileModule(orc::SimpleCompiler &compiler, Module &module); std::unique_ptr<Module> parseBitcode(MemoryBufferRef bufferRef, LLVMContext &context); }

LLVM 3.9 LLVM 8.0

slide-27
SLIDE 27

Multiple LLVM Versions

Symbol Resolver Memory Manager Dynamic Linker Etc.

MCJIT ORC JIT

slide-28
SLIDE 28

Multiple LLVM Versions

Symbol Resolver Memory Manager Dynamic Linker Etc.

MCJIT ORC JIT Native JIT

slide-29
SLIDE 29

Multiple LLVM Versions

Function * CloneFunction(Function *F, ValueToValueMapTy &VMap, ClonedCodeInfo *CodeInfo);

LLVM 3.8 LLVM 3.9+

Function * CloneFunction(const Function *F, ValueToValueMapTy &VMap, bool ModuleLevelChanges, ClonedCodeInfo *CodeInfo);

slide-30
SLIDE 30

Multiple LLVM Versions

Function * CloneFunction(Function *F, ValueToValueMapTy &VMap, ClonedCodeInfo *CodeInfo);

LLVM 3.8 LLVM 3.9+

Function * CloneFunction(const Function *F, ValueToValueMapTy &VMap, bool ModuleLevelChanges, ClonedCodeInfo *CodeInfo);

Module Module Original Function Original Function Cloned Function Cloned Function

slide-31
SLIDE 31

Sources vs Binaries

Precompiled LLVM LLVM Sources Fast compile time

✅ ❌

Debugging

❌ ✅

Asserts

❌ ✅

slide-32
SLIDE 32

Sources vs Binaries

if (EXISTS ${PATH_TO_LLVM}/CMakeLists.txt) add_subdirectory(${PATH_TO_LLVM} llvm-build-dir) # LLVM_INCUDE_DIRS ??? # LLVM_VERSION ??? else() ... endif()

slide-33
SLIDE 33

Sources vs Binaries

if (EXISTS ${PATH_TO_LLVM}/CMakeLists.txt) add_subdirectory(${PATH_TO_LLVM} llvm-build-dir) get_target_property(LLVM_INCLUDE_DIRS LLVMSupport INCLUDE_DIRECTORIES) # LLVM_VERSION ??? else() ... endif()

slide-34
SLIDE 34

Sources vs Binaries

if (EXISTS ${PATH_TO_LLVM}/CMakeLists.txt) add_subdirectory(${PATH_TO_LLVM} llvm-build-dir) get_target_property(LLVM_INCLUDE_DIRS LLVMSupport INCLUDE_DIRECTORIES) string(REGEX MATCH "LLVM_VERSION ([0-9]+.[0-9]+.[0-9]+)" LLVM_VERSION ${PATH_TO_LLVM}/CMakeLists.txt) else() ... endif()

slide-35
SLIDE 35

Memory Management

std::vector<std::unique_ptr<llvm::Module>> modules; LLVMContext context; auto module = loadModule("foo.bc", context); modules.push_back(std::move(module));

slide-36
SLIDE 36

Memory Management

LLVMContext context; std::vector<std::unique_ptr<llvm::Module>> modules; auto module = loadModule("foo.bc", context); modules.push_back(std::move(module));

slide-37
SLIDE 37

Memory Management

LLVMContext context; for (auto x : something) { auto module = loadModule("foo.bc", context); doSomethingWithModule(module); /// the module is destroyed, right? }

slide-38
SLIDE 38

Memory Management

LLVMContext context; for (auto x : something) { LLVMContext localContext; auto module = loadModule("foo.bc", localContext); doSomethingWithModule(module); /// the module is destroyed, right? right! }

slide-39
SLIDE 39

Parallelization

slide-40
SLIDE 40

Parallelization

  • LLVMContext
  • Module
  • Function
  • Block
  • Instruction
  • TargetMachine
  • orc::SimpleCompiler
  • CodeGen
slide-41
SLIDE 41

Parallelization

Thread 1 / LLVMContext 1 Thread 2 / LLVMContext 2 Loading Bitcode Analysis Transformation

Module #1

F1 F2 F3

Module #2

F4 F5 F6

Module #3

F7 F8

slide-42
SLIDE 42

Parallelization

Thread 1 / LLVMContext 1 Thread 2 / LLVMContext 2 Loading Bitcode Analysis Transformation

Module #1

F1 F2 F3

Module #2

F4 F5 F6

Module #3

F7 F8 F1 F2 F3 F4 F5 F6 F7 F8

slide-43
SLIDE 43

Parallelization

Thread 1 / LLVMContext 1 Thread 2 / LLVMContext 2 Loading Bitcode Analysis Transformation

Module #1

F1 F2 F3

Module #2

F4 F5 F6

Module #3

F7 F8 F1 F2 F3 F4 F5 F6 F7 F8 F1 F2 F3 F4 F5 F6 F7 F8

slide-44
SLIDE 44

Parallelization

seconds 0,125 0,25 0,375 0,5 Analysis Transformation

Sequential Parallel

slide-45
SLIDE 45

Parallelization

seconds 1,5 3 4,5 6 Loading Analysis Transformation

Sequential Parallel

slide-46
SLIDE 46

Parallelization

seconds 40 80 120 160 Loading Analysis Transformation Total

Sequential Parallel

slide-47
SLIDE 47

Getting Bitcode

https://github.com/travitch/whole-program-llvm

slide-48
SLIDE 48

Getting Bitcode

Compiler Flags Object File Executable

slide-49
SLIDE 49

Getting Bitcode

Compiler Flags Object File Executable

  • emit-llvm

LLVM Bitcode N/A

slide-50
SLIDE 50

Getting Bitcode

Compiler Flags Object File Executable

  • emit-llvm

LLVM Bitcode N/A

  • flto

LLVM Bitcode Machine Code

slide-51
SLIDE 51

Getting Bitcode

Compiler Flags Object File Executable

  • emit-llvm

LLVM Bitcode N/A

  • flto

LLVM Bitcode Machine Code

  • fembed-bitcode

LLVM Bitcode + Machine Code LLVM Bitcode + Machine Code

slide-52
SLIDE 52
  • fembed-bitcode

https://github.com/JDevlieghere/LibEBC → https://github.com/AlexDenisov/LibEBC

slide-53
SLIDE 53
  • fembed-bitcode

cmake -G Ninja \

  • DCMAKE_C_FLAGS=-embed-bitcode \
  • DCMAKE_CXX_FLAGS=-fembed-bitcode ..

ninja ADTTests ebcutil -e unittests/ADT/ADTTests mull-cxx unittests/ADT/ADTTests

slide-54
SLIDE 54

Multi-OS Support

slide-55
SLIDE 55

Multi-OS Support

config.vm.define "debian" do |cfg| cfg.vm.box = "debian/stretch64" cfg.vm.provision "ansible" do |ansible| ansible.playbook = "debian-playbook.yaml" end end config.vm.define "ubuntu" do |cfg| cfg.vm.box = "ubuntu/xenial64" cfg.vm.provision "ansible" do |ansible| ansible.playbook = "ubuntu-playbook.yaml" end end

slide-56
SLIDE 56

Multi-OS Support

tasks:

  • name: Prepare Working Directory
  • name: Install Required Packages
  • name: Install LLVM
  • name: Build Mull
  • name: Integration Tests
slide-57
SLIDE 57

Multi-OS Support

slide-58
SLIDE 58

Thank you!

Alex Denisov alex@lowlevelbits.org https://twitter.com/1101_debian