(工具) 交叉编译 gperftools及使用

交叉编译gperftools及使用

sudo apt-get install kcachegrind  # 导出为 callgrind 格式时需要
sudo apt install doxygen-latex
sudo apt install graphviz

# 编译 pprof
go install github.com/google/pprof@latest

1. 交叉编译gperftools

# https://github.com/gperftools/gperftools/releases/tag/gperftools-2.9.1
CC=arm-linux-gnueabihf-gcc CXX=arm-linux-gnueabihf-g++ cmake .. -Dgperftools_enable_libunwind=OFF -Dgperftools_enable_frame_pointers=ON -DCMAKE_INSTALL_PREFIX=/home/hxf0223/tmp/gperftools/bbb
make -j && make install

2. ARM 测量待测试项目

2.1. 添加链接：libprofiler.so

file(GLOB SRCS *.cc)
ADD_EXECUTABLE(eigen_arm_test ${SRCS})

# SET(EXECUTABLE_OUTPUT_PATH ${PROJECT_SOURCE_DIR}/bin)
# target_link_libraries(eigen_arm_test gtest glog pthread)
target_link_libraries(eigen_arm_test dcl_utils pthread "-Wl,--no-as-needed" profiler "-Wl,--as-needed")

2.2. 在ARM板卡上运行CPUPROFILE

# 生成 eigen_test.prof
CPUPROFILE=eigen_test.prof CPUPROFILE_FREQUENCY=500 ./eigen_arm_test

2.3. 转换.prof文件为pdf文件

sudo apt install graphviz ghostscript
sudo apt-get install kcachegrind # 转换为callgrind之后使用kcachegrind查看

拷贝ARM板卡上生成eigen_test.prof文件到PC主机上交叉编译生成的eigen_arm_test目录。

直接运行转换命令会导致找不到ARM测试程序依赖库文件：

/usr/local/arm-linux-gnueabihf-8.3/bin/arm-linux-gnueabihf-objdump: '/lib/arm-linux-gnueabihf/libc-2.28.so': No such file
/usr/local/arm-linux-gnueabihf-8.3/bin/arm-linux-gnueabihf-objdump: '/lib/arm-linux-gnueabihf/libc-2.28.so': No such file
/usr/local/arm-linux-gnueabihf-8.3/bin/arm-linux-gnueabihf-objdump: '/lib/arm-linux-gnueabihf/libc-2.28.so': No such file
/usr/local/arm-linux-gnueabihf-8.3/bin/arm-linux-gnueabihf-objdump: '/lib/arm-linux-gnueabihf/libgcc_s.so.1': No such file
/usr/local/arm-linux-gnueabihf-8.3/bin/arm-linux-gnueabihf-objdump: '/lib/arm-linux-gnueabihf/libgcc_s.so.1': No such file
/usr/local/arm-linux-gnueabihf-8.3/bin/arm-linux-gnueabihf-objdump: '/lib/arm-linux-gnueabihf/libgcc_s.so.1': No such file
/usr/local/arm-linux-gnueabihf-8.3/bin/arm-linux-gnueabihf-objdump: '/lib/arm-linux-gnueabihf/libm-2.28.so': No such file
/usr/local/arm-linux-gnueabihf-8.3/bin/arm-linux-gnueabihf-objdump: '/lib/arm-linux-gnueabihf/libm-2.28.so': No such file
/usr/local/arm-linux-gnueabihf-8.3/bin/arm-linux-gnueabihf-objdump: '/lib/arm-linux-gnueabihf/libm-2.28.so': No such file
/usr/local/arm-linux-gnueabihf-8.3/bin/arm-linux-gnueabihf-objdump: '/usr/lib/arm-linux-gnueabihf/libstdc++.so.6.0.25': No such file
/usr/local/arm-linux-gnueabihf-8.3/bin/arm-linux-gnueabihf-objdump: '/usr/lib/arm-linux-gnueabihf/libstdc++.so.6.0.25': No such file
/usr/local/arm-linux-gnueabihf-8.3/bin/arm-linux-gnueabihf-objdump: '/usr/lib/arm-linux-gnueabihf/libstdc++.so.6.0.25': No such file
/usr/local/arm-linux-gnueabihf-8.3/bin/arm-linux-gnueabihf-objdump: /usr/local/lib/libprofiler.so.0.5.4: file format not recognized
/usr/local/arm-linux-gnueabihf-8.3/bin/arm-linux-gnueabihf-objdump: /usr/local/lib/libprofiler.so.0.5.4: file format not recognized
/usr/local/arm-linux-gnueabihf-8.3/bin/arm-linux-gnueabihf-objdump: /usr/local/lib/libprofiler.so.0.5.4: file format not recognized
/usr/local/arm-linux-gnueabihf-8.3/bin/arm-linux-gnueabihf-objdump: '/lib/arm-linux-gnueabihf/libpthread-2.28.so': No such file
/usr/local/arm-linux-gnueabihf-8.3/bin/arm-linux-gnueabihf-objdump: '/lib/arm-linux-gnueabihf/libpthread-2.28.so': No such file
/usr/local/arm-linux-gnueabihf-8.3/bin/arm-linux-gnueabihf-objdump: '/lib/arm-linux-gnueabihf/libpthread-2.28.so': No such file
/usr/local/arm-linux-gnueabihf-8.3/bin/arm-linux-gnueabihf-objdump: '/lib/arm-linux-gnueabihf/ld-2.28.so': No such file
/usr/local/arm-linux-gnueabihf-8.3/bin/arm-linux-gnueabihf-objdump: '/lib/arm-linux-gnueabihf/ld-2.28.so': No such file
/usr/local/arm-linux-gnueabihf-8.3/bin/arm-linux-gnueabihf-objdump: '/lib/arm-linux-gnueabihf/ld-2.28.so': No such file
/usr/local/arm-linux-gnueabihf-8.3/bin/arm-linux-gnueabihf-addr2line: '/usr/lib/arm-linux-gnueabihf/libstdc++.so.6.0.25': No such file
/usr/local/arm-linux-gnueabihf-8.3/bin/arm-linux-gnueabihf-addr2line: '/lib/arm-linux-gnueabihf/libgcc_s.so.1': No such file
/usr/local/arm-linux-gnueabihf-8.3/bin/arm-linux-gnueabihf-addr2line: '/lib/arm-linux-gnueabihf/libc-2.28.so': No such file

解决办法是，在主机上创建一个目录，在此目录下创建名称跟缺失的路径一样的软链接（该缺失的路径为ARM板卡上的路径）：

mkdir ~/tmp/fake_root && cd ~/tmp/fake_root
mkdir -p lib usr lib_gcc/lib
ln -s /usr/local/arm-linux-gnueabihf-8.3/arm-linux-gnueabihf/libc/lib/ lib/arm-linux-gnueabihf
ln -s /usr/local/arm-linux-gnueabihf-8.3/arm-linux-gnueabihf/libc/usr/lib usr/lib
ln -s /usr/local/arm-linux-gnueabihf-8.3/arm-linux-gnueabihf/lib lib_gcc/lib/arm-linux-gnueabihf

从.prof文件生成pdf文件：

pprof --tools=/usr/local/arm-linux-gnueabihf-8.3/bin/arm-linux-gnueabihf- --lib_prefix=/home/hxf0223/tmp/fake_bbb_root,/home/hxf0223/tmp/fake_bbb_root/lib_gcc,/home/hxf0223/tmp/gperftools/bbb/lib ./eigen_arm_test eigen_test.prof --pdf > eigen_test.pdf

lib_prefix添加了三个目录：ARM运行时目录，arm-gcc相关lib目录，及交叉编译libprofiler.so目录。

注意：lib_prefix只能添加绝对路径。

2.4. PC下转换命令格式

# 生成pdf格式的性能报告（层次调用节点有向图）
pprof --pdf ./eigen_arm_test eigen_test.prof > profile_eigen_x86.pdf
# pprof ----callgrind ./eigen_arm_test eigen_test.prof  > profile_eigen_x86.callgrind

3. 在项目中使用ProfilerStart

链接使用libprofiler.so, 在代码中添加ProfilerStart, ProfilerStop。

#include <gperftools/profiler.h>

ProfilerStart("test_capture.prof");
// 待分析代码
ProfilerStop();

运行编译的可执行程序，得到test_capture.prof文件。随后从.prof文件生成报告。

4. Ubuntu x86_64编译 gperf-tools

使用如下链接，utoconf编译安装即可：

https://github.com/libunwind/libunwind.git
https://github.com/gperftools/gperftools.git

1. libunwind 按照官方说明编译即可；
2. gperftools 使用默认configure即可。如果libunwind安装到其他目录，gperftools使用 ./configure CFLAGS=inc_dir LDFLAGS=lib_dir；

5. 内存 Profile

export LD_PRELOAD=/usr/local/lib/libtcmalloc.so
export HEAPPROFILE=/home/q/perf-result/
export HEAP_PROFILE_ALLOCATION_INTERVAL=2000000000

• 使用google perf工具来排查堆外内存占用

6. 参考

• gperftools github io
• Analyze program performance with gperftools
• 用gperftools对C/C++程序进行profile

7. 其他

• Linux Trace Tool NG