How to install shared libraries to embedded Linux

I'm writing some C++17 code for a Digilent Zybo Zynq-7000 (ARMv7 Cortex-A9). I'm using the arm-linux-gnueabihf GCC 8.3 toolchain. I downloaded the compiler and the sysroot.

I can compile and run the project using the -static linker flag, but it results in huge binaries, and I run into internal compiler errors if I enable LTO.

When I try to compile it using shared libraries, I'm unable to run it on the target device, even after copying all .so files in the sysroot-glibc-8.3-2019.03-x86_64-arm-linux-gnueabi/lib and usr/lib folders.

The important part of my Dockerfile with the build environment:

RUN wget https://developer.arm.com/-/media/Files/downloads/gnu-a/8.3-2019.03/binrel/gcc-arm-8.3-2019.03-x86_64-arm-linux-gnueabihf.tar.xz
RUN tar xf gcc-arm-8.3-2019.03-x86_64-arm-linux-gnueabihf.tar.xz && \
    rm gcc-arm-8.3-2019.03-x86_64-arm-linux-gnueabihf.tar.xz
ENV PATH="${PATH}:/home/develop/gcc-arm-8.3-2019.03-x86_64-arm-linux-gnueabihf/bin"
RUN wget https://developer.arm.com/-/media/Files/downloads/gnu-a/8.3-2019.03/binrel/sysroot/sysroot-glibc-8.3-2019.03-x86_64-arm-linux-gnueabi.tar.xz
RUN mkdir /home/develop/sysroot-glibc-8.3-2019.03-x86_64-arm-linux-gnueabi
WORKDIR /home/develop/sysroot-glibc-8.3-2019.03-x86_64-arm-linux-gnueabi
RUN tar xf ../sysroot-glibc-8.3-2019.03-x86_64-arm-linux-gnueabi.tar.xz && \
    rm ../sysroot-glibc-8.3-2019.03-x86_64-arm-linux-gnueabi.tar.xz

(Complete version on GitHub)

My CMake Toolchain file:

SET(CMAKE_SYSTEM_NAME Linux)
SET(CMAKE_SYSTEM_VERSION 1)

# Specify the cross compiler
SET(CMAKE_C_COMPILER   arm-linux-gnueabihf-gcc)
SET(CMAKE_CXX_COMPILER arm-linux-gnueabihf-g++)

# Where is the target environment
SET(CMAKE_FIND_ROOT_PATH /home/develop/sysroot-glibc-8.3-2019.03-x86_64-arm-linux-gnueabi)

# Search for programs in the build host directories
SET(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
# for libraries and headers in the target directories
SET(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
SET(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)

SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} \
-mcpu=cortex-a9 \
-mfpu=neon -mfloat-abi=hard -ftree-vectorize -mvectorize-with-neon-quad" 
CACHE STRING "" FORCE)

# Link all libraries statically
# SET(CMAKE_EXE_LINKER_FLAGS " -static"
# CACHE STRING "" FORCE)

The shared libraries needed by my executable are:

$ arm-linux-gnueabihf-readelf -d bin/test-crypto
Dynamic section at offset 0x5cef0 contains 29 entries:
  Tag        Type                         Name/Value
 0x00000001 (NEEDED)                     Shared library: [libpthread.so.0]
 0x00000001 (NEEDED)                     Shared library: [libstdc++.so.6]
 0x00000001 (NEEDED)                     Shared library: [libm.so.6]
 0x00000001 (NEEDED)                     Shared library: [libgcc_s.so.1]
 0x00000001 (NEEDED)                     Shared library: [libc.so.6]

When I try to run the executable on the target, I get the following error, because it can't find the necessary libraries:

$ /media/test-crypto 
/bin/sh: /media/test-crypto: not found

$ ldd /media/test-crypto
checking sub-depends for 'not found'
checking sub-depends for '/usr/lib/libstdc++.so.6'
checking sub-depends for 'not found'
checking sub-depends for '/lib/libgcc_s.so.1'
checking sub-depends for 'not found'
checking sub-depends for '/lib/libm.so.1'
        libc.so.1 => /lib/libc.so.1 (0xb6ea7000)
        ld-uClibc.so.1 => /lib/ld-uClibc.so.1 (0xb6f06000)
checking sub-depends for '/lib/libc.so.1'
        ld-uClibc.so.1 => /lib/ld-uClibc.so.1 (0xb6f88000)
        libpthread.so.0 => not found (0x00000000)
        libstdc++.so.6 => /usr/lib/libstdc++.so.6 (0x00000000)
        libm.so.6 => not found (0x00000000)
        libgcc_s.so.1 => /lib/libgcc_s.so.1 (0x00000000)
        libc.so.6 => not found (0x00000000)
        libm.so.1 => /lib/libm.so.1 (0x00000000)
        libc.so.1 => /lib/libc.so.1 (0x00000000)
        /lib/ld-uClibc.so.1 => /lib/ld-uClibc.so.1 (0x00000000)
        /lib/ld-uClibc.so.1 => /lib/ld-uClibc.so.1 (0x00000000)

I checked the /lib and /usr/lib folders on the target, and /usr/lib/libstdc++.so.6 and /lib/libgcc_s.so.1 are there, the other three are not.

When I copy all libraries from the GCC sysroot folder to the SD card, and add them to the LD_LIBRARY_PATH, the ldd output changes, and it seems to find all libraries, but when I try to run the executable, I still get the same error:

$ LD_LIBRARY_PATH=/media/lib ldd /media/test-crypto 
checking sub-depends for '/media/lib/libpthread.so.0'
checking sub-depends for '/media/lib/libstdc++.so.6'
checking sub-depends for '/media/lib/libm.so.6'
checking sub-depends for '/media/lib/libgcc_s.so.1'
checking sub-depends for '/media/lib/libc.so.6'
        libpthread.so.0 => /media/lib/libpthread.so.0 (0x00000000)
        libstdc++.so.6 => /media/lib/libstdc++.so.6 (0x00000000)
        libm.so.6 => /media/lib/libm.so.6 (0x00000000)
        libgcc_s.so.1 => /media/lib/libgcc_s.so.1 (0x00000000)
        libc.so.6 => /media/lib/libc.so.6 (0x00000000)
        /lib/ld-linux.so.3 => /lib/ld-linux.so.3 (0x00000000)
        /lib/ld-linux.so.3 => /lib/ld-linux.so.3 (0x00000000)

$ LD_LIBRARY_PATH=/media/lib /media/test-crypto 
/bin/sh: /media/test-crypto: not found

I have tried creating symbolic links from the libraries to /lib and /usr/lib using ln -s /media/lib/* /lib, but that didn't work.

I cannot add the GCC sysroot to the rootfs.cpio archive of the Linux image we were given, because it's too large, and then it no longer boots when I add too many files. The FAT32 file system of the SD card doesn't support symlinks, and many of the shared libraries are symlinks, so maybe that's a problem. I've tried using ext2 and ext4 instead of FAT, but this is not supported by the development board, it simply doesn't boot if I try anything else than FAT. Adding a second partition for just the libraries didn't work either.

What is the best way to install the necessary libraries on the SD card so I can run my C/C++ programs?