diff options
Diffstat (limited to 'lib/mlibc/options/rtdl/generic')
| -rw-r--r-- | lib/mlibc/options/rtdl/generic/linker.cpp | 1872 | ||||
| -rw-r--r-- | lib/mlibc/options/rtdl/generic/linker.hpp | 402 | ||||
| -rw-r--r-- | lib/mlibc/options/rtdl/generic/main.cpp | 844 |
3 files changed, 3118 insertions, 0 deletions
diff --git a/lib/mlibc/options/rtdl/generic/linker.cpp b/lib/mlibc/options/rtdl/generic/linker.cpp new file mode 100644 index 0000000..a519c35 --- /dev/null +++ b/lib/mlibc/options/rtdl/generic/linker.cpp @@ -0,0 +1,1872 @@ +#include <mlibc/arch-defs.hpp> +#include <stdint.h> +#include <string.h> + +// keep a list of optional generic relocation types +enum { + R_OFFSET = (uintptr_t) -1, +}; + + +#include <frg/manual_box.hpp> +#include <frg/small_vector.hpp> +#include <mlibc/allocator.hpp> +#include <mlibc/debug.hpp> +#include <mlibc/rtdl-sysdeps.hpp> +#include <mlibc/rtdl-abi.hpp> +#include <mlibc/thread.hpp> +#include <abi-bits/fcntl.h> +#include <internal-config.h> + +#include "elf.hpp" +#include "linker.hpp" + +#if !MLIBC_MMAP_ALLOCATE_DSO +uintptr_t libraryBase = 0x41000000; +#endif + +constexpr bool verbose = false; +constexpr bool stillSlightlyVerbose = false; +constexpr bool logBaseAddresses = false; +constexpr bool logRpath = false; +constexpr bool logLdPath = false; +constexpr bool eagerBinding = true; + +#if defined(__x86_64__) || defined(__i386__) +constexpr inline bool tlsAboveTp = false; +#elif defined(__aarch64__) +constexpr inline bool tlsAboveTp = true; +#elif defined(__riscv) +constexpr inline bool tlsAboveTp = true; +#else +# error Unknown architecture +#endif + +extern DebugInterface globalDebugInterface; +extern uintptr_t __stack_chk_guard; + +extern frg::manual_box<frg::small_vector<frg::string_view, 4, MemoryAllocator>> libraryPaths; +extern frg::manual_box<frg::vector<frg::string_view, MemoryAllocator>> preloads; + +#if MLIBC_STATIC_BUILD +extern "C" size_t __init_array_start[]; +extern "C" size_t __init_array_end[]; +extern "C" size_t __preinit_array_start[]; +extern "C" size_t __preinit_array_end[]; +#endif + +size_t tlsMaxAlignment = 16; + +// This is the global "resolution timestamp" (RTS) counter. +// It is incremented each time __dlapi_open() (i.e. dlopen()) is called. +// Each DSO stores its objectRts (i.e. RTS at the time the object was loaded). +// DSOs in the global scope also store a globalRts (i.e. RTS at the time the +// object became global). This mechanism is used to determine which +// part of the global scope is considered for symbol resolution. +uint64_t rtsCounter = 2; + +bool trySeek(int fd, int64_t offset) { + off_t noff; + return mlibc::sys_seek(fd, offset, SEEK_SET, &noff) == 0; +} + +bool tryReadExactly(int fd, void *data, size_t length) { + size_t offset = 0; + while(offset < length) { + ssize_t chunk; + if(mlibc::sys_read(fd, reinterpret_cast<char *>(data) + offset, + length - offset, &chunk)) + return false; + __ensure(chunk > 0); + offset += chunk; + } + __ensure(offset == length); + return true; +} + +void closeOrDie(int fd) { + if(mlibc::sys_close(fd)) + __ensure(!"sys_close() failed"); +} + +uintptr_t alignUp(uintptr_t address, size_t align) { + return (address + align - 1) & ~(align - 1); +} + +// -------------------------------------------------------- +// ObjectRepository +// -------------------------------------------------------- + +ObjectRepository::ObjectRepository() +: loadedObjects{getAllocator()}, + _nameMap{frg::hash<frg::string_view>{}, getAllocator()} {} + +SharedObject *ObjectRepository::injectObjectFromDts(frg::string_view name, + frg::string<MemoryAllocator> path, uintptr_t base_address, + elf_dyn *dynamic, uint64_t rts) { + __ensure(!findLoadedObject(name)); + + auto object = frg::construct<SharedObject>(getAllocator(), + name.data(), std::move(path), false, globalScope.get(), rts); + object->baseAddress = base_address; + object->dynamic = dynamic; + _parseDynamic(object); + + _addLoadedObject(object); + _discoverDependencies(object, globalScope.get(), rts); + + return object; +} + +SharedObject *ObjectRepository::injectObjectFromPhdrs(frg::string_view name, + frg::string<MemoryAllocator> path, void *phdr_pointer, + size_t phdr_entry_size, size_t num_phdrs, void *entry_pointer, + uint64_t rts) { + __ensure(!findLoadedObject(name)); + + auto object = frg::construct<SharedObject>(getAllocator(), + name.data(), std::move(path), true, globalScope.get(), rts); + _fetchFromPhdrs(object, phdr_pointer, phdr_entry_size, num_phdrs, entry_pointer); + _parseDynamic(object); + + _addLoadedObject(object); + _discoverDependencies(object, globalScope.get(), rts); + + return object; +} + +SharedObject *ObjectRepository::injectStaticObject(frg::string_view name, + frg::string<MemoryAllocator> path, void *phdr_pointer, + size_t phdr_entry_size, size_t num_phdrs, void *entry_pointer, + uint64_t rts) { + __ensure(!findLoadedObject(name)); + auto object = frg::construct<SharedObject>(getAllocator(), + name.data(), std::move(path), true, globalScope.get(), rts); + _fetchFromPhdrs(object, phdr_pointer, phdr_entry_size, num_phdrs, entry_pointer); + +#if MLIBC_STATIC_BUILD + object->initArray = reinterpret_cast<InitFuncPtr*>(__init_array_start); + object->initArraySize = static_cast<size_t>((uintptr_t)__init_array_end - + (uintptr_t)__init_array_start); + object->preInitArray = reinterpret_cast<InitFuncPtr*>(__preinit_array_start); + object->preInitArraySize = static_cast<size_t>((uintptr_t)__preinit_array_end - + (uintptr_t)__preinit_array_start); +#endif + + _addLoadedObject(object); + + return object; +} + +frg::expected<LinkerError, SharedObject *> ObjectRepository::requestObjectWithName(frg::string_view name, + SharedObject *origin, Scope *localScope, bool createScope, uint64_t rts) { + if (auto obj = findLoadedObject(name)) + return obj; + + auto tryToOpen = [&] (const char *path) { + int fd; + if(auto x = mlibc::sys_open(path, O_RDONLY, 0, &fd); x) { + return -1; + } + return fd; + }; + + // TODO(arsen): this process can probably undergo heavy optimization, by + // preprocessing the rpath only once on parse + auto processRpath = [&] (frg::string_view path) { + frg::string<MemoryAllocator> sPath { getAllocator() }; + if (path.starts_with("$ORIGIN")) { + frg::string_view dirname = origin->path; + auto lastsl = dirname.find_last('/'); + if (lastsl != size_t(-1)) { + dirname = dirname.sub_string(0, lastsl); + } else { + dirname = "."; + } + sPath = frg::string<MemoryAllocator>{ getAllocator(), dirname }; + sPath += path.sub_string(7, path.size() - 7); + } else { + sPath = frg::string<MemoryAllocator>{ getAllocator(), path }; + } + if (sPath[sPath.size() - 1] != '/') { + sPath += '/'; + } + sPath += name; + if (logRpath) + mlibc::infoLogger() << "rtdl: trying in rpath " << sPath << frg::endlog; + int fd = tryToOpen(sPath.data()); + if (logRpath && fd >= 0) + mlibc::infoLogger() << "rtdl: found in rpath" << frg::endlog; + return frg::tuple { fd, std::move(sPath) }; + }; + + frg::string<MemoryAllocator> chosenPath { getAllocator() }; + int fd = -1; + if (origin && origin->runPath) { + size_t start = 0; + size_t idx = 0; + frg::string_view rpath { origin->runPath }; + auto next = [&] () { + idx = rpath.find_first(':', start); + if (idx == (size_t)-1) + idx = rpath.size(); + }; + for (next(); idx < rpath.size(); next()) { + auto path = rpath.sub_string(start, idx - start); + start = idx + 1; + auto [fd_, fullPath] = processRpath(path); + if (fd_ != -1) { + fd = fd_; + chosenPath = std::move(fullPath); + break; + } + } + if (fd == -1) { + auto path = rpath.sub_string(start, rpath.size() - start); + auto [fd_, fullPath] = processRpath(path); + if (fd_ != -1) { + fd = fd_; + chosenPath = std::move(fullPath); + } + } + } else if (logRpath) { + mlibc::infoLogger() << "rtdl: no rpath set for object" << frg::endlog; + } + + for(size_t i = 0; i < libraryPaths->size() && fd == -1; i++) { + auto ldPath = (*libraryPaths)[i]; + auto path = frg::string<MemoryAllocator>{getAllocator(), ldPath} + '/' + name; + if(logLdPath) + mlibc::infoLogger() << "rtdl: Trying to load " << name << " from ldpath " << ldPath << "/" << frg::endlog; + fd = tryToOpen(path.data()); + if(fd >= 0) { + chosenPath = std::move(path); + break; + } + } + if(fd == -1) + return LinkerError::notFound; + + if (createScope) { + __ensure(localScope == nullptr); + + // TODO: Free this when the scope is no longer needed. + localScope = frg::construct<Scope>(getAllocator()); + } + + __ensure(localScope != nullptr); + + auto object = frg::construct<SharedObject>(getAllocator(), + name.data(), std::move(chosenPath), false, localScope, rts); + + auto result = _fetchFromFile(object, fd); + closeOrDie(fd); + if(!result) { + frg::destruct(getAllocator(), object); + return result.error(); + } + + _parseDynamic(object); + + _addLoadedObject(object); + _discoverDependencies(object, localScope, rts); + + return object; +} + +frg::expected<LinkerError, SharedObject *> ObjectRepository::requestObjectAtPath(frg::string_view path, + Scope *localScope, bool createScope, uint64_t rts) { + // TODO: Support SONAME correctly. + auto lastSlash = path.find_last('/') + 1; + auto name = path; + if (!lastSlash) { + name = name.sub_string(lastSlash, path.size() - lastSlash); + } + if (auto obj = findLoadedObject(name)) + return obj; + + if (createScope) { + __ensure(localScope == nullptr); + + // TODO: Free this when the scope is no longer needed. + localScope = frg::construct<Scope>(getAllocator()); + } + + __ensure(localScope != nullptr); + + auto object = frg::construct<SharedObject>(getAllocator(), + name.data(), path.data(), false, localScope, rts); + + frg::string<MemoryAllocator> no_prefix(getAllocator(), path); + + int fd; + if(mlibc::sys_open((no_prefix + '\0').data(), O_RDONLY, 0, &fd)) { + frg::destruct(getAllocator(), object); + return LinkerError::notFound; + } + auto result = _fetchFromFile(object, fd); + closeOrDie(fd); + if(!result) { + frg::destruct(getAllocator(), object); + return result.error(); + } + + _parseDynamic(object); + + _addLoadedObject(object); + _discoverDependencies(object, localScope, rts); + + return object; +} + +SharedObject *ObjectRepository::findCaller(void *addr) { + uintptr_t target = reinterpret_cast<uintptr_t>(addr); + + for (auto [name, object] : _nameMap) { + // Search all PT_LOAD segments for the specified address. + for(size_t j = 0; j < object->phdrCount; j++) { + auto phdr = (elf_phdr *)((uintptr_t)object->phdrPointer + j * object->phdrEntrySize); + if (phdr->p_type == PT_LOAD) { + uintptr_t start = object->baseAddress + phdr->p_vaddr; + uintptr_t end = start + phdr->p_memsz; + if (start <= target && target < end) + return object; + } + } + } + + return nullptr; +} + +SharedObject *ObjectRepository::findLoadedObject(frg::string_view name) { + auto it = _nameMap.get(name); + if (it) + return *it; + + for (auto object : loadedObjects) { + // See if any object has a matching SONAME. + if (object->soName && name == object->soName) + return object; + } + + // TODO: We should also look at the device and inode here as a fallback. + return nullptr; +} + +// -------------------------------------------------------- +// ObjectRepository: Fetching methods. +// -------------------------------------------------------- + +void ObjectRepository::_fetchFromPhdrs(SharedObject *object, void *phdr_pointer, + size_t phdr_entry_size, size_t phdr_count, void *entry_pointer) { + __ensure(object->isMainObject); + object->phdrPointer = phdr_pointer; + object->phdrEntrySize = phdr_entry_size; + object->phdrCount = phdr_count; + if(verbose) + mlibc::infoLogger() << "rtdl: Loading " << object->name << frg::endlog; + + // Note: the entry pointer is absolute and not relative to the base address. + object->entry = entry_pointer; + + frg::optional<ptrdiff_t> dynamic_offset; + frg::optional<ptrdiff_t> tls_offset; + + // segments are already mapped, so we just have to find the dynamic section + for(size_t i = 0; i < phdr_count; i++) { + auto phdr = (elf_phdr *)((uintptr_t)phdr_pointer + i * phdr_entry_size); + switch(phdr->p_type) { + case PT_PHDR: + // Determine the executable's base address (in the PIE case) by comparing + // the PHDR segment's load address against it's address in the ELF file. + object->baseAddress = reinterpret_cast<uintptr_t>(phdr_pointer) - phdr->p_vaddr; + if(verbose) + mlibc::infoLogger() << "rtdl: Executable is loaded at " + << (void *)object->baseAddress << frg::endlog; + break; + case PT_DYNAMIC: + dynamic_offset = phdr->p_vaddr; + break; + case PT_TLS: { + object->tlsSegmentSize = phdr->p_memsz; + object->tlsAlignment = phdr->p_align; + object->tlsImageSize = phdr->p_filesz; + tls_offset = phdr->p_vaddr; + break; + case PT_INTERP: + object->interpreterPath = frg::string<MemoryAllocator>{ + (char*)(object->baseAddress + phdr->p_vaddr), + getAllocator() + }; + } break; + default: + //FIXME warn about unknown phdrs + break; + } + } + + if(dynamic_offset) + object->dynamic = (elf_dyn *)(object->baseAddress + *dynamic_offset); + if(tls_offset) + object->tlsImagePtr = (void *)(object->baseAddress + *tls_offset); +} + + +frg::expected<LinkerError, void> ObjectRepository::_fetchFromFile(SharedObject *object, int fd) { + __ensure(!object->isMainObject); + + // read the elf file header + elf_ehdr ehdr; + if(!tryReadExactly(fd, &ehdr, sizeof(elf_ehdr))) + return LinkerError::fileTooShort; + + if(ehdr.e_ident[0] != 0x7F + || ehdr.e_ident[1] != 'E' + || ehdr.e_ident[2] != 'L' + || ehdr.e_ident[3] != 'F') + return LinkerError::notElf; + + if((ehdr.e_type != ET_EXEC && ehdr.e_type != ET_DYN) + || ehdr.e_machine != ELF_MACHINE + || ehdr.e_ident[EI_CLASS] != ELF_CLASS) + return LinkerError::wrongElfType; + + // read the elf program headers + auto phdr_buffer = (char *)getAllocator().allocate(ehdr.e_phnum * ehdr.e_phentsize); + if(!phdr_buffer) + return LinkerError::outOfMemory; + + if(!trySeek(fd, ehdr.e_phoff)) { + getAllocator().deallocate(phdr_buffer, ehdr.e_phnum * ehdr.e_phentsize); + return LinkerError::invalidProgramHeader; + } + if(!tryReadExactly(fd, phdr_buffer, ehdr.e_phnum * ehdr.e_phentsize)) { + getAllocator().deallocate(phdr_buffer, ehdr.e_phnum * ehdr.e_phentsize); + return LinkerError::invalidProgramHeader; + } + + object->phdrPointer = phdr_buffer; + object->phdrCount = ehdr.e_phnum; + object->phdrEntrySize = ehdr.e_phentsize; + + // Allocate virtual address space for the DSO. + constexpr size_t hugeSize = 0x200000; + + uintptr_t highest_address = 0; + for(int i = 0; i < ehdr.e_phnum; i++) { + auto phdr = (elf_phdr *)(phdr_buffer + i * ehdr.e_phentsize); + + if(phdr->p_type != PT_LOAD) + continue; + + auto limit = phdr->p_vaddr + phdr->p_memsz; + if(limit > highest_address) + highest_address = limit; + } + + __ensure(!(object->baseAddress & (hugeSize - 1))); + + highest_address = (highest_address + mlibc::page_size - 1) & ~(mlibc::page_size - 1); + +#if MLIBC_MMAP_ALLOCATE_DSO + void *mappedAddr = nullptr; + + if (mlibc::sys_vm_map(nullptr, + highest_address - object->baseAddress, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0, &mappedAddr)) { + mlibc::infoLogger() << "sys_vm_map failed when allocating address space for DSO \"" + << object->name << "\"" + << ", base " << (void *)object->baseAddress + << ", requested " << (highest_address - object->baseAddress) << " bytes" + << frg::endlog; + getAllocator().deallocate(phdr_buffer, ehdr.e_phnum * ehdr.e_phentsize); + return LinkerError::outOfMemory; + } + + object->baseAddress = reinterpret_cast<uintptr_t>(mappedAddr); +#else + object->baseAddress = libraryBase; + libraryBase += (highest_address + (hugeSize - 1)) & ~(hugeSize - 1); +#endif + + if(verbose || logBaseAddresses) + mlibc::infoLogger() << "rtdl: Loading " << object->name + << " at " << (void *)object->baseAddress << frg::endlog; + + // Load all segments. + constexpr size_t pageSize = 0x1000; + for(int i = 0; i < ehdr.e_phnum; i++) { + auto phdr = (elf_phdr *)(phdr_buffer + i * ehdr.e_phentsize); + + if(phdr->p_type == PT_LOAD) { + size_t misalign = phdr->p_vaddr & (pageSize - 1); + __ensure(phdr->p_memsz > 0); + __ensure(phdr->p_memsz >= phdr->p_filesz); + + // If the following condition is violated, we cannot use mmap() the segment; + // however, GCC only generates ELF files that satisfy this. + __ensure(misalign == (phdr->p_offset & (pageSize - 1))); + + auto map_address = object->baseAddress + phdr->p_vaddr - misalign; + auto backed_map_size = (phdr->p_filesz + misalign + pageSize - 1) & ~(pageSize - 1); + auto total_map_size = (phdr->p_memsz + misalign + pageSize - 1) & ~(pageSize - 1); + + int prot = 0; + if(phdr->p_flags & PF_R) + prot |= PROT_READ; + if(phdr->p_flags & PF_W) + prot |= PROT_WRITE; + if(phdr->p_flags & PF_X) + prot |= PROT_EXEC; + + #if MLIBC_MAP_DSO_SEGMENTS + void *map_pointer; + if(mlibc::sys_vm_map(reinterpret_cast<void *>(map_address), + backed_map_size, prot | PROT_WRITE, + MAP_PRIVATE | MAP_FIXED, fd, phdr->p_offset - misalign, &map_pointer)) + __ensure(!"sys_vm_map failed"); + if(total_map_size > backed_map_size) + if(mlibc::sys_vm_map(reinterpret_cast<void *>(map_address + backed_map_size), + total_map_size - backed_map_size, prot | PROT_WRITE, + MAP_PRIVATE | MAP_FIXED | MAP_ANONYMOUS, -1, 0, &map_pointer)) + __ensure(!"sys_vm_map failed"); + + if(mlibc::sys_vm_readahead) + if(mlibc::sys_vm_readahead(reinterpret_cast<void *>(map_address), + backed_map_size)) + mlibc::infoLogger() << "mlibc: sys_vm_readahead() failed in ld.so" + << frg::endlog; + + // Clear the trailing area at the end of the backed mapping. + // We do not clear the leading area; programs are not supposed to access it. + memset(reinterpret_cast<void *>(map_address + misalign + phdr->p_filesz), + 0, phdr->p_memsz - phdr->p_filesz); + #else + (void)backed_map_size; + + void *map_pointer; + if(mlibc::sys_vm_map(reinterpret_cast<void *>(map_address), + total_map_size, prot | PROT_WRITE, + MAP_PRIVATE | MAP_FIXED | MAP_ANONYMOUS, -1, 0, &map_pointer)) + __ensure(!"sys_vm_map failed"); + + __ensure(trySeek(fd, phdr->p_offset)); + __ensure(tryReadExactly(fd, reinterpret_cast<char *>(map_address) + misalign, + phdr->p_filesz)); + #endif + // Take care of removing superfluous permissions. + if(mlibc::sys_vm_protect && ((prot & PROT_WRITE) == 0)) + if(mlibc::sys_vm_protect(map_pointer, total_map_size, prot)) + mlibc::infoLogger() << "mlibc: sys_vm_protect() failed in ld.so" << frg::endlog; + }else if(phdr->p_type == PT_TLS) { + object->tlsSegmentSize = phdr->p_memsz; + object->tlsAlignment = phdr->p_align; + object->tlsImageSize = phdr->p_filesz; + object->tlsImagePtr = (void *)(object->baseAddress + phdr->p_vaddr); + }else if(phdr->p_type == PT_DYNAMIC) { + object->dynamic = (elf_dyn *)(object->baseAddress + phdr->p_vaddr); + }else if(phdr->p_type == PT_INTERP + || phdr->p_type == PT_PHDR + || phdr->p_type == PT_NOTE + || phdr->p_type == PT_RISCV_ATTRIBUTES + || phdr->p_type == PT_GNU_EH_FRAME + || phdr->p_type == PT_GNU_RELRO + || phdr->p_type == PT_GNU_STACK + || phdr->p_type == PT_GNU_PROPERTY) { + // ignore the phdr + }else{ + mlibc::panicLogger() << "Unexpected PHDR type 0x" + << frg::hex_fmt(phdr->p_type) << " in DSO " << object->name << frg::endlog; + } + } + + return frg::success; +} + +// -------------------------------------------------------- +// ObjectRepository: Parsing methods. +// -------------------------------------------------------- + +void ObjectRepository::_parseDynamic(SharedObject *object) { + if(!object->dynamic) + mlibc::infoLogger() << "ldso: Object '" << object->name + << "' does not have a dynamic section" << frg::endlog; + __ensure(object->dynamic); + + // Fix up these offsets to addresses after the loop, since the + // addresses depend on the value of DT_STRTAB. + frg::optional<ptrdiff_t> runpath_offset; + /* If true, ignore the RPATH. */ + bool runpath_found = false; + frg::optional<ptrdiff_t> soname_offset; + + for(size_t i = 0; object->dynamic[i].d_tag != DT_NULL; i++) { + elf_dyn *dynamic = &object->dynamic[i]; + switch(dynamic->d_tag) { + // handle hash table, symbol table and string table + case DT_HASH: + object->hashStyle = HashStyle::systemV; + object->hashTableOffset = dynamic->d_un.d_ptr; + break; + case DT_GNU_HASH: + object->hashStyle = HashStyle::gnu; + object->hashTableOffset = dynamic->d_un.d_ptr; + break; + case DT_STRTAB: + object->stringTableOffset = dynamic->d_un.d_ptr; + break; + case DT_STRSZ: + break; // we don't need the size of the string table + case DT_SYMTAB: + object->symbolTableOffset = dynamic->d_un.d_ptr; + break; + case DT_SYMENT: + __ensure(dynamic->d_un.d_val == sizeof(elf_sym)); + break; + // handle lazy relocation table + case DT_PLTGOT: + object->globalOffsetTable = (void **)(object->baseAddress + + dynamic->d_un.d_ptr); + break; + case DT_JMPREL: + object->lazyRelocTableOffset = dynamic->d_un.d_ptr; + break; + case DT_PLTRELSZ: + object->lazyTableSize = dynamic->d_un.d_val; + break; + case DT_PLTREL: + if(dynamic->d_un.d_val == DT_RELA) { + object->lazyExplicitAddend = true; + }else{ + __ensure(dynamic->d_un.d_val == DT_REL); + object->lazyExplicitAddend = false; + } + break; + // TODO: Implement this correctly! + case DT_SYMBOLIC: + object->symbolicResolution = true; + break; + case DT_BIND_NOW: + object->eagerBinding = true; + break; + case DT_FLAGS: { + if(dynamic->d_un.d_val & DF_SYMBOLIC) + object->symbolicResolution = true; + if(dynamic->d_un.d_val & DF_STATIC_TLS) + object->haveStaticTls = true; + if(dynamic->d_un.d_val & DF_BIND_NOW) + object->eagerBinding = true; + + auto ignored = DF_BIND_NOW | DF_SYMBOLIC | DF_STATIC_TLS; +#ifdef __riscv + // Work around https://sourceware.org/bugzilla/show_bug.cgi?id=24673. + ignored |= DF_TEXTREL; +#else + if(dynamic->d_un.d_val & DF_TEXTREL) + mlibc::panicLogger() << "\e[31mrtdl: DF_TEXTREL is unimplemented" << frg::endlog; +#endif + if(dynamic->d_un.d_val & ~ignored) + mlibc::infoLogger() << "\e[31mrtdl: DT_FLAGS(" << frg::hex_fmt{dynamic->d_un.d_val & ~ignored} + << ") is not implemented correctly!\e[39m" + << frg::endlog; + } break; + case DT_FLAGS_1: + if(dynamic->d_un.d_val & DF_1_NOW) + object->eagerBinding = true; + // The DF_1_PIE flag is informational only. It is used by e.g file(1). + // The DF_1_NODELETE flag has a similar effect to RTLD_NODELETE, both of which we + // ignore because we don't implement dlclose(). + if(dynamic->d_un.d_val & ~(DF_1_NOW | DF_1_PIE | DF_1_NODELETE)) + mlibc::infoLogger() << "\e[31mrtdl: DT_FLAGS_1(" << frg::hex_fmt{dynamic->d_un.d_val} + << ") is not implemented correctly!\e[39m" + << frg::endlog; + break; + case DT_RPATH: + if (runpath_found) { + /* Ignore RPATH if RUNPATH was present. */ + break; + } + [[fallthrough]]; + case DT_RUNPATH: + runpath_found = dynamic->d_tag == DT_RUNPATH; + runpath_offset = dynamic->d_un.d_val; + break; + case DT_INIT: + if(dynamic->d_un.d_ptr != 0) + object->initPtr = (InitFuncPtr)(object->baseAddress + dynamic->d_un.d_ptr); + break; + case DT_INIT_ARRAY: + if(dynamic->d_un.d_ptr != 0) + object->initArray = (InitFuncPtr *)(object->baseAddress + dynamic->d_un.d_ptr); + break; + case DT_INIT_ARRAYSZ: + object->initArraySize = dynamic->d_un.d_val; + break; + case DT_PREINIT_ARRAY: + if(dynamic->d_un.d_ptr != 0) { + // Only the main object is allowed pre-initializers. + __ensure(object->isMainObject); + object->preInitArray = (InitFuncPtr *)(object->baseAddress + dynamic->d_un.d_ptr); + } + break; + case DT_PREINIT_ARRAYSZ: + // Only the main object is allowed pre-initializers. + __ensure(object->isMainObject); + object->preInitArraySize = dynamic->d_un.d_val; + break; + case DT_DEBUG: +#if ELF_CLASS == ELFCLASS32 + dynamic->d_un.d_val = reinterpret_cast<Elf32_Word>(&globalDebugInterface); +#elif ELF_CLASS == ELFCLASS64 + dynamic->d_un.d_val = reinterpret_cast<Elf64_Xword>(&globalDebugInterface); +#endif + break; + case DT_SONAME: + soname_offset = dynamic->d_un.d_val; + break; + // ignore unimportant tags + case DT_NEEDED: // we handle this later + case DT_FINI: case DT_FINI_ARRAY: case DT_FINI_ARRAYSZ: + case DT_RELA: case DT_RELASZ: case DT_RELAENT: case DT_RELACOUNT: + case DT_REL: case DT_RELSZ: case DT_RELENT: case DT_RELCOUNT: + case DT_RELR: case DT_RELRSZ: case DT_RELRENT: + case DT_VERSYM: + case DT_VERDEF: case DT_VERDEFNUM: + case DT_VERNEED: case DT_VERNEEDNUM: +#ifdef __riscv + case DT_TEXTREL: // Work around https://sourceware.org/bugzilla/show_bug.cgi?id=24673. +#endif + break; + case DT_TLSDESC_PLT: case DT_TLSDESC_GOT: + break; + default: + // Ignore unknown entries in the os-specific area as we don't use them. + if(dynamic->d_tag < DT_LOOS || dynamic->d_tag > DT_HIOS) { + mlibc::panicLogger() << "Unexpected dynamic entry " + << (void *)dynamic->d_tag << " in object" << frg::endlog; + } + } + } + + if(runpath_offset) { + object->runPath = reinterpret_cast<const char *>(object->baseAddress + + object->stringTableOffset + *runpath_offset); + } + if(soname_offset) { + object->soName = reinterpret_cast<const char *>(object->baseAddress + + object->stringTableOffset + *soname_offset); + } +} + +void ObjectRepository::_discoverDependencies(SharedObject *object, + Scope *localScope, uint64_t rts) { + if(object->isMainObject) { + for(auto preload : *preloads) { + frg::expected<LinkerError, SharedObject *> libraryResult; + if (preload.find_first('/') == size_t(-1)) { + libraryResult = requestObjectWithName(preload, object, globalScope.get(), false, 1); + } else { + libraryResult = requestObjectAtPath(preload, globalScope.get(), false, 1); + } + if(!libraryResult) + mlibc::panicLogger() << "rtdl: Could not load preload " << preload << frg::endlog; + + if(verbose) + mlibc::infoLogger() << "rtdl: Preloading " << preload << frg::endlog; + + object->dependencies.push_back(libraryResult.value()); + } + } + + // Load required dynamic libraries. + for(size_t i = 0; object->dynamic[i].d_tag != DT_NULL; i++) { + elf_dyn *dynamic = &object->dynamic[i]; + if(dynamic->d_tag != DT_NEEDED) + continue; + + const char *library_str = (const char *)(object->baseAddress + + object->stringTableOffset + dynamic->d_un.d_val); + + auto library = requestObjectWithName(frg::string_view{library_str}, + object, localScope, false, rts); + if(!library) + mlibc::panicLogger() << "Could not satisfy dependency " << library_str << frg::endlog; + object->dependencies.push(library.value()); + } +} + +void ObjectRepository::_addLoadedObject(SharedObject *object) { + _nameMap.insert(object->name, object); + loadedObjects.push_back(object); +} + +// -------------------------------------------------------- +// SharedObject +// -------------------------------------------------------- + +SharedObject::SharedObject(const char *name, frg::string<MemoryAllocator> path, + bool is_main_object, Scope *local_scope, uint64_t object_rts) + : name(name, getAllocator()), path(std::move(path)), + interpreterPath(getAllocator()), soName(nullptr), + isMainObject(is_main_object), objectRts(object_rts), inLinkMap(false), + baseAddress(0), localScope(local_scope), dynamic(nullptr), + globalOffsetTable(nullptr), entry(nullptr), tlsSegmentSize(0), + tlsAlignment(0), tlsImageSize(0), tlsImagePtr(nullptr), + tlsInitialized(false), hashTableOffset(0), symbolTableOffset(0), + stringTableOffset(0), lazyRelocTableOffset(0), lazyTableSize(0), + lazyExplicitAddend(false), symbolicResolution(false), + eagerBinding(false), haveStaticTls(false), + dependencies(getAllocator()), tlsModel(TlsModel::null), + tlsOffset(0), globalRts(0), wasLinked(false), + scheduledForInit(false), onInitStack(false), + wasInitialized(false) { } + +SharedObject::SharedObject(const char *name, const char *path, + bool is_main_object, Scope *localScope, uint64_t object_rts) + : SharedObject(name, + frg::string<MemoryAllocator> { path, getAllocator() }, + is_main_object, localScope, object_rts) {} + +void processLateRelocation(Relocation rel) { + // resolve the symbol if there is a symbol + frg::optional<ObjectSymbol> p; + if(rel.symbol_index()) { + auto symbol = (elf_sym *)(rel.object()->baseAddress + rel.object()->symbolTableOffset + + rel.symbol_index() * sizeof(elf_sym)); + ObjectSymbol r(rel.object(), symbol); + + p = Scope::resolveGlobalOrLocal(*globalScope, rel.object()->localScope, + r.getString(), rel.object()->objectRts, Scope::resolveCopy); + } + + switch(rel.type()) { + case R_COPY: + __ensure(p); + memcpy(rel.destination(), (void *)p->virtualAddress(), p->symbol()->st_size); + break; + +// TODO: R_IRELATIVE also exists on other architectures but will likely need a different implementation. +#if defined(__x86_64__) || defined(__i386__) + case R_IRELATIVE: { + uintptr_t addr = rel.object()->baseAddress + rel.addend_rel(); + auto* fn = reinterpret_cast<uintptr_t (*)()>(addr); + rel.relocate(fn()); + } break; +#elif defined(__aarch64__) + case R_IRELATIVE: { + uintptr_t addr = rel.object()->baseAddress + rel.addend_rel(); + auto* fn = reinterpret_cast<uintptr_t (*)(uint64_t)>(addr); + // TODO: the function should get passed AT_HWCAP value. + rel.relocate(fn(0)); + } break; +#endif + + default: + break; + } +} + +void processLateRelocations(SharedObject *object) { + frg::optional<uintptr_t> rel_offset; + frg::optional<size_t> rel_length; + + frg::optional<uintptr_t> rela_offset; + frg::optional<size_t> rela_length; + + for(size_t i = 0; object->dynamic[i].d_tag != DT_NULL; i++) { + elf_dyn *dynamic = &object->dynamic[i]; + + switch(dynamic->d_tag) { + case DT_REL: + rel_offset = dynamic->d_un.d_ptr; + break; + case DT_RELSZ: + rel_length = dynamic->d_un.d_val; + break; + case DT_RELENT: + __ensure(dynamic->d_un.d_val == sizeof(elf_rel)); + break; + case DT_RELA: + rela_offset = dynamic->d_un.d_ptr; + break; + case DT_RELASZ: + rela_length = dynamic->d_un.d_val; + break; + case DT_RELAENT: + __ensure(dynamic->d_un.d_val == sizeof(elf_rela)); + break; + } + } + + if(rela_offset && rela_length) { + for(size_t offset = 0; offset < *rela_length; offset += sizeof(elf_rela)) { + auto reloc = (elf_rela *)(object->baseAddress + *rela_offset + offset); + auto r = Relocation(object, reloc); + processLateRelocation(r); + } + } else if(rel_offset && rel_length) { + for(size_t offset = 0; offset < *rel_length; offset += sizeof(elf_rel)) { + auto reloc = (elf_rel *)(object->baseAddress + *rel_offset + offset); + auto r = Relocation(object, reloc); + processLateRelocation(r); + } + }else{ + __ensure(!rela_offset && !rela_length); + __ensure(!rel_offset && !rel_length); + } +} + +void doInitialize(SharedObject *object) { + __ensure(object->wasLinked); + __ensure(!object->wasInitialized); + + // if the object has dependencies we initialize them first + for(size_t i = 0; i < object->dependencies.size(); i++) + __ensure(object->dependencies[i]->wasInitialized); + + if(verbose) + mlibc::infoLogger() << "rtdl: Initialize " << object->name << frg::endlog; + + if(verbose) + mlibc::infoLogger() << "rtdl: Running DT_INIT function" << frg::endlog; + if(object->initPtr != nullptr) + object->initPtr(); + + if(verbose) + mlibc::infoLogger() << "rtdl: Running DT_INIT_ARRAY functions" << frg::endlog; + __ensure((object->initArraySize % sizeof(InitFuncPtr)) == 0); + for(size_t i = 0; i < object->initArraySize / sizeof(InitFuncPtr); i++) + object->initArray[i](); + + if(verbose) + mlibc::infoLogger() << "rtdl: Object initialization complete" << frg::endlog; + object->wasInitialized = true; +} + +// -------------------------------------------------------- +// RuntimeTlsMap +// -------------------------------------------------------- + +RuntimeTlsMap::RuntimeTlsMap() +: initialPtr{0}, initialLimit{0}, indices{getAllocator()} { } + +void initTlsObjects(Tcb *tcb, const frg::vector<SharedObject *, MemoryAllocator> &objects, bool checkInitialized) { + // Initialize TLS segments that follow the static model. + for(auto object : objects) { + if(object->tlsModel == TlsModel::initial) { + if(checkInitialized && object->tlsInitialized) + continue; + + char *tcb_ptr = reinterpret_cast<char *>(tcb); + auto tls_ptr = tcb_ptr + object->tlsOffset; + memset(tls_ptr, 0, object->tlsSegmentSize); + memcpy(tls_ptr, object->tlsImagePtr, object->tlsImageSize); + + if (verbose) { + mlibc::infoLogger() << "rtdl: wrote tls image at " << (void *)tls_ptr + << ", size = 0x" << frg::hex_fmt{object->tlsSegmentSize} << frg::endlog; + } + + if (checkInitialized) + object->tlsInitialized = true; + } + } +} + +Tcb *allocateTcb() { + size_t tlsInitialSize = runtimeTlsMap->initialLimit; + + // To make sure that both the TCB and TLS data are sufficiently aligned, allocate + // slightly more than necessary and adjust alignment afterwards. + size_t alignOverhead = frg::max(alignof(Tcb), tlsMaxAlignment); + size_t allocSize = tlsInitialSize + sizeof(Tcb) + alignOverhead; + auto allocation = reinterpret_cast<uintptr_t>(getAllocator().allocate(allocSize)); + memset(reinterpret_cast<void *>(allocation), 0, allocSize); + + uintptr_t tlsAddress, tcbAddress; + if constexpr (tlsAboveTp) { + // Here we must satisfy two requirements of the TCB and the TLS data: + // 1. One should follow the other immediately in memory. We do this so that + // we can simply add or subtract sizeof(Tcb) to obtain the address of the other. + // 2. Both should be sufficiently aligned. + // To do this, we will fix whichever address has stricter alignment requirements, and + // derive the other from it. + if (tlsMaxAlignment > alignof(Tcb)) { + tlsAddress = alignUp(allocation + sizeof(Tcb), tlsMaxAlignment); + tcbAddress = tlsAddress - sizeof(Tcb); + } else { + tcbAddress = alignUp(allocation, alignof(Tcb)); + tlsAddress = tcbAddress + sizeof(Tcb); + } + __ensure((tlsAddress & (tlsMaxAlignment - 1)) == 0); + __ensure(tlsAddress == tcbAddress + sizeof(Tcb)); + } else { + // The TCB should be aligned such that the preceding blocks are aligned too. + tcbAddress = alignUp(allocation + tlsInitialSize, alignOverhead); + tlsAddress = tcbAddress - tlsInitialSize; + } + __ensure((tcbAddress & (alignof(Tcb) - 1)) == 0); + + if (verbose) { + mlibc::infoLogger() << "rtdl: tcb allocated at " << (void *)tcbAddress + << ", size = 0x" << frg::hex_fmt{sizeof(Tcb)} << frg::endlog; + mlibc::infoLogger() << "rtdl: tls allocated at " << (void *)tlsAddress + << ", size = 0x" << frg::hex_fmt{tlsInitialSize} << frg::endlog; + } + + Tcb *tcb_ptr = new ((char *)tcbAddress) Tcb; + tcb_ptr->selfPointer = tcb_ptr; + + tcb_ptr->stackCanary = __stack_chk_guard; + tcb_ptr->cancelBits = tcbCancelEnableBit; + tcb_ptr->didExit = 0; + tcb_ptr->isJoinable = 1; + memset(&tcb_ptr->returnValue, 0, sizeof(tcb_ptr->returnValue)); + tcb_ptr->localKeys = frg::construct<frg::array<Tcb::LocalKey, PTHREAD_KEYS_MAX>>(getAllocator()); + tcb_ptr->dtvSize = runtimeTlsMap->indices.size(); + tcb_ptr->dtvPointers = frg::construct_n<void *>(getAllocator(), runtimeTlsMap->indices.size()); + memset(tcb_ptr->dtvPointers, 0, sizeof(void *) * runtimeTlsMap->indices.size()); + for(size_t i = 0; i < runtimeTlsMap->indices.size(); ++i) { + auto object = runtimeTlsMap->indices[i]; + if(object->tlsModel != TlsModel::initial) + continue; + tcb_ptr->dtvPointers[i] = reinterpret_cast<char *>(tcb_ptr) + object->tlsOffset; + } + + return tcb_ptr; +} + +void *accessDtv(SharedObject *object) { + Tcb *tcb_ptr = mlibc::get_current_tcb(); + + // We might need to reallocate the DTV. + if(object->tlsIndex >= tcb_ptr->dtvSize) { + // TODO: need to protect runtimeTlsMap against concurrent access. + auto ndtv = frg::construct_n<void *>(getAllocator(), runtimeTlsMap->indices.size()); + memset(ndtv, 0, sizeof(void *) * runtimeTlsMap->indices.size()); + memcpy(ndtv, tcb_ptr->dtvPointers, sizeof(void *) * tcb_ptr->dtvSize); + frg::destruct_n(getAllocator(), tcb_ptr->dtvPointers, tcb_ptr->dtvSize); + tcb_ptr->dtvSize = runtimeTlsMap->indices.size(); + tcb_ptr->dtvPointers = ndtv; + } + + // We might need to fill in a new DTV entry. + if(!tcb_ptr->dtvPointers[object->tlsIndex]) { + __ensure(object->tlsModel == TlsModel::dynamic); + + auto buffer = getAllocator().allocate(object->tlsSegmentSize); + __ensure(!(reinterpret_cast<uintptr_t>(buffer) & (object->tlsAlignment - 1))); + memset(buffer, 0, object->tlsSegmentSize); + memcpy(buffer, object->tlsImagePtr, object->tlsImageSize); + tcb_ptr->dtvPointers[object->tlsIndex] = buffer; + + if (verbose) { + mlibc::infoLogger() << "rtdl: accessDtv wrote tls image at " << buffer + << ", size = 0x" << frg::hex_fmt{object->tlsSegmentSize} << frg::endlog; + } + } + + return (void *)((char *)tcb_ptr->dtvPointers[object->tlsIndex] + TLS_DTV_OFFSET); +} + +void *tryAccessDtv(SharedObject *object) { + Tcb *tcb_ptr = mlibc::get_current_tcb(); + + if (object->tlsIndex >= tcb_ptr->dtvSize) + return nullptr; + if (!tcb_ptr->dtvPointers[object->tlsIndex]) + return nullptr; + + return (void *)((char *)tcb_ptr->dtvPointers[object->tlsIndex] + TLS_DTV_OFFSET); +} + +// -------------------------------------------------------- +// ObjectSymbol +// -------------------------------------------------------- + +ObjectSymbol::ObjectSymbol(SharedObject *object, const elf_sym *symbol) +: _object(object), _symbol(symbol) { } + +const char *ObjectSymbol::getString() { + __ensure(_symbol->st_name != 0); + return (const char *)(_object->baseAddress + + _object->stringTableOffset + _symbol->st_name); +} + +uintptr_t ObjectSymbol::virtualAddress() { + auto bind = ELF_ST_BIND(_symbol->st_info); + __ensure(bind == STB_GLOBAL || bind == STB_WEAK || bind == STB_GNU_UNIQUE); + __ensure(_symbol->st_shndx != SHN_UNDEF); + return _object->baseAddress + _symbol->st_value; +} + +// -------------------------------------------------------- +// Scope +// -------------------------------------------------------- + +uint32_t elf64Hash(frg::string_view string) { + uint32_t h = 0, g; + + for(size_t i = 0; i < string.size(); ++i) { + h = (h << 4) + (uint32_t)string[i]; + g = h & 0xF0000000; + if(g) + h ^= g >> 24; + h &= 0x0FFFFFFF; + } + + return h; +} + +uint32_t gnuHash(frg::string_view string) { + uint32_t h = 5381; + for(size_t i = 0; i < string.size(); ++i) + h = (h << 5) + h + string[i]; + return h; +} + +// TODO: move this to some namespace or class? +frg::optional<ObjectSymbol> resolveInObject(SharedObject *object, frg::string_view string) { + // Checks if the symbol can be used to satisfy the dependency. + auto eligible = [&] (ObjectSymbol cand) { + if(cand.symbol()->st_shndx == SHN_UNDEF) + return false; + + auto bind = ELF_ST_BIND(cand.symbol()->st_info); + if(bind != STB_GLOBAL && bind != STB_WEAK && bind != STB_GNU_UNIQUE) + return false; + + return true; + }; + + if (object->hashStyle == HashStyle::systemV) { + auto hash_table = (Elf64_Word *)(object->baseAddress + object->hashTableOffset); + Elf64_Word num_buckets = hash_table[0]; + auto bucket = elf64Hash(string) % num_buckets; + + auto index = hash_table[2 + bucket]; + while(index != 0) { + ObjectSymbol cand{object, (elf_sym *)(object->baseAddress + + object->symbolTableOffset + index * sizeof(elf_sym))}; + if(eligible(cand) && frg::string_view{cand.getString()} == string) + return cand; + + index = hash_table[2 + num_buckets + index]; + } + + return frg::optional<ObjectSymbol>{}; + }else{ + __ensure(object->hashStyle == HashStyle::gnu); + + struct GnuTable { + uint32_t nBuckets; + uint32_t symbolOffset; + uint32_t bloomSize; + uint32_t bloomShift; + }; + + auto hash_table = reinterpret_cast<const GnuTable *>(object->baseAddress + + object->hashTableOffset); + auto buckets = reinterpret_cast<const uint32_t *>(object->baseAddress + + object->hashTableOffset + sizeof(GnuTable) + + hash_table->bloomSize * sizeof(elf_addr)); + auto chains = reinterpret_cast<const uint32_t *>(object->baseAddress + + object->hashTableOffset + sizeof(GnuTable) + + hash_table->bloomSize * sizeof(elf_addr) + + hash_table->nBuckets * sizeof(uint32_t)); + + // TODO: Use the bloom filter. + + // The symbols of a given bucket are contiguous in the table. + auto hash = gnuHash(string); + auto index = buckets[hash % hash_table->nBuckets]; + + if(!index) + return frg::optional<ObjectSymbol>{}; + + while(true) { + // chains[] contains an array of hashes, parallel to the symbol table. + auto chash = chains[index - hash_table->symbolOffset]; + if ((chash & ~1) == (hash & ~1)) { + ObjectSymbol cand{object, (elf_sym *)(object->baseAddress + + object->symbolTableOffset + index * sizeof(elf_sym))}; + if(eligible(cand) && frg::string_view{cand.getString()} == string) + return cand; + } + + // If we hit the end of the chain, the symbol is not present. + if(chash & 1) + return frg::optional<ObjectSymbol>{}; + index++; + } + } +} + +frg::optional<ObjectSymbol> Scope::_resolveNext(frg::string_view string, + SharedObject *target) { + // Skip objects until we find the target, and only look for symbols after that. + size_t i; + for (i = 0; i < _objects.size(); i++) { + if (_objects[i] == target) + break; + } + + if (i == _objects.size()) { + mlibc::infoLogger() << "rtdl: object passed to Scope::resolveAfter was not found" << frg::endlog; + return frg::optional<ObjectSymbol>(); + } + + for (i = i + 1; i < _objects.size(); i++) { + if(_objects[i]->isMainObject) + continue; + + frg::optional<ObjectSymbol> p = resolveInObject(_objects[i], string); + if(p) + return p; + } + + return frg::optional<ObjectSymbol>(); +} + +Scope::Scope(bool isGlobal) +: isGlobal{isGlobal}, _objects(getAllocator()) { } + +void Scope::appendObject(SharedObject *object) { + // Don't insert duplicates. + for (auto obj : _objects) { + if (obj == object) + return; + } + + _objects.push(object); +} + +frg::optional<ObjectSymbol> Scope::resolveGlobalOrLocal(Scope &globalScope, + Scope *localScope, frg::string_view string, uint64_t skipRts, ResolveFlags flags) { + auto sym = globalScope.resolveSymbol(string, skipRts, flags | skipGlobalAfterRts); + if(!sym && localScope) + sym = localScope->resolveSymbol(string, skipRts, flags | skipGlobalAfterRts); + return sym; +} + +frg::optional<ObjectSymbol> Scope::resolveGlobalOrLocalNext(Scope &globalScope, + Scope *localScope, frg::string_view string, SharedObject *origin) { + auto sym = globalScope._resolveNext(string, origin); + if(!sym && localScope) { + sym = localScope->_resolveNext(string, origin); + } + return sym; +} + +// TODO: let this return uintptr_t +frg::optional<ObjectSymbol> Scope::resolveSymbol(frg::string_view string, + uint64_t skipRts, ResolveFlags flags) { + for (auto object : _objects) { + if((flags & resolveCopy) && object->isMainObject) + continue; + if((flags & skipGlobalAfterRts) && object->globalRts > skipRts) { + // globalRts should be monotone increasing for objects in the global scope, + // so as an optimization we can break early here. + // TODO: If we implement DT_SYMBOLIC, this assumption fails. + if(isGlobal) + break; + else + continue; + } + + frg::optional<ObjectSymbol> p = resolveInObject(object, string); + if(p) + return p; + } + + return frg::optional<ObjectSymbol>(); +} + +// -------------------------------------------------------- +// Loader +// -------------------------------------------------------- + +Loader::Loader(Scope *scope, SharedObject *mainExecutable, bool is_initial_link, uint64_t rts) +: _mainExecutable{mainExecutable}, _loadScope{scope}, _isInitialLink{is_initial_link}, + _linkRts{rts}, _linkBfs{getAllocator()}, _initQueue{getAllocator()} { } + +void Loader::_buildLinkBfs(SharedObject *root) { + __ensure(_linkBfs.size() == 0); + + struct Token {}; + using Set = frg::hash_map<SharedObject *, Token, + frg::hash<SharedObject *>, MemoryAllocator>; + Set set{frg::hash<SharedObject *>{}, getAllocator()}; + _linkBfs.push(root); + + // Loop over indices (not iterators) here: We are adding elements in the loop! + for(size_t i = 0; i < _linkBfs.size(); i++) { + auto current = _linkBfs[i]; + + // At this point the object is loaded and we can fill in its debug struct, + // the linked list fields will be filled later. + current->linkMap.base = current->baseAddress; + current->linkMap.name = current->path.data(); + current->linkMap.dynv = current->dynamic; + + __ensure((current->tlsAlignment & (current->tlsAlignment - 1)) == 0); + + if (_isInitialLink && current->tlsAlignment > tlsMaxAlignment) { + tlsMaxAlignment = current->tlsAlignment; + } + + for (auto dep : current->dependencies) { + if (!set.get(dep)) { + set.insert(dep, Token{}); + _linkBfs.push(dep); + } + } + } +} + +void Loader::linkObjects(SharedObject *root) { + _buildLinkBfs(root); + _buildTlsMaps(); + + // Promote objects to the desired scope. + for(auto object : _linkBfs) { + if (object->globalRts == 0 && _loadScope->isGlobal) + object->globalRts = _linkRts; + + _loadScope->appendObject(object); + } + + // Process regular relocations. + for(auto object : _linkBfs) { + // Some objects have already been linked before. + if(object->objectRts < _linkRts) + continue; + + if(object->dynamic == nullptr) + continue; + + if(verbose) + mlibc::infoLogger() << "rtdl: Linking " << object->name << frg::endlog; + + __ensure(!object->wasLinked); + + // TODO: Support this. + if(object->symbolicResolution) + mlibc::infoLogger() << "\e[31mrtdl: DT_SYMBOLIC is not implemented correctly!\e[39m" + << frg::endlog; + + _processStaticRelocations(object); + _processLazyRelocations(object); + } + + // Process copy relocations. + for(auto object : _linkBfs) { + if(!object->isMainObject) + continue; + + // Some objects have already been linked before. + if(object->objectRts < _linkRts) + continue; + + if(object->dynamic == nullptr) + continue; + + processLateRelocations(object); + } + + for(auto object : _linkBfs) { + object->wasLinked = true; + + if(object->inLinkMap) + continue; + + auto linkMap = reinterpret_cast<LinkMap*>(globalDebugInterface.head); + + object->linkMap.prev = linkMap; + object->linkMap.next = linkMap->next; + if(linkMap->next) + linkMap->next->prev = &(object->linkMap); + linkMap->next = &(object->linkMap); + object->inLinkMap = true; + } +} + +void Loader::_buildTlsMaps() { + if(_isInitialLink) { + __ensure(runtimeTlsMap->initialPtr == 0); + __ensure(runtimeTlsMap->initialLimit == 0); + + __ensure(!_linkBfs.empty()); + __ensure(_linkBfs.front()->isMainObject); + + for(auto object : _linkBfs) { + __ensure(object->tlsModel == TlsModel::null); + + if(object->tlsSegmentSize == 0) + continue; + + // Allocate an index for the object. + object->tlsIndex = runtimeTlsMap->indices.size(); + runtimeTlsMap->indices.push_back(object); + + object->tlsModel = TlsModel::initial; + + if constexpr (tlsAboveTp) { + // As per the comment in allocateTcb(), we may simply add sizeof(Tcb) to + // reach the TLS data. + object->tlsOffset = runtimeTlsMap->initialPtr + sizeof(Tcb); + runtimeTlsMap->initialPtr += object->tlsSegmentSize; + + size_t misalign = runtimeTlsMap->initialPtr & (object->tlsAlignment - 1); + if(misalign) + runtimeTlsMap->initialPtr += object->tlsAlignment - misalign; + } else { + runtimeTlsMap->initialPtr += object->tlsSegmentSize; + + size_t misalign = runtimeTlsMap->initialPtr & (object->tlsAlignment - 1); + if(misalign) + runtimeTlsMap->initialPtr += object->tlsAlignment - misalign; + + object->tlsOffset = -runtimeTlsMap->initialPtr; + } + + if(verbose) + mlibc::infoLogger() << "rtdl: TLS of " << object->name + << " mapped to 0x" << frg::hex_fmt{object->tlsOffset} + << ", size: " << object->tlsSegmentSize + << ", alignment: " << object->tlsAlignment << frg::endlog; + } + + // Reserve some additional space for future libraries. + runtimeTlsMap->initialLimit = runtimeTlsMap->initialPtr + 64; + }else{ + for(auto object : _linkBfs) { + if(object->tlsModel != TlsModel::null) + continue; + if(object->tlsSegmentSize == 0) + continue; + + // Allocate an index for the object. + object->tlsIndex = runtimeTlsMap->indices.size(); + runtimeTlsMap->indices.push_back(object); + + // There are some libraries (e.g. Mesa) that require static TLS even though + // they expect to be dynamically loaded. + if(object->haveStaticTls) { + auto ptr = runtimeTlsMap->initialPtr + object->tlsSegmentSize; + size_t misalign = ptr & (object->tlsAlignment - 1); + if(misalign) + ptr += object->tlsAlignment - misalign; + + if(ptr > runtimeTlsMap->initialLimit) + mlibc::panicLogger() << "rtdl: Static TLS space exhausted while while" + " allocating TLS for " << object->name << frg::endlog; + + object->tlsModel = TlsModel::initial; + + if constexpr (tlsAboveTp) { + size_t tcbSize = ((sizeof(Tcb) + tlsMaxAlignment - 1) & ~(tlsMaxAlignment - 1)); + + object->tlsOffset = runtimeTlsMap->initialPtr + tcbSize; + runtimeTlsMap->initialPtr = ptr; + } else { + runtimeTlsMap->initialPtr = ptr; + object->tlsOffset = -runtimeTlsMap->initialPtr; + } + + if(verbose) + mlibc::infoLogger() << "rtdl: TLS of " << object->name + << " mapped to 0x" << frg::hex_fmt{object->tlsOffset} + << ", size: " << object->tlsSegmentSize + << ", alignment: " << object->tlsAlignment << frg::endlog; + }else{ + object->tlsModel = TlsModel::dynamic; + } + } + } +} + +void Loader::initObjects() { + initTlsObjects(mlibc::get_current_tcb(), _linkBfs, true); + + if (_mainExecutable && _mainExecutable->preInitArray) { + if (verbose) + mlibc::infoLogger() << "rtdl: Running DT_PREINIT_ARRAY functions" << frg::endlog; + + __ensure(_mainExecutable->isMainObject); + __ensure(!_mainExecutable->wasInitialized); + __ensure((_mainExecutable->preInitArraySize % sizeof(InitFuncPtr)) == 0); + for(size_t i = 0; i < _mainExecutable->preInitArraySize / sizeof(InitFuncPtr); i++) + _mainExecutable->preInitArray[i](); + } + + // Convert the breadth-first representation to a depth-first post-order representation, + // so that every object is initialized *after* its dependencies. + for(auto object : _linkBfs) { + if(!object->scheduledForInit) + _scheduleInit(object); + } + + for(auto object : _initQueue) { + if(!object->wasInitialized) + doInitialize(object); + } +} + +// TODO: Use an explicit vector to reduce stack usage to O(1)? +void Loader::_scheduleInit(SharedObject *object) { + // Here we detect cyclic dependencies. + __ensure(!object->onInitStack); + object->onInitStack = true; + + __ensure(!object->scheduledForInit); + object->scheduledForInit = true; + + for(size_t i = 0; i < object->dependencies.size(); i++) { + if(!object->dependencies[i]->scheduledForInit) + _scheduleInit(object->dependencies[i]); + } + + _initQueue.push(object); + object->onInitStack = false; +} + +void Loader::_processRelocations(Relocation &rel) { + // copy and irelative relocations have to be performed after all other relocations + if(rel.type() == R_COPY || rel.type() == R_IRELATIVE) + return; + + // resolve the symbol if there is a symbol + frg::optional<ObjectSymbol> p; + if(rel.symbol_index()) { + auto symbol = (elf_sym *)(rel.object()->baseAddress + rel.object()->symbolTableOffset + + rel.symbol_index() * sizeof(elf_sym)); + ObjectSymbol r(rel.object(), symbol); + + p = Scope::resolveGlobalOrLocal(*globalScope, rel.object()->localScope, + r.getString(), rel.object()->objectRts, 0); + if(!p) { + if(ELF_ST_BIND(symbol->st_info) != STB_WEAK) + mlibc::panicLogger() << "Unresolved load-time symbol " + << r.getString() << " in object " << rel.object()->name << frg::endlog; + + if(verbose) + mlibc::infoLogger() << "rtdl: Unresolved weak load-time symbol " + << r.getString() << " in object " << rel.object()->name << frg::endlog; + } + } + + switch(rel.type()) { + case R_NONE: + break; + + case R_JUMP_SLOT: { + __ensure(!rel.addend_norel()); + uintptr_t symbol_addr = p ? p->virtualAddress() : 0; + rel.relocate(symbol_addr); + } break; + +#if !defined(__riscv) + // on some architectures, R_GLOB_DAT can be defined to other relocations + case R_GLOB_DAT: { + __ensure(rel.symbol_index()); + uintptr_t symbol_addr = p ? p->virtualAddress() : 0; + rel.relocate(symbol_addr + rel.addend_norel()); + } break; +#endif + + case R_ABSOLUTE: { + __ensure(rel.symbol_index()); + uintptr_t symbol_addr = p ? p->virtualAddress() : 0; + rel.relocate(symbol_addr + rel.addend_rel()); + } break; + + case R_RELATIVE: { + __ensure(!rel.symbol_index()); + rel.relocate(rel.object()->baseAddress + rel.addend_rel()); + } break; + + // DTPMOD and DTPREL are dynamic TLS relocations (for __tls_get_addr()). + // TPOFF is a relocation to the initial TLS model. + case R_TLS_DTPMOD: { + // sets the first `sizeof(uintptr_t)` bytes of `struct __abi_tls_entry` + // this means that we can just use the `SharedObject *` to resolve whatever we need + __ensure(!rel.addend_rel()); + if(rel.symbol_index()) { + __ensure(p); + rel.relocate(elf_addr(p->object())); + }else{ + if(stillSlightlyVerbose) + mlibc::infoLogger() << "rtdl: Warning: TLS_DTPMOD64 with no symbol in object " + << rel.object()->name << frg::endlog; + rel.relocate(elf_addr(rel.object())); + } + } break; + case R_TLS_DTPREL: { + __ensure(rel.symbol_index()); + __ensure(p); + rel.relocate(p->symbol()->st_value + rel.addend_rel() - TLS_DTV_OFFSET); + } break; + case R_TLS_TPREL: { + uintptr_t off = rel.addend_rel(); + uintptr_t tls_offset = 0; + + if(rel.symbol_index()) { + __ensure(p); + if(p->object()->tlsModel != TlsModel::initial) + mlibc::panicLogger() << "rtdl: In object " << rel.object()->name + << ": Static TLS relocation to symbol " << p->getString() + << " in dynamically loaded object " + << p->object()->name << frg::endlog; + off += p->symbol()->st_value; + tls_offset = p->object()->tlsOffset; + }else{ + if(stillSlightlyVerbose) + mlibc::infoLogger() << "rtdl: Warning: TPOFF64 with no symbol" + " in object " << rel.object()->name << frg::endlog; + if(rel.object()->tlsModel != TlsModel::initial) + mlibc::panicLogger() << "rtdl: In object " << rel.object()->name + << ": Static TLS relocation to dynamically loaded object " + << rel.object()->name << frg::endlog; + tls_offset = rel.object()->tlsOffset; + } + + if constexpr (tlsAboveTp) { + off += tls_offset - sizeof(Tcb); + } else { + off += tls_offset; + } + + rel.relocate(off); + } break; + default: + mlibc::panicLogger() << "Unexpected relocation type " + << (void *) rel.type() << frg::endlog; + } +} + +void Loader::_processStaticRelocations(SharedObject *object) { + frg::optional<uintptr_t> rela_offset; + frg::optional<size_t> rela_length; + + frg::optional<uintptr_t> rel_offset; + frg::optional<size_t> rel_length; + + frg::optional<uintptr_t> relr_offset; + frg::optional<size_t> relr_length; + + for(size_t i = 0; object->dynamic[i].d_tag != DT_NULL; i++) { + elf_dyn *dynamic = &object->dynamic[i]; + + switch(dynamic->d_tag) { + case DT_RELA: + rela_offset = dynamic->d_un.d_ptr; + break; + case DT_RELASZ: + rela_length = dynamic->d_un.d_val; + break; + case DT_RELAENT: + __ensure(dynamic->d_un.d_val == sizeof(elf_rela)); + break; + case DT_REL: + rel_offset = dynamic->d_un.d_ptr; + break; + case DT_RELSZ: + rel_length = dynamic->d_un.d_val; + break; + case DT_RELENT: + __ensure(dynamic->d_un.d_val == sizeof(elf_rel)); + break; + case DT_RELR: + relr_offset = dynamic->d_un.d_ptr; + break; + case DT_RELRSZ: + relr_length = dynamic->d_un.d_val; + break; + case DT_RELRENT: + __ensure(dynamic->d_un.d_val == sizeof(elf_relr)); + break; + } + } + + if(rela_offset && rela_length) { + __ensure(!rel_offset && !rel_length); + + for(size_t offset = 0; offset < *rela_length; offset += sizeof(elf_rela)) { + auto reloc = (elf_rela *)(object->baseAddress + *rela_offset + offset); + auto r = Relocation(object, reloc); + + _processRelocations(r); + } + }else if(rel_offset && rel_length) { + __ensure(!rela_offset && !rela_length); + + for(size_t offset = 0; offset < *rel_length; offset += sizeof(elf_rel)) { + auto reloc = (elf_rel *)(object->baseAddress + *rel_offset + offset); + auto r = Relocation(object, reloc); + + _processRelocations(r); + } + } + + if(relr_offset && relr_length) { + elf_addr *addr = nullptr; + + for(size_t offset = 0; offset < *relr_length; offset += sizeof(elf_relr)) { + auto entry = *(elf_relr *)(object->baseAddress + *relr_offset + offset); + + // Even entry indicates the beginning address. + if(!(entry & 1)) { + addr = (elf_addr *)(object->baseAddress + entry); + __ensure(addr); + *addr++ += object->baseAddress; + }else { + // Odd entry indicates entry is a bitmap of the subsequent locations to be relocated. + for(int i = 0; entry; ++i) { + if(entry & 1) { + addr[i] += object->baseAddress; + } + entry >>= 1; + } + + // Each entry describes at max 63 (on 64bit) or 31 (on 32bit) subsequent locations. + addr += CHAR_BIT * sizeof(elf_relr) - 1; + } + } + } +} + +// TODO: TLSDESC relocations aren't aarch64 specific +#ifdef __aarch64__ +extern "C" void *__mlibcTlsdescStatic(void *); +extern "C" void *__mlibcTlsdescDynamic(void *); +#endif + +void Loader::_processLazyRelocations(SharedObject *object) { + if(object->globalOffsetTable == nullptr) { + __ensure(object->lazyRelocTableOffset == 0); + return; + } + object->globalOffsetTable[1] = object; + object->globalOffsetTable[2] = (void *)&pltRelocateStub; + + if(!object->lazyTableSize) + return; + + // adjust the addresses of JUMP_SLOT relocations + __ensure(object->lazyExplicitAddend.has_value()); + size_t rel_size = (*object->lazyExplicitAddend) ? sizeof(elf_rela) : sizeof(elf_rel); + + for(size_t offset = 0; offset < object->lazyTableSize; offset += rel_size) { + elf_info type; + elf_info symbol_index; + + uintptr_t rel_addr; + uintptr_t addend [[maybe_unused]] = 0; + + if(*object->lazyExplicitAddend) { + auto reloc = (elf_rela *)(object->baseAddress + object->lazyRelocTableOffset + offset); + type = ELF_R_TYPE(reloc->r_info); + symbol_index = ELF_R_SYM(reloc->r_info); + rel_addr = object->baseAddress + reloc->r_offset; + addend = reloc->r_addend; + } else { + auto reloc = (elf_rel *)(object->baseAddress + object->lazyRelocTableOffset + offset); + type = ELF_R_TYPE(reloc->r_info); + symbol_index = ELF_R_SYM(reloc->r_info); + rel_addr = object->baseAddress + reloc->r_offset; + } + + switch (type) { + case R_JUMP_SLOT: + if(eagerBinding) { + auto symbol = (elf_sym *)(object->baseAddress + object->symbolTableOffset + + symbol_index * sizeof(elf_sym)); + ObjectSymbol r(object, symbol); + auto p = Scope::resolveGlobalOrLocal(*globalScope, object->localScope, r.getString(), object->objectRts, 0); + + if(!p) { + if(ELF_ST_BIND(symbol->st_info) != STB_WEAK) + mlibc::panicLogger() << "rtdl: Unresolved JUMP_SLOT symbol " + << r.getString() << " in object " << object->name << frg::endlog; + + if(verbose) + mlibc::infoLogger() << "rtdl: Unresolved weak JUMP_SLOT symbol " + << r.getString() << " in object " << object->name << frg::endlog; + *((uintptr_t *)rel_addr) = 0; + }else{ + *((uintptr_t *)rel_addr) = p->virtualAddress(); + } + }else{ + *((uintptr_t *)rel_addr) += object->baseAddress; + } + break; +#if defined(__x86_64__) + case R_X86_64_IRELATIVE: { + auto ptr = object->baseAddress + addend; + auto target = reinterpret_cast<uintptr_t (*)(void)>(ptr)(); + *((uintptr_t *)rel_addr) = target; + break; + } +#endif +// TODO: TLSDESC relocations aren't aarch64 specific +#if defined(__aarch64__) + case R_AARCH64_TLSDESC: { + size_t symValue = 0; + SharedObject *target = nullptr; + + if (symbol_index) { + auto symbol = (elf_sym *)(object->baseAddress + object->symbolTableOffset + + symbol_index * sizeof(elf_sym)); + ObjectSymbol r(object, symbol); + auto p = Scope::resolveGlobalOrLocal(*globalScope, object->localScope, r.getString(), object->objectRts, 0); + + if (!p) { + __ensure(ELF_ST_BIND(symbol->st_info) != STB_WEAK); + mlibc::panicLogger() << "rtdl: Unresolved TLSDESC for symbol " + << r.getString() << " in object " << object->name << frg::endlog; + } else { + target = p->object(); + if (p->symbol()) + symValue = p->symbol()->st_value; + } + } else { + target = object; + } + + __ensure(target); + + if (target->tlsModel == TlsModel::initial) { + ((uint64_t *)rel_addr)[0] = reinterpret_cast<uintptr_t>(&__mlibcTlsdescStatic); + // TODO: guard the subtraction of TCB size with `if constexpr (tlsAboveTp)` + // for the arch-generic case + __ensure(tlsAboveTp == true); + ((uint64_t *)rel_addr)[1] = symValue + target->tlsOffset + addend - sizeof(Tcb); + } else { + struct TlsdescData { + uintptr_t tlsIndex; + uintptr_t addend; + }; + + // Access DTV for object to force the entry to be allocated and initialized + accessDtv(target); + + __ensure(target->tlsIndex < mlibc::get_current_tcb()->dtvSize); + + // TODO: We should free this when the DSO gets destroyed + auto data = frg::construct<TlsdescData>(getAllocator()); + data->tlsIndex = target->tlsIndex; + data->addend = symValue + addend; + + ((uint64_t *)rel_addr)[0] = reinterpret_cast<uintptr_t>(&__mlibcTlsdescDynamic); + ((uint64_t *)rel_addr)[1] = reinterpret_cast<uintptr_t>(data); + } + } break; +#endif + default: + mlibc::panicLogger() << "unimplemented lazy relocation type " << type << frg::endlog; + break; + } + } +} + diff --git a/lib/mlibc/options/rtdl/generic/linker.hpp b/lib/mlibc/options/rtdl/generic/linker.hpp new file mode 100644 index 0000000..ad84ca9 --- /dev/null +++ b/lib/mlibc/options/rtdl/generic/linker.hpp @@ -0,0 +1,402 @@ + +#include <frg/hash_map.hpp> +#include <frg/optional.hpp> +#include <frg/string.hpp> +#include <frg/vector.hpp> +#include <frg/expected.hpp> +#include <mlibc/allocator.hpp> +#include <mlibc/tcb.hpp> + +#include "elf.hpp" + +struct ObjectRepository; +struct Scope; +struct Loader; +struct SharedObject; + +extern uint64_t rtsCounter; + +enum class TlsModel { + null, + initial, + dynamic +}; + +enum class LinkerError { + success, + notFound, + fileTooShort, + notElf, + wrongElfType, + outOfMemory, + invalidProgramHeader +}; + +// -------------------------------------------------------- +// ObjectRepository +// -------------------------------------------------------- + +struct ObjectRepository { + ObjectRepository(); + + ObjectRepository(const ObjectRepository &) = delete; + + ObjectRepository &operator= (const ObjectRepository &) = delete; + + // This is primarily used to create a SharedObject for the RTDL itself. + SharedObject *injectObjectFromDts(frg::string_view name, + frg::string<MemoryAllocator> path, + uintptr_t base_address, elf_dyn *dynamic, uint64_t rts); + + // This is used to create a SharedObject for the executable that we want to link. + SharedObject *injectObjectFromPhdrs(frg::string_view name, + frg::string<MemoryAllocator> path, void *phdr_pointer, + size_t phdr_entry_size, size_t num_phdrs, void *entry_pointer, + uint64_t rts); + + SharedObject *injectStaticObject(frg::string_view name, + frg::string<MemoryAllocator> path, void *phdr_pointer, + size_t phdr_entry_size, size_t num_phdrs, void *entry_pointer, + uint64_t rts); + + frg::expected<LinkerError, SharedObject *> requestObjectWithName(frg::string_view name, + SharedObject *origin, Scope *localScope, bool createScope, uint64_t rts); + + frg::expected<LinkerError, SharedObject *> requestObjectAtPath(frg::string_view path, + Scope *localScope, bool createScope, uint64_t rts); + + SharedObject *findCaller(void *address); + + SharedObject *findLoadedObject(frg::string_view name); + + // Used by dl_iterate_phdr: stores objects in the order they are loaded. + frg::vector<SharedObject *, MemoryAllocator> loadedObjects; + +private: + void _fetchFromPhdrs(SharedObject *object, void *phdr_pointer, + size_t phdr_entry_size, size_t num_phdrs, void *entry_pointer); + + frg::expected<LinkerError, void> _fetchFromFile(SharedObject *object, int fd); + + void _parseDynamic(SharedObject *object); + + void _discoverDependencies(SharedObject *object, Scope *localScope, uint64_t rts); + + void _addLoadedObject(SharedObject *object); + + frg::hash_map<frg::string_view, SharedObject *, + frg::hash<frg::string_view>, MemoryAllocator> _nameMap; +}; + +// -------------------------------------------------------- +// SharedObject +// -------------------------------------------------------- + +enum class HashStyle { + none, + systemV, + gnu +}; + +using InitFuncPtr = void (*)(); + +// The ABI of this struct is fixed by GDB +struct DebugInterface { + int ver; + void *head; + void (*brk)(void); + int state; + void *base; +}; + +// The ABI of this struct is fixed by GDB +struct LinkMap { + uintptr_t base = 0; + const char *name = nullptr; + elf_dyn *dynv = nullptr; + LinkMap *next = nullptr, *prev = nullptr; +}; + +struct SharedObject { + // path is copied + SharedObject(const char *name, frg::string<MemoryAllocator> path, + bool is_main_object, Scope *localScope, uint64_t object_rts); + + SharedObject(const char *name, const char *path, bool is_main_object, + Scope *localScope, uint64_t object_rts); + + frg::string<MemoryAllocator> name; + frg::string<MemoryAllocator> path; + frg::string<MemoryAllocator> interpreterPath; + const char *soName; + bool isMainObject; + uint64_t objectRts; + + // link map for debugging + LinkMap linkMap; + bool inLinkMap; + + // base address this shared object was loaded to + uintptr_t baseAddress; + + Scope *localScope; + + // pointers to the dynamic table, GOT and entry point + elf_dyn *dynamic = nullptr; + void **globalOffsetTable; + void *entry; + + // object initialization information + InitFuncPtr initPtr = nullptr; + InitFuncPtr *initArray = nullptr; + InitFuncPtr *preInitArray = nullptr; + size_t initArraySize = 0; + size_t preInitArraySize = 0; + + + // TODO: read this from the PHDR + size_t tlsSegmentSize, tlsAlignment, tlsImageSize; + void *tlsImagePtr; + bool tlsInitialized; + + // symbol and string table of this shared object + HashStyle hashStyle = HashStyle::none; + uintptr_t hashTableOffset; + uintptr_t symbolTableOffset; + uintptr_t stringTableOffset; + + const char *runPath = nullptr; + + // save the lazy JUMP_SLOT relocation table + uintptr_t lazyRelocTableOffset; + size_t lazyTableSize; + frg::optional<bool> lazyExplicitAddend; + + bool symbolicResolution; + bool eagerBinding; + bool haveStaticTls; + + // vector of dependencies + frg::vector<SharedObject *, MemoryAllocator> dependencies; + + TlsModel tlsModel; + size_t tlsIndex; + size_t tlsOffset; + + uint64_t globalRts; + bool wasLinked; + + bool scheduledForInit; + bool onInitStack; + bool wasInitialized; + + // PHDR related stuff, we only set these for the main executable + void *phdrPointer = nullptr; + size_t phdrEntrySize = 0; + size_t phdrCount = 0; +}; + +struct Relocation { + Relocation(SharedObject *object, elf_rela *r) + : object_{object}, type_{Addend::Explicit} { + offset_ = r->r_offset; + info_ = r->r_info; + addend_ = r->r_addend; + } + + Relocation(SharedObject *object, elf_rel *r) + : object_{object}, type_{Addend::Implicit} { + offset_ = r->r_offset; + info_ = r->r_info; + } + + SharedObject *object() { + return object_; + } + + elf_info type() const { + return ELF_R_TYPE(info_); + } + + elf_info symbol_index() const { + return ELF_R_SYM(info_); + } + + elf_addr addend_rel() { + switch(type_) { + case Addend::Explicit: + return addend_; + case Addend::Implicit: { + auto ptr = reinterpret_cast<elf_addr *>(object_->baseAddress + offset_); + return *ptr; + } + } + __builtin_unreachable(); + } + + elf_addr addend_norel() { + switch(type_) { + case Addend::Explicit: + return addend_; + case Addend::Implicit: + return 0; + } + __builtin_unreachable(); + } + + void *destination() { + return reinterpret_cast<void *>(object_->baseAddress + offset_); + } + + void relocate(elf_addr addr) { + auto ptr = destination(); + memcpy(ptr, &addr, sizeof(addr)); + } + +private: + enum class Addend { + Implicit, + Explicit + }; + + SharedObject *object_; + Addend type_; + + elf_addr offset_; + elf_info info_; + elf_addend addend_ = 0; +}; + +void processCopyRelocations(SharedObject *object); + +// -------------------------------------------------------- +// RuntimeTlsMap +// -------------------------------------------------------- + +struct RuntimeTlsMap { + RuntimeTlsMap(); + + // Amount of initialLimit that has already been allocated. + size_t initialPtr; + + // Size of the inital TLS segment. + size_t initialLimit; + + // TLS indices. + frg::vector<SharedObject *, MemoryAllocator> indices; +}; + +extern frg::manual_box<RuntimeTlsMap> runtimeTlsMap; + +Tcb *allocateTcb(); +void initTlsObjects(Tcb *tcb, const frg::vector<SharedObject *, MemoryAllocator> &objects, bool checkInitialized); +void *accessDtv(SharedObject *object); +// Tries to access the DTV, if not allocated, or object doesn't have +// PT_TLS, return nullptr. +void *tryAccessDtv(SharedObject *object); + +// -------------------------------------------------------- +// ObjectSymbol +// -------------------------------------------------------- + +struct ObjectSymbol { + ObjectSymbol(SharedObject *object, const elf_sym *symbol); + + SharedObject *object() { + return _object; + } + + const elf_sym *symbol() { + return _symbol; + } + + const char *getString(); + + uintptr_t virtualAddress(); + +private: + SharedObject *_object; + const elf_sym *_symbol; +}; + +frg::optional<ObjectSymbol> resolveInObject(SharedObject *object, frg::string_view string); + +// -------------------------------------------------------- +// Scope +// -------------------------------------------------------- + +struct Scope { + using ResolveFlags = uint32_t; + static inline constexpr ResolveFlags resolveCopy = 1; + static inline constexpr ResolveFlags skipGlobalAfterRts = 1 << 1; + + static frg::optional<ObjectSymbol> resolveGlobalOrLocal(Scope &globalScope, + Scope *localScope, frg::string_view string, uint64_t skipRts, ResolveFlags flags); + static frg::optional<ObjectSymbol> resolveGlobalOrLocalNext(Scope &globalScope, + Scope *localScope, frg::string_view string, SharedObject *origin); + + Scope(bool isGlobal = false); + + void appendObject(SharedObject *object); + + frg::optional<ObjectSymbol> resolveSymbol(frg::string_view string, uint64_t skipRts, ResolveFlags flags); + + bool isGlobal; + +private: + frg::optional<ObjectSymbol> _resolveNext(frg::string_view string, SharedObject *target); +public: // TODO: Make this private again. (Was made public for __dlapi_reverse()). + frg::vector<SharedObject *, MemoryAllocator> _objects; +}; + +extern frg::manual_box<Scope> globalScope; + +// -------------------------------------------------------- +// Loader +// -------------------------------------------------------- + +class Loader { +public: + Loader(Scope *scope, SharedObject *mainExecutable, bool is_initial_link, uint64_t rts); + +public: + void linkObjects(SharedObject *root); + +private: + void _buildLinkBfs(SharedObject *root); + void _buildTlsMaps(); + + void _processStaticRelocations(SharedObject *object); + void _processLazyRelocations(SharedObject *object); + + void _processRelocations(Relocation &rel); + +public: + void initObjects(); + +private: + void _scheduleInit(SharedObject *object); + +private: + SharedObject *_mainExecutable; + Scope *_loadScope; + bool _isInitialLink; + uint64_t _linkRts; + + frg::vector<SharedObject *, MemoryAllocator> _linkBfs; + + frg::vector<SharedObject *, MemoryAllocator> _initQueue; +}; + +// -------------------------------------------------------- +// Namespace scope functions +// -------------------------------------------------------- + +extern "C" void pltRelocateStub() __attribute__((__visibility__("hidden"))); + +// -------------------------------------------------------- +// RTDL interface +// -------------------------------------------------------- + +void *rtdl_auxvector(); + diff --git a/lib/mlibc/options/rtdl/generic/main.cpp b/lib/mlibc/options/rtdl/generic/main.cpp new file mode 100644 index 0000000..3cff1e4 --- /dev/null +++ b/lib/mlibc/options/rtdl/generic/main.cpp @@ -0,0 +1,844 @@ + +#include <elf.h> +#include <link.h> + +#include <frg/manual_box.hpp> +#include <frg/small_vector.hpp> + +#include <abi-bits/auxv.h> +#include <mlibc/debug.hpp> +#include <mlibc/rtdl-sysdeps.hpp> +#include <mlibc/rtdl-config.hpp> +#include <mlibc/rtdl-abi.hpp> +#include <mlibc/stack_protector.hpp> +#include <internal-config.h> +#include <abi-bits/auxv.h> + +#include "elf.hpp" +#include "linker.hpp" + +#if __MLIBC_POSIX_OPTION +#include <dlfcn.h> +#endif + +#define HIDDEN __attribute__((__visibility__("hidden"))) +#define EXPORT __attribute__((__visibility__("default"))) + +static constexpr bool logEntryExit = false; +static constexpr bool logStartup = false; +static constexpr bool logDlCalls = false; + +#ifndef MLIBC_STATIC_BUILD +extern HIDDEN void *_GLOBAL_OFFSET_TABLE_[]; +extern HIDDEN elf_dyn _DYNAMIC[]; +#endif + +namespace mlibc { + // Declared in options/internal/mlibc/tcb.hpp. + bool tcb_available_flag = false; +} + +mlibc::RtdlConfig rtdlConfig; + +bool ldShowAuxv = false; + +uintptr_t *entryStack; +frg::manual_box<ObjectRepository> initialRepository; +frg::manual_box<Scope> globalScope; + +frg::manual_box<RuntimeTlsMap> runtimeTlsMap; + +// We use a small vector of size 4 to avoid memory allocation for the default library paths +frg::manual_box<frg::small_vector<frg::string_view, 4, MemoryAllocator>> libraryPaths; + +frg::manual_box<frg::vector<frg::string_view, MemoryAllocator>> preloads; + +static SharedObject *executableSO; +extern HIDDEN char __ehdr_start[]; + +// Global debug interface variable +DebugInterface globalDebugInterface; + +#ifndef MLIBC_STATIC_BUILD + +// Use a PC-relative instruction sequence to find our runtime load address. +uintptr_t getLdsoBase() { +#if defined(__x86_64__) || defined(__i386__) || defined(__aarch64__) + // On x86_64, the first GOT entry holds the link-time address of _DYNAMIC. + // TODO: This isn't guaranteed on AArch64, so this might fail with some linkers. + auto linktime_dynamic = reinterpret_cast<uintptr_t>(_GLOBAL_OFFSET_TABLE_[0]); + auto runtime_dynamic = reinterpret_cast<uintptr_t>(_DYNAMIC); + return runtime_dynamic - linktime_dynamic; +#elif defined(__riscv) + return reinterpret_cast<uintptr_t>(&__ehdr_start); +#endif +} + +// Relocates the dynamic linker (i.e. this DSO) itself. +// Assumptions: +// - There are no references to external symbols. +// Note that this code is fragile in the sense that it must not contain relocations itself. +// TODO: Use tooling to verify this at compile time. +extern "C" void relocateSelf() { + size_t rela_offset = 0; + size_t rela_size = 0; + size_t rel_offset = 0; + size_t rel_size = 0; + size_t relr_offset = 0; + size_t relr_size = 0; + for(size_t i = 0; _DYNAMIC[i].d_tag != DT_NULL; i++) { + auto ent = &_DYNAMIC[i]; + switch(ent->d_tag) { + case DT_REL: rel_offset = ent->d_un.d_ptr; break; + case DT_RELSZ: rel_size = ent->d_un.d_val; break; + case DT_RELA: rela_offset = ent->d_un.d_ptr; break; + case DT_RELASZ: rela_size = ent->d_un.d_val; break; + case DT_RELR: relr_offset = ent->d_un.d_ptr; break; + case DT_RELRSZ: relr_size = ent->d_un.d_val; break; + } + } + + auto ldso_base = getLdsoBase(); + + __ensure((rel_offset != 0) ^ (rela_offset != 0)); + + for(size_t disp = 0; disp < rela_size; disp += sizeof(elf_rela)) { + auto reloc = reinterpret_cast<elf_rela *>(ldso_base + rela_offset + disp); + + auto type = ELF_R_TYPE(reloc->r_info); + if(ELF_R_SYM(reloc->r_info)) + __builtin_trap(); + + auto p = reinterpret_cast<uint64_t *>(ldso_base + reloc->r_offset); + switch(type) { + case R_RELATIVE: + *p = ldso_base + reloc->r_addend; + break; + default: + __builtin_trap(); + } + } + + for(size_t disp = 0; disp < rel_size; disp += sizeof(elf_rel)) { + auto reloc = reinterpret_cast<elf_rel *>(ldso_base + rel_offset + disp); + + auto type = ELF_R_TYPE(reloc->r_info); + if(ELF_R_SYM(reloc->r_info)) + __builtin_trap(); + + auto p = reinterpret_cast<uint64_t *>(ldso_base + reloc->r_offset); + switch(type) { + case R_RELATIVE: + *p += ldso_base; + break; + default: + __builtin_trap(); + } + } + + elf_addr *addr = nullptr; + for(size_t disp = 0; disp < relr_size; disp += sizeof(elf_relr)) { + auto entry = *(elf_relr *)(ldso_base + relr_offset + disp); + + // Even entry indicates the beginning address. + if(!(entry & 1)) { + addr = (elf_addr *)(ldso_base + entry); + __ensure(addr); + *addr++ += ldso_base; + }else { + // Odd entry indicates entry is a bitmap of the subsequent locations to be relocated. + for(int i = 0; entry; ++i) { + if(entry & 1) { + addr[i] += ldso_base; + } + entry >>= 1; + } + + // Each entry describes at max 63 (on 64bit) or 31 (on 32bit) subsequent locations. + addr += CHAR_BIT * sizeof(elf_relr) - 1; + } + } +} +#endif + +extern "C" void *lazyRelocate(SharedObject *object, unsigned int rel_index) { + __ensure(object->lazyExplicitAddend); + auto reloc = (elf_rela *)(object->baseAddress + object->lazyRelocTableOffset + + rel_index * sizeof(elf_rela)); + auto type = ELF_R_TYPE(reloc->r_info); + auto symbol_index = ELF_R_SYM(reloc->r_info); + + __ensure(type == R_X86_64_JUMP_SLOT); + __ensure(ELF_CLASS == ELFCLASS64); + + auto symbol = (elf_sym *)(object->baseAddress + object->symbolTableOffset + + symbol_index * sizeof(elf_sym)); + ObjectSymbol r(object, symbol); + auto p = Scope::resolveGlobalOrLocal(*globalScope, object->localScope, r.getString(), object->objectRts, 0); + if(!p) + mlibc::panicLogger() << "Unresolved JUMP_SLOT symbol" << frg::endlog; + + //mlibc::infoLogger() << "Lazy relocation to " << symbol_str + // << " resolved to " << pointer << frg::endlog; + + *(uint64_t *)(object->baseAddress + reloc->r_offset) = p->virtualAddress(); + return (void *)p->virtualAddress(); +} + +extern "C" [[ gnu::visibility("default") ]] void *__rtdl_allocateTcb() { + auto tcb = allocateTcb(); + initTlsObjects(tcb, globalScope->_objects, false); + return tcb; +} + +extern "C" { + [[ gnu::visibility("hidden") ]] void dl_debug_state() { + // This function is used to signal changes in the debugging link map, + // GDB just sets a breakpoint on this function and we can call it + // everytime we update the link map. We don't need to implement + // anything besides defining and calling it. + } +} + +extern "C" [[gnu::alias("dl_debug_state"), gnu::visibility("default")]] void _dl_debug_state() noexcept; + +// This symbol can be used by GDB to find the global interface structure +[[ gnu::visibility("default") ]] DebugInterface *_dl_debug_addr = &globalDebugInterface; + +static frg::vector<frg::string_view, MemoryAllocator> parseList(frg::string_view paths, frg::string_view separators) { + frg::vector<frg::string_view, MemoryAllocator> list{getAllocator()}; + + size_t p = 0; + while(p < paths.size()) { + size_t s; // Offset of next colon or end of string. + if(size_t cs = paths.find_first_of(separators, p); cs != size_t(-1)) { + s = cs; + }else{ + s = paths.size(); + } + + auto path = paths.sub_string(p, s - p); + p = s + 1; + + if(path.size() == 0) + continue; + + if(path.ends_with("/")) { + size_t i = path.size() - 1; + while(i > 0 && path[i] == '/') + i--; + path = path.sub_string(0, i + 1); + } + + if(path == "/") + path = ""; + + list.push_back(path); + } + + return list; +} + +extern "C" void *interpreterMain(uintptr_t *entry_stack) { + if(logEntryExit) + mlibc::infoLogger() << "Entering ld.so" << frg::endlog; + entryStack = entry_stack; + runtimeTlsMap.initialize(); + libraryPaths.initialize(getAllocator()); + preloads.initialize(getAllocator()); + + void *phdr_pointer = 0; + size_t phdr_entry_size = 0; + size_t phdr_count = 0; + void *entry_pointer = 0; + void *stack_entropy = nullptr; + + const char *execfn = "(executable)"; + +#ifndef MLIBC_STATIC_BUILD + using ctor_fn = void(*)(void); + + ctor_fn *ldso_ctors = nullptr; + size_t num_ldso_ctors = 0; + + auto ldso_base = getLdsoBase(); + if(logStartup) { + mlibc::infoLogger() << "ldso: Own base address is: 0x" + << frg::hex_fmt(ldso_base) << frg::endlog; + mlibc::infoLogger() << "ldso: Own dynamic section is at: " << _DYNAMIC << frg::endlog; + } + +#ifdef __x86_64__ + // These entries are reserved on x86_64. + // TODO: Use a fake PLT stub that reports an error message? + _GLOBAL_OFFSET_TABLE_[1] = 0; + _GLOBAL_OFFSET_TABLE_[2] = 0; +#endif + + // Validate our own dynamic section. + // Here, we make sure that the dynamic linker does not need relocations itself. + uintptr_t strtab_offset = 0; + uintptr_t soname_str = 0; + for(size_t i = 0; _DYNAMIC[i].d_tag != DT_NULL; i++) { + auto ent = &_DYNAMIC[i]; + switch(ent->d_tag) { + case DT_STRTAB: strtab_offset = ent->d_un.d_ptr; break; + case DT_SONAME: soname_str = ent->d_un.d_val; break; + case DT_INIT_ARRAY: ldso_ctors = reinterpret_cast<ctor_fn *>(ent->d_un.d_ptr + ldso_base); break; + case DT_INIT_ARRAYSZ: num_ldso_ctors = ent->d_un.d_val / sizeof(ctor_fn); break; + case DT_HASH: + case DT_GNU_HASH: + case DT_STRSZ: + case DT_SYMTAB: + case DT_SYMENT: + case DT_RELA: + case DT_RELASZ: + case DT_RELAENT: + case DT_RELACOUNT: + case DT_DEBUG: + case DT_REL: + case DT_RELSZ: + case DT_RELENT: + case DT_RELCOUNT: + case DT_RELR: + case DT_RELRSZ: + case DT_RELRENT: + continue; + default: + __ensure(!"Unexpected dynamic entry in program interpreter"); + } + } + __ensure(strtab_offset); + __ensure(soname_str); + + // Find the auxiliary vector by skipping args and environment. + auto aux = entryStack; + aux += *aux + 1; // First, we skip argc and all args. + __ensure(!*aux); + aux++; + while(*aux) { // Loop through the environment. + auto env = reinterpret_cast<char *>(*aux); + frg::string_view view{env}; + size_t s = view.find_first('='); + + if(s == size_t(-1)) + mlibc::panicLogger() << "rtdl: environment '" << env << "' is missing a '='" << frg::endlog; + + auto name = view.sub_string(0, s); + auto value = const_cast<char *>(view.data() + s + 1); + + if(name == "LD_SHOW_AUXV" && *value && *value != '0') { + ldShowAuxv = true; + }else if(name == "LD_LIBRARY_PATH" && *value) { + for(auto path : parseList(value, ":;")) + libraryPaths->push_back(path); + }else if(name == "LD_PRELOAD" && *value) { + *preloads = parseList(value, " :"); + } + + aux++; + } + aux++; + + // Add default library paths + libraryPaths->push_back("/lib"); + libraryPaths->push_back("/lib64"); + libraryPaths->push_back("/usr/lib"); + libraryPaths->push_back("/usr/lib64"); + + // Parse the actual vector. + while(true) { + auto value = aux + 1; + if(!(*aux)) + break; + + if(ldShowAuxv) { + switch(*aux) { + case AT_PHDR: mlibc::infoLogger() << "AT_PHDR: 0x" << frg::hex_fmt{*value} << frg::endlog; break; + case AT_PHENT: mlibc::infoLogger() << "AT_PHENT: " << *value << frg::endlog; break; + case AT_PHNUM: mlibc::infoLogger() << "AT_PHNUM: " << *value << frg::endlog; break; + case AT_ENTRY: mlibc::infoLogger() << "AT_ENTRY: 0x" << frg::hex_fmt{*value} << frg::endlog; break; + case AT_PAGESZ: mlibc::infoLogger() << "AT_PAGESZ: " << *value << frg::endlog; break; + case AT_BASE: mlibc::infoLogger() << "AT_BASE: 0x" << frg::hex_fmt{*value} << frg::endlog; break; + case AT_FLAGS: mlibc::infoLogger() << "AT_FLAGS: 0x" << frg::hex_fmt{*value} << frg::endlog; break; + case AT_NOTELF: mlibc::infoLogger() << "AT_NOTELF: " << frg::hex_fmt{*value} << frg::endlog; break; + case AT_UID: mlibc::infoLogger() << "AT_UID: " << *value << frg::endlog; break; + case AT_EUID: mlibc::infoLogger() << "AT_EUID: " << *value << frg::endlog; break; + case AT_GID: mlibc::infoLogger() << "AT_GID: " << *value << frg::endlog; break; + case AT_EGID: mlibc::infoLogger() << "AT_EGID: " << *value << frg::endlog; break; +#ifdef AT_PLATFORM + case AT_PLATFORM: mlibc::infoLogger() << "AT_PLATFORM: " << reinterpret_cast<const char *>(*value) << frg::endlog; break; +#endif +#ifdef AT_HWCAP + case AT_HWCAP: mlibc::infoLogger() << "AT_HWCAP: " << frg::hex_fmt{*value} << frg::endlog; break; +#endif +#ifdef AT_CLKTCK + case AT_CLKTCK: mlibc::infoLogger() << "AT_CLKTCK: " << *value << frg::endlog; break; +#endif +#ifdef AT_FPUCW + case AT_FPUCW: mlibc::infoLogger() << "AT_FPUCW: " << frg::hex_fmt{*value} << frg::endlog; break; +#endif +#ifdef AT_SECURE + case AT_SECURE: mlibc::infoLogger() << "AT_SECURE: " << *value << frg::endlog; break; +#endif +#ifdef AT_RANDOM + case AT_RANDOM: mlibc::infoLogger() << "AT_RANDOM: 0x" << frg::hex_fmt{*value} << frg::endlog; break; +#endif +#ifdef AT_EXECFN + case AT_EXECFN: mlibc::infoLogger() << "AT_EXECFN: " << reinterpret_cast<const char *>(*value) << frg::endlog; break; +#endif +#ifdef AT_SYSINFO_EHDR + case AT_SYSINFO_EHDR: mlibc::infoLogger() << "AT_SYSINFO_EHDR: 0x" << frg::hex_fmt{*value} << frg::endlog; break; +#endif + } + } + + // TODO: Whitelist auxiliary vector entries here? + switch(*aux) { + case AT_PHDR: phdr_pointer = reinterpret_cast<void *>(*value); break; + case AT_PHENT: phdr_entry_size = *value; break; + case AT_PHNUM: phdr_count = *value; break; + case AT_ENTRY: entry_pointer = reinterpret_cast<void *>(*value); break; + case AT_EXECFN: execfn = reinterpret_cast<const char *>(*value); break; + case AT_RANDOM: stack_entropy = reinterpret_cast<void*>(*value); break; + case AT_SECURE: rtdlConfig.secureRequired = reinterpret_cast<uintptr_t>(*value); break; + } + + aux += 2; + } + globalDebugInterface.base = reinterpret_cast<void*>(ldso_base); + +// This is here because libgcc will add a global constructor on glibc Linux +// (which is what it believes we are due to the aarch64-linux-gnu toolchain) +// in order to check if LSE atomics are supported. +// +// This is not necessary on a custom Linux toolchain and is purely an artifact of +// using the host toolchain. +#if defined(__aarch64__) && defined(__gnu_linux__) + for (size_t i = 0; i < num_ldso_ctors; i++) { + if(logStartup) + mlibc::infoLogger() << "ldso: Running own constructor at " + << reinterpret_cast<void *>(ldso_ctors[i]) + << frg::endlog; + ldso_ctors[i](); + } +#else + if (num_ldso_ctors > 0) { + mlibc::panicLogger() << "ldso: Found unexpected own global constructor(s), init_array starts at: " + << ldso_ctors + << frg::endlog; + } +#endif + +#else + auto ehdr = reinterpret_cast<elf_ehdr*>(__ehdr_start); + phdr_pointer = reinterpret_cast<void*>((uintptr_t)ehdr->e_phoff + (uintptr_t)ehdr); + phdr_entry_size = ehdr->e_phentsize; + phdr_count = ehdr->e_phnum; + entry_pointer = reinterpret_cast<void*>(ehdr->e_entry); +#endif + __ensure(phdr_pointer); + __ensure(entry_pointer); + + if(logStartup) + mlibc::infoLogger() << "ldso: Executable PHDRs are at " << phdr_pointer + << frg::endlog; + + // perform the initial dynamic linking + initialRepository.initialize(); + + globalScope.initialize(true); + + // Add the dynamic linker, as well as the exectuable to the repository. +#ifndef MLIBC_STATIC_BUILD + auto ldso_soname = reinterpret_cast<const char *>(ldso_base + strtab_offset + soname_str); + auto ldso = initialRepository->injectObjectFromDts(ldso_soname, + frg::string<MemoryAllocator> { getAllocator() }, + ldso_base, _DYNAMIC, 1); + ldso->phdrPointer = phdr_pointer; + ldso->phdrCount = phdr_count; + ldso->phdrEntrySize = phdr_entry_size; + + // TODO: support non-zero base addresses? + executableSO = initialRepository->injectObjectFromPhdrs(execfn, + frg::string<MemoryAllocator> { execfn, getAllocator() }, + phdr_pointer, phdr_entry_size, phdr_count, entry_pointer, 1); + + // We can't initialise the ldso object after the executable SO, + // so we have to set the ldso path after loading both. + ldso->path = executableSO->interpreterPath; + +#else + executableSO = initialRepository->injectStaticObject(execfn, + frg::string<MemoryAllocator>{ execfn, getAllocator() }, + phdr_pointer, phdr_entry_size, phdr_count, entry_pointer, 1); + globalDebugInterface.base = (void*)executableSO->baseAddress; +#endif + + globalDebugInterface.head = &executableSO->linkMap; + executableSO->inLinkMap = true; + Loader linker{globalScope.get(), executableSO, true, 1}; + linker.linkObjects(executableSO); + + mlibc::initStackGuard(stack_entropy); + + auto tcb = allocateTcb(); + if(mlibc::sys_tcb_set(tcb)) + __ensure(!"sys_tcb_set() failed"); + tcb->tid = mlibc::this_tid(); + mlibc::tcb_available_flag = true; + + globalDebugInterface.ver = 1; + globalDebugInterface.brk = &dl_debug_state; + globalDebugInterface.state = 0; + dl_debug_state(); + + linker.initObjects(); + + if(logEntryExit) + mlibc::infoLogger() << "Leaving ld.so, jump to " + << (void *)executableSO->entry << frg::endlog; + return executableSO->entry; +} + +const char *lastError; + +extern "C" [[ gnu::visibility("default") ]] uintptr_t *__dlapi_entrystack() { + return entryStack; +} + +extern "C" [[ gnu::visibility("default") ]] +const char *__dlapi_error() { + auto error = lastError; + lastError = nullptr; + return error; +} + +extern "C" [[ gnu::visibility("default") ]] +void *__dlapi_get_tls(struct __abi_tls_entry *entry) { + return reinterpret_cast<char *>(accessDtv(entry->object)) + entry->offset; +} + +extern "C" [[ gnu::visibility("default") ]] +const mlibc::RtdlConfig &__dlapi_get_config() { + return rtdlConfig; +} + +#if __MLIBC_POSIX_OPTION + +extern "C" [[ gnu::visibility("default") ]] +void *__dlapi_open(const char *file, int flags, void *returnAddress) { + if (logDlCalls) + mlibc::infoLogger() << "rtdl: __dlapi_open(" << (file ? file : "nullptr") << ")" << frg::endlog; + + if (flags & RTLD_DEEPBIND) + mlibc::infoLogger() << "rtdl: dlopen(RTLD_DEEPBIND) is unsupported" << frg::endlog; + + if(!file) + return executableSO; + + // TODO: Thread-safety! + auto rts = rtsCounter++; + + SharedObject *object; + if (flags & RTLD_NOLOAD) { + object = initialRepository->findLoadedObject(file); + if (object && object->globalRts == 0 && (flags & RTLD_GLOBAL)) { + // The object was opened with RTLD_LOCAL, but we are called with RTLD_NOLOAD | RTLD_GLOBAL. + // According to the man page, we should promote to the global scope here. + object->globalRts = rts; + globalScope->appendObject(object); + } + } else { + bool isGlobal = flags & RTLD_GLOBAL; + Scope *newScope = isGlobal ? globalScope.get() : nullptr; + + frg::expected<LinkerError, SharedObject *> objectResult; + if (frg::string_view{file}.find_first('/') == size_t(-1)) { + // In order to know which RUNPATH / RPATH to process, we must find the calling object. + SharedObject *origin = initialRepository->findCaller(returnAddress); + if (!origin) { + mlibc::panicLogger() << "rtdl: unable to determine calling object of dlopen " + << "(ra = " << returnAddress << ")" << frg::endlog; + } + + objectResult = initialRepository->requestObjectWithName(file, origin, newScope, !isGlobal, rts); + } else { + objectResult = initialRepository->requestObjectAtPath(file, newScope, !isGlobal, rts); + } + + if(!objectResult) { + switch (objectResult.error()) { + case LinkerError::success: + __builtin_unreachable(); + case LinkerError::notFound: + lastError = "Cannot locate requested DSO"; + break; + case LinkerError::fileTooShort: + lastError = "File too short"; + break; + case LinkerError::notElf: + lastError = "File is not an ELF file"; + break; + case LinkerError::wrongElfType: + lastError = "File has wrong ELF type"; + break; + case LinkerError::outOfMemory: + lastError = "Out of memory"; + break; + case LinkerError::invalidProgramHeader: + lastError = "File has invalid program header"; + break; + } + return nullptr; + } + object = objectResult.value(); + + Loader linker{object->localScope, nullptr, false, rts}; + linker.linkObjects(object); + linker.initObjects(); + } + + dl_debug_state(); + + return object; +} + +extern "C" [[ gnu::visibility("default") ]] +void *__dlapi_resolve(void *handle, const char *string, void *returnAddress) { + if (logDlCalls) { + const char *name; + bool quote = false; + if (handle == RTLD_DEFAULT) { + name = "RTLD_DEFAULT"; + } else if (handle == RTLD_NEXT) { + name = "RTLD_NEXT"; + } else { + name = ((SharedObject *)handle)->name.data(); + quote = true; + } + + mlibc::infoLogger() << "rtdl: __dlapi_resolve(" << (quote ? "\"" : "") << name + << (quote ? "\"" : "") << ", \"" << string << "\")" << frg::endlog; + } + + frg::optional<ObjectSymbol> target; + + if (handle == RTLD_DEFAULT) { + target = globalScope->resolveSymbol(string, 0, 0); + } else if (handle == RTLD_NEXT) { + SharedObject *origin = initialRepository->findCaller(returnAddress); + if (!origin) { + mlibc::panicLogger() << "rtdl: unable to determine calling object of dlsym " + << "(ra = " << returnAddress << ")" << frg::endlog; + } + + target = Scope::resolveGlobalOrLocalNext(*globalScope, origin->localScope, string, origin); + } else { + // POSIX does not unambiguously state how dlsym() is supposed to work; it just + // states that "The symbol resolution algorithm used shall be dependency order + // as described in dlopen()". + // + // Linux libc's lookup the symbol in the given DSO and all of its dependencies + // in breadth-first order. That is also what we implement here. + // + // Note that this *differs* from the algorithm that is used for relocations + // (since the algorithm used for relocations takes (i) the global scope, + // and (ii) the local scope of the DSO into account (which can contain more objects + // than just the dependencies of the DSO, if the DSO was loaded as a dependency + // of a dlopen()ed DSO). + + frg::vector<SharedObject *, MemoryAllocator> queue{getAllocator()}; + + struct Token { }; + frg::hash_map< + SharedObject *, Token, + frg::hash<SharedObject *>, MemoryAllocator + > visited{frg::hash<SharedObject *>{}, getAllocator()}; + + auto root = reinterpret_cast<SharedObject *>(handle); + visited.insert(root, Token{}); + queue.push_back(root); + + for(size_t i = 0; i < queue.size(); i++) { + auto current = queue[i]; + + target = resolveInObject(current, string); + if(target) + break; + + for(auto dep : current->dependencies) { + if(visited.get(dep)) + continue; + visited.insert(dep, Token{}); + queue.push_back(dep); + } + } + } + + if (!target) { + if (logDlCalls) + mlibc::infoLogger() << "rtdl: could not resolve \"" << string << "\"" << frg::endlog; + + lastError = "Cannot resolve requested symbol"; + return nullptr; + } + return reinterpret_cast<void *>(target->virtualAddress()); +} + +struct __dlapi_symbol { + const char *file; + void *base; + const char *symbol; + void *address; +}; + +extern "C" [[ gnu::visibility("default") ]] +int __dlapi_reverse(const void *ptr, __dlapi_symbol *info) { + if (logDlCalls) + mlibc::infoLogger() << "rtdl: __dlapi_reverse(" << ptr << ")" << frg::endlog; + + for(size_t i = 0; i < initialRepository->loadedObjects.size(); i++) { + auto object = initialRepository->loadedObjects[i]; + + auto eligible = [&] (ObjectSymbol cand) { + if(cand.symbol()->st_shndx == SHN_UNDEF) + return false; + + auto bind = ELF_ST_BIND(cand.symbol()->st_info); + if(bind != STB_GLOBAL && bind != STB_WEAK) + return false; + + return true; + }; + + auto hash_table = (Elf64_Word *)(object->baseAddress + object->hashTableOffset); + auto num_symbols = hash_table[1]; + for(size_t i = 0; i < num_symbols; i++) { + ObjectSymbol cand{object, (elf_sym *)(object->baseAddress + + object->symbolTableOffset + i * sizeof(elf_sym))}; + if(eligible(cand) && cand.virtualAddress() == reinterpret_cast<uintptr_t>(ptr)) { + if (logDlCalls) + mlibc::infoLogger() << "rtdl: Found symbol " << cand.getString() << " in object " + << object->path << frg::endlog; + + info->file = object->path.data(); + info->base = reinterpret_cast<void *>(object->baseAddress); + info->symbol = cand.getString(); + info->address = reinterpret_cast<void *>(cand.virtualAddress()); + return 0; + } + } + } + + // Not found, find the DSO it should be in. + for(size_t i = 0; i < initialRepository->loadedObjects.size(); i++) { + auto object = initialRepository->loadedObjects[i]; + + for(size_t j = 0; j < object->phdrCount; j++) { + auto phdr = (elf_phdr *)((uintptr_t)object->phdrPointer + j * object->phdrEntrySize); + if(phdr->p_type != PT_LOAD) { + continue; + } + uintptr_t start = object->baseAddress + phdr->p_vaddr; + uintptr_t end = start + phdr->p_memsz; + if(reinterpret_cast<uintptr_t>(ptr) >= start && reinterpret_cast<uintptr_t>(ptr) < end) { + mlibc::infoLogger() << "rtdl: Found DSO " << object->path << frg::endlog; + info->file = object->path.data(); + info->base = reinterpret_cast<void *>(object->baseAddress); + info->symbol = nullptr; + info->address = 0; + return 0; + } + } + } + + if (logDlCalls) + mlibc::infoLogger() << "rtdl: Could not find symbol in __dlapi_reverse()" << frg::endlog; + + return -1; +} + +extern "C" [[ gnu::visibility("default") ]] +int __dlapi_close(void *) { + if (logDlCalls) + mlibc::infoLogger() << "mlibc: dlclose() is a no-op" << frg::endlog; + return 0; +} + +#endif + +extern "C" [[ gnu::visibility("default") ]] +int __dlapi_iterate_phdr(int (*callback)(struct dl_phdr_info *, size_t, void*), void *data) { + int last_return = 0; + for (auto object : initialRepository->loadedObjects) { + struct dl_phdr_info info; + info.dlpi_addr = object->baseAddress; + info.dlpi_name = object->name.data(); + + if(object->isMainObject) { + info.dlpi_name = ""; + } else { + info.dlpi_name = object->name.data(); + } + info.dlpi_phdr = static_cast<ElfW(Phdr)*>(object->phdrPointer); + info.dlpi_phnum = object->phdrCount; + info.dlpi_adds = rtsCounter; + info.dlpi_subs = 0; // TODO(geert): implement dlclose(). + if (object->tlsModel != TlsModel::null) + info.dlpi_tls_modid = object->tlsIndex; + else + info.dlpi_tls_modid = 0; + info.dlpi_tls_data = tryAccessDtv(object); + + last_return = callback(&info, sizeof(struct dl_phdr_info), data); + if(last_return) + return last_return; + } + + return last_return; +} + +extern "C" [[ gnu::visibility("default") ]] +void __dlapi_enter(uintptr_t *entry_stack) { +#if MLIBC_STATIC_BUILD + interpreterMain(entry_stack); +#else + (void)entry_stack; +#endif +} + +// XXX(qookie): +// This is here because libgcc will call into __getauxval on glibc Linux +// (which is what it believes we are due to the aarch64-linux-gnu toolchain) +// in order to find AT_HWCAP to discover if LSE atomics are supported. +// +// This is not necessary on a custom Linux toolchain and is purely an artifact of +// using the host toolchain. + +// __gnu_linux__ is the define checked by libgcc +#if defined(__aarch64__) && defined(__gnu_linux__) && !defined(MLIBC_STATIC_BUILD) + +extern "C" unsigned long __getauxval(unsigned long type) { + // Find the auxiliary vector by skipping args and environment. + auto aux = entryStack; + aux += *aux + 1; // Skip argc and all arguments + __ensure(!*aux); + aux++; + while(*aux) // Now, we skip the environment. + aux++; + aux++; + + // Parse the auxiliary vector. + while(true) { + auto value = aux + 1; + if(*aux == AT_NULL) { + return 0; + }else if(*aux == type) { + return *value; + } + aux += 2; + } +} + +#endif |