diff options
Diffstat (limited to 'lib/mlibc/options/rtdl/generic')
| -rw-r--r-- | lib/mlibc/options/rtdl/generic/linker.cpp | 1872 | ||||
| -rw-r--r-- | lib/mlibc/options/rtdl/generic/linker.hpp | 402 | ||||
| -rw-r--r-- | lib/mlibc/options/rtdl/generic/main.cpp | 844 |
3 files changed, 0 insertions, 3118 deletions
diff --git a/lib/mlibc/options/rtdl/generic/linker.cpp b/lib/mlibc/options/rtdl/generic/linker.cpp deleted file mode 100644 index a519c35..0000000 --- a/lib/mlibc/options/rtdl/generic/linker.cpp +++ /dev/null @@ -1,1872 +0,0 @@ -#include <mlibc/arch-defs.hpp> -#include <stdint.h> -#include <string.h> - -// keep a list of optional generic relocation types -enum { - R_OFFSET = (uintptr_t) -1, -}; - - -#include <frg/manual_box.hpp> -#include <frg/small_vector.hpp> -#include <mlibc/allocator.hpp> -#include <mlibc/debug.hpp> -#include <mlibc/rtdl-sysdeps.hpp> -#include <mlibc/rtdl-abi.hpp> -#include <mlibc/thread.hpp> -#include <abi-bits/fcntl.h> -#include <internal-config.h> - -#include "elf.hpp" -#include "linker.hpp" - -#if !MLIBC_MMAP_ALLOCATE_DSO -uintptr_t libraryBase = 0x41000000; -#endif - -constexpr bool verbose = false; -constexpr bool stillSlightlyVerbose = false; -constexpr bool logBaseAddresses = false; -constexpr bool logRpath = false; -constexpr bool logLdPath = false; -constexpr bool eagerBinding = true; - -#if defined(__x86_64__) || defined(__i386__) -constexpr inline bool tlsAboveTp = false; -#elif defined(__aarch64__) -constexpr inline bool tlsAboveTp = true; -#elif defined(__riscv) -constexpr inline bool tlsAboveTp = true; -#else -# error Unknown architecture -#endif - -extern DebugInterface globalDebugInterface; -extern uintptr_t __stack_chk_guard; - -extern frg::manual_box<frg::small_vector<frg::string_view, 4, MemoryAllocator>> libraryPaths; -extern frg::manual_box<frg::vector<frg::string_view, MemoryAllocator>> preloads; - -#if MLIBC_STATIC_BUILD -extern "C" size_t __init_array_start[]; -extern "C" size_t __init_array_end[]; -extern "C" size_t __preinit_array_start[]; -extern "C" size_t __preinit_array_end[]; -#endif - -size_t tlsMaxAlignment = 16; - -// This is the global "resolution timestamp" (RTS) counter. -// It is incremented each time __dlapi_open() (i.e. dlopen()) is called. -// Each DSO stores its objectRts (i.e. RTS at the time the object was loaded). -// DSOs in the global scope also store a globalRts (i.e. RTS at the time the -// object became global). This mechanism is used to determine which -// part of the global scope is considered for symbol resolution. -uint64_t rtsCounter = 2; - -bool trySeek(int fd, int64_t offset) { - off_t noff; - return mlibc::sys_seek(fd, offset, SEEK_SET, &noff) == 0; -} - -bool tryReadExactly(int fd, void *data, size_t length) { - size_t offset = 0; - while(offset < length) { - ssize_t chunk; - if(mlibc::sys_read(fd, reinterpret_cast<char *>(data) + offset, - length - offset, &chunk)) - return false; - __ensure(chunk > 0); - offset += chunk; - } - __ensure(offset == length); - return true; -} - -void closeOrDie(int fd) { - if(mlibc::sys_close(fd)) - __ensure(!"sys_close() failed"); -} - -uintptr_t alignUp(uintptr_t address, size_t align) { - return (address + align - 1) & ~(align - 1); -} - -// -------------------------------------------------------- -// ObjectRepository -// -------------------------------------------------------- - -ObjectRepository::ObjectRepository() -: loadedObjects{getAllocator()}, - _nameMap{frg::hash<frg::string_view>{}, getAllocator()} {} - -SharedObject *ObjectRepository::injectObjectFromDts(frg::string_view name, - frg::string<MemoryAllocator> path, uintptr_t base_address, - elf_dyn *dynamic, uint64_t rts) { - __ensure(!findLoadedObject(name)); - - auto object = frg::construct<SharedObject>(getAllocator(), - name.data(), std::move(path), false, globalScope.get(), rts); - object->baseAddress = base_address; - object->dynamic = dynamic; - _parseDynamic(object); - - _addLoadedObject(object); - _discoverDependencies(object, globalScope.get(), rts); - - return object; -} - -SharedObject *ObjectRepository::injectObjectFromPhdrs(frg::string_view name, - frg::string<MemoryAllocator> path, void *phdr_pointer, - size_t phdr_entry_size, size_t num_phdrs, void *entry_pointer, - uint64_t rts) { - __ensure(!findLoadedObject(name)); - - auto object = frg::construct<SharedObject>(getAllocator(), - name.data(), std::move(path), true, globalScope.get(), rts); - _fetchFromPhdrs(object, phdr_pointer, phdr_entry_size, num_phdrs, entry_pointer); - _parseDynamic(object); - - _addLoadedObject(object); - _discoverDependencies(object, globalScope.get(), rts); - - return object; -} - -SharedObject *ObjectRepository::injectStaticObject(frg::string_view name, - frg::string<MemoryAllocator> path, void *phdr_pointer, - size_t phdr_entry_size, size_t num_phdrs, void *entry_pointer, - uint64_t rts) { - __ensure(!findLoadedObject(name)); - auto object = frg::construct<SharedObject>(getAllocator(), - name.data(), std::move(path), true, globalScope.get(), rts); - _fetchFromPhdrs(object, phdr_pointer, phdr_entry_size, num_phdrs, entry_pointer); - -#if MLIBC_STATIC_BUILD - object->initArray = reinterpret_cast<InitFuncPtr*>(__init_array_start); - object->initArraySize = static_cast<size_t>((uintptr_t)__init_array_end - - (uintptr_t)__init_array_start); - object->preInitArray = reinterpret_cast<InitFuncPtr*>(__preinit_array_start); - object->preInitArraySize = static_cast<size_t>((uintptr_t)__preinit_array_end - - (uintptr_t)__preinit_array_start); -#endif - - _addLoadedObject(object); - - return object; -} - -frg::expected<LinkerError, SharedObject *> ObjectRepository::requestObjectWithName(frg::string_view name, - SharedObject *origin, Scope *localScope, bool createScope, uint64_t rts) { - if (auto obj = findLoadedObject(name)) - return obj; - - auto tryToOpen = [&] (const char *path) { - int fd; - if(auto x = mlibc::sys_open(path, O_RDONLY, 0, &fd); x) { - return -1; - } - return fd; - }; - - // TODO(arsen): this process can probably undergo heavy optimization, by - // preprocessing the rpath only once on parse - auto processRpath = [&] (frg::string_view path) { - frg::string<MemoryAllocator> sPath { getAllocator() }; - if (path.starts_with("$ORIGIN")) { - frg::string_view dirname = origin->path; - auto lastsl = dirname.find_last('/'); - if (lastsl != size_t(-1)) { - dirname = dirname.sub_string(0, lastsl); - } else { - dirname = "."; - } - sPath = frg::string<MemoryAllocator>{ getAllocator(), dirname }; - sPath += path.sub_string(7, path.size() - 7); - } else { - sPath = frg::string<MemoryAllocator>{ getAllocator(), path }; - } - if (sPath[sPath.size() - 1] != '/') { - sPath += '/'; - } - sPath += name; - if (logRpath) - mlibc::infoLogger() << "rtdl: trying in rpath " << sPath << frg::endlog; - int fd = tryToOpen(sPath.data()); - if (logRpath && fd >= 0) - mlibc::infoLogger() << "rtdl: found in rpath" << frg::endlog; - return frg::tuple { fd, std::move(sPath) }; - }; - - frg::string<MemoryAllocator> chosenPath { getAllocator() }; - int fd = -1; - if (origin && origin->runPath) { - size_t start = 0; - size_t idx = 0; - frg::string_view rpath { origin->runPath }; - auto next = [&] () { - idx = rpath.find_first(':', start); - if (idx == (size_t)-1) - idx = rpath.size(); - }; - for (next(); idx < rpath.size(); next()) { - auto path = rpath.sub_string(start, idx - start); - start = idx + 1; - auto [fd_, fullPath] = processRpath(path); - if (fd_ != -1) { - fd = fd_; - chosenPath = std::move(fullPath); - break; - } - } - if (fd == -1) { - auto path = rpath.sub_string(start, rpath.size() - start); - auto [fd_, fullPath] = processRpath(path); - if (fd_ != -1) { - fd = fd_; - chosenPath = std::move(fullPath); - } - } - } else if (logRpath) { - mlibc::infoLogger() << "rtdl: no rpath set for object" << frg::endlog; - } - - for(size_t i = 0; i < libraryPaths->size() && fd == -1; i++) { - auto ldPath = (*libraryPaths)[i]; - auto path = frg::string<MemoryAllocator>{getAllocator(), ldPath} + '/' + name; - if(logLdPath) - mlibc::infoLogger() << "rtdl: Trying to load " << name << " from ldpath " << ldPath << "/" << frg::endlog; - fd = tryToOpen(path.data()); - if(fd >= 0) { - chosenPath = std::move(path); - break; - } - } - if(fd == -1) - return LinkerError::notFound; - - if (createScope) { - __ensure(localScope == nullptr); - - // TODO: Free this when the scope is no longer needed. - localScope = frg::construct<Scope>(getAllocator()); - } - - __ensure(localScope != nullptr); - - auto object = frg::construct<SharedObject>(getAllocator(), - name.data(), std::move(chosenPath), false, localScope, rts); - - auto result = _fetchFromFile(object, fd); - closeOrDie(fd); - if(!result) { - frg::destruct(getAllocator(), object); - return result.error(); - } - - _parseDynamic(object); - - _addLoadedObject(object); - _discoverDependencies(object, localScope, rts); - - return object; -} - -frg::expected<LinkerError, SharedObject *> ObjectRepository::requestObjectAtPath(frg::string_view path, - Scope *localScope, bool createScope, uint64_t rts) { - // TODO: Support SONAME correctly. - auto lastSlash = path.find_last('/') + 1; - auto name = path; - if (!lastSlash) { - name = name.sub_string(lastSlash, path.size() - lastSlash); - } - if (auto obj = findLoadedObject(name)) - return obj; - - if (createScope) { - __ensure(localScope == nullptr); - - // TODO: Free this when the scope is no longer needed. - localScope = frg::construct<Scope>(getAllocator()); - } - - __ensure(localScope != nullptr); - - auto object = frg::construct<SharedObject>(getAllocator(), - name.data(), path.data(), false, localScope, rts); - - frg::string<MemoryAllocator> no_prefix(getAllocator(), path); - - int fd; - if(mlibc::sys_open((no_prefix + '\0').data(), O_RDONLY, 0, &fd)) { - frg::destruct(getAllocator(), object); - return LinkerError::notFound; - } - auto result = _fetchFromFile(object, fd); - closeOrDie(fd); - if(!result) { - frg::destruct(getAllocator(), object); - return result.error(); - } - - _parseDynamic(object); - - _addLoadedObject(object); - _discoverDependencies(object, localScope, rts); - - return object; -} - -SharedObject *ObjectRepository::findCaller(void *addr) { - uintptr_t target = reinterpret_cast<uintptr_t>(addr); - - for (auto [name, object] : _nameMap) { - // Search all PT_LOAD segments for the specified address. - for(size_t j = 0; j < object->phdrCount; j++) { - auto phdr = (elf_phdr *)((uintptr_t)object->phdrPointer + j * object->phdrEntrySize); - if (phdr->p_type == PT_LOAD) { - uintptr_t start = object->baseAddress + phdr->p_vaddr; - uintptr_t end = start + phdr->p_memsz; - if (start <= target && target < end) - return object; - } - } - } - - return nullptr; -} - -SharedObject *ObjectRepository::findLoadedObject(frg::string_view name) { - auto it = _nameMap.get(name); - if (it) - return *it; - - for (auto object : loadedObjects) { - // See if any object has a matching SONAME. - if (object->soName && name == object->soName) - return object; - } - - // TODO: We should also look at the device and inode here as a fallback. - return nullptr; -} - -// -------------------------------------------------------- -// ObjectRepository: Fetching methods. -// -------------------------------------------------------- - -void ObjectRepository::_fetchFromPhdrs(SharedObject *object, void *phdr_pointer, - size_t phdr_entry_size, size_t phdr_count, void *entry_pointer) { - __ensure(object->isMainObject); - object->phdrPointer = phdr_pointer; - object->phdrEntrySize = phdr_entry_size; - object->phdrCount = phdr_count; - if(verbose) - mlibc::infoLogger() << "rtdl: Loading " << object->name << frg::endlog; - - // Note: the entry pointer is absolute and not relative to the base address. - object->entry = entry_pointer; - - frg::optional<ptrdiff_t> dynamic_offset; - frg::optional<ptrdiff_t> tls_offset; - - // segments are already mapped, so we just have to find the dynamic section - for(size_t i = 0; i < phdr_count; i++) { - auto phdr = (elf_phdr *)((uintptr_t)phdr_pointer + i * phdr_entry_size); - switch(phdr->p_type) { - case PT_PHDR: - // Determine the executable's base address (in the PIE case) by comparing - // the PHDR segment's load address against it's address in the ELF file. - object->baseAddress = reinterpret_cast<uintptr_t>(phdr_pointer) - phdr->p_vaddr; - if(verbose) - mlibc::infoLogger() << "rtdl: Executable is loaded at " - << (void *)object->baseAddress << frg::endlog; - break; - case PT_DYNAMIC: - dynamic_offset = phdr->p_vaddr; - break; - case PT_TLS: { - object->tlsSegmentSize = phdr->p_memsz; - object->tlsAlignment = phdr->p_align; - object->tlsImageSize = phdr->p_filesz; - tls_offset = phdr->p_vaddr; - break; - case PT_INTERP: - object->interpreterPath = frg::string<MemoryAllocator>{ - (char*)(object->baseAddress + phdr->p_vaddr), - getAllocator() - }; - } break; - default: - //FIXME warn about unknown phdrs - break; - } - } - - if(dynamic_offset) - object->dynamic = (elf_dyn *)(object->baseAddress + *dynamic_offset); - if(tls_offset) - object->tlsImagePtr = (void *)(object->baseAddress + *tls_offset); -} - - -frg::expected<LinkerError, void> ObjectRepository::_fetchFromFile(SharedObject *object, int fd) { - __ensure(!object->isMainObject); - - // read the elf file header - elf_ehdr ehdr; - if(!tryReadExactly(fd, &ehdr, sizeof(elf_ehdr))) - return LinkerError::fileTooShort; - - if(ehdr.e_ident[0] != 0x7F - || ehdr.e_ident[1] != 'E' - || ehdr.e_ident[2] != 'L' - || ehdr.e_ident[3] != 'F') - return LinkerError::notElf; - - if((ehdr.e_type != ET_EXEC && ehdr.e_type != ET_DYN) - || ehdr.e_machine != ELF_MACHINE - || ehdr.e_ident[EI_CLASS] != ELF_CLASS) - return LinkerError::wrongElfType; - - // read the elf program headers - auto phdr_buffer = (char *)getAllocator().allocate(ehdr.e_phnum * ehdr.e_phentsize); - if(!phdr_buffer) - return LinkerError::outOfMemory; - - if(!trySeek(fd, ehdr.e_phoff)) { - getAllocator().deallocate(phdr_buffer, ehdr.e_phnum * ehdr.e_phentsize); - return LinkerError::invalidProgramHeader; - } - if(!tryReadExactly(fd, phdr_buffer, ehdr.e_phnum * ehdr.e_phentsize)) { - getAllocator().deallocate(phdr_buffer, ehdr.e_phnum * ehdr.e_phentsize); - return LinkerError::invalidProgramHeader; - } - - object->phdrPointer = phdr_buffer; - object->phdrCount = ehdr.e_phnum; - object->phdrEntrySize = ehdr.e_phentsize; - - // Allocate virtual address space for the DSO. - constexpr size_t hugeSize = 0x200000; - - uintptr_t highest_address = 0; - for(int i = 0; i < ehdr.e_phnum; i++) { - auto phdr = (elf_phdr *)(phdr_buffer + i * ehdr.e_phentsize); - - if(phdr->p_type != PT_LOAD) - continue; - - auto limit = phdr->p_vaddr + phdr->p_memsz; - if(limit > highest_address) - highest_address = limit; - } - - __ensure(!(object->baseAddress & (hugeSize - 1))); - - highest_address = (highest_address + mlibc::page_size - 1) & ~(mlibc::page_size - 1); - -#if MLIBC_MMAP_ALLOCATE_DSO - void *mappedAddr = nullptr; - - if (mlibc::sys_vm_map(nullptr, - highest_address - object->baseAddress, PROT_NONE, - MAP_PRIVATE | MAP_ANONYMOUS, -1, 0, &mappedAddr)) { - mlibc::infoLogger() << "sys_vm_map failed when allocating address space for DSO \"" - << object->name << "\"" - << ", base " << (void *)object->baseAddress - << ", requested " << (highest_address - object->baseAddress) << " bytes" - << frg::endlog; - getAllocator().deallocate(phdr_buffer, ehdr.e_phnum * ehdr.e_phentsize); - return LinkerError::outOfMemory; - } - - object->baseAddress = reinterpret_cast<uintptr_t>(mappedAddr); -#else - object->baseAddress = libraryBase; - libraryBase += (highest_address + (hugeSize - 1)) & ~(hugeSize - 1); -#endif - - if(verbose || logBaseAddresses) - mlibc::infoLogger() << "rtdl: Loading " << object->name - << " at " << (void *)object->baseAddress << frg::endlog; - - // Load all segments. - constexpr size_t pageSize = 0x1000; - for(int i = 0; i < ehdr.e_phnum; i++) { - auto phdr = (elf_phdr *)(phdr_buffer + i * ehdr.e_phentsize); - - if(phdr->p_type == PT_LOAD) { - size_t misalign = phdr->p_vaddr & (pageSize - 1); - __ensure(phdr->p_memsz > 0); - __ensure(phdr->p_memsz >= phdr->p_filesz); - - // If the following condition is violated, we cannot use mmap() the segment; - // however, GCC only generates ELF files that satisfy this. - __ensure(misalign == (phdr->p_offset & (pageSize - 1))); - - auto map_address = object->baseAddress + phdr->p_vaddr - misalign; - auto backed_map_size = (phdr->p_filesz + misalign + pageSize - 1) & ~(pageSize - 1); - auto total_map_size = (phdr->p_memsz + misalign + pageSize - 1) & ~(pageSize - 1); - - int prot = 0; - if(phdr->p_flags & PF_R) - prot |= PROT_READ; - if(phdr->p_flags & PF_W) - prot |= PROT_WRITE; - if(phdr->p_flags & PF_X) - prot |= PROT_EXEC; - - #if MLIBC_MAP_DSO_SEGMENTS - void *map_pointer; - if(mlibc::sys_vm_map(reinterpret_cast<void *>(map_address), - backed_map_size, prot | PROT_WRITE, - MAP_PRIVATE | MAP_FIXED, fd, phdr->p_offset - misalign, &map_pointer)) - __ensure(!"sys_vm_map failed"); - if(total_map_size > backed_map_size) - if(mlibc::sys_vm_map(reinterpret_cast<void *>(map_address + backed_map_size), - total_map_size - backed_map_size, prot | PROT_WRITE, - MAP_PRIVATE | MAP_FIXED | MAP_ANONYMOUS, -1, 0, &map_pointer)) - __ensure(!"sys_vm_map failed"); - - if(mlibc::sys_vm_readahead) - if(mlibc::sys_vm_readahead(reinterpret_cast<void *>(map_address), - backed_map_size)) - mlibc::infoLogger() << "mlibc: sys_vm_readahead() failed in ld.so" - << frg::endlog; - - // Clear the trailing area at the end of the backed mapping. - // We do not clear the leading area; programs are not supposed to access it. - memset(reinterpret_cast<void *>(map_address + misalign + phdr->p_filesz), - 0, phdr->p_memsz - phdr->p_filesz); - #else - (void)backed_map_size; - - void *map_pointer; - if(mlibc::sys_vm_map(reinterpret_cast<void *>(map_address), - total_map_size, prot | PROT_WRITE, - MAP_PRIVATE | MAP_FIXED | MAP_ANONYMOUS, -1, 0, &map_pointer)) - __ensure(!"sys_vm_map failed"); - - __ensure(trySeek(fd, phdr->p_offset)); - __ensure(tryReadExactly(fd, reinterpret_cast<char *>(map_address) + misalign, - phdr->p_filesz)); - #endif - // Take care of removing superfluous permissions. - if(mlibc::sys_vm_protect && ((prot & PROT_WRITE) == 0)) - if(mlibc::sys_vm_protect(map_pointer, total_map_size, prot)) - mlibc::infoLogger() << "mlibc: sys_vm_protect() failed in ld.so" << frg::endlog; - }else if(phdr->p_type == PT_TLS) { - object->tlsSegmentSize = phdr->p_memsz; - object->tlsAlignment = phdr->p_align; - object->tlsImageSize = phdr->p_filesz; - object->tlsImagePtr = (void *)(object->baseAddress + phdr->p_vaddr); - }else if(phdr->p_type == PT_DYNAMIC) { - object->dynamic = (elf_dyn *)(object->baseAddress + phdr->p_vaddr); - }else if(phdr->p_type == PT_INTERP - || phdr->p_type == PT_PHDR - || phdr->p_type == PT_NOTE - || phdr->p_type == PT_RISCV_ATTRIBUTES - || phdr->p_type == PT_GNU_EH_FRAME - || phdr->p_type == PT_GNU_RELRO - || phdr->p_type == PT_GNU_STACK - || phdr->p_type == PT_GNU_PROPERTY) { - // ignore the phdr - }else{ - mlibc::panicLogger() << "Unexpected PHDR type 0x" - << frg::hex_fmt(phdr->p_type) << " in DSO " << object->name << frg::endlog; - } - } - - return frg::success; -} - -// -------------------------------------------------------- -// ObjectRepository: Parsing methods. -// -------------------------------------------------------- - -void ObjectRepository::_parseDynamic(SharedObject *object) { - if(!object->dynamic) - mlibc::infoLogger() << "ldso: Object '" << object->name - << "' does not have a dynamic section" << frg::endlog; - __ensure(object->dynamic); - - // Fix up these offsets to addresses after the loop, since the - // addresses depend on the value of DT_STRTAB. - frg::optional<ptrdiff_t> runpath_offset; - /* If true, ignore the RPATH. */ - bool runpath_found = false; - frg::optional<ptrdiff_t> soname_offset; - - for(size_t i = 0; object->dynamic[i].d_tag != DT_NULL; i++) { - elf_dyn *dynamic = &object->dynamic[i]; - switch(dynamic->d_tag) { - // handle hash table, symbol table and string table - case DT_HASH: - object->hashStyle = HashStyle::systemV; - object->hashTableOffset = dynamic->d_un.d_ptr; - break; - case DT_GNU_HASH: - object->hashStyle = HashStyle::gnu; - object->hashTableOffset = dynamic->d_un.d_ptr; - break; - case DT_STRTAB: - object->stringTableOffset = dynamic->d_un.d_ptr; - break; - case DT_STRSZ: - break; // we don't need the size of the string table - case DT_SYMTAB: - object->symbolTableOffset = dynamic->d_un.d_ptr; - break; - case DT_SYMENT: - __ensure(dynamic->d_un.d_val == sizeof(elf_sym)); - break; - // handle lazy relocation table - case DT_PLTGOT: - object->globalOffsetTable = (void **)(object->baseAddress - + dynamic->d_un.d_ptr); - break; - case DT_JMPREL: - object->lazyRelocTableOffset = dynamic->d_un.d_ptr; - break; - case DT_PLTRELSZ: - object->lazyTableSize = dynamic->d_un.d_val; - break; - case DT_PLTREL: - if(dynamic->d_un.d_val == DT_RELA) { - object->lazyExplicitAddend = true; - }else{ - __ensure(dynamic->d_un.d_val == DT_REL); - object->lazyExplicitAddend = false; - } - break; - // TODO: Implement this correctly! - case DT_SYMBOLIC: - object->symbolicResolution = true; - break; - case DT_BIND_NOW: - object->eagerBinding = true; - break; - case DT_FLAGS: { - if(dynamic->d_un.d_val & DF_SYMBOLIC) - object->symbolicResolution = true; - if(dynamic->d_un.d_val & DF_STATIC_TLS) - object->haveStaticTls = true; - if(dynamic->d_un.d_val & DF_BIND_NOW) - object->eagerBinding = true; - - auto ignored = DF_BIND_NOW | DF_SYMBOLIC | DF_STATIC_TLS; -#ifdef __riscv - // Work around https://sourceware.org/bugzilla/show_bug.cgi?id=24673. - ignored |= DF_TEXTREL; -#else - if(dynamic->d_un.d_val & DF_TEXTREL) - mlibc::panicLogger() << "\e[31mrtdl: DF_TEXTREL is unimplemented" << frg::endlog; -#endif - if(dynamic->d_un.d_val & ~ignored) - mlibc::infoLogger() << "\e[31mrtdl: DT_FLAGS(" << frg::hex_fmt{dynamic->d_un.d_val & ~ignored} - << ") is not implemented correctly!\e[39m" - << frg::endlog; - } break; - case DT_FLAGS_1: - if(dynamic->d_un.d_val & DF_1_NOW) - object->eagerBinding = true; - // The DF_1_PIE flag is informational only. It is used by e.g file(1). - // The DF_1_NODELETE flag has a similar effect to RTLD_NODELETE, both of which we - // ignore because we don't implement dlclose(). - if(dynamic->d_un.d_val & ~(DF_1_NOW | DF_1_PIE | DF_1_NODELETE)) - mlibc::infoLogger() << "\e[31mrtdl: DT_FLAGS_1(" << frg::hex_fmt{dynamic->d_un.d_val} - << ") is not implemented correctly!\e[39m" - << frg::endlog; - break; - case DT_RPATH: - if (runpath_found) { - /* Ignore RPATH if RUNPATH was present. */ - break; - } - [[fallthrough]]; - case DT_RUNPATH: - runpath_found = dynamic->d_tag == DT_RUNPATH; - runpath_offset = dynamic->d_un.d_val; - break; - case DT_INIT: - if(dynamic->d_un.d_ptr != 0) - object->initPtr = (InitFuncPtr)(object->baseAddress + dynamic->d_un.d_ptr); - break; - case DT_INIT_ARRAY: - if(dynamic->d_un.d_ptr != 0) - object->initArray = (InitFuncPtr *)(object->baseAddress + dynamic->d_un.d_ptr); - break; - case DT_INIT_ARRAYSZ: - object->initArraySize = dynamic->d_un.d_val; - break; - case DT_PREINIT_ARRAY: - if(dynamic->d_un.d_ptr != 0) { - // Only the main object is allowed pre-initializers. - __ensure(object->isMainObject); - object->preInitArray = (InitFuncPtr *)(object->baseAddress + dynamic->d_un.d_ptr); - } - break; - case DT_PREINIT_ARRAYSZ: - // Only the main object is allowed pre-initializers. - __ensure(object->isMainObject); - object->preInitArraySize = dynamic->d_un.d_val; - break; - case DT_DEBUG: -#if ELF_CLASS == ELFCLASS32 - dynamic->d_un.d_val = reinterpret_cast<Elf32_Word>(&globalDebugInterface); -#elif ELF_CLASS == ELFCLASS64 - dynamic->d_un.d_val = reinterpret_cast<Elf64_Xword>(&globalDebugInterface); -#endif - break; - case DT_SONAME: - soname_offset = dynamic->d_un.d_val; - break; - // ignore unimportant tags - case DT_NEEDED: // we handle this later - case DT_FINI: case DT_FINI_ARRAY: case DT_FINI_ARRAYSZ: - case DT_RELA: case DT_RELASZ: case DT_RELAENT: case DT_RELACOUNT: - case DT_REL: case DT_RELSZ: case DT_RELENT: case DT_RELCOUNT: - case DT_RELR: case DT_RELRSZ: case DT_RELRENT: - case DT_VERSYM: - case DT_VERDEF: case DT_VERDEFNUM: - case DT_VERNEED: case DT_VERNEEDNUM: -#ifdef __riscv - case DT_TEXTREL: // Work around https://sourceware.org/bugzilla/show_bug.cgi?id=24673. -#endif - break; - case DT_TLSDESC_PLT: case DT_TLSDESC_GOT: - break; - default: - // Ignore unknown entries in the os-specific area as we don't use them. - if(dynamic->d_tag < DT_LOOS || dynamic->d_tag > DT_HIOS) { - mlibc::panicLogger() << "Unexpected dynamic entry " - << (void *)dynamic->d_tag << " in object" << frg::endlog; - } - } - } - - if(runpath_offset) { - object->runPath = reinterpret_cast<const char *>(object->baseAddress - + object->stringTableOffset + *runpath_offset); - } - if(soname_offset) { - object->soName = reinterpret_cast<const char *>(object->baseAddress - + object->stringTableOffset + *soname_offset); - } -} - -void ObjectRepository::_discoverDependencies(SharedObject *object, - Scope *localScope, uint64_t rts) { - if(object->isMainObject) { - for(auto preload : *preloads) { - frg::expected<LinkerError, SharedObject *> libraryResult; - if (preload.find_first('/') == size_t(-1)) { - libraryResult = requestObjectWithName(preload, object, globalScope.get(), false, 1); - } else { - libraryResult = requestObjectAtPath(preload, globalScope.get(), false, 1); - } - if(!libraryResult) - mlibc::panicLogger() << "rtdl: Could not load preload " << preload << frg::endlog; - - if(verbose) - mlibc::infoLogger() << "rtdl: Preloading " << preload << frg::endlog; - - object->dependencies.push_back(libraryResult.value()); - } - } - - // Load required dynamic libraries. - for(size_t i = 0; object->dynamic[i].d_tag != DT_NULL; i++) { - elf_dyn *dynamic = &object->dynamic[i]; - if(dynamic->d_tag != DT_NEEDED) - continue; - - const char *library_str = (const char *)(object->baseAddress - + object->stringTableOffset + dynamic->d_un.d_val); - - auto library = requestObjectWithName(frg::string_view{library_str}, - object, localScope, false, rts); - if(!library) - mlibc::panicLogger() << "Could not satisfy dependency " << library_str << frg::endlog; - object->dependencies.push(library.value()); - } -} - -void ObjectRepository::_addLoadedObject(SharedObject *object) { - _nameMap.insert(object->name, object); - loadedObjects.push_back(object); -} - -// -------------------------------------------------------- -// SharedObject -// -------------------------------------------------------- - -SharedObject::SharedObject(const char *name, frg::string<MemoryAllocator> path, - bool is_main_object, Scope *local_scope, uint64_t object_rts) - : name(name, getAllocator()), path(std::move(path)), - interpreterPath(getAllocator()), soName(nullptr), - isMainObject(is_main_object), objectRts(object_rts), inLinkMap(false), - baseAddress(0), localScope(local_scope), dynamic(nullptr), - globalOffsetTable(nullptr), entry(nullptr), tlsSegmentSize(0), - tlsAlignment(0), tlsImageSize(0), tlsImagePtr(nullptr), - tlsInitialized(false), hashTableOffset(0), symbolTableOffset(0), - stringTableOffset(0), lazyRelocTableOffset(0), lazyTableSize(0), - lazyExplicitAddend(false), symbolicResolution(false), - eagerBinding(false), haveStaticTls(false), - dependencies(getAllocator()), tlsModel(TlsModel::null), - tlsOffset(0), globalRts(0), wasLinked(false), - scheduledForInit(false), onInitStack(false), - wasInitialized(false) { } - -SharedObject::SharedObject(const char *name, const char *path, - bool is_main_object, Scope *localScope, uint64_t object_rts) - : SharedObject(name, - frg::string<MemoryAllocator> { path, getAllocator() }, - is_main_object, localScope, object_rts) {} - -void processLateRelocation(Relocation rel) { - // resolve the symbol if there is a symbol - frg::optional<ObjectSymbol> p; - if(rel.symbol_index()) { - auto symbol = (elf_sym *)(rel.object()->baseAddress + rel.object()->symbolTableOffset - + rel.symbol_index() * sizeof(elf_sym)); - ObjectSymbol r(rel.object(), symbol); - - p = Scope::resolveGlobalOrLocal(*globalScope, rel.object()->localScope, - r.getString(), rel.object()->objectRts, Scope::resolveCopy); - } - - switch(rel.type()) { - case R_COPY: - __ensure(p); - memcpy(rel.destination(), (void *)p->virtualAddress(), p->symbol()->st_size); - break; - -// TODO: R_IRELATIVE also exists on other architectures but will likely need a different implementation. -#if defined(__x86_64__) || defined(__i386__) - case R_IRELATIVE: { - uintptr_t addr = rel.object()->baseAddress + rel.addend_rel(); - auto* fn = reinterpret_cast<uintptr_t (*)()>(addr); - rel.relocate(fn()); - } break; -#elif defined(__aarch64__) - case R_IRELATIVE: { - uintptr_t addr = rel.object()->baseAddress + rel.addend_rel(); - auto* fn = reinterpret_cast<uintptr_t (*)(uint64_t)>(addr); - // TODO: the function should get passed AT_HWCAP value. - rel.relocate(fn(0)); - } break; -#endif - - default: - break; - } -} - -void processLateRelocations(SharedObject *object) { - frg::optional<uintptr_t> rel_offset; - frg::optional<size_t> rel_length; - - frg::optional<uintptr_t> rela_offset; - frg::optional<size_t> rela_length; - - for(size_t i = 0; object->dynamic[i].d_tag != DT_NULL; i++) { - elf_dyn *dynamic = &object->dynamic[i]; - - switch(dynamic->d_tag) { - case DT_REL: - rel_offset = dynamic->d_un.d_ptr; - break; - case DT_RELSZ: - rel_length = dynamic->d_un.d_val; - break; - case DT_RELENT: - __ensure(dynamic->d_un.d_val == sizeof(elf_rel)); - break; - case DT_RELA: - rela_offset = dynamic->d_un.d_ptr; - break; - case DT_RELASZ: - rela_length = dynamic->d_un.d_val; - break; - case DT_RELAENT: - __ensure(dynamic->d_un.d_val == sizeof(elf_rela)); - break; - } - } - - if(rela_offset && rela_length) { - for(size_t offset = 0; offset < *rela_length; offset += sizeof(elf_rela)) { - auto reloc = (elf_rela *)(object->baseAddress + *rela_offset + offset); - auto r = Relocation(object, reloc); - processLateRelocation(r); - } - } else if(rel_offset && rel_length) { - for(size_t offset = 0; offset < *rel_length; offset += sizeof(elf_rel)) { - auto reloc = (elf_rel *)(object->baseAddress + *rel_offset + offset); - auto r = Relocation(object, reloc); - processLateRelocation(r); - } - }else{ - __ensure(!rela_offset && !rela_length); - __ensure(!rel_offset && !rel_length); - } -} - -void doInitialize(SharedObject *object) { - __ensure(object->wasLinked); - __ensure(!object->wasInitialized); - - // if the object has dependencies we initialize them first - for(size_t i = 0; i < object->dependencies.size(); i++) - __ensure(object->dependencies[i]->wasInitialized); - - if(verbose) - mlibc::infoLogger() << "rtdl: Initialize " << object->name << frg::endlog; - - if(verbose) - mlibc::infoLogger() << "rtdl: Running DT_INIT function" << frg::endlog; - if(object->initPtr != nullptr) - object->initPtr(); - - if(verbose) - mlibc::infoLogger() << "rtdl: Running DT_INIT_ARRAY functions" << frg::endlog; - __ensure((object->initArraySize % sizeof(InitFuncPtr)) == 0); - for(size_t i = 0; i < object->initArraySize / sizeof(InitFuncPtr); i++) - object->initArray[i](); - - if(verbose) - mlibc::infoLogger() << "rtdl: Object initialization complete" << frg::endlog; - object->wasInitialized = true; -} - -// -------------------------------------------------------- -// RuntimeTlsMap -// -------------------------------------------------------- - -RuntimeTlsMap::RuntimeTlsMap() -: initialPtr{0}, initialLimit{0}, indices{getAllocator()} { } - -void initTlsObjects(Tcb *tcb, const frg::vector<SharedObject *, MemoryAllocator> &objects, bool checkInitialized) { - // Initialize TLS segments that follow the static model. - for(auto object : objects) { - if(object->tlsModel == TlsModel::initial) { - if(checkInitialized && object->tlsInitialized) - continue; - - char *tcb_ptr = reinterpret_cast<char *>(tcb); - auto tls_ptr = tcb_ptr + object->tlsOffset; - memset(tls_ptr, 0, object->tlsSegmentSize); - memcpy(tls_ptr, object->tlsImagePtr, object->tlsImageSize); - - if (verbose) { - mlibc::infoLogger() << "rtdl: wrote tls image at " << (void *)tls_ptr - << ", size = 0x" << frg::hex_fmt{object->tlsSegmentSize} << frg::endlog; - } - - if (checkInitialized) - object->tlsInitialized = true; - } - } -} - -Tcb *allocateTcb() { - size_t tlsInitialSize = runtimeTlsMap->initialLimit; - - // To make sure that both the TCB and TLS data are sufficiently aligned, allocate - // slightly more than necessary and adjust alignment afterwards. - size_t alignOverhead = frg::max(alignof(Tcb), tlsMaxAlignment); - size_t allocSize = tlsInitialSize + sizeof(Tcb) + alignOverhead; - auto allocation = reinterpret_cast<uintptr_t>(getAllocator().allocate(allocSize)); - memset(reinterpret_cast<void *>(allocation), 0, allocSize); - - uintptr_t tlsAddress, tcbAddress; - if constexpr (tlsAboveTp) { - // Here we must satisfy two requirements of the TCB and the TLS data: - // 1. One should follow the other immediately in memory. We do this so that - // we can simply add or subtract sizeof(Tcb) to obtain the address of the other. - // 2. Both should be sufficiently aligned. - // To do this, we will fix whichever address has stricter alignment requirements, and - // derive the other from it. - if (tlsMaxAlignment > alignof(Tcb)) { - tlsAddress = alignUp(allocation + sizeof(Tcb), tlsMaxAlignment); - tcbAddress = tlsAddress - sizeof(Tcb); - } else { - tcbAddress = alignUp(allocation, alignof(Tcb)); - tlsAddress = tcbAddress + sizeof(Tcb); - } - __ensure((tlsAddress & (tlsMaxAlignment - 1)) == 0); - __ensure(tlsAddress == tcbAddress + sizeof(Tcb)); - } else { - // The TCB should be aligned such that the preceding blocks are aligned too. - tcbAddress = alignUp(allocation + tlsInitialSize, alignOverhead); - tlsAddress = tcbAddress - tlsInitialSize; - } - __ensure((tcbAddress & (alignof(Tcb) - 1)) == 0); - - if (verbose) { - mlibc::infoLogger() << "rtdl: tcb allocated at " << (void *)tcbAddress - << ", size = 0x" << frg::hex_fmt{sizeof(Tcb)} << frg::endlog; - mlibc::infoLogger() << "rtdl: tls allocated at " << (void *)tlsAddress - << ", size = 0x" << frg::hex_fmt{tlsInitialSize} << frg::endlog; - } - - Tcb *tcb_ptr = new ((char *)tcbAddress) Tcb; - tcb_ptr->selfPointer = tcb_ptr; - - tcb_ptr->stackCanary = __stack_chk_guard; - tcb_ptr->cancelBits = tcbCancelEnableBit; - tcb_ptr->didExit = 0; - tcb_ptr->isJoinable = 1; - memset(&tcb_ptr->returnValue, 0, sizeof(tcb_ptr->returnValue)); - tcb_ptr->localKeys = frg::construct<frg::array<Tcb::LocalKey, PTHREAD_KEYS_MAX>>(getAllocator()); - tcb_ptr->dtvSize = runtimeTlsMap->indices.size(); - tcb_ptr->dtvPointers = frg::construct_n<void *>(getAllocator(), runtimeTlsMap->indices.size()); - memset(tcb_ptr->dtvPointers, 0, sizeof(void *) * runtimeTlsMap->indices.size()); - for(size_t i = 0; i < runtimeTlsMap->indices.size(); ++i) { - auto object = runtimeTlsMap->indices[i]; - if(object->tlsModel != TlsModel::initial) - continue; - tcb_ptr->dtvPointers[i] = reinterpret_cast<char *>(tcb_ptr) + object->tlsOffset; - } - - return tcb_ptr; -} - -void *accessDtv(SharedObject *object) { - Tcb *tcb_ptr = mlibc::get_current_tcb(); - - // We might need to reallocate the DTV. - if(object->tlsIndex >= tcb_ptr->dtvSize) { - // TODO: need to protect runtimeTlsMap against concurrent access. - auto ndtv = frg::construct_n<void *>(getAllocator(), runtimeTlsMap->indices.size()); - memset(ndtv, 0, sizeof(void *) * runtimeTlsMap->indices.size()); - memcpy(ndtv, tcb_ptr->dtvPointers, sizeof(void *) * tcb_ptr->dtvSize); - frg::destruct_n(getAllocator(), tcb_ptr->dtvPointers, tcb_ptr->dtvSize); - tcb_ptr->dtvSize = runtimeTlsMap->indices.size(); - tcb_ptr->dtvPointers = ndtv; - } - - // We might need to fill in a new DTV entry. - if(!tcb_ptr->dtvPointers[object->tlsIndex]) { - __ensure(object->tlsModel == TlsModel::dynamic); - - auto buffer = getAllocator().allocate(object->tlsSegmentSize); - __ensure(!(reinterpret_cast<uintptr_t>(buffer) & (object->tlsAlignment - 1))); - memset(buffer, 0, object->tlsSegmentSize); - memcpy(buffer, object->tlsImagePtr, object->tlsImageSize); - tcb_ptr->dtvPointers[object->tlsIndex] = buffer; - - if (verbose) { - mlibc::infoLogger() << "rtdl: accessDtv wrote tls image at " << buffer - << ", size = 0x" << frg::hex_fmt{object->tlsSegmentSize} << frg::endlog; - } - } - - return (void *)((char *)tcb_ptr->dtvPointers[object->tlsIndex] + TLS_DTV_OFFSET); -} - -void *tryAccessDtv(SharedObject *object) { - Tcb *tcb_ptr = mlibc::get_current_tcb(); - - if (object->tlsIndex >= tcb_ptr->dtvSize) - return nullptr; - if (!tcb_ptr->dtvPointers[object->tlsIndex]) - return nullptr; - - return (void *)((char *)tcb_ptr->dtvPointers[object->tlsIndex] + TLS_DTV_OFFSET); -} - -// -------------------------------------------------------- -// ObjectSymbol -// -------------------------------------------------------- - -ObjectSymbol::ObjectSymbol(SharedObject *object, const elf_sym *symbol) -: _object(object), _symbol(symbol) { } - -const char *ObjectSymbol::getString() { - __ensure(_symbol->st_name != 0); - return (const char *)(_object->baseAddress - + _object->stringTableOffset + _symbol->st_name); -} - -uintptr_t ObjectSymbol::virtualAddress() { - auto bind = ELF_ST_BIND(_symbol->st_info); - __ensure(bind == STB_GLOBAL || bind == STB_WEAK || bind == STB_GNU_UNIQUE); - __ensure(_symbol->st_shndx != SHN_UNDEF); - return _object->baseAddress + _symbol->st_value; -} - -// -------------------------------------------------------- -// Scope -// -------------------------------------------------------- - -uint32_t elf64Hash(frg::string_view string) { - uint32_t h = 0, g; - - for(size_t i = 0; i < string.size(); ++i) { - h = (h << 4) + (uint32_t)string[i]; - g = h & 0xF0000000; - if(g) - h ^= g >> 24; - h &= 0x0FFFFFFF; - } - - return h; -} - -uint32_t gnuHash(frg::string_view string) { - uint32_t h = 5381; - for(size_t i = 0; i < string.size(); ++i) - h = (h << 5) + h + string[i]; - return h; -} - -// TODO: move this to some namespace or class? -frg::optional<ObjectSymbol> resolveInObject(SharedObject *object, frg::string_view string) { - // Checks if the symbol can be used to satisfy the dependency. - auto eligible = [&] (ObjectSymbol cand) { - if(cand.symbol()->st_shndx == SHN_UNDEF) - return false; - - auto bind = ELF_ST_BIND(cand.symbol()->st_info); - if(bind != STB_GLOBAL && bind != STB_WEAK && bind != STB_GNU_UNIQUE) - return false; - - return true; - }; - - if (object->hashStyle == HashStyle::systemV) { - auto hash_table = (Elf64_Word *)(object->baseAddress + object->hashTableOffset); - Elf64_Word num_buckets = hash_table[0]; - auto bucket = elf64Hash(string) % num_buckets; - - auto index = hash_table[2 + bucket]; - while(index != 0) { - ObjectSymbol cand{object, (elf_sym *)(object->baseAddress - + object->symbolTableOffset + index * sizeof(elf_sym))}; - if(eligible(cand) && frg::string_view{cand.getString()} == string) - return cand; - - index = hash_table[2 + num_buckets + index]; - } - - return frg::optional<ObjectSymbol>{}; - }else{ - __ensure(object->hashStyle == HashStyle::gnu); - - struct GnuTable { - uint32_t nBuckets; - uint32_t symbolOffset; - uint32_t bloomSize; - uint32_t bloomShift; - }; - - auto hash_table = reinterpret_cast<const GnuTable *>(object->baseAddress - + object->hashTableOffset); - auto buckets = reinterpret_cast<const uint32_t *>(object->baseAddress - + object->hashTableOffset + sizeof(GnuTable) - + hash_table->bloomSize * sizeof(elf_addr)); - auto chains = reinterpret_cast<const uint32_t *>(object->baseAddress - + object->hashTableOffset + sizeof(GnuTable) - + hash_table->bloomSize * sizeof(elf_addr) - + hash_table->nBuckets * sizeof(uint32_t)); - - // TODO: Use the bloom filter. - - // The symbols of a given bucket are contiguous in the table. - auto hash = gnuHash(string); - auto index = buckets[hash % hash_table->nBuckets]; - - if(!index) - return frg::optional<ObjectSymbol>{}; - - while(true) { - // chains[] contains an array of hashes, parallel to the symbol table. - auto chash = chains[index - hash_table->symbolOffset]; - if ((chash & ~1) == (hash & ~1)) { - ObjectSymbol cand{object, (elf_sym *)(object->baseAddress - + object->symbolTableOffset + index * sizeof(elf_sym))}; - if(eligible(cand) && frg::string_view{cand.getString()} == string) - return cand; - } - - // If we hit the end of the chain, the symbol is not present. - if(chash & 1) - return frg::optional<ObjectSymbol>{}; - index++; - } - } -} - -frg::optional<ObjectSymbol> Scope::_resolveNext(frg::string_view string, - SharedObject *target) { - // Skip objects until we find the target, and only look for symbols after that. - size_t i; - for (i = 0; i < _objects.size(); i++) { - if (_objects[i] == target) - break; - } - - if (i == _objects.size()) { - mlibc::infoLogger() << "rtdl: object passed to Scope::resolveAfter was not found" << frg::endlog; - return frg::optional<ObjectSymbol>(); - } - - for (i = i + 1; i < _objects.size(); i++) { - if(_objects[i]->isMainObject) - continue; - - frg::optional<ObjectSymbol> p = resolveInObject(_objects[i], string); - if(p) - return p; - } - - return frg::optional<ObjectSymbol>(); -} - -Scope::Scope(bool isGlobal) -: isGlobal{isGlobal}, _objects(getAllocator()) { } - -void Scope::appendObject(SharedObject *object) { - // Don't insert duplicates. - for (auto obj : _objects) { - if (obj == object) - return; - } - - _objects.push(object); -} - -frg::optional<ObjectSymbol> Scope::resolveGlobalOrLocal(Scope &globalScope, - Scope *localScope, frg::string_view string, uint64_t skipRts, ResolveFlags flags) { - auto sym = globalScope.resolveSymbol(string, skipRts, flags | skipGlobalAfterRts); - if(!sym && localScope) - sym = localScope->resolveSymbol(string, skipRts, flags | skipGlobalAfterRts); - return sym; -} - -frg::optional<ObjectSymbol> Scope::resolveGlobalOrLocalNext(Scope &globalScope, - Scope *localScope, frg::string_view string, SharedObject *origin) { - auto sym = globalScope._resolveNext(string, origin); - if(!sym && localScope) { - sym = localScope->_resolveNext(string, origin); - } - return sym; -} - -// TODO: let this return uintptr_t -frg::optional<ObjectSymbol> Scope::resolveSymbol(frg::string_view string, - uint64_t skipRts, ResolveFlags flags) { - for (auto object : _objects) { - if((flags & resolveCopy) && object->isMainObject) - continue; - if((flags & skipGlobalAfterRts) && object->globalRts > skipRts) { - // globalRts should be monotone increasing for objects in the global scope, - // so as an optimization we can break early here. - // TODO: If we implement DT_SYMBOLIC, this assumption fails. - if(isGlobal) - break; - else - continue; - } - - frg::optional<ObjectSymbol> p = resolveInObject(object, string); - if(p) - return p; - } - - return frg::optional<ObjectSymbol>(); -} - -// -------------------------------------------------------- -// Loader -// -------------------------------------------------------- - -Loader::Loader(Scope *scope, SharedObject *mainExecutable, bool is_initial_link, uint64_t rts) -: _mainExecutable{mainExecutable}, _loadScope{scope}, _isInitialLink{is_initial_link}, - _linkRts{rts}, _linkBfs{getAllocator()}, _initQueue{getAllocator()} { } - -void Loader::_buildLinkBfs(SharedObject *root) { - __ensure(_linkBfs.size() == 0); - - struct Token {}; - using Set = frg::hash_map<SharedObject *, Token, - frg::hash<SharedObject *>, MemoryAllocator>; - Set set{frg::hash<SharedObject *>{}, getAllocator()}; - _linkBfs.push(root); - - // Loop over indices (not iterators) here: We are adding elements in the loop! - for(size_t i = 0; i < _linkBfs.size(); i++) { - auto current = _linkBfs[i]; - - // At this point the object is loaded and we can fill in its debug struct, - // the linked list fields will be filled later. - current->linkMap.base = current->baseAddress; - current->linkMap.name = current->path.data(); - current->linkMap.dynv = current->dynamic; - - __ensure((current->tlsAlignment & (current->tlsAlignment - 1)) == 0); - - if (_isInitialLink && current->tlsAlignment > tlsMaxAlignment) { - tlsMaxAlignment = current->tlsAlignment; - } - - for (auto dep : current->dependencies) { - if (!set.get(dep)) { - set.insert(dep, Token{}); - _linkBfs.push(dep); - } - } - } -} - -void Loader::linkObjects(SharedObject *root) { - _buildLinkBfs(root); - _buildTlsMaps(); - - // Promote objects to the desired scope. - for(auto object : _linkBfs) { - if (object->globalRts == 0 && _loadScope->isGlobal) - object->globalRts = _linkRts; - - _loadScope->appendObject(object); - } - - // Process regular relocations. - for(auto object : _linkBfs) { - // Some objects have already been linked before. - if(object->objectRts < _linkRts) - continue; - - if(object->dynamic == nullptr) - continue; - - if(verbose) - mlibc::infoLogger() << "rtdl: Linking " << object->name << frg::endlog; - - __ensure(!object->wasLinked); - - // TODO: Support this. - if(object->symbolicResolution) - mlibc::infoLogger() << "\e[31mrtdl: DT_SYMBOLIC is not implemented correctly!\e[39m" - << frg::endlog; - - _processStaticRelocations(object); - _processLazyRelocations(object); - } - - // Process copy relocations. - for(auto object : _linkBfs) { - if(!object->isMainObject) - continue; - - // Some objects have already been linked before. - if(object->objectRts < _linkRts) - continue; - - if(object->dynamic == nullptr) - continue; - - processLateRelocations(object); - } - - for(auto object : _linkBfs) { - object->wasLinked = true; - - if(object->inLinkMap) - continue; - - auto linkMap = reinterpret_cast<LinkMap*>(globalDebugInterface.head); - - object->linkMap.prev = linkMap; - object->linkMap.next = linkMap->next; - if(linkMap->next) - linkMap->next->prev = &(object->linkMap); - linkMap->next = &(object->linkMap); - object->inLinkMap = true; - } -} - -void Loader::_buildTlsMaps() { - if(_isInitialLink) { - __ensure(runtimeTlsMap->initialPtr == 0); - __ensure(runtimeTlsMap->initialLimit == 0); - - __ensure(!_linkBfs.empty()); - __ensure(_linkBfs.front()->isMainObject); - - for(auto object : _linkBfs) { - __ensure(object->tlsModel == TlsModel::null); - - if(object->tlsSegmentSize == 0) - continue; - - // Allocate an index for the object. - object->tlsIndex = runtimeTlsMap->indices.size(); - runtimeTlsMap->indices.push_back(object); - - object->tlsModel = TlsModel::initial; - - if constexpr (tlsAboveTp) { - // As per the comment in allocateTcb(), we may simply add sizeof(Tcb) to - // reach the TLS data. - object->tlsOffset = runtimeTlsMap->initialPtr + sizeof(Tcb); - runtimeTlsMap->initialPtr += object->tlsSegmentSize; - - size_t misalign = runtimeTlsMap->initialPtr & (object->tlsAlignment - 1); - if(misalign) - runtimeTlsMap->initialPtr += object->tlsAlignment - misalign; - } else { - runtimeTlsMap->initialPtr += object->tlsSegmentSize; - - size_t misalign = runtimeTlsMap->initialPtr & (object->tlsAlignment - 1); - if(misalign) - runtimeTlsMap->initialPtr += object->tlsAlignment - misalign; - - object->tlsOffset = -runtimeTlsMap->initialPtr; - } - - if(verbose) - mlibc::infoLogger() << "rtdl: TLS of " << object->name - << " mapped to 0x" << frg::hex_fmt{object->tlsOffset} - << ", size: " << object->tlsSegmentSize - << ", alignment: " << object->tlsAlignment << frg::endlog; - } - - // Reserve some additional space for future libraries. - runtimeTlsMap->initialLimit = runtimeTlsMap->initialPtr + 64; - }else{ - for(auto object : _linkBfs) { - if(object->tlsModel != TlsModel::null) - continue; - if(object->tlsSegmentSize == 0) - continue; - - // Allocate an index for the object. - object->tlsIndex = runtimeTlsMap->indices.size(); - runtimeTlsMap->indices.push_back(object); - - // There are some libraries (e.g. Mesa) that require static TLS even though - // they expect to be dynamically loaded. - if(object->haveStaticTls) { - auto ptr = runtimeTlsMap->initialPtr + object->tlsSegmentSize; - size_t misalign = ptr & (object->tlsAlignment - 1); - if(misalign) - ptr += object->tlsAlignment - misalign; - - if(ptr > runtimeTlsMap->initialLimit) - mlibc::panicLogger() << "rtdl: Static TLS space exhausted while while" - " allocating TLS for " << object->name << frg::endlog; - - object->tlsModel = TlsModel::initial; - - if constexpr (tlsAboveTp) { - size_t tcbSize = ((sizeof(Tcb) + tlsMaxAlignment - 1) & ~(tlsMaxAlignment - 1)); - - object->tlsOffset = runtimeTlsMap->initialPtr + tcbSize; - runtimeTlsMap->initialPtr = ptr; - } else { - runtimeTlsMap->initialPtr = ptr; - object->tlsOffset = -runtimeTlsMap->initialPtr; - } - - if(verbose) - mlibc::infoLogger() << "rtdl: TLS of " << object->name - << " mapped to 0x" << frg::hex_fmt{object->tlsOffset} - << ", size: " << object->tlsSegmentSize - << ", alignment: " << object->tlsAlignment << frg::endlog; - }else{ - object->tlsModel = TlsModel::dynamic; - } - } - } -} - -void Loader::initObjects() { - initTlsObjects(mlibc::get_current_tcb(), _linkBfs, true); - - if (_mainExecutable && _mainExecutable->preInitArray) { - if (verbose) - mlibc::infoLogger() << "rtdl: Running DT_PREINIT_ARRAY functions" << frg::endlog; - - __ensure(_mainExecutable->isMainObject); - __ensure(!_mainExecutable->wasInitialized); - __ensure((_mainExecutable->preInitArraySize % sizeof(InitFuncPtr)) == 0); - for(size_t i = 0; i < _mainExecutable->preInitArraySize / sizeof(InitFuncPtr); i++) - _mainExecutable->preInitArray[i](); - } - - // Convert the breadth-first representation to a depth-first post-order representation, - // so that every object is initialized *after* its dependencies. - for(auto object : _linkBfs) { - if(!object->scheduledForInit) - _scheduleInit(object); - } - - for(auto object : _initQueue) { - if(!object->wasInitialized) - doInitialize(object); - } -} - -// TODO: Use an explicit vector to reduce stack usage to O(1)? -void Loader::_scheduleInit(SharedObject *object) { - // Here we detect cyclic dependencies. - __ensure(!object->onInitStack); - object->onInitStack = true; - - __ensure(!object->scheduledForInit); - object->scheduledForInit = true; - - for(size_t i = 0; i < object->dependencies.size(); i++) { - if(!object->dependencies[i]->scheduledForInit) - _scheduleInit(object->dependencies[i]); - } - - _initQueue.push(object); - object->onInitStack = false; -} - -void Loader::_processRelocations(Relocation &rel) { - // copy and irelative relocations have to be performed after all other relocations - if(rel.type() == R_COPY || rel.type() == R_IRELATIVE) - return; - - // resolve the symbol if there is a symbol - frg::optional<ObjectSymbol> p; - if(rel.symbol_index()) { - auto symbol = (elf_sym *)(rel.object()->baseAddress + rel.object()->symbolTableOffset - + rel.symbol_index() * sizeof(elf_sym)); - ObjectSymbol r(rel.object(), symbol); - - p = Scope::resolveGlobalOrLocal(*globalScope, rel.object()->localScope, - r.getString(), rel.object()->objectRts, 0); - if(!p) { - if(ELF_ST_BIND(symbol->st_info) != STB_WEAK) - mlibc::panicLogger() << "Unresolved load-time symbol " - << r.getString() << " in object " << rel.object()->name << frg::endlog; - - if(verbose) - mlibc::infoLogger() << "rtdl: Unresolved weak load-time symbol " - << r.getString() << " in object " << rel.object()->name << frg::endlog; - } - } - - switch(rel.type()) { - case R_NONE: - break; - - case R_JUMP_SLOT: { - __ensure(!rel.addend_norel()); - uintptr_t symbol_addr = p ? p->virtualAddress() : 0; - rel.relocate(symbol_addr); - } break; - -#if !defined(__riscv) - // on some architectures, R_GLOB_DAT can be defined to other relocations - case R_GLOB_DAT: { - __ensure(rel.symbol_index()); - uintptr_t symbol_addr = p ? p->virtualAddress() : 0; - rel.relocate(symbol_addr + rel.addend_norel()); - } break; -#endif - - case R_ABSOLUTE: { - __ensure(rel.symbol_index()); - uintptr_t symbol_addr = p ? p->virtualAddress() : 0; - rel.relocate(symbol_addr + rel.addend_rel()); - } break; - - case R_RELATIVE: { - __ensure(!rel.symbol_index()); - rel.relocate(rel.object()->baseAddress + rel.addend_rel()); - } break; - - // DTPMOD and DTPREL are dynamic TLS relocations (for __tls_get_addr()). - // TPOFF is a relocation to the initial TLS model. - case R_TLS_DTPMOD: { - // sets the first `sizeof(uintptr_t)` bytes of `struct __abi_tls_entry` - // this means that we can just use the `SharedObject *` to resolve whatever we need - __ensure(!rel.addend_rel()); - if(rel.symbol_index()) { - __ensure(p); - rel.relocate(elf_addr(p->object())); - }else{ - if(stillSlightlyVerbose) - mlibc::infoLogger() << "rtdl: Warning: TLS_DTPMOD64 with no symbol in object " - << rel.object()->name << frg::endlog; - rel.relocate(elf_addr(rel.object())); - } - } break; - case R_TLS_DTPREL: { - __ensure(rel.symbol_index()); - __ensure(p); - rel.relocate(p->symbol()->st_value + rel.addend_rel() - TLS_DTV_OFFSET); - } break; - case R_TLS_TPREL: { - uintptr_t off = rel.addend_rel(); - uintptr_t tls_offset = 0; - - if(rel.symbol_index()) { - __ensure(p); - if(p->object()->tlsModel != TlsModel::initial) - mlibc::panicLogger() << "rtdl: In object " << rel.object()->name - << ": Static TLS relocation to symbol " << p->getString() - << " in dynamically loaded object " - << p->object()->name << frg::endlog; - off += p->symbol()->st_value; - tls_offset = p->object()->tlsOffset; - }else{ - if(stillSlightlyVerbose) - mlibc::infoLogger() << "rtdl: Warning: TPOFF64 with no symbol" - " in object " << rel.object()->name << frg::endlog; - if(rel.object()->tlsModel != TlsModel::initial) - mlibc::panicLogger() << "rtdl: In object " << rel.object()->name - << ": Static TLS relocation to dynamically loaded object " - << rel.object()->name << frg::endlog; - tls_offset = rel.object()->tlsOffset; - } - - if constexpr (tlsAboveTp) { - off += tls_offset - sizeof(Tcb); - } else { - off += tls_offset; - } - - rel.relocate(off); - } break; - default: - mlibc::panicLogger() << "Unexpected relocation type " - << (void *) rel.type() << frg::endlog; - } -} - -void Loader::_processStaticRelocations(SharedObject *object) { - frg::optional<uintptr_t> rela_offset; - frg::optional<size_t> rela_length; - - frg::optional<uintptr_t> rel_offset; - frg::optional<size_t> rel_length; - - frg::optional<uintptr_t> relr_offset; - frg::optional<size_t> relr_length; - - for(size_t i = 0; object->dynamic[i].d_tag != DT_NULL; i++) { - elf_dyn *dynamic = &object->dynamic[i]; - - switch(dynamic->d_tag) { - case DT_RELA: - rela_offset = dynamic->d_un.d_ptr; - break; - case DT_RELASZ: - rela_length = dynamic->d_un.d_val; - break; - case DT_RELAENT: - __ensure(dynamic->d_un.d_val == sizeof(elf_rela)); - break; - case DT_REL: - rel_offset = dynamic->d_un.d_ptr; - break; - case DT_RELSZ: - rel_length = dynamic->d_un.d_val; - break; - case DT_RELENT: - __ensure(dynamic->d_un.d_val == sizeof(elf_rel)); - break; - case DT_RELR: - relr_offset = dynamic->d_un.d_ptr; - break; - case DT_RELRSZ: - relr_length = dynamic->d_un.d_val; - break; - case DT_RELRENT: - __ensure(dynamic->d_un.d_val == sizeof(elf_relr)); - break; - } - } - - if(rela_offset && rela_length) { - __ensure(!rel_offset && !rel_length); - - for(size_t offset = 0; offset < *rela_length; offset += sizeof(elf_rela)) { - auto reloc = (elf_rela *)(object->baseAddress + *rela_offset + offset); - auto r = Relocation(object, reloc); - - _processRelocations(r); - } - }else if(rel_offset && rel_length) { - __ensure(!rela_offset && !rela_length); - - for(size_t offset = 0; offset < *rel_length; offset += sizeof(elf_rel)) { - auto reloc = (elf_rel *)(object->baseAddress + *rel_offset + offset); - auto r = Relocation(object, reloc); - - _processRelocations(r); - } - } - - if(relr_offset && relr_length) { - elf_addr *addr = nullptr; - - for(size_t offset = 0; offset < *relr_length; offset += sizeof(elf_relr)) { - auto entry = *(elf_relr *)(object->baseAddress + *relr_offset + offset); - - // Even entry indicates the beginning address. - if(!(entry & 1)) { - addr = (elf_addr *)(object->baseAddress + entry); - __ensure(addr); - *addr++ += object->baseAddress; - }else { - // Odd entry indicates entry is a bitmap of the subsequent locations to be relocated. - for(int i = 0; entry; ++i) { - if(entry & 1) { - addr[i] += object->baseAddress; - } - entry >>= 1; - } - - // Each entry describes at max 63 (on 64bit) or 31 (on 32bit) subsequent locations. - addr += CHAR_BIT * sizeof(elf_relr) - 1; - } - } - } -} - -// TODO: TLSDESC relocations aren't aarch64 specific -#ifdef __aarch64__ -extern "C" void *__mlibcTlsdescStatic(void *); -extern "C" void *__mlibcTlsdescDynamic(void *); -#endif - -void Loader::_processLazyRelocations(SharedObject *object) { - if(object->globalOffsetTable == nullptr) { - __ensure(object->lazyRelocTableOffset == 0); - return; - } - object->globalOffsetTable[1] = object; - object->globalOffsetTable[2] = (void *)&pltRelocateStub; - - if(!object->lazyTableSize) - return; - - // adjust the addresses of JUMP_SLOT relocations - __ensure(object->lazyExplicitAddend.has_value()); - size_t rel_size = (*object->lazyExplicitAddend) ? sizeof(elf_rela) : sizeof(elf_rel); - - for(size_t offset = 0; offset < object->lazyTableSize; offset += rel_size) { - elf_info type; - elf_info symbol_index; - - uintptr_t rel_addr; - uintptr_t addend [[maybe_unused]] = 0; - - if(*object->lazyExplicitAddend) { - auto reloc = (elf_rela *)(object->baseAddress + object->lazyRelocTableOffset + offset); - type = ELF_R_TYPE(reloc->r_info); - symbol_index = ELF_R_SYM(reloc->r_info); - rel_addr = object->baseAddress + reloc->r_offset; - addend = reloc->r_addend; - } else { - auto reloc = (elf_rel *)(object->baseAddress + object->lazyRelocTableOffset + offset); - type = ELF_R_TYPE(reloc->r_info); - symbol_index = ELF_R_SYM(reloc->r_info); - rel_addr = object->baseAddress + reloc->r_offset; - } - - switch (type) { - case R_JUMP_SLOT: - if(eagerBinding) { - auto symbol = (elf_sym *)(object->baseAddress + object->symbolTableOffset - + symbol_index * sizeof(elf_sym)); - ObjectSymbol r(object, symbol); - auto p = Scope::resolveGlobalOrLocal(*globalScope, object->localScope, r.getString(), object->objectRts, 0); - - if(!p) { - if(ELF_ST_BIND(symbol->st_info) != STB_WEAK) - mlibc::panicLogger() << "rtdl: Unresolved JUMP_SLOT symbol " - << r.getString() << " in object " << object->name << frg::endlog; - - if(verbose) - mlibc::infoLogger() << "rtdl: Unresolved weak JUMP_SLOT symbol " - << r.getString() << " in object " << object->name << frg::endlog; - *((uintptr_t *)rel_addr) = 0; - }else{ - *((uintptr_t *)rel_addr) = p->virtualAddress(); - } - }else{ - *((uintptr_t *)rel_addr) += object->baseAddress; - } - break; -#if defined(__x86_64__) - case R_X86_64_IRELATIVE: { - auto ptr = object->baseAddress + addend; - auto target = reinterpret_cast<uintptr_t (*)(void)>(ptr)(); - *((uintptr_t *)rel_addr) = target; - break; - } -#endif -// TODO: TLSDESC relocations aren't aarch64 specific -#if defined(__aarch64__) - case R_AARCH64_TLSDESC: { - size_t symValue = 0; - SharedObject *target = nullptr; - - if (symbol_index) { - auto symbol = (elf_sym *)(object->baseAddress + object->symbolTableOffset - + symbol_index * sizeof(elf_sym)); - ObjectSymbol r(object, symbol); - auto p = Scope::resolveGlobalOrLocal(*globalScope, object->localScope, r.getString(), object->objectRts, 0); - - if (!p) { - __ensure(ELF_ST_BIND(symbol->st_info) != STB_WEAK); - mlibc::panicLogger() << "rtdl: Unresolved TLSDESC for symbol " - << r.getString() << " in object " << object->name << frg::endlog; - } else { - target = p->object(); - if (p->symbol()) - symValue = p->symbol()->st_value; - } - } else { - target = object; - } - - __ensure(target); - - if (target->tlsModel == TlsModel::initial) { - ((uint64_t *)rel_addr)[0] = reinterpret_cast<uintptr_t>(&__mlibcTlsdescStatic); - // TODO: guard the subtraction of TCB size with `if constexpr (tlsAboveTp)` - // for the arch-generic case - __ensure(tlsAboveTp == true); - ((uint64_t *)rel_addr)[1] = symValue + target->tlsOffset + addend - sizeof(Tcb); - } else { - struct TlsdescData { - uintptr_t tlsIndex; - uintptr_t addend; - }; - - // Access DTV for object to force the entry to be allocated and initialized - accessDtv(target); - - __ensure(target->tlsIndex < mlibc::get_current_tcb()->dtvSize); - - // TODO: We should free this when the DSO gets destroyed - auto data = frg::construct<TlsdescData>(getAllocator()); - data->tlsIndex = target->tlsIndex; - data->addend = symValue + addend; - - ((uint64_t *)rel_addr)[0] = reinterpret_cast<uintptr_t>(&__mlibcTlsdescDynamic); - ((uint64_t *)rel_addr)[1] = reinterpret_cast<uintptr_t>(data); - } - } break; -#endif - default: - mlibc::panicLogger() << "unimplemented lazy relocation type " << type << frg::endlog; - break; - } - } -} - diff --git a/lib/mlibc/options/rtdl/generic/linker.hpp b/lib/mlibc/options/rtdl/generic/linker.hpp deleted file mode 100644 index ad84ca9..0000000 --- a/lib/mlibc/options/rtdl/generic/linker.hpp +++ /dev/null @@ -1,402 +0,0 @@ - -#include <frg/hash_map.hpp> -#include <frg/optional.hpp> -#include <frg/string.hpp> -#include <frg/vector.hpp> -#include <frg/expected.hpp> -#include <mlibc/allocator.hpp> -#include <mlibc/tcb.hpp> - -#include "elf.hpp" - -struct ObjectRepository; -struct Scope; -struct Loader; -struct SharedObject; - -extern uint64_t rtsCounter; - -enum class TlsModel { - null, - initial, - dynamic -}; - -enum class LinkerError { - success, - notFound, - fileTooShort, - notElf, - wrongElfType, - outOfMemory, - invalidProgramHeader -}; - -// -------------------------------------------------------- -// ObjectRepository -// -------------------------------------------------------- - -struct ObjectRepository { - ObjectRepository(); - - ObjectRepository(const ObjectRepository &) = delete; - - ObjectRepository &operator= (const ObjectRepository &) = delete; - - // This is primarily used to create a SharedObject for the RTDL itself. - SharedObject *injectObjectFromDts(frg::string_view name, - frg::string<MemoryAllocator> path, - uintptr_t base_address, elf_dyn *dynamic, uint64_t rts); - - // This is used to create a SharedObject for the executable that we want to link. - SharedObject *injectObjectFromPhdrs(frg::string_view name, - frg::string<MemoryAllocator> path, void *phdr_pointer, - size_t phdr_entry_size, size_t num_phdrs, void *entry_pointer, - uint64_t rts); - - SharedObject *injectStaticObject(frg::string_view name, - frg::string<MemoryAllocator> path, void *phdr_pointer, - size_t phdr_entry_size, size_t num_phdrs, void *entry_pointer, - uint64_t rts); - - frg::expected<LinkerError, SharedObject *> requestObjectWithName(frg::string_view name, - SharedObject *origin, Scope *localScope, bool createScope, uint64_t rts); - - frg::expected<LinkerError, SharedObject *> requestObjectAtPath(frg::string_view path, - Scope *localScope, bool createScope, uint64_t rts); - - SharedObject *findCaller(void *address); - - SharedObject *findLoadedObject(frg::string_view name); - - // Used by dl_iterate_phdr: stores objects in the order they are loaded. - frg::vector<SharedObject *, MemoryAllocator> loadedObjects; - -private: - void _fetchFromPhdrs(SharedObject *object, void *phdr_pointer, - size_t phdr_entry_size, size_t num_phdrs, void *entry_pointer); - - frg::expected<LinkerError, void> _fetchFromFile(SharedObject *object, int fd); - - void _parseDynamic(SharedObject *object); - - void _discoverDependencies(SharedObject *object, Scope *localScope, uint64_t rts); - - void _addLoadedObject(SharedObject *object); - - frg::hash_map<frg::string_view, SharedObject *, - frg::hash<frg::string_view>, MemoryAllocator> _nameMap; -}; - -// -------------------------------------------------------- -// SharedObject -// -------------------------------------------------------- - -enum class HashStyle { - none, - systemV, - gnu -}; - -using InitFuncPtr = void (*)(); - -// The ABI of this struct is fixed by GDB -struct DebugInterface { - int ver; - void *head; - void (*brk)(void); - int state; - void *base; -}; - -// The ABI of this struct is fixed by GDB -struct LinkMap { - uintptr_t base = 0; - const char *name = nullptr; - elf_dyn *dynv = nullptr; - LinkMap *next = nullptr, *prev = nullptr; -}; - -struct SharedObject { - // path is copied - SharedObject(const char *name, frg::string<MemoryAllocator> path, - bool is_main_object, Scope *localScope, uint64_t object_rts); - - SharedObject(const char *name, const char *path, bool is_main_object, - Scope *localScope, uint64_t object_rts); - - frg::string<MemoryAllocator> name; - frg::string<MemoryAllocator> path; - frg::string<MemoryAllocator> interpreterPath; - const char *soName; - bool isMainObject; - uint64_t objectRts; - - // link map for debugging - LinkMap linkMap; - bool inLinkMap; - - // base address this shared object was loaded to - uintptr_t baseAddress; - - Scope *localScope; - - // pointers to the dynamic table, GOT and entry point - elf_dyn *dynamic = nullptr; - void **globalOffsetTable; - void *entry; - - // object initialization information - InitFuncPtr initPtr = nullptr; - InitFuncPtr *initArray = nullptr; - InitFuncPtr *preInitArray = nullptr; - size_t initArraySize = 0; - size_t preInitArraySize = 0; - - - // TODO: read this from the PHDR - size_t tlsSegmentSize, tlsAlignment, tlsImageSize; - void *tlsImagePtr; - bool tlsInitialized; - - // symbol and string table of this shared object - HashStyle hashStyle = HashStyle::none; - uintptr_t hashTableOffset; - uintptr_t symbolTableOffset; - uintptr_t stringTableOffset; - - const char *runPath = nullptr; - - // save the lazy JUMP_SLOT relocation table - uintptr_t lazyRelocTableOffset; - size_t lazyTableSize; - frg::optional<bool> lazyExplicitAddend; - - bool symbolicResolution; - bool eagerBinding; - bool haveStaticTls; - - // vector of dependencies - frg::vector<SharedObject *, MemoryAllocator> dependencies; - - TlsModel tlsModel; - size_t tlsIndex; - size_t tlsOffset; - - uint64_t globalRts; - bool wasLinked; - - bool scheduledForInit; - bool onInitStack; - bool wasInitialized; - - // PHDR related stuff, we only set these for the main executable - void *phdrPointer = nullptr; - size_t phdrEntrySize = 0; - size_t phdrCount = 0; -}; - -struct Relocation { - Relocation(SharedObject *object, elf_rela *r) - : object_{object}, type_{Addend::Explicit} { - offset_ = r->r_offset; - info_ = r->r_info; - addend_ = r->r_addend; - } - - Relocation(SharedObject *object, elf_rel *r) - : object_{object}, type_{Addend::Implicit} { - offset_ = r->r_offset; - info_ = r->r_info; - } - - SharedObject *object() { - return object_; - } - - elf_info type() const { - return ELF_R_TYPE(info_); - } - - elf_info symbol_index() const { - return ELF_R_SYM(info_); - } - - elf_addr addend_rel() { - switch(type_) { - case Addend::Explicit: - return addend_; - case Addend::Implicit: { - auto ptr = reinterpret_cast<elf_addr *>(object_->baseAddress + offset_); - return *ptr; - } - } - __builtin_unreachable(); - } - - elf_addr addend_norel() { - switch(type_) { - case Addend::Explicit: - return addend_; - case Addend::Implicit: - return 0; - } - __builtin_unreachable(); - } - - void *destination() { - return reinterpret_cast<void *>(object_->baseAddress + offset_); - } - - void relocate(elf_addr addr) { - auto ptr = destination(); - memcpy(ptr, &addr, sizeof(addr)); - } - -private: - enum class Addend { - Implicit, - Explicit - }; - - SharedObject *object_; - Addend type_; - - elf_addr offset_; - elf_info info_; - elf_addend addend_ = 0; -}; - -void processCopyRelocations(SharedObject *object); - -// -------------------------------------------------------- -// RuntimeTlsMap -// -------------------------------------------------------- - -struct RuntimeTlsMap { - RuntimeTlsMap(); - - // Amount of initialLimit that has already been allocated. - size_t initialPtr; - - // Size of the inital TLS segment. - size_t initialLimit; - - // TLS indices. - frg::vector<SharedObject *, MemoryAllocator> indices; -}; - -extern frg::manual_box<RuntimeTlsMap> runtimeTlsMap; - -Tcb *allocateTcb(); -void initTlsObjects(Tcb *tcb, const frg::vector<SharedObject *, MemoryAllocator> &objects, bool checkInitialized); -void *accessDtv(SharedObject *object); -// Tries to access the DTV, if not allocated, or object doesn't have -// PT_TLS, return nullptr. -void *tryAccessDtv(SharedObject *object); - -// -------------------------------------------------------- -// ObjectSymbol -// -------------------------------------------------------- - -struct ObjectSymbol { - ObjectSymbol(SharedObject *object, const elf_sym *symbol); - - SharedObject *object() { - return _object; - } - - const elf_sym *symbol() { - return _symbol; - } - - const char *getString(); - - uintptr_t virtualAddress(); - -private: - SharedObject *_object; - const elf_sym *_symbol; -}; - -frg::optional<ObjectSymbol> resolveInObject(SharedObject *object, frg::string_view string); - -// -------------------------------------------------------- -// Scope -// -------------------------------------------------------- - -struct Scope { - using ResolveFlags = uint32_t; - static inline constexpr ResolveFlags resolveCopy = 1; - static inline constexpr ResolveFlags skipGlobalAfterRts = 1 << 1; - - static frg::optional<ObjectSymbol> resolveGlobalOrLocal(Scope &globalScope, - Scope *localScope, frg::string_view string, uint64_t skipRts, ResolveFlags flags); - static frg::optional<ObjectSymbol> resolveGlobalOrLocalNext(Scope &globalScope, - Scope *localScope, frg::string_view string, SharedObject *origin); - - Scope(bool isGlobal = false); - - void appendObject(SharedObject *object); - - frg::optional<ObjectSymbol> resolveSymbol(frg::string_view string, uint64_t skipRts, ResolveFlags flags); - - bool isGlobal; - -private: - frg::optional<ObjectSymbol> _resolveNext(frg::string_view string, SharedObject *target); -public: // TODO: Make this private again. (Was made public for __dlapi_reverse()). - frg::vector<SharedObject *, MemoryAllocator> _objects; -}; - -extern frg::manual_box<Scope> globalScope; - -// -------------------------------------------------------- -// Loader -// -------------------------------------------------------- - -class Loader { -public: - Loader(Scope *scope, SharedObject *mainExecutable, bool is_initial_link, uint64_t rts); - -public: - void linkObjects(SharedObject *root); - -private: - void _buildLinkBfs(SharedObject *root); - void _buildTlsMaps(); - - void _processStaticRelocations(SharedObject *object); - void _processLazyRelocations(SharedObject *object); - - void _processRelocations(Relocation &rel); - -public: - void initObjects(); - -private: - void _scheduleInit(SharedObject *object); - -private: - SharedObject *_mainExecutable; - Scope *_loadScope; - bool _isInitialLink; - uint64_t _linkRts; - - frg::vector<SharedObject *, MemoryAllocator> _linkBfs; - - frg::vector<SharedObject *, MemoryAllocator> _initQueue; -}; - -// -------------------------------------------------------- -// Namespace scope functions -// -------------------------------------------------------- - -extern "C" void pltRelocateStub() __attribute__((__visibility__("hidden"))); - -// -------------------------------------------------------- -// RTDL interface -// -------------------------------------------------------- - -void *rtdl_auxvector(); - diff --git a/lib/mlibc/options/rtdl/generic/main.cpp b/lib/mlibc/options/rtdl/generic/main.cpp deleted file mode 100644 index 3cff1e4..0000000 --- a/lib/mlibc/options/rtdl/generic/main.cpp +++ /dev/null @@ -1,844 +0,0 @@ - -#include <elf.h> -#include <link.h> - -#include <frg/manual_box.hpp> -#include <frg/small_vector.hpp> - -#include <abi-bits/auxv.h> -#include <mlibc/debug.hpp> -#include <mlibc/rtdl-sysdeps.hpp> -#include <mlibc/rtdl-config.hpp> -#include <mlibc/rtdl-abi.hpp> -#include <mlibc/stack_protector.hpp> -#include <internal-config.h> -#include <abi-bits/auxv.h> - -#include "elf.hpp" -#include "linker.hpp" - -#if __MLIBC_POSIX_OPTION -#include <dlfcn.h> -#endif - -#define HIDDEN __attribute__((__visibility__("hidden"))) -#define EXPORT __attribute__((__visibility__("default"))) - -static constexpr bool logEntryExit = false; -static constexpr bool logStartup = false; -static constexpr bool logDlCalls = false; - -#ifndef MLIBC_STATIC_BUILD -extern HIDDEN void *_GLOBAL_OFFSET_TABLE_[]; -extern HIDDEN elf_dyn _DYNAMIC[]; -#endif - -namespace mlibc { - // Declared in options/internal/mlibc/tcb.hpp. - bool tcb_available_flag = false; -} - -mlibc::RtdlConfig rtdlConfig; - -bool ldShowAuxv = false; - -uintptr_t *entryStack; -frg::manual_box<ObjectRepository> initialRepository; -frg::manual_box<Scope> globalScope; - -frg::manual_box<RuntimeTlsMap> runtimeTlsMap; - -// We use a small vector of size 4 to avoid memory allocation for the default library paths -frg::manual_box<frg::small_vector<frg::string_view, 4, MemoryAllocator>> libraryPaths; - -frg::manual_box<frg::vector<frg::string_view, MemoryAllocator>> preloads; - -static SharedObject *executableSO; -extern HIDDEN char __ehdr_start[]; - -// Global debug interface variable -DebugInterface globalDebugInterface; - -#ifndef MLIBC_STATIC_BUILD - -// Use a PC-relative instruction sequence to find our runtime load address. -uintptr_t getLdsoBase() { -#if defined(__x86_64__) || defined(__i386__) || defined(__aarch64__) - // On x86_64, the first GOT entry holds the link-time address of _DYNAMIC. - // TODO: This isn't guaranteed on AArch64, so this might fail with some linkers. - auto linktime_dynamic = reinterpret_cast<uintptr_t>(_GLOBAL_OFFSET_TABLE_[0]); - auto runtime_dynamic = reinterpret_cast<uintptr_t>(_DYNAMIC); - return runtime_dynamic - linktime_dynamic; -#elif defined(__riscv) - return reinterpret_cast<uintptr_t>(&__ehdr_start); -#endif -} - -// Relocates the dynamic linker (i.e. this DSO) itself. -// Assumptions: -// - There are no references to external symbols. -// Note that this code is fragile in the sense that it must not contain relocations itself. -// TODO: Use tooling to verify this at compile time. -extern "C" void relocateSelf() { - size_t rela_offset = 0; - size_t rela_size = 0; - size_t rel_offset = 0; - size_t rel_size = 0; - size_t relr_offset = 0; - size_t relr_size = 0; - for(size_t i = 0; _DYNAMIC[i].d_tag != DT_NULL; i++) { - auto ent = &_DYNAMIC[i]; - switch(ent->d_tag) { - case DT_REL: rel_offset = ent->d_un.d_ptr; break; - case DT_RELSZ: rel_size = ent->d_un.d_val; break; - case DT_RELA: rela_offset = ent->d_un.d_ptr; break; - case DT_RELASZ: rela_size = ent->d_un.d_val; break; - case DT_RELR: relr_offset = ent->d_un.d_ptr; break; - case DT_RELRSZ: relr_size = ent->d_un.d_val; break; - } - } - - auto ldso_base = getLdsoBase(); - - __ensure((rel_offset != 0) ^ (rela_offset != 0)); - - for(size_t disp = 0; disp < rela_size; disp += sizeof(elf_rela)) { - auto reloc = reinterpret_cast<elf_rela *>(ldso_base + rela_offset + disp); - - auto type = ELF_R_TYPE(reloc->r_info); - if(ELF_R_SYM(reloc->r_info)) - __builtin_trap(); - - auto p = reinterpret_cast<uint64_t *>(ldso_base + reloc->r_offset); - switch(type) { - case R_RELATIVE: - *p = ldso_base + reloc->r_addend; - break; - default: - __builtin_trap(); - } - } - - for(size_t disp = 0; disp < rel_size; disp += sizeof(elf_rel)) { - auto reloc = reinterpret_cast<elf_rel *>(ldso_base + rel_offset + disp); - - auto type = ELF_R_TYPE(reloc->r_info); - if(ELF_R_SYM(reloc->r_info)) - __builtin_trap(); - - auto p = reinterpret_cast<uint64_t *>(ldso_base + reloc->r_offset); - switch(type) { - case R_RELATIVE: - *p += ldso_base; - break; - default: - __builtin_trap(); - } - } - - elf_addr *addr = nullptr; - for(size_t disp = 0; disp < relr_size; disp += sizeof(elf_relr)) { - auto entry = *(elf_relr *)(ldso_base + relr_offset + disp); - - // Even entry indicates the beginning address. - if(!(entry & 1)) { - addr = (elf_addr *)(ldso_base + entry); - __ensure(addr); - *addr++ += ldso_base; - }else { - // Odd entry indicates entry is a bitmap of the subsequent locations to be relocated. - for(int i = 0; entry; ++i) { - if(entry & 1) { - addr[i] += ldso_base; - } - entry >>= 1; - } - - // Each entry describes at max 63 (on 64bit) or 31 (on 32bit) subsequent locations. - addr += CHAR_BIT * sizeof(elf_relr) - 1; - } - } -} -#endif - -extern "C" void *lazyRelocate(SharedObject *object, unsigned int rel_index) { - __ensure(object->lazyExplicitAddend); - auto reloc = (elf_rela *)(object->baseAddress + object->lazyRelocTableOffset - + rel_index * sizeof(elf_rela)); - auto type = ELF_R_TYPE(reloc->r_info); - auto symbol_index = ELF_R_SYM(reloc->r_info); - - __ensure(type == R_X86_64_JUMP_SLOT); - __ensure(ELF_CLASS == ELFCLASS64); - - auto symbol = (elf_sym *)(object->baseAddress + object->symbolTableOffset - + symbol_index * sizeof(elf_sym)); - ObjectSymbol r(object, symbol); - auto p = Scope::resolveGlobalOrLocal(*globalScope, object->localScope, r.getString(), object->objectRts, 0); - if(!p) - mlibc::panicLogger() << "Unresolved JUMP_SLOT symbol" << frg::endlog; - - //mlibc::infoLogger() << "Lazy relocation to " << symbol_str - // << " resolved to " << pointer << frg::endlog; - - *(uint64_t *)(object->baseAddress + reloc->r_offset) = p->virtualAddress(); - return (void *)p->virtualAddress(); -} - -extern "C" [[ gnu::visibility("default") ]] void *__rtdl_allocateTcb() { - auto tcb = allocateTcb(); - initTlsObjects(tcb, globalScope->_objects, false); - return tcb; -} - -extern "C" { - [[ gnu::visibility("hidden") ]] void dl_debug_state() { - // This function is used to signal changes in the debugging link map, - // GDB just sets a breakpoint on this function and we can call it - // everytime we update the link map. We don't need to implement - // anything besides defining and calling it. - } -} - -extern "C" [[gnu::alias("dl_debug_state"), gnu::visibility("default")]] void _dl_debug_state() noexcept; - -// This symbol can be used by GDB to find the global interface structure -[[ gnu::visibility("default") ]] DebugInterface *_dl_debug_addr = &globalDebugInterface; - -static frg::vector<frg::string_view, MemoryAllocator> parseList(frg::string_view paths, frg::string_view separators) { - frg::vector<frg::string_view, MemoryAllocator> list{getAllocator()}; - - size_t p = 0; - while(p < paths.size()) { - size_t s; // Offset of next colon or end of string. - if(size_t cs = paths.find_first_of(separators, p); cs != size_t(-1)) { - s = cs; - }else{ - s = paths.size(); - } - - auto path = paths.sub_string(p, s - p); - p = s + 1; - - if(path.size() == 0) - continue; - - if(path.ends_with("/")) { - size_t i = path.size() - 1; - while(i > 0 && path[i] == '/') - i--; - path = path.sub_string(0, i + 1); - } - - if(path == "/") - path = ""; - - list.push_back(path); - } - - return list; -} - -extern "C" void *interpreterMain(uintptr_t *entry_stack) { - if(logEntryExit) - mlibc::infoLogger() << "Entering ld.so" << frg::endlog; - entryStack = entry_stack; - runtimeTlsMap.initialize(); - libraryPaths.initialize(getAllocator()); - preloads.initialize(getAllocator()); - - void *phdr_pointer = 0; - size_t phdr_entry_size = 0; - size_t phdr_count = 0; - void *entry_pointer = 0; - void *stack_entropy = nullptr; - - const char *execfn = "(executable)"; - -#ifndef MLIBC_STATIC_BUILD - using ctor_fn = void(*)(void); - - ctor_fn *ldso_ctors = nullptr; - size_t num_ldso_ctors = 0; - - auto ldso_base = getLdsoBase(); - if(logStartup) { - mlibc::infoLogger() << "ldso: Own base address is: 0x" - << frg::hex_fmt(ldso_base) << frg::endlog; - mlibc::infoLogger() << "ldso: Own dynamic section is at: " << _DYNAMIC << frg::endlog; - } - -#ifdef __x86_64__ - // These entries are reserved on x86_64. - // TODO: Use a fake PLT stub that reports an error message? - _GLOBAL_OFFSET_TABLE_[1] = 0; - _GLOBAL_OFFSET_TABLE_[2] = 0; -#endif - - // Validate our own dynamic section. - // Here, we make sure that the dynamic linker does not need relocations itself. - uintptr_t strtab_offset = 0; - uintptr_t soname_str = 0; - for(size_t i = 0; _DYNAMIC[i].d_tag != DT_NULL; i++) { - auto ent = &_DYNAMIC[i]; - switch(ent->d_tag) { - case DT_STRTAB: strtab_offset = ent->d_un.d_ptr; break; - case DT_SONAME: soname_str = ent->d_un.d_val; break; - case DT_INIT_ARRAY: ldso_ctors = reinterpret_cast<ctor_fn *>(ent->d_un.d_ptr + ldso_base); break; - case DT_INIT_ARRAYSZ: num_ldso_ctors = ent->d_un.d_val / sizeof(ctor_fn); break; - case DT_HASH: - case DT_GNU_HASH: - case DT_STRSZ: - case DT_SYMTAB: - case DT_SYMENT: - case DT_RELA: - case DT_RELASZ: - case DT_RELAENT: - case DT_RELACOUNT: - case DT_DEBUG: - case DT_REL: - case DT_RELSZ: - case DT_RELENT: - case DT_RELCOUNT: - case DT_RELR: - case DT_RELRSZ: - case DT_RELRENT: - continue; - default: - __ensure(!"Unexpected dynamic entry in program interpreter"); - } - } - __ensure(strtab_offset); - __ensure(soname_str); - - // Find the auxiliary vector by skipping args and environment. - auto aux = entryStack; - aux += *aux + 1; // First, we skip argc and all args. - __ensure(!*aux); - aux++; - while(*aux) { // Loop through the environment. - auto env = reinterpret_cast<char *>(*aux); - frg::string_view view{env}; - size_t s = view.find_first('='); - - if(s == size_t(-1)) - mlibc::panicLogger() << "rtdl: environment '" << env << "' is missing a '='" << frg::endlog; - - auto name = view.sub_string(0, s); - auto value = const_cast<char *>(view.data() + s + 1); - - if(name == "LD_SHOW_AUXV" && *value && *value != '0') { - ldShowAuxv = true; - }else if(name == "LD_LIBRARY_PATH" && *value) { - for(auto path : parseList(value, ":;")) - libraryPaths->push_back(path); - }else if(name == "LD_PRELOAD" && *value) { - *preloads = parseList(value, " :"); - } - - aux++; - } - aux++; - - // Add default library paths - libraryPaths->push_back("/lib"); - libraryPaths->push_back("/lib64"); - libraryPaths->push_back("/usr/lib"); - libraryPaths->push_back("/usr/lib64"); - - // Parse the actual vector. - while(true) { - auto value = aux + 1; - if(!(*aux)) - break; - - if(ldShowAuxv) { - switch(*aux) { - case AT_PHDR: mlibc::infoLogger() << "AT_PHDR: 0x" << frg::hex_fmt{*value} << frg::endlog; break; - case AT_PHENT: mlibc::infoLogger() << "AT_PHENT: " << *value << frg::endlog; break; - case AT_PHNUM: mlibc::infoLogger() << "AT_PHNUM: " << *value << frg::endlog; break; - case AT_ENTRY: mlibc::infoLogger() << "AT_ENTRY: 0x" << frg::hex_fmt{*value} << frg::endlog; break; - case AT_PAGESZ: mlibc::infoLogger() << "AT_PAGESZ: " << *value << frg::endlog; break; - case AT_BASE: mlibc::infoLogger() << "AT_BASE: 0x" << frg::hex_fmt{*value} << frg::endlog; break; - case AT_FLAGS: mlibc::infoLogger() << "AT_FLAGS: 0x" << frg::hex_fmt{*value} << frg::endlog; break; - case AT_NOTELF: mlibc::infoLogger() << "AT_NOTELF: " << frg::hex_fmt{*value} << frg::endlog; break; - case AT_UID: mlibc::infoLogger() << "AT_UID: " << *value << frg::endlog; break; - case AT_EUID: mlibc::infoLogger() << "AT_EUID: " << *value << frg::endlog; break; - case AT_GID: mlibc::infoLogger() << "AT_GID: " << *value << frg::endlog; break; - case AT_EGID: mlibc::infoLogger() << "AT_EGID: " << *value << frg::endlog; break; -#ifdef AT_PLATFORM - case AT_PLATFORM: mlibc::infoLogger() << "AT_PLATFORM: " << reinterpret_cast<const char *>(*value) << frg::endlog; break; -#endif -#ifdef AT_HWCAP - case AT_HWCAP: mlibc::infoLogger() << "AT_HWCAP: " << frg::hex_fmt{*value} << frg::endlog; break; -#endif -#ifdef AT_CLKTCK - case AT_CLKTCK: mlibc::infoLogger() << "AT_CLKTCK: " << *value << frg::endlog; break; -#endif -#ifdef AT_FPUCW - case AT_FPUCW: mlibc::infoLogger() << "AT_FPUCW: " << frg::hex_fmt{*value} << frg::endlog; break; -#endif -#ifdef AT_SECURE - case AT_SECURE: mlibc::infoLogger() << "AT_SECURE: " << *value << frg::endlog; break; -#endif -#ifdef AT_RANDOM - case AT_RANDOM: mlibc::infoLogger() << "AT_RANDOM: 0x" << frg::hex_fmt{*value} << frg::endlog; break; -#endif -#ifdef AT_EXECFN - case AT_EXECFN: mlibc::infoLogger() << "AT_EXECFN: " << reinterpret_cast<const char *>(*value) << frg::endlog; break; -#endif -#ifdef AT_SYSINFO_EHDR - case AT_SYSINFO_EHDR: mlibc::infoLogger() << "AT_SYSINFO_EHDR: 0x" << frg::hex_fmt{*value} << frg::endlog; break; -#endif - } - } - - // TODO: Whitelist auxiliary vector entries here? - switch(*aux) { - case AT_PHDR: phdr_pointer = reinterpret_cast<void *>(*value); break; - case AT_PHENT: phdr_entry_size = *value; break; - case AT_PHNUM: phdr_count = *value; break; - case AT_ENTRY: entry_pointer = reinterpret_cast<void *>(*value); break; - case AT_EXECFN: execfn = reinterpret_cast<const char *>(*value); break; - case AT_RANDOM: stack_entropy = reinterpret_cast<void*>(*value); break; - case AT_SECURE: rtdlConfig.secureRequired = reinterpret_cast<uintptr_t>(*value); break; - } - - aux += 2; - } - globalDebugInterface.base = reinterpret_cast<void*>(ldso_base); - -// This is here because libgcc will add a global constructor on glibc Linux -// (which is what it believes we are due to the aarch64-linux-gnu toolchain) -// in order to check if LSE atomics are supported. -// -// This is not necessary on a custom Linux toolchain and is purely an artifact of -// using the host toolchain. -#if defined(__aarch64__) && defined(__gnu_linux__) - for (size_t i = 0; i < num_ldso_ctors; i++) { - if(logStartup) - mlibc::infoLogger() << "ldso: Running own constructor at " - << reinterpret_cast<void *>(ldso_ctors[i]) - << frg::endlog; - ldso_ctors[i](); - } -#else - if (num_ldso_ctors > 0) { - mlibc::panicLogger() << "ldso: Found unexpected own global constructor(s), init_array starts at: " - << ldso_ctors - << frg::endlog; - } -#endif - -#else - auto ehdr = reinterpret_cast<elf_ehdr*>(__ehdr_start); - phdr_pointer = reinterpret_cast<void*>((uintptr_t)ehdr->e_phoff + (uintptr_t)ehdr); - phdr_entry_size = ehdr->e_phentsize; - phdr_count = ehdr->e_phnum; - entry_pointer = reinterpret_cast<void*>(ehdr->e_entry); -#endif - __ensure(phdr_pointer); - __ensure(entry_pointer); - - if(logStartup) - mlibc::infoLogger() << "ldso: Executable PHDRs are at " << phdr_pointer - << frg::endlog; - - // perform the initial dynamic linking - initialRepository.initialize(); - - globalScope.initialize(true); - - // Add the dynamic linker, as well as the exectuable to the repository. -#ifndef MLIBC_STATIC_BUILD - auto ldso_soname = reinterpret_cast<const char *>(ldso_base + strtab_offset + soname_str); - auto ldso = initialRepository->injectObjectFromDts(ldso_soname, - frg::string<MemoryAllocator> { getAllocator() }, - ldso_base, _DYNAMIC, 1); - ldso->phdrPointer = phdr_pointer; - ldso->phdrCount = phdr_count; - ldso->phdrEntrySize = phdr_entry_size; - - // TODO: support non-zero base addresses? - executableSO = initialRepository->injectObjectFromPhdrs(execfn, - frg::string<MemoryAllocator> { execfn, getAllocator() }, - phdr_pointer, phdr_entry_size, phdr_count, entry_pointer, 1); - - // We can't initialise the ldso object after the executable SO, - // so we have to set the ldso path after loading both. - ldso->path = executableSO->interpreterPath; - -#else - executableSO = initialRepository->injectStaticObject(execfn, - frg::string<MemoryAllocator>{ execfn, getAllocator() }, - phdr_pointer, phdr_entry_size, phdr_count, entry_pointer, 1); - globalDebugInterface.base = (void*)executableSO->baseAddress; -#endif - - globalDebugInterface.head = &executableSO->linkMap; - executableSO->inLinkMap = true; - Loader linker{globalScope.get(), executableSO, true, 1}; - linker.linkObjects(executableSO); - - mlibc::initStackGuard(stack_entropy); - - auto tcb = allocateTcb(); - if(mlibc::sys_tcb_set(tcb)) - __ensure(!"sys_tcb_set() failed"); - tcb->tid = mlibc::this_tid(); - mlibc::tcb_available_flag = true; - - globalDebugInterface.ver = 1; - globalDebugInterface.brk = &dl_debug_state; - globalDebugInterface.state = 0; - dl_debug_state(); - - linker.initObjects(); - - if(logEntryExit) - mlibc::infoLogger() << "Leaving ld.so, jump to " - << (void *)executableSO->entry << frg::endlog; - return executableSO->entry; -} - -const char *lastError; - -extern "C" [[ gnu::visibility("default") ]] uintptr_t *__dlapi_entrystack() { - return entryStack; -} - -extern "C" [[ gnu::visibility("default") ]] -const char *__dlapi_error() { - auto error = lastError; - lastError = nullptr; - return error; -} - -extern "C" [[ gnu::visibility("default") ]] -void *__dlapi_get_tls(struct __abi_tls_entry *entry) { - return reinterpret_cast<char *>(accessDtv(entry->object)) + entry->offset; -} - -extern "C" [[ gnu::visibility("default") ]] -const mlibc::RtdlConfig &__dlapi_get_config() { - return rtdlConfig; -} - -#if __MLIBC_POSIX_OPTION - -extern "C" [[ gnu::visibility("default") ]] -void *__dlapi_open(const char *file, int flags, void *returnAddress) { - if (logDlCalls) - mlibc::infoLogger() << "rtdl: __dlapi_open(" << (file ? file : "nullptr") << ")" << frg::endlog; - - if (flags & RTLD_DEEPBIND) - mlibc::infoLogger() << "rtdl: dlopen(RTLD_DEEPBIND) is unsupported" << frg::endlog; - - if(!file) - return executableSO; - - // TODO: Thread-safety! - auto rts = rtsCounter++; - - SharedObject *object; - if (flags & RTLD_NOLOAD) { - object = initialRepository->findLoadedObject(file); - if (object && object->globalRts == 0 && (flags & RTLD_GLOBAL)) { - // The object was opened with RTLD_LOCAL, but we are called with RTLD_NOLOAD | RTLD_GLOBAL. - // According to the man page, we should promote to the global scope here. - object->globalRts = rts; - globalScope->appendObject(object); - } - } else { - bool isGlobal = flags & RTLD_GLOBAL; - Scope *newScope = isGlobal ? globalScope.get() : nullptr; - - frg::expected<LinkerError, SharedObject *> objectResult; - if (frg::string_view{file}.find_first('/') == size_t(-1)) { - // In order to know which RUNPATH / RPATH to process, we must find the calling object. - SharedObject *origin = initialRepository->findCaller(returnAddress); - if (!origin) { - mlibc::panicLogger() << "rtdl: unable to determine calling object of dlopen " - << "(ra = " << returnAddress << ")" << frg::endlog; - } - - objectResult = initialRepository->requestObjectWithName(file, origin, newScope, !isGlobal, rts); - } else { - objectResult = initialRepository->requestObjectAtPath(file, newScope, !isGlobal, rts); - } - - if(!objectResult) { - switch (objectResult.error()) { - case LinkerError::success: - __builtin_unreachable(); - case LinkerError::notFound: - lastError = "Cannot locate requested DSO"; - break; - case LinkerError::fileTooShort: - lastError = "File too short"; - break; - case LinkerError::notElf: - lastError = "File is not an ELF file"; - break; - case LinkerError::wrongElfType: - lastError = "File has wrong ELF type"; - break; - case LinkerError::outOfMemory: - lastError = "Out of memory"; - break; - case LinkerError::invalidProgramHeader: - lastError = "File has invalid program header"; - break; - } - return nullptr; - } - object = objectResult.value(); - - Loader linker{object->localScope, nullptr, false, rts}; - linker.linkObjects(object); - linker.initObjects(); - } - - dl_debug_state(); - - return object; -} - -extern "C" [[ gnu::visibility("default") ]] -void *__dlapi_resolve(void *handle, const char *string, void *returnAddress) { - if (logDlCalls) { - const char *name; - bool quote = false; - if (handle == RTLD_DEFAULT) { - name = "RTLD_DEFAULT"; - } else if (handle == RTLD_NEXT) { - name = "RTLD_NEXT"; - } else { - name = ((SharedObject *)handle)->name.data(); - quote = true; - } - - mlibc::infoLogger() << "rtdl: __dlapi_resolve(" << (quote ? "\"" : "") << name - << (quote ? "\"" : "") << ", \"" << string << "\")" << frg::endlog; - } - - frg::optional<ObjectSymbol> target; - - if (handle == RTLD_DEFAULT) { - target = globalScope->resolveSymbol(string, 0, 0); - } else if (handle == RTLD_NEXT) { - SharedObject *origin = initialRepository->findCaller(returnAddress); - if (!origin) { - mlibc::panicLogger() << "rtdl: unable to determine calling object of dlsym " - << "(ra = " << returnAddress << ")" << frg::endlog; - } - - target = Scope::resolveGlobalOrLocalNext(*globalScope, origin->localScope, string, origin); - } else { - // POSIX does not unambiguously state how dlsym() is supposed to work; it just - // states that "The symbol resolution algorithm used shall be dependency order - // as described in dlopen()". - // - // Linux libc's lookup the symbol in the given DSO and all of its dependencies - // in breadth-first order. That is also what we implement here. - // - // Note that this *differs* from the algorithm that is used for relocations - // (since the algorithm used for relocations takes (i) the global scope, - // and (ii) the local scope of the DSO into account (which can contain more objects - // than just the dependencies of the DSO, if the DSO was loaded as a dependency - // of a dlopen()ed DSO). - - frg::vector<SharedObject *, MemoryAllocator> queue{getAllocator()}; - - struct Token { }; - frg::hash_map< - SharedObject *, Token, - frg::hash<SharedObject *>, MemoryAllocator - > visited{frg::hash<SharedObject *>{}, getAllocator()}; - - auto root = reinterpret_cast<SharedObject *>(handle); - visited.insert(root, Token{}); - queue.push_back(root); - - for(size_t i = 0; i < queue.size(); i++) { - auto current = queue[i]; - - target = resolveInObject(current, string); - if(target) - break; - - for(auto dep : current->dependencies) { - if(visited.get(dep)) - continue; - visited.insert(dep, Token{}); - queue.push_back(dep); - } - } - } - - if (!target) { - if (logDlCalls) - mlibc::infoLogger() << "rtdl: could not resolve \"" << string << "\"" << frg::endlog; - - lastError = "Cannot resolve requested symbol"; - return nullptr; - } - return reinterpret_cast<void *>(target->virtualAddress()); -} - -struct __dlapi_symbol { - const char *file; - void *base; - const char *symbol; - void *address; -}; - -extern "C" [[ gnu::visibility("default") ]] -int __dlapi_reverse(const void *ptr, __dlapi_symbol *info) { - if (logDlCalls) - mlibc::infoLogger() << "rtdl: __dlapi_reverse(" << ptr << ")" << frg::endlog; - - for(size_t i = 0; i < initialRepository->loadedObjects.size(); i++) { - auto object = initialRepository->loadedObjects[i]; - - auto eligible = [&] (ObjectSymbol cand) { - if(cand.symbol()->st_shndx == SHN_UNDEF) - return false; - - auto bind = ELF_ST_BIND(cand.symbol()->st_info); - if(bind != STB_GLOBAL && bind != STB_WEAK) - return false; - - return true; - }; - - auto hash_table = (Elf64_Word *)(object->baseAddress + object->hashTableOffset); - auto num_symbols = hash_table[1]; - for(size_t i = 0; i < num_symbols; i++) { - ObjectSymbol cand{object, (elf_sym *)(object->baseAddress - + object->symbolTableOffset + i * sizeof(elf_sym))}; - if(eligible(cand) && cand.virtualAddress() == reinterpret_cast<uintptr_t>(ptr)) { - if (logDlCalls) - mlibc::infoLogger() << "rtdl: Found symbol " << cand.getString() << " in object " - << object->path << frg::endlog; - - info->file = object->path.data(); - info->base = reinterpret_cast<void *>(object->baseAddress); - info->symbol = cand.getString(); - info->address = reinterpret_cast<void *>(cand.virtualAddress()); - return 0; - } - } - } - - // Not found, find the DSO it should be in. - for(size_t i = 0; i < initialRepository->loadedObjects.size(); i++) { - auto object = initialRepository->loadedObjects[i]; - - for(size_t j = 0; j < object->phdrCount; j++) { - auto phdr = (elf_phdr *)((uintptr_t)object->phdrPointer + j * object->phdrEntrySize); - if(phdr->p_type != PT_LOAD) { - continue; - } - uintptr_t start = object->baseAddress + phdr->p_vaddr; - uintptr_t end = start + phdr->p_memsz; - if(reinterpret_cast<uintptr_t>(ptr) >= start && reinterpret_cast<uintptr_t>(ptr) < end) { - mlibc::infoLogger() << "rtdl: Found DSO " << object->path << frg::endlog; - info->file = object->path.data(); - info->base = reinterpret_cast<void *>(object->baseAddress); - info->symbol = nullptr; - info->address = 0; - return 0; - } - } - } - - if (logDlCalls) - mlibc::infoLogger() << "rtdl: Could not find symbol in __dlapi_reverse()" << frg::endlog; - - return -1; -} - -extern "C" [[ gnu::visibility("default") ]] -int __dlapi_close(void *) { - if (logDlCalls) - mlibc::infoLogger() << "mlibc: dlclose() is a no-op" << frg::endlog; - return 0; -} - -#endif - -extern "C" [[ gnu::visibility("default") ]] -int __dlapi_iterate_phdr(int (*callback)(struct dl_phdr_info *, size_t, void*), void *data) { - int last_return = 0; - for (auto object : initialRepository->loadedObjects) { - struct dl_phdr_info info; - info.dlpi_addr = object->baseAddress; - info.dlpi_name = object->name.data(); - - if(object->isMainObject) { - info.dlpi_name = ""; - } else { - info.dlpi_name = object->name.data(); - } - info.dlpi_phdr = static_cast<ElfW(Phdr)*>(object->phdrPointer); - info.dlpi_phnum = object->phdrCount; - info.dlpi_adds = rtsCounter; - info.dlpi_subs = 0; // TODO(geert): implement dlclose(). - if (object->tlsModel != TlsModel::null) - info.dlpi_tls_modid = object->tlsIndex; - else - info.dlpi_tls_modid = 0; - info.dlpi_tls_data = tryAccessDtv(object); - - last_return = callback(&info, sizeof(struct dl_phdr_info), data); - if(last_return) - return last_return; - } - - return last_return; -} - -extern "C" [[ gnu::visibility("default") ]] -void __dlapi_enter(uintptr_t *entry_stack) { -#if MLIBC_STATIC_BUILD - interpreterMain(entry_stack); -#else - (void)entry_stack; -#endif -} - -// XXX(qookie): -// This is here because libgcc will call into __getauxval on glibc Linux -// (which is what it believes we are due to the aarch64-linux-gnu toolchain) -// in order to find AT_HWCAP to discover if LSE atomics are supported. -// -// This is not necessary on a custom Linux toolchain and is purely an artifact of -// using the host toolchain. - -// __gnu_linux__ is the define checked by libgcc -#if defined(__aarch64__) && defined(__gnu_linux__) && !defined(MLIBC_STATIC_BUILD) - -extern "C" unsigned long __getauxval(unsigned long type) { - // Find the auxiliary vector by skipping args and environment. - auto aux = entryStack; - aux += *aux + 1; // Skip argc and all arguments - __ensure(!*aux); - aux++; - while(*aux) // Now, we skip the environment. - aux++; - aux++; - - // Parse the auxiliary vector. - while(true) { - auto value = aux + 1; - if(*aux == AT_NULL) { - return 0; - }else if(*aux == type) { - return *value; - } - aux += 2; - } -} - -#endif |