aboutsummaryrefslogtreecommitdiff
path: root/lib/mlibc/options/rtdl/generic
diff options
context:
space:
mode:
authorIan Moffett <ian@osmora.org>2024-03-07 17:28:52 -0500
committerIan Moffett <ian@osmora.org>2024-03-07 18:24:51 -0500
commitf5e48e94a2f4d4bbd6e5628c7f2afafc6dbcc459 (patch)
tree93b156621dc0303816b37f60ba88051b702d92f6 /lib/mlibc/options/rtdl/generic
parentbd5969fc876a10b18613302db7087ef3c40f18e1 (diff)
build: Build mlibc + add distclean target
Signed-off-by: Ian Moffett <ian@osmora.org>
Diffstat (limited to 'lib/mlibc/options/rtdl/generic')
-rw-r--r--lib/mlibc/options/rtdl/generic/linker.cpp1872
-rw-r--r--lib/mlibc/options/rtdl/generic/linker.hpp402
-rw-r--r--lib/mlibc/options/rtdl/generic/main.cpp844
3 files changed, 0 insertions, 3118 deletions
diff --git a/lib/mlibc/options/rtdl/generic/linker.cpp b/lib/mlibc/options/rtdl/generic/linker.cpp
deleted file mode 100644
index a519c35..0000000
--- a/lib/mlibc/options/rtdl/generic/linker.cpp
+++ /dev/null
@@ -1,1872 +0,0 @@
-#include <mlibc/arch-defs.hpp>
-#include <stdint.h>
-#include <string.h>
-
-// keep a list of optional generic relocation types
-enum {
- R_OFFSET = (uintptr_t) -1,
-};
-
-
-#include <frg/manual_box.hpp>
-#include <frg/small_vector.hpp>
-#include <mlibc/allocator.hpp>
-#include <mlibc/debug.hpp>
-#include <mlibc/rtdl-sysdeps.hpp>
-#include <mlibc/rtdl-abi.hpp>
-#include <mlibc/thread.hpp>
-#include <abi-bits/fcntl.h>
-#include <internal-config.h>
-
-#include "elf.hpp"
-#include "linker.hpp"
-
-#if !MLIBC_MMAP_ALLOCATE_DSO
-uintptr_t libraryBase = 0x41000000;
-#endif
-
-constexpr bool verbose = false;
-constexpr bool stillSlightlyVerbose = false;
-constexpr bool logBaseAddresses = false;
-constexpr bool logRpath = false;
-constexpr bool logLdPath = false;
-constexpr bool eagerBinding = true;
-
-#if defined(__x86_64__) || defined(__i386__)
-constexpr inline bool tlsAboveTp = false;
-#elif defined(__aarch64__)
-constexpr inline bool tlsAboveTp = true;
-#elif defined(__riscv)
-constexpr inline bool tlsAboveTp = true;
-#else
-# error Unknown architecture
-#endif
-
-extern DebugInterface globalDebugInterface;
-extern uintptr_t __stack_chk_guard;
-
-extern frg::manual_box<frg::small_vector<frg::string_view, 4, MemoryAllocator>> libraryPaths;
-extern frg::manual_box<frg::vector<frg::string_view, MemoryAllocator>> preloads;
-
-#if MLIBC_STATIC_BUILD
-extern "C" size_t __init_array_start[];
-extern "C" size_t __init_array_end[];
-extern "C" size_t __preinit_array_start[];
-extern "C" size_t __preinit_array_end[];
-#endif
-
-size_t tlsMaxAlignment = 16;
-
-// This is the global "resolution timestamp" (RTS) counter.
-// It is incremented each time __dlapi_open() (i.e. dlopen()) is called.
-// Each DSO stores its objectRts (i.e. RTS at the time the object was loaded).
-// DSOs in the global scope also store a globalRts (i.e. RTS at the time the
-// object became global). This mechanism is used to determine which
-// part of the global scope is considered for symbol resolution.
-uint64_t rtsCounter = 2;
-
-bool trySeek(int fd, int64_t offset) {
- off_t noff;
- return mlibc::sys_seek(fd, offset, SEEK_SET, &noff) == 0;
-}
-
-bool tryReadExactly(int fd, void *data, size_t length) {
- size_t offset = 0;
- while(offset < length) {
- ssize_t chunk;
- if(mlibc::sys_read(fd, reinterpret_cast<char *>(data) + offset,
- length - offset, &chunk))
- return false;
- __ensure(chunk > 0);
- offset += chunk;
- }
- __ensure(offset == length);
- return true;
-}
-
-void closeOrDie(int fd) {
- if(mlibc::sys_close(fd))
- __ensure(!"sys_close() failed");
-}
-
-uintptr_t alignUp(uintptr_t address, size_t align) {
- return (address + align - 1) & ~(align - 1);
-}
-
-// --------------------------------------------------------
-// ObjectRepository
-// --------------------------------------------------------
-
-ObjectRepository::ObjectRepository()
-: loadedObjects{getAllocator()},
- _nameMap{frg::hash<frg::string_view>{}, getAllocator()} {}
-
-SharedObject *ObjectRepository::injectObjectFromDts(frg::string_view name,
- frg::string<MemoryAllocator> path, uintptr_t base_address,
- elf_dyn *dynamic, uint64_t rts) {
- __ensure(!findLoadedObject(name));
-
- auto object = frg::construct<SharedObject>(getAllocator(),
- name.data(), std::move(path), false, globalScope.get(), rts);
- object->baseAddress = base_address;
- object->dynamic = dynamic;
- _parseDynamic(object);
-
- _addLoadedObject(object);
- _discoverDependencies(object, globalScope.get(), rts);
-
- return object;
-}
-
-SharedObject *ObjectRepository::injectObjectFromPhdrs(frg::string_view name,
- frg::string<MemoryAllocator> path, void *phdr_pointer,
- size_t phdr_entry_size, size_t num_phdrs, void *entry_pointer,
- uint64_t rts) {
- __ensure(!findLoadedObject(name));
-
- auto object = frg::construct<SharedObject>(getAllocator(),
- name.data(), std::move(path), true, globalScope.get(), rts);
- _fetchFromPhdrs(object, phdr_pointer, phdr_entry_size, num_phdrs, entry_pointer);
- _parseDynamic(object);
-
- _addLoadedObject(object);
- _discoverDependencies(object, globalScope.get(), rts);
-
- return object;
-}
-
-SharedObject *ObjectRepository::injectStaticObject(frg::string_view name,
- frg::string<MemoryAllocator> path, void *phdr_pointer,
- size_t phdr_entry_size, size_t num_phdrs, void *entry_pointer,
- uint64_t rts) {
- __ensure(!findLoadedObject(name));
- auto object = frg::construct<SharedObject>(getAllocator(),
- name.data(), std::move(path), true, globalScope.get(), rts);
- _fetchFromPhdrs(object, phdr_pointer, phdr_entry_size, num_phdrs, entry_pointer);
-
-#if MLIBC_STATIC_BUILD
- object->initArray = reinterpret_cast<InitFuncPtr*>(__init_array_start);
- object->initArraySize = static_cast<size_t>((uintptr_t)__init_array_end -
- (uintptr_t)__init_array_start);
- object->preInitArray = reinterpret_cast<InitFuncPtr*>(__preinit_array_start);
- object->preInitArraySize = static_cast<size_t>((uintptr_t)__preinit_array_end -
- (uintptr_t)__preinit_array_start);
-#endif
-
- _addLoadedObject(object);
-
- return object;
-}
-
-frg::expected<LinkerError, SharedObject *> ObjectRepository::requestObjectWithName(frg::string_view name,
- SharedObject *origin, Scope *localScope, bool createScope, uint64_t rts) {
- if (auto obj = findLoadedObject(name))
- return obj;
-
- auto tryToOpen = [&] (const char *path) {
- int fd;
- if(auto x = mlibc::sys_open(path, O_RDONLY, 0, &fd); x) {
- return -1;
- }
- return fd;
- };
-
- // TODO(arsen): this process can probably undergo heavy optimization, by
- // preprocessing the rpath only once on parse
- auto processRpath = [&] (frg::string_view path) {
- frg::string<MemoryAllocator> sPath { getAllocator() };
- if (path.starts_with("$ORIGIN")) {
- frg::string_view dirname = origin->path;
- auto lastsl = dirname.find_last('/');
- if (lastsl != size_t(-1)) {
- dirname = dirname.sub_string(0, lastsl);
- } else {
- dirname = ".";
- }
- sPath = frg::string<MemoryAllocator>{ getAllocator(), dirname };
- sPath += path.sub_string(7, path.size() - 7);
- } else {
- sPath = frg::string<MemoryAllocator>{ getAllocator(), path };
- }
- if (sPath[sPath.size() - 1] != '/') {
- sPath += '/';
- }
- sPath += name;
- if (logRpath)
- mlibc::infoLogger() << "rtdl: trying in rpath " << sPath << frg::endlog;
- int fd = tryToOpen(sPath.data());
- if (logRpath && fd >= 0)
- mlibc::infoLogger() << "rtdl: found in rpath" << frg::endlog;
- return frg::tuple { fd, std::move(sPath) };
- };
-
- frg::string<MemoryAllocator> chosenPath { getAllocator() };
- int fd = -1;
- if (origin && origin->runPath) {
- size_t start = 0;
- size_t idx = 0;
- frg::string_view rpath { origin->runPath };
- auto next = [&] () {
- idx = rpath.find_first(':', start);
- if (idx == (size_t)-1)
- idx = rpath.size();
- };
- for (next(); idx < rpath.size(); next()) {
- auto path = rpath.sub_string(start, idx - start);
- start = idx + 1;
- auto [fd_, fullPath] = processRpath(path);
- if (fd_ != -1) {
- fd = fd_;
- chosenPath = std::move(fullPath);
- break;
- }
- }
- if (fd == -1) {
- auto path = rpath.sub_string(start, rpath.size() - start);
- auto [fd_, fullPath] = processRpath(path);
- if (fd_ != -1) {
- fd = fd_;
- chosenPath = std::move(fullPath);
- }
- }
- } else if (logRpath) {
- mlibc::infoLogger() << "rtdl: no rpath set for object" << frg::endlog;
- }
-
- for(size_t i = 0; i < libraryPaths->size() && fd == -1; i++) {
- auto ldPath = (*libraryPaths)[i];
- auto path = frg::string<MemoryAllocator>{getAllocator(), ldPath} + '/' + name;
- if(logLdPath)
- mlibc::infoLogger() << "rtdl: Trying to load " << name << " from ldpath " << ldPath << "/" << frg::endlog;
- fd = tryToOpen(path.data());
- if(fd >= 0) {
- chosenPath = std::move(path);
- break;
- }
- }
- if(fd == -1)
- return LinkerError::notFound;
-
- if (createScope) {
- __ensure(localScope == nullptr);
-
- // TODO: Free this when the scope is no longer needed.
- localScope = frg::construct<Scope>(getAllocator());
- }
-
- __ensure(localScope != nullptr);
-
- auto object = frg::construct<SharedObject>(getAllocator(),
- name.data(), std::move(chosenPath), false, localScope, rts);
-
- auto result = _fetchFromFile(object, fd);
- closeOrDie(fd);
- if(!result) {
- frg::destruct(getAllocator(), object);
- return result.error();
- }
-
- _parseDynamic(object);
-
- _addLoadedObject(object);
- _discoverDependencies(object, localScope, rts);
-
- return object;
-}
-
-frg::expected<LinkerError, SharedObject *> ObjectRepository::requestObjectAtPath(frg::string_view path,
- Scope *localScope, bool createScope, uint64_t rts) {
- // TODO: Support SONAME correctly.
- auto lastSlash = path.find_last('/') + 1;
- auto name = path;
- if (!lastSlash) {
- name = name.sub_string(lastSlash, path.size() - lastSlash);
- }
- if (auto obj = findLoadedObject(name))
- return obj;
-
- if (createScope) {
- __ensure(localScope == nullptr);
-
- // TODO: Free this when the scope is no longer needed.
- localScope = frg::construct<Scope>(getAllocator());
- }
-
- __ensure(localScope != nullptr);
-
- auto object = frg::construct<SharedObject>(getAllocator(),
- name.data(), path.data(), false, localScope, rts);
-
- frg::string<MemoryAllocator> no_prefix(getAllocator(), path);
-
- int fd;
- if(mlibc::sys_open((no_prefix + '\0').data(), O_RDONLY, 0, &fd)) {
- frg::destruct(getAllocator(), object);
- return LinkerError::notFound;
- }
- auto result = _fetchFromFile(object, fd);
- closeOrDie(fd);
- if(!result) {
- frg::destruct(getAllocator(), object);
- return result.error();
- }
-
- _parseDynamic(object);
-
- _addLoadedObject(object);
- _discoverDependencies(object, localScope, rts);
-
- return object;
-}
-
-SharedObject *ObjectRepository::findCaller(void *addr) {
- uintptr_t target = reinterpret_cast<uintptr_t>(addr);
-
- for (auto [name, object] : _nameMap) {
- // Search all PT_LOAD segments for the specified address.
- for(size_t j = 0; j < object->phdrCount; j++) {
- auto phdr = (elf_phdr *)((uintptr_t)object->phdrPointer + j * object->phdrEntrySize);
- if (phdr->p_type == PT_LOAD) {
- uintptr_t start = object->baseAddress + phdr->p_vaddr;
- uintptr_t end = start + phdr->p_memsz;
- if (start <= target && target < end)
- return object;
- }
- }
- }
-
- return nullptr;
-}
-
-SharedObject *ObjectRepository::findLoadedObject(frg::string_view name) {
- auto it = _nameMap.get(name);
- if (it)
- return *it;
-
- for (auto object : loadedObjects) {
- // See if any object has a matching SONAME.
- if (object->soName && name == object->soName)
- return object;
- }
-
- // TODO: We should also look at the device and inode here as a fallback.
- return nullptr;
-}
-
-// --------------------------------------------------------
-// ObjectRepository: Fetching methods.
-// --------------------------------------------------------
-
-void ObjectRepository::_fetchFromPhdrs(SharedObject *object, void *phdr_pointer,
- size_t phdr_entry_size, size_t phdr_count, void *entry_pointer) {
- __ensure(object->isMainObject);
- object->phdrPointer = phdr_pointer;
- object->phdrEntrySize = phdr_entry_size;
- object->phdrCount = phdr_count;
- if(verbose)
- mlibc::infoLogger() << "rtdl: Loading " << object->name << frg::endlog;
-
- // Note: the entry pointer is absolute and not relative to the base address.
- object->entry = entry_pointer;
-
- frg::optional<ptrdiff_t> dynamic_offset;
- frg::optional<ptrdiff_t> tls_offset;
-
- // segments are already mapped, so we just have to find the dynamic section
- for(size_t i = 0; i < phdr_count; i++) {
- auto phdr = (elf_phdr *)((uintptr_t)phdr_pointer + i * phdr_entry_size);
- switch(phdr->p_type) {
- case PT_PHDR:
- // Determine the executable's base address (in the PIE case) by comparing
- // the PHDR segment's load address against it's address in the ELF file.
- object->baseAddress = reinterpret_cast<uintptr_t>(phdr_pointer) - phdr->p_vaddr;
- if(verbose)
- mlibc::infoLogger() << "rtdl: Executable is loaded at "
- << (void *)object->baseAddress << frg::endlog;
- break;
- case PT_DYNAMIC:
- dynamic_offset = phdr->p_vaddr;
- break;
- case PT_TLS: {
- object->tlsSegmentSize = phdr->p_memsz;
- object->tlsAlignment = phdr->p_align;
- object->tlsImageSize = phdr->p_filesz;
- tls_offset = phdr->p_vaddr;
- break;
- case PT_INTERP:
- object->interpreterPath = frg::string<MemoryAllocator>{
- (char*)(object->baseAddress + phdr->p_vaddr),
- getAllocator()
- };
- } break;
- default:
- //FIXME warn about unknown phdrs
- break;
- }
- }
-
- if(dynamic_offset)
- object->dynamic = (elf_dyn *)(object->baseAddress + *dynamic_offset);
- if(tls_offset)
- object->tlsImagePtr = (void *)(object->baseAddress + *tls_offset);
-}
-
-
-frg::expected<LinkerError, void> ObjectRepository::_fetchFromFile(SharedObject *object, int fd) {
- __ensure(!object->isMainObject);
-
- // read the elf file header
- elf_ehdr ehdr;
- if(!tryReadExactly(fd, &ehdr, sizeof(elf_ehdr)))
- return LinkerError::fileTooShort;
-
- if(ehdr.e_ident[0] != 0x7F
- || ehdr.e_ident[1] != 'E'
- || ehdr.e_ident[2] != 'L'
- || ehdr.e_ident[3] != 'F')
- return LinkerError::notElf;
-
- if((ehdr.e_type != ET_EXEC && ehdr.e_type != ET_DYN)
- || ehdr.e_machine != ELF_MACHINE
- || ehdr.e_ident[EI_CLASS] != ELF_CLASS)
- return LinkerError::wrongElfType;
-
- // read the elf program headers
- auto phdr_buffer = (char *)getAllocator().allocate(ehdr.e_phnum * ehdr.e_phentsize);
- if(!phdr_buffer)
- return LinkerError::outOfMemory;
-
- if(!trySeek(fd, ehdr.e_phoff)) {
- getAllocator().deallocate(phdr_buffer, ehdr.e_phnum * ehdr.e_phentsize);
- return LinkerError::invalidProgramHeader;
- }
- if(!tryReadExactly(fd, phdr_buffer, ehdr.e_phnum * ehdr.e_phentsize)) {
- getAllocator().deallocate(phdr_buffer, ehdr.e_phnum * ehdr.e_phentsize);
- return LinkerError::invalidProgramHeader;
- }
-
- object->phdrPointer = phdr_buffer;
- object->phdrCount = ehdr.e_phnum;
- object->phdrEntrySize = ehdr.e_phentsize;
-
- // Allocate virtual address space for the DSO.
- constexpr size_t hugeSize = 0x200000;
-
- uintptr_t highest_address = 0;
- for(int i = 0; i < ehdr.e_phnum; i++) {
- auto phdr = (elf_phdr *)(phdr_buffer + i * ehdr.e_phentsize);
-
- if(phdr->p_type != PT_LOAD)
- continue;
-
- auto limit = phdr->p_vaddr + phdr->p_memsz;
- if(limit > highest_address)
- highest_address = limit;
- }
-
- __ensure(!(object->baseAddress & (hugeSize - 1)));
-
- highest_address = (highest_address + mlibc::page_size - 1) & ~(mlibc::page_size - 1);
-
-#if MLIBC_MMAP_ALLOCATE_DSO
- void *mappedAddr = nullptr;
-
- if (mlibc::sys_vm_map(nullptr,
- highest_address - object->baseAddress, PROT_NONE,
- MAP_PRIVATE | MAP_ANONYMOUS, -1, 0, &mappedAddr)) {
- mlibc::infoLogger() << "sys_vm_map failed when allocating address space for DSO \""
- << object->name << "\""
- << ", base " << (void *)object->baseAddress
- << ", requested " << (highest_address - object->baseAddress) << " bytes"
- << frg::endlog;
- getAllocator().deallocate(phdr_buffer, ehdr.e_phnum * ehdr.e_phentsize);
- return LinkerError::outOfMemory;
- }
-
- object->baseAddress = reinterpret_cast<uintptr_t>(mappedAddr);
-#else
- object->baseAddress = libraryBase;
- libraryBase += (highest_address + (hugeSize - 1)) & ~(hugeSize - 1);
-#endif
-
- if(verbose || logBaseAddresses)
- mlibc::infoLogger() << "rtdl: Loading " << object->name
- << " at " << (void *)object->baseAddress << frg::endlog;
-
- // Load all segments.
- constexpr size_t pageSize = 0x1000;
- for(int i = 0; i < ehdr.e_phnum; i++) {
- auto phdr = (elf_phdr *)(phdr_buffer + i * ehdr.e_phentsize);
-
- if(phdr->p_type == PT_LOAD) {
- size_t misalign = phdr->p_vaddr & (pageSize - 1);
- __ensure(phdr->p_memsz > 0);
- __ensure(phdr->p_memsz >= phdr->p_filesz);
-
- // If the following condition is violated, we cannot use mmap() the segment;
- // however, GCC only generates ELF files that satisfy this.
- __ensure(misalign == (phdr->p_offset & (pageSize - 1)));
-
- auto map_address = object->baseAddress + phdr->p_vaddr - misalign;
- auto backed_map_size = (phdr->p_filesz + misalign + pageSize - 1) & ~(pageSize - 1);
- auto total_map_size = (phdr->p_memsz + misalign + pageSize - 1) & ~(pageSize - 1);
-
- int prot = 0;
- if(phdr->p_flags & PF_R)
- prot |= PROT_READ;
- if(phdr->p_flags & PF_W)
- prot |= PROT_WRITE;
- if(phdr->p_flags & PF_X)
- prot |= PROT_EXEC;
-
- #if MLIBC_MAP_DSO_SEGMENTS
- void *map_pointer;
- if(mlibc::sys_vm_map(reinterpret_cast<void *>(map_address),
- backed_map_size, prot | PROT_WRITE,
- MAP_PRIVATE | MAP_FIXED, fd, phdr->p_offset - misalign, &map_pointer))
- __ensure(!"sys_vm_map failed");
- if(total_map_size > backed_map_size)
- if(mlibc::sys_vm_map(reinterpret_cast<void *>(map_address + backed_map_size),
- total_map_size - backed_map_size, prot | PROT_WRITE,
- MAP_PRIVATE | MAP_FIXED | MAP_ANONYMOUS, -1, 0, &map_pointer))
- __ensure(!"sys_vm_map failed");
-
- if(mlibc::sys_vm_readahead)
- if(mlibc::sys_vm_readahead(reinterpret_cast<void *>(map_address),
- backed_map_size))
- mlibc::infoLogger() << "mlibc: sys_vm_readahead() failed in ld.so"
- << frg::endlog;
-
- // Clear the trailing area at the end of the backed mapping.
- // We do not clear the leading area; programs are not supposed to access it.
- memset(reinterpret_cast<void *>(map_address + misalign + phdr->p_filesz),
- 0, phdr->p_memsz - phdr->p_filesz);
- #else
- (void)backed_map_size;
-
- void *map_pointer;
- if(mlibc::sys_vm_map(reinterpret_cast<void *>(map_address),
- total_map_size, prot | PROT_WRITE,
- MAP_PRIVATE | MAP_FIXED | MAP_ANONYMOUS, -1, 0, &map_pointer))
- __ensure(!"sys_vm_map failed");
-
- __ensure(trySeek(fd, phdr->p_offset));
- __ensure(tryReadExactly(fd, reinterpret_cast<char *>(map_address) + misalign,
- phdr->p_filesz));
- #endif
- // Take care of removing superfluous permissions.
- if(mlibc::sys_vm_protect && ((prot & PROT_WRITE) == 0))
- if(mlibc::sys_vm_protect(map_pointer, total_map_size, prot))
- mlibc::infoLogger() << "mlibc: sys_vm_protect() failed in ld.so" << frg::endlog;
- }else if(phdr->p_type == PT_TLS) {
- object->tlsSegmentSize = phdr->p_memsz;
- object->tlsAlignment = phdr->p_align;
- object->tlsImageSize = phdr->p_filesz;
- object->tlsImagePtr = (void *)(object->baseAddress + phdr->p_vaddr);
- }else if(phdr->p_type == PT_DYNAMIC) {
- object->dynamic = (elf_dyn *)(object->baseAddress + phdr->p_vaddr);
- }else if(phdr->p_type == PT_INTERP
- || phdr->p_type == PT_PHDR
- || phdr->p_type == PT_NOTE
- || phdr->p_type == PT_RISCV_ATTRIBUTES
- || phdr->p_type == PT_GNU_EH_FRAME
- || phdr->p_type == PT_GNU_RELRO
- || phdr->p_type == PT_GNU_STACK
- || phdr->p_type == PT_GNU_PROPERTY) {
- // ignore the phdr
- }else{
- mlibc::panicLogger() << "Unexpected PHDR type 0x"
- << frg::hex_fmt(phdr->p_type) << " in DSO " << object->name << frg::endlog;
- }
- }
-
- return frg::success;
-}
-
-// --------------------------------------------------------
-// ObjectRepository: Parsing methods.
-// --------------------------------------------------------
-
-void ObjectRepository::_parseDynamic(SharedObject *object) {
- if(!object->dynamic)
- mlibc::infoLogger() << "ldso: Object '" << object->name
- << "' does not have a dynamic section" << frg::endlog;
- __ensure(object->dynamic);
-
- // Fix up these offsets to addresses after the loop, since the
- // addresses depend on the value of DT_STRTAB.
- frg::optional<ptrdiff_t> runpath_offset;
- /* If true, ignore the RPATH. */
- bool runpath_found = false;
- frg::optional<ptrdiff_t> soname_offset;
-
- for(size_t i = 0; object->dynamic[i].d_tag != DT_NULL; i++) {
- elf_dyn *dynamic = &object->dynamic[i];
- switch(dynamic->d_tag) {
- // handle hash table, symbol table and string table
- case DT_HASH:
- object->hashStyle = HashStyle::systemV;
- object->hashTableOffset = dynamic->d_un.d_ptr;
- break;
- case DT_GNU_HASH:
- object->hashStyle = HashStyle::gnu;
- object->hashTableOffset = dynamic->d_un.d_ptr;
- break;
- case DT_STRTAB:
- object->stringTableOffset = dynamic->d_un.d_ptr;
- break;
- case DT_STRSZ:
- break; // we don't need the size of the string table
- case DT_SYMTAB:
- object->symbolTableOffset = dynamic->d_un.d_ptr;
- break;
- case DT_SYMENT:
- __ensure(dynamic->d_un.d_val == sizeof(elf_sym));
- break;
- // handle lazy relocation table
- case DT_PLTGOT:
- object->globalOffsetTable = (void **)(object->baseAddress
- + dynamic->d_un.d_ptr);
- break;
- case DT_JMPREL:
- object->lazyRelocTableOffset = dynamic->d_un.d_ptr;
- break;
- case DT_PLTRELSZ:
- object->lazyTableSize = dynamic->d_un.d_val;
- break;
- case DT_PLTREL:
- if(dynamic->d_un.d_val == DT_RELA) {
- object->lazyExplicitAddend = true;
- }else{
- __ensure(dynamic->d_un.d_val == DT_REL);
- object->lazyExplicitAddend = false;
- }
- break;
- // TODO: Implement this correctly!
- case DT_SYMBOLIC:
- object->symbolicResolution = true;
- break;
- case DT_BIND_NOW:
- object->eagerBinding = true;
- break;
- case DT_FLAGS: {
- if(dynamic->d_un.d_val & DF_SYMBOLIC)
- object->symbolicResolution = true;
- if(dynamic->d_un.d_val & DF_STATIC_TLS)
- object->haveStaticTls = true;
- if(dynamic->d_un.d_val & DF_BIND_NOW)
- object->eagerBinding = true;
-
- auto ignored = DF_BIND_NOW | DF_SYMBOLIC | DF_STATIC_TLS;
-#ifdef __riscv
- // Work around https://sourceware.org/bugzilla/show_bug.cgi?id=24673.
- ignored |= DF_TEXTREL;
-#else
- if(dynamic->d_un.d_val & DF_TEXTREL)
- mlibc::panicLogger() << "\e[31mrtdl: DF_TEXTREL is unimplemented" << frg::endlog;
-#endif
- if(dynamic->d_un.d_val & ~ignored)
- mlibc::infoLogger() << "\e[31mrtdl: DT_FLAGS(" << frg::hex_fmt{dynamic->d_un.d_val & ~ignored}
- << ") is not implemented correctly!\e[39m"
- << frg::endlog;
- } break;
- case DT_FLAGS_1:
- if(dynamic->d_un.d_val & DF_1_NOW)
- object->eagerBinding = true;
- // The DF_1_PIE flag is informational only. It is used by e.g file(1).
- // The DF_1_NODELETE flag has a similar effect to RTLD_NODELETE, both of which we
- // ignore because we don't implement dlclose().
- if(dynamic->d_un.d_val & ~(DF_1_NOW | DF_1_PIE | DF_1_NODELETE))
- mlibc::infoLogger() << "\e[31mrtdl: DT_FLAGS_1(" << frg::hex_fmt{dynamic->d_un.d_val}
- << ") is not implemented correctly!\e[39m"
- << frg::endlog;
- break;
- case DT_RPATH:
- if (runpath_found) {
- /* Ignore RPATH if RUNPATH was present. */
- break;
- }
- [[fallthrough]];
- case DT_RUNPATH:
- runpath_found = dynamic->d_tag == DT_RUNPATH;
- runpath_offset = dynamic->d_un.d_val;
- break;
- case DT_INIT:
- if(dynamic->d_un.d_ptr != 0)
- object->initPtr = (InitFuncPtr)(object->baseAddress + dynamic->d_un.d_ptr);
- break;
- case DT_INIT_ARRAY:
- if(dynamic->d_un.d_ptr != 0)
- object->initArray = (InitFuncPtr *)(object->baseAddress + dynamic->d_un.d_ptr);
- break;
- case DT_INIT_ARRAYSZ:
- object->initArraySize = dynamic->d_un.d_val;
- break;
- case DT_PREINIT_ARRAY:
- if(dynamic->d_un.d_ptr != 0) {
- // Only the main object is allowed pre-initializers.
- __ensure(object->isMainObject);
- object->preInitArray = (InitFuncPtr *)(object->baseAddress + dynamic->d_un.d_ptr);
- }
- break;
- case DT_PREINIT_ARRAYSZ:
- // Only the main object is allowed pre-initializers.
- __ensure(object->isMainObject);
- object->preInitArraySize = dynamic->d_un.d_val;
- break;
- case DT_DEBUG:
-#if ELF_CLASS == ELFCLASS32
- dynamic->d_un.d_val = reinterpret_cast<Elf32_Word>(&globalDebugInterface);
-#elif ELF_CLASS == ELFCLASS64
- dynamic->d_un.d_val = reinterpret_cast<Elf64_Xword>(&globalDebugInterface);
-#endif
- break;
- case DT_SONAME:
- soname_offset = dynamic->d_un.d_val;
- break;
- // ignore unimportant tags
- case DT_NEEDED: // we handle this later
- case DT_FINI: case DT_FINI_ARRAY: case DT_FINI_ARRAYSZ:
- case DT_RELA: case DT_RELASZ: case DT_RELAENT: case DT_RELACOUNT:
- case DT_REL: case DT_RELSZ: case DT_RELENT: case DT_RELCOUNT:
- case DT_RELR: case DT_RELRSZ: case DT_RELRENT:
- case DT_VERSYM:
- case DT_VERDEF: case DT_VERDEFNUM:
- case DT_VERNEED: case DT_VERNEEDNUM:
-#ifdef __riscv
- case DT_TEXTREL: // Work around https://sourceware.org/bugzilla/show_bug.cgi?id=24673.
-#endif
- break;
- case DT_TLSDESC_PLT: case DT_TLSDESC_GOT:
- break;
- default:
- // Ignore unknown entries in the os-specific area as we don't use them.
- if(dynamic->d_tag < DT_LOOS || dynamic->d_tag > DT_HIOS) {
- mlibc::panicLogger() << "Unexpected dynamic entry "
- << (void *)dynamic->d_tag << " in object" << frg::endlog;
- }
- }
- }
-
- if(runpath_offset) {
- object->runPath = reinterpret_cast<const char *>(object->baseAddress
- + object->stringTableOffset + *runpath_offset);
- }
- if(soname_offset) {
- object->soName = reinterpret_cast<const char *>(object->baseAddress
- + object->stringTableOffset + *soname_offset);
- }
-}
-
-void ObjectRepository::_discoverDependencies(SharedObject *object,
- Scope *localScope, uint64_t rts) {
- if(object->isMainObject) {
- for(auto preload : *preloads) {
- frg::expected<LinkerError, SharedObject *> libraryResult;
- if (preload.find_first('/') == size_t(-1)) {
- libraryResult = requestObjectWithName(preload, object, globalScope.get(), false, 1);
- } else {
- libraryResult = requestObjectAtPath(preload, globalScope.get(), false, 1);
- }
- if(!libraryResult)
- mlibc::panicLogger() << "rtdl: Could not load preload " << preload << frg::endlog;
-
- if(verbose)
- mlibc::infoLogger() << "rtdl: Preloading " << preload << frg::endlog;
-
- object->dependencies.push_back(libraryResult.value());
- }
- }
-
- // Load required dynamic libraries.
- for(size_t i = 0; object->dynamic[i].d_tag != DT_NULL; i++) {
- elf_dyn *dynamic = &object->dynamic[i];
- if(dynamic->d_tag != DT_NEEDED)
- continue;
-
- const char *library_str = (const char *)(object->baseAddress
- + object->stringTableOffset + dynamic->d_un.d_val);
-
- auto library = requestObjectWithName(frg::string_view{library_str},
- object, localScope, false, rts);
- if(!library)
- mlibc::panicLogger() << "Could not satisfy dependency " << library_str << frg::endlog;
- object->dependencies.push(library.value());
- }
-}
-
-void ObjectRepository::_addLoadedObject(SharedObject *object) {
- _nameMap.insert(object->name, object);
- loadedObjects.push_back(object);
-}
-
-// --------------------------------------------------------
-// SharedObject
-// --------------------------------------------------------
-
-SharedObject::SharedObject(const char *name, frg::string<MemoryAllocator> path,
- bool is_main_object, Scope *local_scope, uint64_t object_rts)
- : name(name, getAllocator()), path(std::move(path)),
- interpreterPath(getAllocator()), soName(nullptr),
- isMainObject(is_main_object), objectRts(object_rts), inLinkMap(false),
- baseAddress(0), localScope(local_scope), dynamic(nullptr),
- globalOffsetTable(nullptr), entry(nullptr), tlsSegmentSize(0),
- tlsAlignment(0), tlsImageSize(0), tlsImagePtr(nullptr),
- tlsInitialized(false), hashTableOffset(0), symbolTableOffset(0),
- stringTableOffset(0), lazyRelocTableOffset(0), lazyTableSize(0),
- lazyExplicitAddend(false), symbolicResolution(false),
- eagerBinding(false), haveStaticTls(false),
- dependencies(getAllocator()), tlsModel(TlsModel::null),
- tlsOffset(0), globalRts(0), wasLinked(false),
- scheduledForInit(false), onInitStack(false),
- wasInitialized(false) { }
-
-SharedObject::SharedObject(const char *name, const char *path,
- bool is_main_object, Scope *localScope, uint64_t object_rts)
- : SharedObject(name,
- frg::string<MemoryAllocator> { path, getAllocator() },
- is_main_object, localScope, object_rts) {}
-
-void processLateRelocation(Relocation rel) {
- // resolve the symbol if there is a symbol
- frg::optional<ObjectSymbol> p;
- if(rel.symbol_index()) {
- auto symbol = (elf_sym *)(rel.object()->baseAddress + rel.object()->symbolTableOffset
- + rel.symbol_index() * sizeof(elf_sym));
- ObjectSymbol r(rel.object(), symbol);
-
- p = Scope::resolveGlobalOrLocal(*globalScope, rel.object()->localScope,
- r.getString(), rel.object()->objectRts, Scope::resolveCopy);
- }
-
- switch(rel.type()) {
- case R_COPY:
- __ensure(p);
- memcpy(rel.destination(), (void *)p->virtualAddress(), p->symbol()->st_size);
- break;
-
-// TODO: R_IRELATIVE also exists on other architectures but will likely need a different implementation.
-#if defined(__x86_64__) || defined(__i386__)
- case R_IRELATIVE: {
- uintptr_t addr = rel.object()->baseAddress + rel.addend_rel();
- auto* fn = reinterpret_cast<uintptr_t (*)()>(addr);
- rel.relocate(fn());
- } break;
-#elif defined(__aarch64__)
- case R_IRELATIVE: {
- uintptr_t addr = rel.object()->baseAddress + rel.addend_rel();
- auto* fn = reinterpret_cast<uintptr_t (*)(uint64_t)>(addr);
- // TODO: the function should get passed AT_HWCAP value.
- rel.relocate(fn(0));
- } break;
-#endif
-
- default:
- break;
- }
-}
-
-void processLateRelocations(SharedObject *object) {
- frg::optional<uintptr_t> rel_offset;
- frg::optional<size_t> rel_length;
-
- frg::optional<uintptr_t> rela_offset;
- frg::optional<size_t> rela_length;
-
- for(size_t i = 0; object->dynamic[i].d_tag != DT_NULL; i++) {
- elf_dyn *dynamic = &object->dynamic[i];
-
- switch(dynamic->d_tag) {
- case DT_REL:
- rel_offset = dynamic->d_un.d_ptr;
- break;
- case DT_RELSZ:
- rel_length = dynamic->d_un.d_val;
- break;
- case DT_RELENT:
- __ensure(dynamic->d_un.d_val == sizeof(elf_rel));
- break;
- case DT_RELA:
- rela_offset = dynamic->d_un.d_ptr;
- break;
- case DT_RELASZ:
- rela_length = dynamic->d_un.d_val;
- break;
- case DT_RELAENT:
- __ensure(dynamic->d_un.d_val == sizeof(elf_rela));
- break;
- }
- }
-
- if(rela_offset && rela_length) {
- for(size_t offset = 0; offset < *rela_length; offset += sizeof(elf_rela)) {
- auto reloc = (elf_rela *)(object->baseAddress + *rela_offset + offset);
- auto r = Relocation(object, reloc);
- processLateRelocation(r);
- }
- } else if(rel_offset && rel_length) {
- for(size_t offset = 0; offset < *rel_length; offset += sizeof(elf_rel)) {
- auto reloc = (elf_rel *)(object->baseAddress + *rel_offset + offset);
- auto r = Relocation(object, reloc);
- processLateRelocation(r);
- }
- }else{
- __ensure(!rela_offset && !rela_length);
- __ensure(!rel_offset && !rel_length);
- }
-}
-
-void doInitialize(SharedObject *object) {
- __ensure(object->wasLinked);
- __ensure(!object->wasInitialized);
-
- // if the object has dependencies we initialize them first
- for(size_t i = 0; i < object->dependencies.size(); i++)
- __ensure(object->dependencies[i]->wasInitialized);
-
- if(verbose)
- mlibc::infoLogger() << "rtdl: Initialize " << object->name << frg::endlog;
-
- if(verbose)
- mlibc::infoLogger() << "rtdl: Running DT_INIT function" << frg::endlog;
- if(object->initPtr != nullptr)
- object->initPtr();
-
- if(verbose)
- mlibc::infoLogger() << "rtdl: Running DT_INIT_ARRAY functions" << frg::endlog;
- __ensure((object->initArraySize % sizeof(InitFuncPtr)) == 0);
- for(size_t i = 0; i < object->initArraySize / sizeof(InitFuncPtr); i++)
- object->initArray[i]();
-
- if(verbose)
- mlibc::infoLogger() << "rtdl: Object initialization complete" << frg::endlog;
- object->wasInitialized = true;
-}
-
-// --------------------------------------------------------
-// RuntimeTlsMap
-// --------------------------------------------------------
-
-RuntimeTlsMap::RuntimeTlsMap()
-: initialPtr{0}, initialLimit{0}, indices{getAllocator()} { }
-
-void initTlsObjects(Tcb *tcb, const frg::vector<SharedObject *, MemoryAllocator> &objects, bool checkInitialized) {
- // Initialize TLS segments that follow the static model.
- for(auto object : objects) {
- if(object->tlsModel == TlsModel::initial) {
- if(checkInitialized && object->tlsInitialized)
- continue;
-
- char *tcb_ptr = reinterpret_cast<char *>(tcb);
- auto tls_ptr = tcb_ptr + object->tlsOffset;
- memset(tls_ptr, 0, object->tlsSegmentSize);
- memcpy(tls_ptr, object->tlsImagePtr, object->tlsImageSize);
-
- if (verbose) {
- mlibc::infoLogger() << "rtdl: wrote tls image at " << (void *)tls_ptr
- << ", size = 0x" << frg::hex_fmt{object->tlsSegmentSize} << frg::endlog;
- }
-
- if (checkInitialized)
- object->tlsInitialized = true;
- }
- }
-}
-
-Tcb *allocateTcb() {
- size_t tlsInitialSize = runtimeTlsMap->initialLimit;
-
- // To make sure that both the TCB and TLS data are sufficiently aligned, allocate
- // slightly more than necessary and adjust alignment afterwards.
- size_t alignOverhead = frg::max(alignof(Tcb), tlsMaxAlignment);
- size_t allocSize = tlsInitialSize + sizeof(Tcb) + alignOverhead;
- auto allocation = reinterpret_cast<uintptr_t>(getAllocator().allocate(allocSize));
- memset(reinterpret_cast<void *>(allocation), 0, allocSize);
-
- uintptr_t tlsAddress, tcbAddress;
- if constexpr (tlsAboveTp) {
- // Here we must satisfy two requirements of the TCB and the TLS data:
- // 1. One should follow the other immediately in memory. We do this so that
- // we can simply add or subtract sizeof(Tcb) to obtain the address of the other.
- // 2. Both should be sufficiently aligned.
- // To do this, we will fix whichever address has stricter alignment requirements, and
- // derive the other from it.
- if (tlsMaxAlignment > alignof(Tcb)) {
- tlsAddress = alignUp(allocation + sizeof(Tcb), tlsMaxAlignment);
- tcbAddress = tlsAddress - sizeof(Tcb);
- } else {
- tcbAddress = alignUp(allocation, alignof(Tcb));
- tlsAddress = tcbAddress + sizeof(Tcb);
- }
- __ensure((tlsAddress & (tlsMaxAlignment - 1)) == 0);
- __ensure(tlsAddress == tcbAddress + sizeof(Tcb));
- } else {
- // The TCB should be aligned such that the preceding blocks are aligned too.
- tcbAddress = alignUp(allocation + tlsInitialSize, alignOverhead);
- tlsAddress = tcbAddress - tlsInitialSize;
- }
- __ensure((tcbAddress & (alignof(Tcb) - 1)) == 0);
-
- if (verbose) {
- mlibc::infoLogger() << "rtdl: tcb allocated at " << (void *)tcbAddress
- << ", size = 0x" << frg::hex_fmt{sizeof(Tcb)} << frg::endlog;
- mlibc::infoLogger() << "rtdl: tls allocated at " << (void *)tlsAddress
- << ", size = 0x" << frg::hex_fmt{tlsInitialSize} << frg::endlog;
- }
-
- Tcb *tcb_ptr = new ((char *)tcbAddress) Tcb;
- tcb_ptr->selfPointer = tcb_ptr;
-
- tcb_ptr->stackCanary = __stack_chk_guard;
- tcb_ptr->cancelBits = tcbCancelEnableBit;
- tcb_ptr->didExit = 0;
- tcb_ptr->isJoinable = 1;
- memset(&tcb_ptr->returnValue, 0, sizeof(tcb_ptr->returnValue));
- tcb_ptr->localKeys = frg::construct<frg::array<Tcb::LocalKey, PTHREAD_KEYS_MAX>>(getAllocator());
- tcb_ptr->dtvSize = runtimeTlsMap->indices.size();
- tcb_ptr->dtvPointers = frg::construct_n<void *>(getAllocator(), runtimeTlsMap->indices.size());
- memset(tcb_ptr->dtvPointers, 0, sizeof(void *) * runtimeTlsMap->indices.size());
- for(size_t i = 0; i < runtimeTlsMap->indices.size(); ++i) {
- auto object = runtimeTlsMap->indices[i];
- if(object->tlsModel != TlsModel::initial)
- continue;
- tcb_ptr->dtvPointers[i] = reinterpret_cast<char *>(tcb_ptr) + object->tlsOffset;
- }
-
- return tcb_ptr;
-}
-
-void *accessDtv(SharedObject *object) {
- Tcb *tcb_ptr = mlibc::get_current_tcb();
-
- // We might need to reallocate the DTV.
- if(object->tlsIndex >= tcb_ptr->dtvSize) {
- // TODO: need to protect runtimeTlsMap against concurrent access.
- auto ndtv = frg::construct_n<void *>(getAllocator(), runtimeTlsMap->indices.size());
- memset(ndtv, 0, sizeof(void *) * runtimeTlsMap->indices.size());
- memcpy(ndtv, tcb_ptr->dtvPointers, sizeof(void *) * tcb_ptr->dtvSize);
- frg::destruct_n(getAllocator(), tcb_ptr->dtvPointers, tcb_ptr->dtvSize);
- tcb_ptr->dtvSize = runtimeTlsMap->indices.size();
- tcb_ptr->dtvPointers = ndtv;
- }
-
- // We might need to fill in a new DTV entry.
- if(!tcb_ptr->dtvPointers[object->tlsIndex]) {
- __ensure(object->tlsModel == TlsModel::dynamic);
-
- auto buffer = getAllocator().allocate(object->tlsSegmentSize);
- __ensure(!(reinterpret_cast<uintptr_t>(buffer) & (object->tlsAlignment - 1)));
- memset(buffer, 0, object->tlsSegmentSize);
- memcpy(buffer, object->tlsImagePtr, object->tlsImageSize);
- tcb_ptr->dtvPointers[object->tlsIndex] = buffer;
-
- if (verbose) {
- mlibc::infoLogger() << "rtdl: accessDtv wrote tls image at " << buffer
- << ", size = 0x" << frg::hex_fmt{object->tlsSegmentSize} << frg::endlog;
- }
- }
-
- return (void *)((char *)tcb_ptr->dtvPointers[object->tlsIndex] + TLS_DTV_OFFSET);
-}
-
-void *tryAccessDtv(SharedObject *object) {
- Tcb *tcb_ptr = mlibc::get_current_tcb();
-
- if (object->tlsIndex >= tcb_ptr->dtvSize)
- return nullptr;
- if (!tcb_ptr->dtvPointers[object->tlsIndex])
- return nullptr;
-
- return (void *)((char *)tcb_ptr->dtvPointers[object->tlsIndex] + TLS_DTV_OFFSET);
-}
-
-// --------------------------------------------------------
-// ObjectSymbol
-// --------------------------------------------------------
-
-ObjectSymbol::ObjectSymbol(SharedObject *object, const elf_sym *symbol)
-: _object(object), _symbol(symbol) { }
-
-const char *ObjectSymbol::getString() {
- __ensure(_symbol->st_name != 0);
- return (const char *)(_object->baseAddress
- + _object->stringTableOffset + _symbol->st_name);
-}
-
-uintptr_t ObjectSymbol::virtualAddress() {
- auto bind = ELF_ST_BIND(_symbol->st_info);
- __ensure(bind == STB_GLOBAL || bind == STB_WEAK || bind == STB_GNU_UNIQUE);
- __ensure(_symbol->st_shndx != SHN_UNDEF);
- return _object->baseAddress + _symbol->st_value;
-}
-
-// --------------------------------------------------------
-// Scope
-// --------------------------------------------------------
-
-uint32_t elf64Hash(frg::string_view string) {
- uint32_t h = 0, g;
-
- for(size_t i = 0; i < string.size(); ++i) {
- h = (h << 4) + (uint32_t)string[i];
- g = h & 0xF0000000;
- if(g)
- h ^= g >> 24;
- h &= 0x0FFFFFFF;
- }
-
- return h;
-}
-
-uint32_t gnuHash(frg::string_view string) {
- uint32_t h = 5381;
- for(size_t i = 0; i < string.size(); ++i)
- h = (h << 5) + h + string[i];
- return h;
-}
-
-// TODO: move this to some namespace or class?
-frg::optional<ObjectSymbol> resolveInObject(SharedObject *object, frg::string_view string) {
- // Checks if the symbol can be used to satisfy the dependency.
- auto eligible = [&] (ObjectSymbol cand) {
- if(cand.symbol()->st_shndx == SHN_UNDEF)
- return false;
-
- auto bind = ELF_ST_BIND(cand.symbol()->st_info);
- if(bind != STB_GLOBAL && bind != STB_WEAK && bind != STB_GNU_UNIQUE)
- return false;
-
- return true;
- };
-
- if (object->hashStyle == HashStyle::systemV) {
- auto hash_table = (Elf64_Word *)(object->baseAddress + object->hashTableOffset);
- Elf64_Word num_buckets = hash_table[0];
- auto bucket = elf64Hash(string) % num_buckets;
-
- auto index = hash_table[2 + bucket];
- while(index != 0) {
- ObjectSymbol cand{object, (elf_sym *)(object->baseAddress
- + object->symbolTableOffset + index * sizeof(elf_sym))};
- if(eligible(cand) && frg::string_view{cand.getString()} == string)
- return cand;
-
- index = hash_table[2 + num_buckets + index];
- }
-
- return frg::optional<ObjectSymbol>{};
- }else{
- __ensure(object->hashStyle == HashStyle::gnu);
-
- struct GnuTable {
- uint32_t nBuckets;
- uint32_t symbolOffset;
- uint32_t bloomSize;
- uint32_t bloomShift;
- };
-
- auto hash_table = reinterpret_cast<const GnuTable *>(object->baseAddress
- + object->hashTableOffset);
- auto buckets = reinterpret_cast<const uint32_t *>(object->baseAddress
- + object->hashTableOffset + sizeof(GnuTable)
- + hash_table->bloomSize * sizeof(elf_addr));
- auto chains = reinterpret_cast<const uint32_t *>(object->baseAddress
- + object->hashTableOffset + sizeof(GnuTable)
- + hash_table->bloomSize * sizeof(elf_addr)
- + hash_table->nBuckets * sizeof(uint32_t));
-
- // TODO: Use the bloom filter.
-
- // The symbols of a given bucket are contiguous in the table.
- auto hash = gnuHash(string);
- auto index = buckets[hash % hash_table->nBuckets];
-
- if(!index)
- return frg::optional<ObjectSymbol>{};
-
- while(true) {
- // chains[] contains an array of hashes, parallel to the symbol table.
- auto chash = chains[index - hash_table->symbolOffset];
- if ((chash & ~1) == (hash & ~1)) {
- ObjectSymbol cand{object, (elf_sym *)(object->baseAddress
- + object->symbolTableOffset + index * sizeof(elf_sym))};
- if(eligible(cand) && frg::string_view{cand.getString()} == string)
- return cand;
- }
-
- // If we hit the end of the chain, the symbol is not present.
- if(chash & 1)
- return frg::optional<ObjectSymbol>{};
- index++;
- }
- }
-}
-
-frg::optional<ObjectSymbol> Scope::_resolveNext(frg::string_view string,
- SharedObject *target) {
- // Skip objects until we find the target, and only look for symbols after that.
- size_t i;
- for (i = 0; i < _objects.size(); i++) {
- if (_objects[i] == target)
- break;
- }
-
- if (i == _objects.size()) {
- mlibc::infoLogger() << "rtdl: object passed to Scope::resolveAfter was not found" << frg::endlog;
- return frg::optional<ObjectSymbol>();
- }
-
- for (i = i + 1; i < _objects.size(); i++) {
- if(_objects[i]->isMainObject)
- continue;
-
- frg::optional<ObjectSymbol> p = resolveInObject(_objects[i], string);
- if(p)
- return p;
- }
-
- return frg::optional<ObjectSymbol>();
-}
-
-Scope::Scope(bool isGlobal)
-: isGlobal{isGlobal}, _objects(getAllocator()) { }
-
-void Scope::appendObject(SharedObject *object) {
- // Don't insert duplicates.
- for (auto obj : _objects) {
- if (obj == object)
- return;
- }
-
- _objects.push(object);
-}
-
-frg::optional<ObjectSymbol> Scope::resolveGlobalOrLocal(Scope &globalScope,
- Scope *localScope, frg::string_view string, uint64_t skipRts, ResolveFlags flags) {
- auto sym = globalScope.resolveSymbol(string, skipRts, flags | skipGlobalAfterRts);
- if(!sym && localScope)
- sym = localScope->resolveSymbol(string, skipRts, flags | skipGlobalAfterRts);
- return sym;
-}
-
-frg::optional<ObjectSymbol> Scope::resolveGlobalOrLocalNext(Scope &globalScope,
- Scope *localScope, frg::string_view string, SharedObject *origin) {
- auto sym = globalScope._resolveNext(string, origin);
- if(!sym && localScope) {
- sym = localScope->_resolveNext(string, origin);
- }
- return sym;
-}
-
-// TODO: let this return uintptr_t
-frg::optional<ObjectSymbol> Scope::resolveSymbol(frg::string_view string,
- uint64_t skipRts, ResolveFlags flags) {
- for (auto object : _objects) {
- if((flags & resolveCopy) && object->isMainObject)
- continue;
- if((flags & skipGlobalAfterRts) && object->globalRts > skipRts) {
- // globalRts should be monotone increasing for objects in the global scope,
- // so as an optimization we can break early here.
- // TODO: If we implement DT_SYMBOLIC, this assumption fails.
- if(isGlobal)
- break;
- else
- continue;
- }
-
- frg::optional<ObjectSymbol> p = resolveInObject(object, string);
- if(p)
- return p;
- }
-
- return frg::optional<ObjectSymbol>();
-}
-
-// --------------------------------------------------------
-// Loader
-// --------------------------------------------------------
-
-Loader::Loader(Scope *scope, SharedObject *mainExecutable, bool is_initial_link, uint64_t rts)
-: _mainExecutable{mainExecutable}, _loadScope{scope}, _isInitialLink{is_initial_link},
- _linkRts{rts}, _linkBfs{getAllocator()}, _initQueue{getAllocator()} { }
-
-void Loader::_buildLinkBfs(SharedObject *root) {
- __ensure(_linkBfs.size() == 0);
-
- struct Token {};
- using Set = frg::hash_map<SharedObject *, Token,
- frg::hash<SharedObject *>, MemoryAllocator>;
- Set set{frg::hash<SharedObject *>{}, getAllocator()};
- _linkBfs.push(root);
-
- // Loop over indices (not iterators) here: We are adding elements in the loop!
- for(size_t i = 0; i < _linkBfs.size(); i++) {
- auto current = _linkBfs[i];
-
- // At this point the object is loaded and we can fill in its debug struct,
- // the linked list fields will be filled later.
- current->linkMap.base = current->baseAddress;
- current->linkMap.name = current->path.data();
- current->linkMap.dynv = current->dynamic;
-
- __ensure((current->tlsAlignment & (current->tlsAlignment - 1)) == 0);
-
- if (_isInitialLink && current->tlsAlignment > tlsMaxAlignment) {
- tlsMaxAlignment = current->tlsAlignment;
- }
-
- for (auto dep : current->dependencies) {
- if (!set.get(dep)) {
- set.insert(dep, Token{});
- _linkBfs.push(dep);
- }
- }
- }
-}
-
-void Loader::linkObjects(SharedObject *root) {
- _buildLinkBfs(root);
- _buildTlsMaps();
-
- // Promote objects to the desired scope.
- for(auto object : _linkBfs) {
- if (object->globalRts == 0 && _loadScope->isGlobal)
- object->globalRts = _linkRts;
-
- _loadScope->appendObject(object);
- }
-
- // Process regular relocations.
- for(auto object : _linkBfs) {
- // Some objects have already been linked before.
- if(object->objectRts < _linkRts)
- continue;
-
- if(object->dynamic == nullptr)
- continue;
-
- if(verbose)
- mlibc::infoLogger() << "rtdl: Linking " << object->name << frg::endlog;
-
- __ensure(!object->wasLinked);
-
- // TODO: Support this.
- if(object->symbolicResolution)
- mlibc::infoLogger() << "\e[31mrtdl: DT_SYMBOLIC is not implemented correctly!\e[39m"
- << frg::endlog;
-
- _processStaticRelocations(object);
- _processLazyRelocations(object);
- }
-
- // Process copy relocations.
- for(auto object : _linkBfs) {
- if(!object->isMainObject)
- continue;
-
- // Some objects have already been linked before.
- if(object->objectRts < _linkRts)
- continue;
-
- if(object->dynamic == nullptr)
- continue;
-
- processLateRelocations(object);
- }
-
- for(auto object : _linkBfs) {
- object->wasLinked = true;
-
- if(object->inLinkMap)
- continue;
-
- auto linkMap = reinterpret_cast<LinkMap*>(globalDebugInterface.head);
-
- object->linkMap.prev = linkMap;
- object->linkMap.next = linkMap->next;
- if(linkMap->next)
- linkMap->next->prev = &(object->linkMap);
- linkMap->next = &(object->linkMap);
- object->inLinkMap = true;
- }
-}
-
-void Loader::_buildTlsMaps() {
- if(_isInitialLink) {
- __ensure(runtimeTlsMap->initialPtr == 0);
- __ensure(runtimeTlsMap->initialLimit == 0);
-
- __ensure(!_linkBfs.empty());
- __ensure(_linkBfs.front()->isMainObject);
-
- for(auto object : _linkBfs) {
- __ensure(object->tlsModel == TlsModel::null);
-
- if(object->tlsSegmentSize == 0)
- continue;
-
- // Allocate an index for the object.
- object->tlsIndex = runtimeTlsMap->indices.size();
- runtimeTlsMap->indices.push_back(object);
-
- object->tlsModel = TlsModel::initial;
-
- if constexpr (tlsAboveTp) {
- // As per the comment in allocateTcb(), we may simply add sizeof(Tcb) to
- // reach the TLS data.
- object->tlsOffset = runtimeTlsMap->initialPtr + sizeof(Tcb);
- runtimeTlsMap->initialPtr += object->tlsSegmentSize;
-
- size_t misalign = runtimeTlsMap->initialPtr & (object->tlsAlignment - 1);
- if(misalign)
- runtimeTlsMap->initialPtr += object->tlsAlignment - misalign;
- } else {
- runtimeTlsMap->initialPtr += object->tlsSegmentSize;
-
- size_t misalign = runtimeTlsMap->initialPtr & (object->tlsAlignment - 1);
- if(misalign)
- runtimeTlsMap->initialPtr += object->tlsAlignment - misalign;
-
- object->tlsOffset = -runtimeTlsMap->initialPtr;
- }
-
- if(verbose)
- mlibc::infoLogger() << "rtdl: TLS of " << object->name
- << " mapped to 0x" << frg::hex_fmt{object->tlsOffset}
- << ", size: " << object->tlsSegmentSize
- << ", alignment: " << object->tlsAlignment << frg::endlog;
- }
-
- // Reserve some additional space for future libraries.
- runtimeTlsMap->initialLimit = runtimeTlsMap->initialPtr + 64;
- }else{
- for(auto object : _linkBfs) {
- if(object->tlsModel != TlsModel::null)
- continue;
- if(object->tlsSegmentSize == 0)
- continue;
-
- // Allocate an index for the object.
- object->tlsIndex = runtimeTlsMap->indices.size();
- runtimeTlsMap->indices.push_back(object);
-
- // There are some libraries (e.g. Mesa) that require static TLS even though
- // they expect to be dynamically loaded.
- if(object->haveStaticTls) {
- auto ptr = runtimeTlsMap->initialPtr + object->tlsSegmentSize;
- size_t misalign = ptr & (object->tlsAlignment - 1);
- if(misalign)
- ptr += object->tlsAlignment - misalign;
-
- if(ptr > runtimeTlsMap->initialLimit)
- mlibc::panicLogger() << "rtdl: Static TLS space exhausted while while"
- " allocating TLS for " << object->name << frg::endlog;
-
- object->tlsModel = TlsModel::initial;
-
- if constexpr (tlsAboveTp) {
- size_t tcbSize = ((sizeof(Tcb) + tlsMaxAlignment - 1) & ~(tlsMaxAlignment - 1));
-
- object->tlsOffset = runtimeTlsMap->initialPtr + tcbSize;
- runtimeTlsMap->initialPtr = ptr;
- } else {
- runtimeTlsMap->initialPtr = ptr;
- object->tlsOffset = -runtimeTlsMap->initialPtr;
- }
-
- if(verbose)
- mlibc::infoLogger() << "rtdl: TLS of " << object->name
- << " mapped to 0x" << frg::hex_fmt{object->tlsOffset}
- << ", size: " << object->tlsSegmentSize
- << ", alignment: " << object->tlsAlignment << frg::endlog;
- }else{
- object->tlsModel = TlsModel::dynamic;
- }
- }
- }
-}
-
-void Loader::initObjects() {
- initTlsObjects(mlibc::get_current_tcb(), _linkBfs, true);
-
- if (_mainExecutable && _mainExecutable->preInitArray) {
- if (verbose)
- mlibc::infoLogger() << "rtdl: Running DT_PREINIT_ARRAY functions" << frg::endlog;
-
- __ensure(_mainExecutable->isMainObject);
- __ensure(!_mainExecutable->wasInitialized);
- __ensure((_mainExecutable->preInitArraySize % sizeof(InitFuncPtr)) == 0);
- for(size_t i = 0; i < _mainExecutable->preInitArraySize / sizeof(InitFuncPtr); i++)
- _mainExecutable->preInitArray[i]();
- }
-
- // Convert the breadth-first representation to a depth-first post-order representation,
- // so that every object is initialized *after* its dependencies.
- for(auto object : _linkBfs) {
- if(!object->scheduledForInit)
- _scheduleInit(object);
- }
-
- for(auto object : _initQueue) {
- if(!object->wasInitialized)
- doInitialize(object);
- }
-}
-
-// TODO: Use an explicit vector to reduce stack usage to O(1)?
-void Loader::_scheduleInit(SharedObject *object) {
- // Here we detect cyclic dependencies.
- __ensure(!object->onInitStack);
- object->onInitStack = true;
-
- __ensure(!object->scheduledForInit);
- object->scheduledForInit = true;
-
- for(size_t i = 0; i < object->dependencies.size(); i++) {
- if(!object->dependencies[i]->scheduledForInit)
- _scheduleInit(object->dependencies[i]);
- }
-
- _initQueue.push(object);
- object->onInitStack = false;
-}
-
-void Loader::_processRelocations(Relocation &rel) {
- // copy and irelative relocations have to be performed after all other relocations
- if(rel.type() == R_COPY || rel.type() == R_IRELATIVE)
- return;
-
- // resolve the symbol if there is a symbol
- frg::optional<ObjectSymbol> p;
- if(rel.symbol_index()) {
- auto symbol = (elf_sym *)(rel.object()->baseAddress + rel.object()->symbolTableOffset
- + rel.symbol_index() * sizeof(elf_sym));
- ObjectSymbol r(rel.object(), symbol);
-
- p = Scope::resolveGlobalOrLocal(*globalScope, rel.object()->localScope,
- r.getString(), rel.object()->objectRts, 0);
- if(!p) {
- if(ELF_ST_BIND(symbol->st_info) != STB_WEAK)
- mlibc::panicLogger() << "Unresolved load-time symbol "
- << r.getString() << " in object " << rel.object()->name << frg::endlog;
-
- if(verbose)
- mlibc::infoLogger() << "rtdl: Unresolved weak load-time symbol "
- << r.getString() << " in object " << rel.object()->name << frg::endlog;
- }
- }
-
- switch(rel.type()) {
- case R_NONE:
- break;
-
- case R_JUMP_SLOT: {
- __ensure(!rel.addend_norel());
- uintptr_t symbol_addr = p ? p->virtualAddress() : 0;
- rel.relocate(symbol_addr);
- } break;
-
-#if !defined(__riscv)
- // on some architectures, R_GLOB_DAT can be defined to other relocations
- case R_GLOB_DAT: {
- __ensure(rel.symbol_index());
- uintptr_t symbol_addr = p ? p->virtualAddress() : 0;
- rel.relocate(symbol_addr + rel.addend_norel());
- } break;
-#endif
-
- case R_ABSOLUTE: {
- __ensure(rel.symbol_index());
- uintptr_t symbol_addr = p ? p->virtualAddress() : 0;
- rel.relocate(symbol_addr + rel.addend_rel());
- } break;
-
- case R_RELATIVE: {
- __ensure(!rel.symbol_index());
- rel.relocate(rel.object()->baseAddress + rel.addend_rel());
- } break;
-
- // DTPMOD and DTPREL are dynamic TLS relocations (for __tls_get_addr()).
- // TPOFF is a relocation to the initial TLS model.
- case R_TLS_DTPMOD: {
- // sets the first `sizeof(uintptr_t)` bytes of `struct __abi_tls_entry`
- // this means that we can just use the `SharedObject *` to resolve whatever we need
- __ensure(!rel.addend_rel());
- if(rel.symbol_index()) {
- __ensure(p);
- rel.relocate(elf_addr(p->object()));
- }else{
- if(stillSlightlyVerbose)
- mlibc::infoLogger() << "rtdl: Warning: TLS_DTPMOD64 with no symbol in object "
- << rel.object()->name << frg::endlog;
- rel.relocate(elf_addr(rel.object()));
- }
- } break;
- case R_TLS_DTPREL: {
- __ensure(rel.symbol_index());
- __ensure(p);
- rel.relocate(p->symbol()->st_value + rel.addend_rel() - TLS_DTV_OFFSET);
- } break;
- case R_TLS_TPREL: {
- uintptr_t off = rel.addend_rel();
- uintptr_t tls_offset = 0;
-
- if(rel.symbol_index()) {
- __ensure(p);
- if(p->object()->tlsModel != TlsModel::initial)
- mlibc::panicLogger() << "rtdl: In object " << rel.object()->name
- << ": Static TLS relocation to symbol " << p->getString()
- << " in dynamically loaded object "
- << p->object()->name << frg::endlog;
- off += p->symbol()->st_value;
- tls_offset = p->object()->tlsOffset;
- }else{
- if(stillSlightlyVerbose)
- mlibc::infoLogger() << "rtdl: Warning: TPOFF64 with no symbol"
- " in object " << rel.object()->name << frg::endlog;
- if(rel.object()->tlsModel != TlsModel::initial)
- mlibc::panicLogger() << "rtdl: In object " << rel.object()->name
- << ": Static TLS relocation to dynamically loaded object "
- << rel.object()->name << frg::endlog;
- tls_offset = rel.object()->tlsOffset;
- }
-
- if constexpr (tlsAboveTp) {
- off += tls_offset - sizeof(Tcb);
- } else {
- off += tls_offset;
- }
-
- rel.relocate(off);
- } break;
- default:
- mlibc::panicLogger() << "Unexpected relocation type "
- << (void *) rel.type() << frg::endlog;
- }
-}
-
-void Loader::_processStaticRelocations(SharedObject *object) {
- frg::optional<uintptr_t> rela_offset;
- frg::optional<size_t> rela_length;
-
- frg::optional<uintptr_t> rel_offset;
- frg::optional<size_t> rel_length;
-
- frg::optional<uintptr_t> relr_offset;
- frg::optional<size_t> relr_length;
-
- for(size_t i = 0; object->dynamic[i].d_tag != DT_NULL; i++) {
- elf_dyn *dynamic = &object->dynamic[i];
-
- switch(dynamic->d_tag) {
- case DT_RELA:
- rela_offset = dynamic->d_un.d_ptr;
- break;
- case DT_RELASZ:
- rela_length = dynamic->d_un.d_val;
- break;
- case DT_RELAENT:
- __ensure(dynamic->d_un.d_val == sizeof(elf_rela));
- break;
- case DT_REL:
- rel_offset = dynamic->d_un.d_ptr;
- break;
- case DT_RELSZ:
- rel_length = dynamic->d_un.d_val;
- break;
- case DT_RELENT:
- __ensure(dynamic->d_un.d_val == sizeof(elf_rel));
- break;
- case DT_RELR:
- relr_offset = dynamic->d_un.d_ptr;
- break;
- case DT_RELRSZ:
- relr_length = dynamic->d_un.d_val;
- break;
- case DT_RELRENT:
- __ensure(dynamic->d_un.d_val == sizeof(elf_relr));
- break;
- }
- }
-
- if(rela_offset && rela_length) {
- __ensure(!rel_offset && !rel_length);
-
- for(size_t offset = 0; offset < *rela_length; offset += sizeof(elf_rela)) {
- auto reloc = (elf_rela *)(object->baseAddress + *rela_offset + offset);
- auto r = Relocation(object, reloc);
-
- _processRelocations(r);
- }
- }else if(rel_offset && rel_length) {
- __ensure(!rela_offset && !rela_length);
-
- for(size_t offset = 0; offset < *rel_length; offset += sizeof(elf_rel)) {
- auto reloc = (elf_rel *)(object->baseAddress + *rel_offset + offset);
- auto r = Relocation(object, reloc);
-
- _processRelocations(r);
- }
- }
-
- if(relr_offset && relr_length) {
- elf_addr *addr = nullptr;
-
- for(size_t offset = 0; offset < *relr_length; offset += sizeof(elf_relr)) {
- auto entry = *(elf_relr *)(object->baseAddress + *relr_offset + offset);
-
- // Even entry indicates the beginning address.
- if(!(entry & 1)) {
- addr = (elf_addr *)(object->baseAddress + entry);
- __ensure(addr);
- *addr++ += object->baseAddress;
- }else {
- // Odd entry indicates entry is a bitmap of the subsequent locations to be relocated.
- for(int i = 0; entry; ++i) {
- if(entry & 1) {
- addr[i] += object->baseAddress;
- }
- entry >>= 1;
- }
-
- // Each entry describes at max 63 (on 64bit) or 31 (on 32bit) subsequent locations.
- addr += CHAR_BIT * sizeof(elf_relr) - 1;
- }
- }
- }
-}
-
-// TODO: TLSDESC relocations aren't aarch64 specific
-#ifdef __aarch64__
-extern "C" void *__mlibcTlsdescStatic(void *);
-extern "C" void *__mlibcTlsdescDynamic(void *);
-#endif
-
-void Loader::_processLazyRelocations(SharedObject *object) {
- if(object->globalOffsetTable == nullptr) {
- __ensure(object->lazyRelocTableOffset == 0);
- return;
- }
- object->globalOffsetTable[1] = object;
- object->globalOffsetTable[2] = (void *)&pltRelocateStub;
-
- if(!object->lazyTableSize)
- return;
-
- // adjust the addresses of JUMP_SLOT relocations
- __ensure(object->lazyExplicitAddend.has_value());
- size_t rel_size = (*object->lazyExplicitAddend) ? sizeof(elf_rela) : sizeof(elf_rel);
-
- for(size_t offset = 0; offset < object->lazyTableSize; offset += rel_size) {
- elf_info type;
- elf_info symbol_index;
-
- uintptr_t rel_addr;
- uintptr_t addend [[maybe_unused]] = 0;
-
- if(*object->lazyExplicitAddend) {
- auto reloc = (elf_rela *)(object->baseAddress + object->lazyRelocTableOffset + offset);
- type = ELF_R_TYPE(reloc->r_info);
- symbol_index = ELF_R_SYM(reloc->r_info);
- rel_addr = object->baseAddress + reloc->r_offset;
- addend = reloc->r_addend;
- } else {
- auto reloc = (elf_rel *)(object->baseAddress + object->lazyRelocTableOffset + offset);
- type = ELF_R_TYPE(reloc->r_info);
- symbol_index = ELF_R_SYM(reloc->r_info);
- rel_addr = object->baseAddress + reloc->r_offset;
- }
-
- switch (type) {
- case R_JUMP_SLOT:
- if(eagerBinding) {
- auto symbol = (elf_sym *)(object->baseAddress + object->symbolTableOffset
- + symbol_index * sizeof(elf_sym));
- ObjectSymbol r(object, symbol);
- auto p = Scope::resolveGlobalOrLocal(*globalScope, object->localScope, r.getString(), object->objectRts, 0);
-
- if(!p) {
- if(ELF_ST_BIND(symbol->st_info) != STB_WEAK)
- mlibc::panicLogger() << "rtdl: Unresolved JUMP_SLOT symbol "
- << r.getString() << " in object " << object->name << frg::endlog;
-
- if(verbose)
- mlibc::infoLogger() << "rtdl: Unresolved weak JUMP_SLOT symbol "
- << r.getString() << " in object " << object->name << frg::endlog;
- *((uintptr_t *)rel_addr) = 0;
- }else{
- *((uintptr_t *)rel_addr) = p->virtualAddress();
- }
- }else{
- *((uintptr_t *)rel_addr) += object->baseAddress;
- }
- break;
-#if defined(__x86_64__)
- case R_X86_64_IRELATIVE: {
- auto ptr = object->baseAddress + addend;
- auto target = reinterpret_cast<uintptr_t (*)(void)>(ptr)();
- *((uintptr_t *)rel_addr) = target;
- break;
- }
-#endif
-// TODO: TLSDESC relocations aren't aarch64 specific
-#if defined(__aarch64__)
- case R_AARCH64_TLSDESC: {
- size_t symValue = 0;
- SharedObject *target = nullptr;
-
- if (symbol_index) {
- auto symbol = (elf_sym *)(object->baseAddress + object->symbolTableOffset
- + symbol_index * sizeof(elf_sym));
- ObjectSymbol r(object, symbol);
- auto p = Scope::resolveGlobalOrLocal(*globalScope, object->localScope, r.getString(), object->objectRts, 0);
-
- if (!p) {
- __ensure(ELF_ST_BIND(symbol->st_info) != STB_WEAK);
- mlibc::panicLogger() << "rtdl: Unresolved TLSDESC for symbol "
- << r.getString() << " in object " << object->name << frg::endlog;
- } else {
- target = p->object();
- if (p->symbol())
- symValue = p->symbol()->st_value;
- }
- } else {
- target = object;
- }
-
- __ensure(target);
-
- if (target->tlsModel == TlsModel::initial) {
- ((uint64_t *)rel_addr)[0] = reinterpret_cast<uintptr_t>(&__mlibcTlsdescStatic);
- // TODO: guard the subtraction of TCB size with `if constexpr (tlsAboveTp)`
- // for the arch-generic case
- __ensure(tlsAboveTp == true);
- ((uint64_t *)rel_addr)[1] = symValue + target->tlsOffset + addend - sizeof(Tcb);
- } else {
- struct TlsdescData {
- uintptr_t tlsIndex;
- uintptr_t addend;
- };
-
- // Access DTV for object to force the entry to be allocated and initialized
- accessDtv(target);
-
- __ensure(target->tlsIndex < mlibc::get_current_tcb()->dtvSize);
-
- // TODO: We should free this when the DSO gets destroyed
- auto data = frg::construct<TlsdescData>(getAllocator());
- data->tlsIndex = target->tlsIndex;
- data->addend = symValue + addend;
-
- ((uint64_t *)rel_addr)[0] = reinterpret_cast<uintptr_t>(&__mlibcTlsdescDynamic);
- ((uint64_t *)rel_addr)[1] = reinterpret_cast<uintptr_t>(data);
- }
- } break;
-#endif
- default:
- mlibc::panicLogger() << "unimplemented lazy relocation type " << type << frg::endlog;
- break;
- }
- }
-}
-
diff --git a/lib/mlibc/options/rtdl/generic/linker.hpp b/lib/mlibc/options/rtdl/generic/linker.hpp
deleted file mode 100644
index ad84ca9..0000000
--- a/lib/mlibc/options/rtdl/generic/linker.hpp
+++ /dev/null
@@ -1,402 +0,0 @@
-
-#include <frg/hash_map.hpp>
-#include <frg/optional.hpp>
-#include <frg/string.hpp>
-#include <frg/vector.hpp>
-#include <frg/expected.hpp>
-#include <mlibc/allocator.hpp>
-#include <mlibc/tcb.hpp>
-
-#include "elf.hpp"
-
-struct ObjectRepository;
-struct Scope;
-struct Loader;
-struct SharedObject;
-
-extern uint64_t rtsCounter;
-
-enum class TlsModel {
- null,
- initial,
- dynamic
-};
-
-enum class LinkerError {
- success,
- notFound,
- fileTooShort,
- notElf,
- wrongElfType,
- outOfMemory,
- invalidProgramHeader
-};
-
-// --------------------------------------------------------
-// ObjectRepository
-// --------------------------------------------------------
-
-struct ObjectRepository {
- ObjectRepository();
-
- ObjectRepository(const ObjectRepository &) = delete;
-
- ObjectRepository &operator= (const ObjectRepository &) = delete;
-
- // This is primarily used to create a SharedObject for the RTDL itself.
- SharedObject *injectObjectFromDts(frg::string_view name,
- frg::string<MemoryAllocator> path,
- uintptr_t base_address, elf_dyn *dynamic, uint64_t rts);
-
- // This is used to create a SharedObject for the executable that we want to link.
- SharedObject *injectObjectFromPhdrs(frg::string_view name,
- frg::string<MemoryAllocator> path, void *phdr_pointer,
- size_t phdr_entry_size, size_t num_phdrs, void *entry_pointer,
- uint64_t rts);
-
- SharedObject *injectStaticObject(frg::string_view name,
- frg::string<MemoryAllocator> path, void *phdr_pointer,
- size_t phdr_entry_size, size_t num_phdrs, void *entry_pointer,
- uint64_t rts);
-
- frg::expected<LinkerError, SharedObject *> requestObjectWithName(frg::string_view name,
- SharedObject *origin, Scope *localScope, bool createScope, uint64_t rts);
-
- frg::expected<LinkerError, SharedObject *> requestObjectAtPath(frg::string_view path,
- Scope *localScope, bool createScope, uint64_t rts);
-
- SharedObject *findCaller(void *address);
-
- SharedObject *findLoadedObject(frg::string_view name);
-
- // Used by dl_iterate_phdr: stores objects in the order they are loaded.
- frg::vector<SharedObject *, MemoryAllocator> loadedObjects;
-
-private:
- void _fetchFromPhdrs(SharedObject *object, void *phdr_pointer,
- size_t phdr_entry_size, size_t num_phdrs, void *entry_pointer);
-
- frg::expected<LinkerError, void> _fetchFromFile(SharedObject *object, int fd);
-
- void _parseDynamic(SharedObject *object);
-
- void _discoverDependencies(SharedObject *object, Scope *localScope, uint64_t rts);
-
- void _addLoadedObject(SharedObject *object);
-
- frg::hash_map<frg::string_view, SharedObject *,
- frg::hash<frg::string_view>, MemoryAllocator> _nameMap;
-};
-
-// --------------------------------------------------------
-// SharedObject
-// --------------------------------------------------------
-
-enum class HashStyle {
- none,
- systemV,
- gnu
-};
-
-using InitFuncPtr = void (*)();
-
-// The ABI of this struct is fixed by GDB
-struct DebugInterface {
- int ver;
- void *head;
- void (*brk)(void);
- int state;
- void *base;
-};
-
-// The ABI of this struct is fixed by GDB
-struct LinkMap {
- uintptr_t base = 0;
- const char *name = nullptr;
- elf_dyn *dynv = nullptr;
- LinkMap *next = nullptr, *prev = nullptr;
-};
-
-struct SharedObject {
- // path is copied
- SharedObject(const char *name, frg::string<MemoryAllocator> path,
- bool is_main_object, Scope *localScope, uint64_t object_rts);
-
- SharedObject(const char *name, const char *path, bool is_main_object,
- Scope *localScope, uint64_t object_rts);
-
- frg::string<MemoryAllocator> name;
- frg::string<MemoryAllocator> path;
- frg::string<MemoryAllocator> interpreterPath;
- const char *soName;
- bool isMainObject;
- uint64_t objectRts;
-
- // link map for debugging
- LinkMap linkMap;
- bool inLinkMap;
-
- // base address this shared object was loaded to
- uintptr_t baseAddress;
-
- Scope *localScope;
-
- // pointers to the dynamic table, GOT and entry point
- elf_dyn *dynamic = nullptr;
- void **globalOffsetTable;
- void *entry;
-
- // object initialization information
- InitFuncPtr initPtr = nullptr;
- InitFuncPtr *initArray = nullptr;
- InitFuncPtr *preInitArray = nullptr;
- size_t initArraySize = 0;
- size_t preInitArraySize = 0;
-
-
- // TODO: read this from the PHDR
- size_t tlsSegmentSize, tlsAlignment, tlsImageSize;
- void *tlsImagePtr;
- bool tlsInitialized;
-
- // symbol and string table of this shared object
- HashStyle hashStyle = HashStyle::none;
- uintptr_t hashTableOffset;
- uintptr_t symbolTableOffset;
- uintptr_t stringTableOffset;
-
- const char *runPath = nullptr;
-
- // save the lazy JUMP_SLOT relocation table
- uintptr_t lazyRelocTableOffset;
- size_t lazyTableSize;
- frg::optional<bool> lazyExplicitAddend;
-
- bool symbolicResolution;
- bool eagerBinding;
- bool haveStaticTls;
-
- // vector of dependencies
- frg::vector<SharedObject *, MemoryAllocator> dependencies;
-
- TlsModel tlsModel;
- size_t tlsIndex;
- size_t tlsOffset;
-
- uint64_t globalRts;
- bool wasLinked;
-
- bool scheduledForInit;
- bool onInitStack;
- bool wasInitialized;
-
- // PHDR related stuff, we only set these for the main executable
- void *phdrPointer = nullptr;
- size_t phdrEntrySize = 0;
- size_t phdrCount = 0;
-};
-
-struct Relocation {
- Relocation(SharedObject *object, elf_rela *r)
- : object_{object}, type_{Addend::Explicit} {
- offset_ = r->r_offset;
- info_ = r->r_info;
- addend_ = r->r_addend;
- }
-
- Relocation(SharedObject *object, elf_rel *r)
- : object_{object}, type_{Addend::Implicit} {
- offset_ = r->r_offset;
- info_ = r->r_info;
- }
-
- SharedObject *object() {
- return object_;
- }
-
- elf_info type() const {
- return ELF_R_TYPE(info_);
- }
-
- elf_info symbol_index() const {
- return ELF_R_SYM(info_);
- }
-
- elf_addr addend_rel() {
- switch(type_) {
- case Addend::Explicit:
- return addend_;
- case Addend::Implicit: {
- auto ptr = reinterpret_cast<elf_addr *>(object_->baseAddress + offset_);
- return *ptr;
- }
- }
- __builtin_unreachable();
- }
-
- elf_addr addend_norel() {
- switch(type_) {
- case Addend::Explicit:
- return addend_;
- case Addend::Implicit:
- return 0;
- }
- __builtin_unreachable();
- }
-
- void *destination() {
- return reinterpret_cast<void *>(object_->baseAddress + offset_);
- }
-
- void relocate(elf_addr addr) {
- auto ptr = destination();
- memcpy(ptr, &addr, sizeof(addr));
- }
-
-private:
- enum class Addend {
- Implicit,
- Explicit
- };
-
- SharedObject *object_;
- Addend type_;
-
- elf_addr offset_;
- elf_info info_;
- elf_addend addend_ = 0;
-};
-
-void processCopyRelocations(SharedObject *object);
-
-// --------------------------------------------------------
-// RuntimeTlsMap
-// --------------------------------------------------------
-
-struct RuntimeTlsMap {
- RuntimeTlsMap();
-
- // Amount of initialLimit that has already been allocated.
- size_t initialPtr;
-
- // Size of the inital TLS segment.
- size_t initialLimit;
-
- // TLS indices.
- frg::vector<SharedObject *, MemoryAllocator> indices;
-};
-
-extern frg::manual_box<RuntimeTlsMap> runtimeTlsMap;
-
-Tcb *allocateTcb();
-void initTlsObjects(Tcb *tcb, const frg::vector<SharedObject *, MemoryAllocator> &objects, bool checkInitialized);
-void *accessDtv(SharedObject *object);
-// Tries to access the DTV, if not allocated, or object doesn't have
-// PT_TLS, return nullptr.
-void *tryAccessDtv(SharedObject *object);
-
-// --------------------------------------------------------
-// ObjectSymbol
-// --------------------------------------------------------
-
-struct ObjectSymbol {
- ObjectSymbol(SharedObject *object, const elf_sym *symbol);
-
- SharedObject *object() {
- return _object;
- }
-
- const elf_sym *symbol() {
- return _symbol;
- }
-
- const char *getString();
-
- uintptr_t virtualAddress();
-
-private:
- SharedObject *_object;
- const elf_sym *_symbol;
-};
-
-frg::optional<ObjectSymbol> resolveInObject(SharedObject *object, frg::string_view string);
-
-// --------------------------------------------------------
-// Scope
-// --------------------------------------------------------
-
-struct Scope {
- using ResolveFlags = uint32_t;
- static inline constexpr ResolveFlags resolveCopy = 1;
- static inline constexpr ResolveFlags skipGlobalAfterRts = 1 << 1;
-
- static frg::optional<ObjectSymbol> resolveGlobalOrLocal(Scope &globalScope,
- Scope *localScope, frg::string_view string, uint64_t skipRts, ResolveFlags flags);
- static frg::optional<ObjectSymbol> resolveGlobalOrLocalNext(Scope &globalScope,
- Scope *localScope, frg::string_view string, SharedObject *origin);
-
- Scope(bool isGlobal = false);
-
- void appendObject(SharedObject *object);
-
- frg::optional<ObjectSymbol> resolveSymbol(frg::string_view string, uint64_t skipRts, ResolveFlags flags);
-
- bool isGlobal;
-
-private:
- frg::optional<ObjectSymbol> _resolveNext(frg::string_view string, SharedObject *target);
-public: // TODO: Make this private again. (Was made public for __dlapi_reverse()).
- frg::vector<SharedObject *, MemoryAllocator> _objects;
-};
-
-extern frg::manual_box<Scope> globalScope;
-
-// --------------------------------------------------------
-// Loader
-// --------------------------------------------------------
-
-class Loader {
-public:
- Loader(Scope *scope, SharedObject *mainExecutable, bool is_initial_link, uint64_t rts);
-
-public:
- void linkObjects(SharedObject *root);
-
-private:
- void _buildLinkBfs(SharedObject *root);
- void _buildTlsMaps();
-
- void _processStaticRelocations(SharedObject *object);
- void _processLazyRelocations(SharedObject *object);
-
- void _processRelocations(Relocation &rel);
-
-public:
- void initObjects();
-
-private:
- void _scheduleInit(SharedObject *object);
-
-private:
- SharedObject *_mainExecutable;
- Scope *_loadScope;
- bool _isInitialLink;
- uint64_t _linkRts;
-
- frg::vector<SharedObject *, MemoryAllocator> _linkBfs;
-
- frg::vector<SharedObject *, MemoryAllocator> _initQueue;
-};
-
-// --------------------------------------------------------
-// Namespace scope functions
-// --------------------------------------------------------
-
-extern "C" void pltRelocateStub() __attribute__((__visibility__("hidden")));
-
-// --------------------------------------------------------
-// RTDL interface
-// --------------------------------------------------------
-
-void *rtdl_auxvector();
-
diff --git a/lib/mlibc/options/rtdl/generic/main.cpp b/lib/mlibc/options/rtdl/generic/main.cpp
deleted file mode 100644
index 3cff1e4..0000000
--- a/lib/mlibc/options/rtdl/generic/main.cpp
+++ /dev/null
@@ -1,844 +0,0 @@
-
-#include <elf.h>
-#include <link.h>
-
-#include <frg/manual_box.hpp>
-#include <frg/small_vector.hpp>
-
-#include <abi-bits/auxv.h>
-#include <mlibc/debug.hpp>
-#include <mlibc/rtdl-sysdeps.hpp>
-#include <mlibc/rtdl-config.hpp>
-#include <mlibc/rtdl-abi.hpp>
-#include <mlibc/stack_protector.hpp>
-#include <internal-config.h>
-#include <abi-bits/auxv.h>
-
-#include "elf.hpp"
-#include "linker.hpp"
-
-#if __MLIBC_POSIX_OPTION
-#include <dlfcn.h>
-#endif
-
-#define HIDDEN __attribute__((__visibility__("hidden")))
-#define EXPORT __attribute__((__visibility__("default")))
-
-static constexpr bool logEntryExit = false;
-static constexpr bool logStartup = false;
-static constexpr bool logDlCalls = false;
-
-#ifndef MLIBC_STATIC_BUILD
-extern HIDDEN void *_GLOBAL_OFFSET_TABLE_[];
-extern HIDDEN elf_dyn _DYNAMIC[];
-#endif
-
-namespace mlibc {
- // Declared in options/internal/mlibc/tcb.hpp.
- bool tcb_available_flag = false;
-}
-
-mlibc::RtdlConfig rtdlConfig;
-
-bool ldShowAuxv = false;
-
-uintptr_t *entryStack;
-frg::manual_box<ObjectRepository> initialRepository;
-frg::manual_box<Scope> globalScope;
-
-frg::manual_box<RuntimeTlsMap> runtimeTlsMap;
-
-// We use a small vector of size 4 to avoid memory allocation for the default library paths
-frg::manual_box<frg::small_vector<frg::string_view, 4, MemoryAllocator>> libraryPaths;
-
-frg::manual_box<frg::vector<frg::string_view, MemoryAllocator>> preloads;
-
-static SharedObject *executableSO;
-extern HIDDEN char __ehdr_start[];
-
-// Global debug interface variable
-DebugInterface globalDebugInterface;
-
-#ifndef MLIBC_STATIC_BUILD
-
-// Use a PC-relative instruction sequence to find our runtime load address.
-uintptr_t getLdsoBase() {
-#if defined(__x86_64__) || defined(__i386__) || defined(__aarch64__)
- // On x86_64, the first GOT entry holds the link-time address of _DYNAMIC.
- // TODO: This isn't guaranteed on AArch64, so this might fail with some linkers.
- auto linktime_dynamic = reinterpret_cast<uintptr_t>(_GLOBAL_OFFSET_TABLE_[0]);
- auto runtime_dynamic = reinterpret_cast<uintptr_t>(_DYNAMIC);
- return runtime_dynamic - linktime_dynamic;
-#elif defined(__riscv)
- return reinterpret_cast<uintptr_t>(&__ehdr_start);
-#endif
-}
-
-// Relocates the dynamic linker (i.e. this DSO) itself.
-// Assumptions:
-// - There are no references to external symbols.
-// Note that this code is fragile in the sense that it must not contain relocations itself.
-// TODO: Use tooling to verify this at compile time.
-extern "C" void relocateSelf() {
- size_t rela_offset = 0;
- size_t rela_size = 0;
- size_t rel_offset = 0;
- size_t rel_size = 0;
- size_t relr_offset = 0;
- size_t relr_size = 0;
- for(size_t i = 0; _DYNAMIC[i].d_tag != DT_NULL; i++) {
- auto ent = &_DYNAMIC[i];
- switch(ent->d_tag) {
- case DT_REL: rel_offset = ent->d_un.d_ptr; break;
- case DT_RELSZ: rel_size = ent->d_un.d_val; break;
- case DT_RELA: rela_offset = ent->d_un.d_ptr; break;
- case DT_RELASZ: rela_size = ent->d_un.d_val; break;
- case DT_RELR: relr_offset = ent->d_un.d_ptr; break;
- case DT_RELRSZ: relr_size = ent->d_un.d_val; break;
- }
- }
-
- auto ldso_base = getLdsoBase();
-
- __ensure((rel_offset != 0) ^ (rela_offset != 0));
-
- for(size_t disp = 0; disp < rela_size; disp += sizeof(elf_rela)) {
- auto reloc = reinterpret_cast<elf_rela *>(ldso_base + rela_offset + disp);
-
- auto type = ELF_R_TYPE(reloc->r_info);
- if(ELF_R_SYM(reloc->r_info))
- __builtin_trap();
-
- auto p = reinterpret_cast<uint64_t *>(ldso_base + reloc->r_offset);
- switch(type) {
- case R_RELATIVE:
- *p = ldso_base + reloc->r_addend;
- break;
- default:
- __builtin_trap();
- }
- }
-
- for(size_t disp = 0; disp < rel_size; disp += sizeof(elf_rel)) {
- auto reloc = reinterpret_cast<elf_rel *>(ldso_base + rel_offset + disp);
-
- auto type = ELF_R_TYPE(reloc->r_info);
- if(ELF_R_SYM(reloc->r_info))
- __builtin_trap();
-
- auto p = reinterpret_cast<uint64_t *>(ldso_base + reloc->r_offset);
- switch(type) {
- case R_RELATIVE:
- *p += ldso_base;
- break;
- default:
- __builtin_trap();
- }
- }
-
- elf_addr *addr = nullptr;
- for(size_t disp = 0; disp < relr_size; disp += sizeof(elf_relr)) {
- auto entry = *(elf_relr *)(ldso_base + relr_offset + disp);
-
- // Even entry indicates the beginning address.
- if(!(entry & 1)) {
- addr = (elf_addr *)(ldso_base + entry);
- __ensure(addr);
- *addr++ += ldso_base;
- }else {
- // Odd entry indicates entry is a bitmap of the subsequent locations to be relocated.
- for(int i = 0; entry; ++i) {
- if(entry & 1) {
- addr[i] += ldso_base;
- }
- entry >>= 1;
- }
-
- // Each entry describes at max 63 (on 64bit) or 31 (on 32bit) subsequent locations.
- addr += CHAR_BIT * sizeof(elf_relr) - 1;
- }
- }
-}
-#endif
-
-extern "C" void *lazyRelocate(SharedObject *object, unsigned int rel_index) {
- __ensure(object->lazyExplicitAddend);
- auto reloc = (elf_rela *)(object->baseAddress + object->lazyRelocTableOffset
- + rel_index * sizeof(elf_rela));
- auto type = ELF_R_TYPE(reloc->r_info);
- auto symbol_index = ELF_R_SYM(reloc->r_info);
-
- __ensure(type == R_X86_64_JUMP_SLOT);
- __ensure(ELF_CLASS == ELFCLASS64);
-
- auto symbol = (elf_sym *)(object->baseAddress + object->symbolTableOffset
- + symbol_index * sizeof(elf_sym));
- ObjectSymbol r(object, symbol);
- auto p = Scope::resolveGlobalOrLocal(*globalScope, object->localScope, r.getString(), object->objectRts, 0);
- if(!p)
- mlibc::panicLogger() << "Unresolved JUMP_SLOT symbol" << frg::endlog;
-
- //mlibc::infoLogger() << "Lazy relocation to " << symbol_str
- // << " resolved to " << pointer << frg::endlog;
-
- *(uint64_t *)(object->baseAddress + reloc->r_offset) = p->virtualAddress();
- return (void *)p->virtualAddress();
-}
-
-extern "C" [[ gnu::visibility("default") ]] void *__rtdl_allocateTcb() {
- auto tcb = allocateTcb();
- initTlsObjects(tcb, globalScope->_objects, false);
- return tcb;
-}
-
-extern "C" {
- [[ gnu::visibility("hidden") ]] void dl_debug_state() {
- // This function is used to signal changes in the debugging link map,
- // GDB just sets a breakpoint on this function and we can call it
- // everytime we update the link map. We don't need to implement
- // anything besides defining and calling it.
- }
-}
-
-extern "C" [[gnu::alias("dl_debug_state"), gnu::visibility("default")]] void _dl_debug_state() noexcept;
-
-// This symbol can be used by GDB to find the global interface structure
-[[ gnu::visibility("default") ]] DebugInterface *_dl_debug_addr = &globalDebugInterface;
-
-static frg::vector<frg::string_view, MemoryAllocator> parseList(frg::string_view paths, frg::string_view separators) {
- frg::vector<frg::string_view, MemoryAllocator> list{getAllocator()};
-
- size_t p = 0;
- while(p < paths.size()) {
- size_t s; // Offset of next colon or end of string.
- if(size_t cs = paths.find_first_of(separators, p); cs != size_t(-1)) {
- s = cs;
- }else{
- s = paths.size();
- }
-
- auto path = paths.sub_string(p, s - p);
- p = s + 1;
-
- if(path.size() == 0)
- continue;
-
- if(path.ends_with("/")) {
- size_t i = path.size() - 1;
- while(i > 0 && path[i] == '/')
- i--;
- path = path.sub_string(0, i + 1);
- }
-
- if(path == "/")
- path = "";
-
- list.push_back(path);
- }
-
- return list;
-}
-
-extern "C" void *interpreterMain(uintptr_t *entry_stack) {
- if(logEntryExit)
- mlibc::infoLogger() << "Entering ld.so" << frg::endlog;
- entryStack = entry_stack;
- runtimeTlsMap.initialize();
- libraryPaths.initialize(getAllocator());
- preloads.initialize(getAllocator());
-
- void *phdr_pointer = 0;
- size_t phdr_entry_size = 0;
- size_t phdr_count = 0;
- void *entry_pointer = 0;
- void *stack_entropy = nullptr;
-
- const char *execfn = "(executable)";
-
-#ifndef MLIBC_STATIC_BUILD
- using ctor_fn = void(*)(void);
-
- ctor_fn *ldso_ctors = nullptr;
- size_t num_ldso_ctors = 0;
-
- auto ldso_base = getLdsoBase();
- if(logStartup) {
- mlibc::infoLogger() << "ldso: Own base address is: 0x"
- << frg::hex_fmt(ldso_base) << frg::endlog;
- mlibc::infoLogger() << "ldso: Own dynamic section is at: " << _DYNAMIC << frg::endlog;
- }
-
-#ifdef __x86_64__
- // These entries are reserved on x86_64.
- // TODO: Use a fake PLT stub that reports an error message?
- _GLOBAL_OFFSET_TABLE_[1] = 0;
- _GLOBAL_OFFSET_TABLE_[2] = 0;
-#endif
-
- // Validate our own dynamic section.
- // Here, we make sure that the dynamic linker does not need relocations itself.
- uintptr_t strtab_offset = 0;
- uintptr_t soname_str = 0;
- for(size_t i = 0; _DYNAMIC[i].d_tag != DT_NULL; i++) {
- auto ent = &_DYNAMIC[i];
- switch(ent->d_tag) {
- case DT_STRTAB: strtab_offset = ent->d_un.d_ptr; break;
- case DT_SONAME: soname_str = ent->d_un.d_val; break;
- case DT_INIT_ARRAY: ldso_ctors = reinterpret_cast<ctor_fn *>(ent->d_un.d_ptr + ldso_base); break;
- case DT_INIT_ARRAYSZ: num_ldso_ctors = ent->d_un.d_val / sizeof(ctor_fn); break;
- case DT_HASH:
- case DT_GNU_HASH:
- case DT_STRSZ:
- case DT_SYMTAB:
- case DT_SYMENT:
- case DT_RELA:
- case DT_RELASZ:
- case DT_RELAENT:
- case DT_RELACOUNT:
- case DT_DEBUG:
- case DT_REL:
- case DT_RELSZ:
- case DT_RELENT:
- case DT_RELCOUNT:
- case DT_RELR:
- case DT_RELRSZ:
- case DT_RELRENT:
- continue;
- default:
- __ensure(!"Unexpected dynamic entry in program interpreter");
- }
- }
- __ensure(strtab_offset);
- __ensure(soname_str);
-
- // Find the auxiliary vector by skipping args and environment.
- auto aux = entryStack;
- aux += *aux + 1; // First, we skip argc and all args.
- __ensure(!*aux);
- aux++;
- while(*aux) { // Loop through the environment.
- auto env = reinterpret_cast<char *>(*aux);
- frg::string_view view{env};
- size_t s = view.find_first('=');
-
- if(s == size_t(-1))
- mlibc::panicLogger() << "rtdl: environment '" << env << "' is missing a '='" << frg::endlog;
-
- auto name = view.sub_string(0, s);
- auto value = const_cast<char *>(view.data() + s + 1);
-
- if(name == "LD_SHOW_AUXV" && *value && *value != '0') {
- ldShowAuxv = true;
- }else if(name == "LD_LIBRARY_PATH" && *value) {
- for(auto path : parseList(value, ":;"))
- libraryPaths->push_back(path);
- }else if(name == "LD_PRELOAD" && *value) {
- *preloads = parseList(value, " :");
- }
-
- aux++;
- }
- aux++;
-
- // Add default library paths
- libraryPaths->push_back("/lib");
- libraryPaths->push_back("/lib64");
- libraryPaths->push_back("/usr/lib");
- libraryPaths->push_back("/usr/lib64");
-
- // Parse the actual vector.
- while(true) {
- auto value = aux + 1;
- if(!(*aux))
- break;
-
- if(ldShowAuxv) {
- switch(*aux) {
- case AT_PHDR: mlibc::infoLogger() << "AT_PHDR: 0x" << frg::hex_fmt{*value} << frg::endlog; break;
- case AT_PHENT: mlibc::infoLogger() << "AT_PHENT: " << *value << frg::endlog; break;
- case AT_PHNUM: mlibc::infoLogger() << "AT_PHNUM: " << *value << frg::endlog; break;
- case AT_ENTRY: mlibc::infoLogger() << "AT_ENTRY: 0x" << frg::hex_fmt{*value} << frg::endlog; break;
- case AT_PAGESZ: mlibc::infoLogger() << "AT_PAGESZ: " << *value << frg::endlog; break;
- case AT_BASE: mlibc::infoLogger() << "AT_BASE: 0x" << frg::hex_fmt{*value} << frg::endlog; break;
- case AT_FLAGS: mlibc::infoLogger() << "AT_FLAGS: 0x" << frg::hex_fmt{*value} << frg::endlog; break;
- case AT_NOTELF: mlibc::infoLogger() << "AT_NOTELF: " << frg::hex_fmt{*value} << frg::endlog; break;
- case AT_UID: mlibc::infoLogger() << "AT_UID: " << *value << frg::endlog; break;
- case AT_EUID: mlibc::infoLogger() << "AT_EUID: " << *value << frg::endlog; break;
- case AT_GID: mlibc::infoLogger() << "AT_GID: " << *value << frg::endlog; break;
- case AT_EGID: mlibc::infoLogger() << "AT_EGID: " << *value << frg::endlog; break;
-#ifdef AT_PLATFORM
- case AT_PLATFORM: mlibc::infoLogger() << "AT_PLATFORM: " << reinterpret_cast<const char *>(*value) << frg::endlog; break;
-#endif
-#ifdef AT_HWCAP
- case AT_HWCAP: mlibc::infoLogger() << "AT_HWCAP: " << frg::hex_fmt{*value} << frg::endlog; break;
-#endif
-#ifdef AT_CLKTCK
- case AT_CLKTCK: mlibc::infoLogger() << "AT_CLKTCK: " << *value << frg::endlog; break;
-#endif
-#ifdef AT_FPUCW
- case AT_FPUCW: mlibc::infoLogger() << "AT_FPUCW: " << frg::hex_fmt{*value} << frg::endlog; break;
-#endif
-#ifdef AT_SECURE
- case AT_SECURE: mlibc::infoLogger() << "AT_SECURE: " << *value << frg::endlog; break;
-#endif
-#ifdef AT_RANDOM
- case AT_RANDOM: mlibc::infoLogger() << "AT_RANDOM: 0x" << frg::hex_fmt{*value} << frg::endlog; break;
-#endif
-#ifdef AT_EXECFN
- case AT_EXECFN: mlibc::infoLogger() << "AT_EXECFN: " << reinterpret_cast<const char *>(*value) << frg::endlog; break;
-#endif
-#ifdef AT_SYSINFO_EHDR
- case AT_SYSINFO_EHDR: mlibc::infoLogger() << "AT_SYSINFO_EHDR: 0x" << frg::hex_fmt{*value} << frg::endlog; break;
-#endif
- }
- }
-
- // TODO: Whitelist auxiliary vector entries here?
- switch(*aux) {
- case AT_PHDR: phdr_pointer = reinterpret_cast<void *>(*value); break;
- case AT_PHENT: phdr_entry_size = *value; break;
- case AT_PHNUM: phdr_count = *value; break;
- case AT_ENTRY: entry_pointer = reinterpret_cast<void *>(*value); break;
- case AT_EXECFN: execfn = reinterpret_cast<const char *>(*value); break;
- case AT_RANDOM: stack_entropy = reinterpret_cast<void*>(*value); break;
- case AT_SECURE: rtdlConfig.secureRequired = reinterpret_cast<uintptr_t>(*value); break;
- }
-
- aux += 2;
- }
- globalDebugInterface.base = reinterpret_cast<void*>(ldso_base);
-
-// This is here because libgcc will add a global constructor on glibc Linux
-// (which is what it believes we are due to the aarch64-linux-gnu toolchain)
-// in order to check if LSE atomics are supported.
-//
-// This is not necessary on a custom Linux toolchain and is purely an artifact of
-// using the host toolchain.
-#if defined(__aarch64__) && defined(__gnu_linux__)
- for (size_t i = 0; i < num_ldso_ctors; i++) {
- if(logStartup)
- mlibc::infoLogger() << "ldso: Running own constructor at "
- << reinterpret_cast<void *>(ldso_ctors[i])
- << frg::endlog;
- ldso_ctors[i]();
- }
-#else
- if (num_ldso_ctors > 0) {
- mlibc::panicLogger() << "ldso: Found unexpected own global constructor(s), init_array starts at: "
- << ldso_ctors
- << frg::endlog;
- }
-#endif
-
-#else
- auto ehdr = reinterpret_cast<elf_ehdr*>(__ehdr_start);
- phdr_pointer = reinterpret_cast<void*>((uintptr_t)ehdr->e_phoff + (uintptr_t)ehdr);
- phdr_entry_size = ehdr->e_phentsize;
- phdr_count = ehdr->e_phnum;
- entry_pointer = reinterpret_cast<void*>(ehdr->e_entry);
-#endif
- __ensure(phdr_pointer);
- __ensure(entry_pointer);
-
- if(logStartup)
- mlibc::infoLogger() << "ldso: Executable PHDRs are at " << phdr_pointer
- << frg::endlog;
-
- // perform the initial dynamic linking
- initialRepository.initialize();
-
- globalScope.initialize(true);
-
- // Add the dynamic linker, as well as the exectuable to the repository.
-#ifndef MLIBC_STATIC_BUILD
- auto ldso_soname = reinterpret_cast<const char *>(ldso_base + strtab_offset + soname_str);
- auto ldso = initialRepository->injectObjectFromDts(ldso_soname,
- frg::string<MemoryAllocator> { getAllocator() },
- ldso_base, _DYNAMIC, 1);
- ldso->phdrPointer = phdr_pointer;
- ldso->phdrCount = phdr_count;
- ldso->phdrEntrySize = phdr_entry_size;
-
- // TODO: support non-zero base addresses?
- executableSO = initialRepository->injectObjectFromPhdrs(execfn,
- frg::string<MemoryAllocator> { execfn, getAllocator() },
- phdr_pointer, phdr_entry_size, phdr_count, entry_pointer, 1);
-
- // We can't initialise the ldso object after the executable SO,
- // so we have to set the ldso path after loading both.
- ldso->path = executableSO->interpreterPath;
-
-#else
- executableSO = initialRepository->injectStaticObject(execfn,
- frg::string<MemoryAllocator>{ execfn, getAllocator() },
- phdr_pointer, phdr_entry_size, phdr_count, entry_pointer, 1);
- globalDebugInterface.base = (void*)executableSO->baseAddress;
-#endif
-
- globalDebugInterface.head = &executableSO->linkMap;
- executableSO->inLinkMap = true;
- Loader linker{globalScope.get(), executableSO, true, 1};
- linker.linkObjects(executableSO);
-
- mlibc::initStackGuard(stack_entropy);
-
- auto tcb = allocateTcb();
- if(mlibc::sys_tcb_set(tcb))
- __ensure(!"sys_tcb_set() failed");
- tcb->tid = mlibc::this_tid();
- mlibc::tcb_available_flag = true;
-
- globalDebugInterface.ver = 1;
- globalDebugInterface.brk = &dl_debug_state;
- globalDebugInterface.state = 0;
- dl_debug_state();
-
- linker.initObjects();
-
- if(logEntryExit)
- mlibc::infoLogger() << "Leaving ld.so, jump to "
- << (void *)executableSO->entry << frg::endlog;
- return executableSO->entry;
-}
-
-const char *lastError;
-
-extern "C" [[ gnu::visibility("default") ]] uintptr_t *__dlapi_entrystack() {
- return entryStack;
-}
-
-extern "C" [[ gnu::visibility("default") ]]
-const char *__dlapi_error() {
- auto error = lastError;
- lastError = nullptr;
- return error;
-}
-
-extern "C" [[ gnu::visibility("default") ]]
-void *__dlapi_get_tls(struct __abi_tls_entry *entry) {
- return reinterpret_cast<char *>(accessDtv(entry->object)) + entry->offset;
-}
-
-extern "C" [[ gnu::visibility("default") ]]
-const mlibc::RtdlConfig &__dlapi_get_config() {
- return rtdlConfig;
-}
-
-#if __MLIBC_POSIX_OPTION
-
-extern "C" [[ gnu::visibility("default") ]]
-void *__dlapi_open(const char *file, int flags, void *returnAddress) {
- if (logDlCalls)
- mlibc::infoLogger() << "rtdl: __dlapi_open(" << (file ? file : "nullptr") << ")" << frg::endlog;
-
- if (flags & RTLD_DEEPBIND)
- mlibc::infoLogger() << "rtdl: dlopen(RTLD_DEEPBIND) is unsupported" << frg::endlog;
-
- if(!file)
- return executableSO;
-
- // TODO: Thread-safety!
- auto rts = rtsCounter++;
-
- SharedObject *object;
- if (flags & RTLD_NOLOAD) {
- object = initialRepository->findLoadedObject(file);
- if (object && object->globalRts == 0 && (flags & RTLD_GLOBAL)) {
- // The object was opened with RTLD_LOCAL, but we are called with RTLD_NOLOAD | RTLD_GLOBAL.
- // According to the man page, we should promote to the global scope here.
- object->globalRts = rts;
- globalScope->appendObject(object);
- }
- } else {
- bool isGlobal = flags & RTLD_GLOBAL;
- Scope *newScope = isGlobal ? globalScope.get() : nullptr;
-
- frg::expected<LinkerError, SharedObject *> objectResult;
- if (frg::string_view{file}.find_first('/') == size_t(-1)) {
- // In order to know which RUNPATH / RPATH to process, we must find the calling object.
- SharedObject *origin = initialRepository->findCaller(returnAddress);
- if (!origin) {
- mlibc::panicLogger() << "rtdl: unable to determine calling object of dlopen "
- << "(ra = " << returnAddress << ")" << frg::endlog;
- }
-
- objectResult = initialRepository->requestObjectWithName(file, origin, newScope, !isGlobal, rts);
- } else {
- objectResult = initialRepository->requestObjectAtPath(file, newScope, !isGlobal, rts);
- }
-
- if(!objectResult) {
- switch (objectResult.error()) {
- case LinkerError::success:
- __builtin_unreachable();
- case LinkerError::notFound:
- lastError = "Cannot locate requested DSO";
- break;
- case LinkerError::fileTooShort:
- lastError = "File too short";
- break;
- case LinkerError::notElf:
- lastError = "File is not an ELF file";
- break;
- case LinkerError::wrongElfType:
- lastError = "File has wrong ELF type";
- break;
- case LinkerError::outOfMemory:
- lastError = "Out of memory";
- break;
- case LinkerError::invalidProgramHeader:
- lastError = "File has invalid program header";
- break;
- }
- return nullptr;
- }
- object = objectResult.value();
-
- Loader linker{object->localScope, nullptr, false, rts};
- linker.linkObjects(object);
- linker.initObjects();
- }
-
- dl_debug_state();
-
- return object;
-}
-
-extern "C" [[ gnu::visibility("default") ]]
-void *__dlapi_resolve(void *handle, const char *string, void *returnAddress) {
- if (logDlCalls) {
- const char *name;
- bool quote = false;
- if (handle == RTLD_DEFAULT) {
- name = "RTLD_DEFAULT";
- } else if (handle == RTLD_NEXT) {
- name = "RTLD_NEXT";
- } else {
- name = ((SharedObject *)handle)->name.data();
- quote = true;
- }
-
- mlibc::infoLogger() << "rtdl: __dlapi_resolve(" << (quote ? "\"" : "") << name
- << (quote ? "\"" : "") << ", \"" << string << "\")" << frg::endlog;
- }
-
- frg::optional<ObjectSymbol> target;
-
- if (handle == RTLD_DEFAULT) {
- target = globalScope->resolveSymbol(string, 0, 0);
- } else if (handle == RTLD_NEXT) {
- SharedObject *origin = initialRepository->findCaller(returnAddress);
- if (!origin) {
- mlibc::panicLogger() << "rtdl: unable to determine calling object of dlsym "
- << "(ra = " << returnAddress << ")" << frg::endlog;
- }
-
- target = Scope::resolveGlobalOrLocalNext(*globalScope, origin->localScope, string, origin);
- } else {
- // POSIX does not unambiguously state how dlsym() is supposed to work; it just
- // states that "The symbol resolution algorithm used shall be dependency order
- // as described in dlopen()".
- //
- // Linux libc's lookup the symbol in the given DSO and all of its dependencies
- // in breadth-first order. That is also what we implement here.
- //
- // Note that this *differs* from the algorithm that is used for relocations
- // (since the algorithm used for relocations takes (i) the global scope,
- // and (ii) the local scope of the DSO into account (which can contain more objects
- // than just the dependencies of the DSO, if the DSO was loaded as a dependency
- // of a dlopen()ed DSO).
-
- frg::vector<SharedObject *, MemoryAllocator> queue{getAllocator()};
-
- struct Token { };
- frg::hash_map<
- SharedObject *, Token,
- frg::hash<SharedObject *>, MemoryAllocator
- > visited{frg::hash<SharedObject *>{}, getAllocator()};
-
- auto root = reinterpret_cast<SharedObject *>(handle);
- visited.insert(root, Token{});
- queue.push_back(root);
-
- for(size_t i = 0; i < queue.size(); i++) {
- auto current = queue[i];
-
- target = resolveInObject(current, string);
- if(target)
- break;
-
- for(auto dep : current->dependencies) {
- if(visited.get(dep))
- continue;
- visited.insert(dep, Token{});
- queue.push_back(dep);
- }
- }
- }
-
- if (!target) {
- if (logDlCalls)
- mlibc::infoLogger() << "rtdl: could not resolve \"" << string << "\"" << frg::endlog;
-
- lastError = "Cannot resolve requested symbol";
- return nullptr;
- }
- return reinterpret_cast<void *>(target->virtualAddress());
-}
-
-struct __dlapi_symbol {
- const char *file;
- void *base;
- const char *symbol;
- void *address;
-};
-
-extern "C" [[ gnu::visibility("default") ]]
-int __dlapi_reverse(const void *ptr, __dlapi_symbol *info) {
- if (logDlCalls)
- mlibc::infoLogger() << "rtdl: __dlapi_reverse(" << ptr << ")" << frg::endlog;
-
- for(size_t i = 0; i < initialRepository->loadedObjects.size(); i++) {
- auto object = initialRepository->loadedObjects[i];
-
- auto eligible = [&] (ObjectSymbol cand) {
- if(cand.symbol()->st_shndx == SHN_UNDEF)
- return false;
-
- auto bind = ELF_ST_BIND(cand.symbol()->st_info);
- if(bind != STB_GLOBAL && bind != STB_WEAK)
- return false;
-
- return true;
- };
-
- auto hash_table = (Elf64_Word *)(object->baseAddress + object->hashTableOffset);
- auto num_symbols = hash_table[1];
- for(size_t i = 0; i < num_symbols; i++) {
- ObjectSymbol cand{object, (elf_sym *)(object->baseAddress
- + object->symbolTableOffset + i * sizeof(elf_sym))};
- if(eligible(cand) && cand.virtualAddress() == reinterpret_cast<uintptr_t>(ptr)) {
- if (logDlCalls)
- mlibc::infoLogger() << "rtdl: Found symbol " << cand.getString() << " in object "
- << object->path << frg::endlog;
-
- info->file = object->path.data();
- info->base = reinterpret_cast<void *>(object->baseAddress);
- info->symbol = cand.getString();
- info->address = reinterpret_cast<void *>(cand.virtualAddress());
- return 0;
- }
- }
- }
-
- // Not found, find the DSO it should be in.
- for(size_t i = 0; i < initialRepository->loadedObjects.size(); i++) {
- auto object = initialRepository->loadedObjects[i];
-
- for(size_t j = 0; j < object->phdrCount; j++) {
- auto phdr = (elf_phdr *)((uintptr_t)object->phdrPointer + j * object->phdrEntrySize);
- if(phdr->p_type != PT_LOAD) {
- continue;
- }
- uintptr_t start = object->baseAddress + phdr->p_vaddr;
- uintptr_t end = start + phdr->p_memsz;
- if(reinterpret_cast<uintptr_t>(ptr) >= start && reinterpret_cast<uintptr_t>(ptr) < end) {
- mlibc::infoLogger() << "rtdl: Found DSO " << object->path << frg::endlog;
- info->file = object->path.data();
- info->base = reinterpret_cast<void *>(object->baseAddress);
- info->symbol = nullptr;
- info->address = 0;
- return 0;
- }
- }
- }
-
- if (logDlCalls)
- mlibc::infoLogger() << "rtdl: Could not find symbol in __dlapi_reverse()" << frg::endlog;
-
- return -1;
-}
-
-extern "C" [[ gnu::visibility("default") ]]
-int __dlapi_close(void *) {
- if (logDlCalls)
- mlibc::infoLogger() << "mlibc: dlclose() is a no-op" << frg::endlog;
- return 0;
-}
-
-#endif
-
-extern "C" [[ gnu::visibility("default") ]]
-int __dlapi_iterate_phdr(int (*callback)(struct dl_phdr_info *, size_t, void*), void *data) {
- int last_return = 0;
- for (auto object : initialRepository->loadedObjects) {
- struct dl_phdr_info info;
- info.dlpi_addr = object->baseAddress;
- info.dlpi_name = object->name.data();
-
- if(object->isMainObject) {
- info.dlpi_name = "";
- } else {
- info.dlpi_name = object->name.data();
- }
- info.dlpi_phdr = static_cast<ElfW(Phdr)*>(object->phdrPointer);
- info.dlpi_phnum = object->phdrCount;
- info.dlpi_adds = rtsCounter;
- info.dlpi_subs = 0; // TODO(geert): implement dlclose().
- if (object->tlsModel != TlsModel::null)
- info.dlpi_tls_modid = object->tlsIndex;
- else
- info.dlpi_tls_modid = 0;
- info.dlpi_tls_data = tryAccessDtv(object);
-
- last_return = callback(&info, sizeof(struct dl_phdr_info), data);
- if(last_return)
- return last_return;
- }
-
- return last_return;
-}
-
-extern "C" [[ gnu::visibility("default") ]]
-void __dlapi_enter(uintptr_t *entry_stack) {
-#if MLIBC_STATIC_BUILD
- interpreterMain(entry_stack);
-#else
- (void)entry_stack;
-#endif
-}
-
-// XXX(qookie):
-// This is here because libgcc will call into __getauxval on glibc Linux
-// (which is what it believes we are due to the aarch64-linux-gnu toolchain)
-// in order to find AT_HWCAP to discover if LSE atomics are supported.
-//
-// This is not necessary on a custom Linux toolchain and is purely an artifact of
-// using the host toolchain.
-
-// __gnu_linux__ is the define checked by libgcc
-#if defined(__aarch64__) && defined(__gnu_linux__) && !defined(MLIBC_STATIC_BUILD)
-
-extern "C" unsigned long __getauxval(unsigned long type) {
- // Find the auxiliary vector by skipping args and environment.
- auto aux = entryStack;
- aux += *aux + 1; // Skip argc and all arguments
- __ensure(!*aux);
- aux++;
- while(*aux) // Now, we skip the environment.
- aux++;
- aux++;
-
- // Parse the auxiliary vector.
- while(true) {
- auto value = aux + 1;
- if(*aux == AT_NULL) {
- return 0;
- }else if(*aux == type) {
- return *value;
- }
- aux += 2;
- }
-}
-
-#endif