1 //===------------- Linux VDSO Implementation --------------------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 #include "src/__support/OSUtil/linux/vdso.h"
9 #include "hdr/link_macros.h"
10 #include "hdr/sys_auxv_macros.h"
11 #include "src/__support/CPP/array.h"
12 #include "src/__support/CPP/optional.h"
13 #include "src/__support/CPP/string_view.h"
14 #include "src/__support/threads/callonce.h"
15 #include "src/__support/threads/linux/futex_word.h"
16 #include "src/errno/libc_errno.h"
17 #include "src/sys/auxv/getauxval.h"
18 #include <linux/auxvec.h>
19
20 // TODO: This is a temporary workaround to avoid including elf.h
21 // Include our own headers for ElfW and friends once we have them.
22 namespace LIBC_NAMESPACE_DECL {
23
24 namespace vdso {
25
26 Symbol::VDSOArray Symbol::global_cache{};
27 CallOnceFlag Symbol::once_flag = callonce_impl::NOT_CALLED;
28
29 namespace {
30 // See https://refspecs.linuxfoundation.org/LSB_1.3.0/gLSB/gLSB/symverdefs.html
31 struct Verdaux {
32 ElfW(Word) vda_name; /* Version or dependency names */
33 ElfW(Word) vda_next; /* Offset in bytes to next verdaux
34 entry */
35 };
36 struct Verdef {
37 ElfW(Half) vd_version; /* Version revision */
38 ElfW(Half) vd_flags; /* Version information */
39 ElfW(Half) vd_ndx; /* Version Index */
40 ElfW(Half) vd_cnt; /* Number of associated aux entries */
41 ElfW(Word) vd_hash; /* Version name hash value */
42 ElfW(Word) vd_aux; /* Offset in bytes to verdaux array */
43 ElfW(Word) vd_next; /* Offset in bytes to next verdef entry */
nextLIBC_NAMESPACE_DECL::vdso::__anond9e5a2140111::Verdef44 Verdef *next() const {
45 if (vd_next == 0)
46 return nullptr;
47 return reinterpret_cast<Verdef *>(reinterpret_cast<uintptr_t>(this) +
48 vd_next);
49 }
auxLIBC_NAMESPACE_DECL::vdso::__anond9e5a2140111::Verdef50 Verdaux *aux() const {
51 return reinterpret_cast<Verdaux *>(reinterpret_cast<uintptr_t>(this) +
52 vd_aux);
53 }
54 };
55
56 // version search procedure specified by
57 // https://refspecs.linuxfoundation.org/LSB_1.3.0/gLSB/gLSB/symversion.html#SYMVERTBL
find_version(Verdef * verdef,ElfW (Half)* versym,const char * strtab,size_t idx)58 cpp::string_view find_version(Verdef *verdef, ElfW(Half) * versym,
59 const char *strtab, size_t idx) {
60 #ifndef VER_FLG_BASE
61 constexpr ElfW(Half) VER_FLG_BASE = 0x1;
62 #endif
63 if (!versym)
64 return "";
65 ElfW(Half) identifier = versym[idx] & 0x7FFF;
66 // iterate through all version definitions
67 for (Verdef *def = verdef; def != nullptr; def = def->next()) {
68 // skip if this is a file-level version
69 if (def->vd_flags & VER_FLG_BASE)
70 continue;
71 // check if the version identifier matches. Highest bit is used to determine
72 // whether the symbol is local. Only lower 15 bits are used for version
73 // identifier.
74 if ((def->vd_ndx & 0x7FFF) == identifier) {
75 Verdaux *aux = def->aux();
76 return strtab + aux->vda_name;
77 }
78 }
79 return "";
80 }
81
shdr_get_symbol_count(ElfW (Shdr)* vdso_shdr,size_t e_shnum)82 size_t shdr_get_symbol_count(ElfW(Shdr) * vdso_shdr, size_t e_shnum) {
83 if (!vdso_shdr)
84 return 0;
85 // iterate all sections until we locate the dynamic symbol section
86 for (size_t i = 0; i < e_shnum; ++i) {
87 // dynamic symbol section is a table section
88 // therefore, the number of entries can be computed as the ratio
89 // of the section size to the size of a single entry
90 if (vdso_shdr[i].sh_type == SHT_DYNSYM)
91 return vdso_shdr[i].sh_size / vdso_shdr[i].sh_entsize;
92 }
93 return 0;
94 }
95
96 struct VDSOSymbolTable {
97 const char *strtab;
98 ElfW(Sym) * symtab;
99 // The following can be nullptr if the vDSO does not have versioning
100 ElfW(Half) * versym;
101 Verdef *verdef;
102
populate_symbol_cacheLIBC_NAMESPACE_DECL::vdso::__anond9e5a2140111::VDSOSymbolTable103 void populate_symbol_cache(Symbol::VDSOArray &symbol_table,
104 size_t symbol_count, ElfW(Addr) vdso_addr) {
105 for (size_t i = 0, e = symbol_table.size(); i < e; ++i) {
106 Symbol sym = i;
107 cpp::string_view name = sym.name();
108 cpp::string_view version = sym.version();
109 if (name.empty())
110 continue;
111
112 for (size_t j = 0; j < symbol_count; ++j) {
113 if (name == strtab + symtab[j].st_name) {
114 // we find a symbol with desired name
115 // now we need to check if it has the right version
116 if (versym && verdef &&
117 version != find_version(verdef, versym, strtab, j))
118 continue;
119
120 // put the symbol address into the symbol table
121 symbol_table[i] =
122 reinterpret_cast<void *>(vdso_addr + symtab[j].st_value);
123 }
124 }
125 }
126 }
127 };
128
129 struct PhdrInfo {
130 ElfW(Addr) vdso_addr;
131 ElfW(Dyn) * vdso_dyn;
fromLIBC_NAMESPACE_DECL::vdso::__anond9e5a2140111::PhdrInfo132 static cpp::optional<PhdrInfo> from(ElfW(Phdr) * vdso_phdr, size_t e_phnum,
133 uintptr_t vdso_ehdr_addr) {
134 constexpr ElfW(Addr) INVALID_ADDR = static_cast<ElfW(Addr)>(-1);
135 ElfW(Addr) vdso_addr = INVALID_ADDR;
136 ElfW(Dyn) *vdso_dyn = nullptr;
137 if (!vdso_phdr)
138 return cpp::nullopt;
139 // iterate through all the program headers until we get the desired pieces
140 for (size_t i = 0; i < e_phnum; ++i) {
141 if (vdso_phdr[i].p_type == PT_DYNAMIC)
142 vdso_dyn = reinterpret_cast<ElfW(Dyn) *>(vdso_ehdr_addr +
143 vdso_phdr[i].p_offset);
144
145 if (vdso_phdr[i].p_type == PT_LOAD)
146 vdso_addr =
147 vdso_ehdr_addr + vdso_phdr[i].p_offset - vdso_phdr[i].p_vaddr;
148
149 if (vdso_addr && vdso_dyn)
150 return PhdrInfo{vdso_addr, vdso_dyn};
151 }
152
153 return cpp::nullopt;
154 }
155
populate_symbol_tableLIBC_NAMESPACE_DECL::vdso::__anond9e5a2140111::PhdrInfo156 cpp::optional<VDSOSymbolTable> populate_symbol_table() {
157 const char *strtab = nullptr;
158 ElfW(Sym) *symtab = nullptr;
159 ElfW(Half) *versym = nullptr;
160 Verdef *verdef = nullptr;
161 for (ElfW(Dyn) *d = vdso_dyn; d->d_tag != DT_NULL; ++d) {
162 switch (d->d_tag) {
163 case DT_STRTAB:
164 strtab = reinterpret_cast<const char *>(vdso_addr + d->d_un.d_ptr);
165 break;
166 case DT_SYMTAB:
167 symtab = reinterpret_cast<ElfW(Sym) *>(vdso_addr + d->d_un.d_ptr);
168 break;
169 case DT_VERSYM:
170 versym = reinterpret_cast<uint16_t *>(vdso_addr + d->d_un.d_ptr);
171 break;
172 case DT_VERDEF:
173 verdef = reinterpret_cast<Verdef *>(vdso_addr + d->d_un.d_ptr);
174 break;
175 }
176 if (strtab && symtab && versym && verdef)
177 break;
178 }
179 if (strtab == nullptr || symtab == nullptr)
180 return cpp::nullopt;
181
182 return VDSOSymbolTable{strtab, symtab, versym, verdef};
183 }
184 };
185 } // namespace
186
initialize_vdso_global_cache()187 void Symbol::initialize_vdso_global_cache() {
188 // first clear the symbol table
189 for (auto &i : global_cache)
190 i = nullptr;
191
192 // get the address of the VDSO, protect errno since getauxval may change
193 // it
194 int errno_backup = libc_errno;
195 uintptr_t vdso_ehdr_addr = getauxval(AT_SYSINFO_EHDR);
196 // Get the memory address of the vDSO ELF header.
197 auto vdso_ehdr = reinterpret_cast<ElfW(Ehdr) *>(vdso_ehdr_addr);
198 // leave the table unpopulated if we don't have vDSO
199 if (vdso_ehdr == nullptr) {
200 libc_errno = errno_backup;
201 return;
202 }
203
204 // locate the section header inside the elf using the section header
205 // offset
206 auto vdso_shdr =
207 reinterpret_cast<ElfW(Shdr) *>(vdso_ehdr_addr + vdso_ehdr->e_shoff);
208 size_t symbol_count = shdr_get_symbol_count(vdso_shdr, vdso_ehdr->e_shnum);
209
210 // early return if no symbol is found
211 if (symbol_count == 0)
212 return;
213
214 // We need to find both the loadable segment and the dynamic linking of
215 // the vDSO. compute vdso_phdr as the program header using the program
216 // header offset
217 ElfW(Phdr) *vdso_phdr =
218 reinterpret_cast<ElfW(Phdr) *>(vdso_ehdr_addr + vdso_ehdr->e_phoff);
219 cpp::optional<PhdrInfo> phdr_info =
220 PhdrInfo::from(vdso_phdr, vdso_ehdr->e_phnum, vdso_ehdr_addr);
221 // early return if either the dynamic linking or the loadable segment is
222 // not found
223 if (!phdr_info.has_value())
224 return;
225
226 // now, locate several more tables inside the dynmaic linking section
227 cpp::optional<VDSOSymbolTable> vdso_symbol_table =
228 phdr_info->populate_symbol_table();
229
230 // early return if we can't find any required fields of the symbol table
231 if (!vdso_symbol_table.has_value())
232 return;
233
234 // finally, populate the global symbol table cache
235 vdso_symbol_table->populate_symbol_cache(global_cache, symbol_count,
236 phdr_info->vdso_addr);
237 }
238 } // namespace vdso
239 } // namespace LIBC_NAMESPACE_DECL
240