[337] | 1 | #define _GNU_SOURCE
|
---|
| 2 | #include "pthread_impl.h"
|
---|
| 3 | #include "stdio_impl.h"
|
---|
| 4 | #include "libc.h"
|
---|
| 5 | #include <sys/mman.h>
|
---|
| 6 | #include <string.h>
|
---|
| 7 | #include <stddef.h>
|
---|
| 8 |
|
---|
| 9 | void *__mmap(void *, size_t, int, int, int, off_t);
|
---|
| 10 | int __munmap(void *, size_t);
|
---|
| 11 | int __mprotect(void *, size_t, int);
|
---|
| 12 |
|
---|
| 13 | static void dummy_0()
|
---|
| 14 | {
|
---|
| 15 | }
|
---|
| 16 | weak_alias(dummy_0, __acquire_ptc);
|
---|
| 17 | weak_alias(dummy_0, __release_ptc);
|
---|
| 18 | weak_alias(dummy_0, __pthread_tsd_run_dtors);
|
---|
| 19 | weak_alias(dummy_0, __do_orphaned_stdio_locks);
|
---|
| 20 | weak_alias(dummy_0, __dl_thread_cleanup);
|
---|
| 21 |
|
---|
| 22 | _Noreturn void __pthread_exit(void *result)
|
---|
| 23 | {
|
---|
| 24 | pthread_t self = __pthread_self();
|
---|
| 25 | sigset_t set;
|
---|
| 26 |
|
---|
| 27 | self->canceldisable = 1;
|
---|
| 28 | self->cancelasync = 0;
|
---|
| 29 | self->result = result;
|
---|
| 30 |
|
---|
| 31 | while (self->cancelbuf) {
|
---|
| 32 | void (*f)(void *) = self->cancelbuf->__f;
|
---|
| 33 | void *x = self->cancelbuf->__x;
|
---|
| 34 | self->cancelbuf = self->cancelbuf->__next;
|
---|
| 35 | f(x);
|
---|
| 36 | }
|
---|
| 37 |
|
---|
| 38 | __pthread_tsd_run_dtors();
|
---|
| 39 |
|
---|
| 40 | __lock(self->exitlock);
|
---|
| 41 |
|
---|
| 42 | /* Mark this thread dead before decrementing count */
|
---|
| 43 | __lock(self->killlock);
|
---|
| 44 | self->dead = 1;
|
---|
| 45 |
|
---|
| 46 | /* Block all signals before decrementing the live thread count.
|
---|
| 47 | * This is important to ensure that dynamically allocated TLS
|
---|
| 48 | * is not under-allocated/over-committed, and possibly for other
|
---|
| 49 | * reasons as well. */
|
---|
| 50 | __block_all_sigs(&set);
|
---|
| 51 |
|
---|
| 52 | /* Wait to unlock the kill lock, which governs functions like
|
---|
| 53 | * pthread_kill which target a thread id, until signals have
|
---|
| 54 | * been blocked. This precludes observation of the thread id
|
---|
| 55 | * as a live thread (with application code running in it) after
|
---|
| 56 | * the thread was reported dead by ESRCH being returned. */
|
---|
| 57 | __unlock(self->killlock);
|
---|
| 58 |
|
---|
| 59 | /* It's impossible to determine whether this is "the last thread"
|
---|
| 60 | * until performing the atomic decrement, since multiple threads
|
---|
| 61 | * could exit at the same time. For the last thread, revert the
|
---|
| 62 | * decrement and unblock signals to give the atexit handlers and
|
---|
| 63 | * stdio cleanup code a consistent state. */
|
---|
| 64 | if (a_fetch_add(&libc.threads_minus_1, -1)==0) {
|
---|
| 65 | libc.threads_minus_1 = 0;
|
---|
| 66 | __restore_sigs(&set);
|
---|
| 67 | exit(0);
|
---|
| 68 | }
|
---|
| 69 |
|
---|
| 70 | /* Process robust list in userspace to handle non-pshared mutexes
|
---|
| 71 | * and the detached thread case where the robust list head will
|
---|
| 72 | * be invalid when the kernel would process it. */
|
---|
| 73 | __vm_lock();
|
---|
| 74 | volatile void *volatile *rp;
|
---|
| 75 | while ((rp=self->robust_list.head) && rp != &self->robust_list.head) {
|
---|
| 76 | pthread_mutex_t *m = (void *)((char *)rp
|
---|
| 77 | - offsetof(pthread_mutex_t, _m_next));
|
---|
| 78 | int waiters = m->_m_waiters;
|
---|
| 79 | int priv = (m->_m_type & 128) ^ 128;
|
---|
| 80 | self->robust_list.pending = rp;
|
---|
| 81 | self->robust_list.head = *rp;
|
---|
| 82 | int cont = a_swap(&m->_m_lock, 0x40000000);
|
---|
| 83 | self->robust_list.pending = 0;
|
---|
| 84 | if (cont < 0 || waiters)
|
---|
| 85 | __wake(&m->_m_lock, 1, priv);
|
---|
| 86 | }
|
---|
| 87 | __vm_unlock();
|
---|
| 88 |
|
---|
| 89 | __do_orphaned_stdio_locks();
|
---|
| 90 | __dl_thread_cleanup();
|
---|
| 91 |
|
---|
| 92 | if (self->detached && self->map_base) {
|
---|
| 93 | /* Detached threads must avoid the kernel clear_child_tid
|
---|
| 94 | * feature, since the virtual address will have been
|
---|
| 95 | * unmapped and possibly already reused by a new mapping
|
---|
| 96 | * at the time the kernel would perform the write. In
|
---|
| 97 | * the case of threads that started out detached, the
|
---|
| 98 | * initial clone flags are correct, but if the thread was
|
---|
| 99 | * detached later (== 2), we need to clear it here. */
|
---|
| 100 | if (self->detached == 2) __syscall(SYS_set_tid_address, 0);
|
---|
| 101 |
|
---|
| 102 | /* Robust list will no longer be valid, and was already
|
---|
| 103 | * processed above, so unregister it with the kernel. */
|
---|
| 104 | if (self->robust_list.off)
|
---|
| 105 | __syscall(SYS_set_robust_list, 0, 3*sizeof(long));
|
---|
| 106 |
|
---|
| 107 | /* Since __unmapself bypasses the normal munmap code path,
|
---|
| 108 | * explicitly wait for vmlock holders first. */
|
---|
| 109 | __vm_wait();
|
---|
| 110 |
|
---|
| 111 | /* The following call unmaps the thread's stack mapping
|
---|
| 112 | * and then exits without touching the stack. */
|
---|
| 113 | __unmapself(self->map_base, self->map_size);
|
---|
| 114 | }
|
---|
| 115 |
|
---|
| 116 | for (;;) __syscall(SYS_exit, 0);
|
---|
| 117 | }
|
---|
| 118 |
|
---|
| 119 | void __do_cleanup_push(struct __ptcb *cb)
|
---|
| 120 | {
|
---|
| 121 | struct pthread *self = __pthread_self();
|
---|
| 122 | cb->__next = self->cancelbuf;
|
---|
| 123 | self->cancelbuf = cb;
|
---|
| 124 | }
|
---|
| 125 |
|
---|
| 126 | void __do_cleanup_pop(struct __ptcb *cb)
|
---|
| 127 | {
|
---|
| 128 | __pthread_self()->cancelbuf = cb->__next;
|
---|
| 129 | }
|
---|
| 130 |
|
---|
| 131 | static int start(void *p)
|
---|
| 132 | {
|
---|
| 133 | pthread_t self = p;
|
---|
| 134 | /* States for startlock:
|
---|
| 135 | * 0 = no need for start sync
|
---|
| 136 | * 1 = waiting for parent to do work
|
---|
| 137 | * 2 = failure in parent, child must abort
|
---|
| 138 | * 3 = success in parent, child must restore sigmask */
|
---|
| 139 | if (self->startlock[0]) {
|
---|
| 140 | __wait(self->startlock, 0, 1, 1);
|
---|
| 141 | if (self->startlock[0] == 2) {
|
---|
| 142 | self->detached = 2;
|
---|
| 143 | pthread_exit(0);
|
---|
| 144 | }
|
---|
| 145 | __restore_sigs(self->sigmask);
|
---|
| 146 | }
|
---|
| 147 | if (self->unblock_cancel)
|
---|
| 148 | __syscall(SYS_rt_sigprocmask, SIG_UNBLOCK,
|
---|
| 149 | SIGPT_SET, 0, _NSIG/8);
|
---|
| 150 | __pthread_exit(self->start(self->start_arg));
|
---|
| 151 | return 0;
|
---|
| 152 | }
|
---|
| 153 |
|
---|
| 154 | static int start_c11(void *p)
|
---|
| 155 | {
|
---|
| 156 | pthread_t self = p;
|
---|
| 157 | int (*start)(void*) = (int(*)(void*)) self->start;
|
---|
| 158 | __pthread_exit((void *)(uintptr_t)start(self->start_arg));
|
---|
| 159 | return 0;
|
---|
| 160 | }
|
---|
| 161 |
|
---|
| 162 | #define ROUND(x) (((x)+PAGE_SIZE-1)&-PAGE_SIZE)
|
---|
| 163 |
|
---|
| 164 | /* pthread_key_create.c overrides this */
|
---|
| 165 | static volatile size_t dummy = 0;
|
---|
| 166 | weak_alias(dummy, __pthread_tsd_size);
|
---|
| 167 | static void *dummy_tsd[1] = { 0 };
|
---|
| 168 | weak_alias(dummy_tsd, __pthread_tsd_main);
|
---|
| 169 |
|
---|
| 170 | volatile int __block_new_threads = 0;
|
---|
| 171 | size_t __default_stacksize = DEFAULT_STACK_SIZE;
|
---|
| 172 | size_t __default_guardsize = DEFAULT_GUARD_SIZE;
|
---|
| 173 |
|
---|
| 174 | static FILE *volatile dummy_file = 0;
|
---|
| 175 | weak_alias(dummy_file, __stdin_used);
|
---|
| 176 | weak_alias(dummy_file, __stdout_used);
|
---|
| 177 | weak_alias(dummy_file, __stderr_used);
|
---|
| 178 |
|
---|
| 179 | static void init_file_lock(FILE *f)
|
---|
| 180 | {
|
---|
| 181 | if (f && f->lock<0) f->lock = 0;
|
---|
| 182 | }
|
---|
| 183 |
|
---|
| 184 | void *__copy_tls(unsigned char *);
|
---|
| 185 |
|
---|
| 186 | int __pthread_create(pthread_t *restrict res, const pthread_attr_t *restrict attrp, void *(*entry)(void *), void *restrict arg)
|
---|
| 187 | {
|
---|
| 188 | int ret, c11 = (attrp == __ATTRP_C11_THREAD);
|
---|
| 189 | size_t size, guard;
|
---|
| 190 | struct pthread *self, *new;
|
---|
| 191 | unsigned char *map = 0, *stack = 0, *tsd = 0, *stack_limit;
|
---|
| 192 | unsigned flags = CLONE_VM | CLONE_FS | CLONE_FILES | CLONE_SIGHAND
|
---|
| 193 | | CLONE_THREAD | CLONE_SYSVSEM | CLONE_SETTLS
|
---|
| 194 | | CLONE_PARENT_SETTID | CLONE_CHILD_CLEARTID | CLONE_DETACHED;
|
---|
| 195 | int do_sched = 0;
|
---|
| 196 | pthread_attr_t attr = { 0 };
|
---|
| 197 |
|
---|
| 198 | if (!libc.can_do_threads) return ENOSYS;
|
---|
| 199 | self = __pthread_self();
|
---|
| 200 | if (!libc.threaded) {
|
---|
| 201 | for (FILE *f=*__ofl_lock(); f; f=f->next)
|
---|
| 202 | init_file_lock(f);
|
---|
| 203 | __ofl_unlock();
|
---|
| 204 | init_file_lock(__stdin_used);
|
---|
| 205 | init_file_lock(__stdout_used);
|
---|
| 206 | init_file_lock(__stderr_used);
|
---|
| 207 | __syscall(SYS_rt_sigprocmask, SIG_UNBLOCK, SIGPT_SET, 0, _NSIG/8);
|
---|
| 208 | self->tsd = (void **)__pthread_tsd_main;
|
---|
| 209 | libc.threaded = 1;
|
---|
| 210 | }
|
---|
| 211 | if (attrp && !c11) attr = *attrp;
|
---|
| 212 |
|
---|
| 213 | __acquire_ptc();
|
---|
| 214 | if (!attrp || c11) {
|
---|
| 215 | attr._a_stacksize = __default_stacksize;
|
---|
| 216 | attr._a_guardsize = __default_guardsize;
|
---|
| 217 | }
|
---|
| 218 |
|
---|
| 219 | if (__block_new_threads) __wait(&__block_new_threads, 0, 1, 1);
|
---|
| 220 |
|
---|
| 221 | if (attr._a_stackaddr) {
|
---|
| 222 | size_t need = libc.tls_size + __pthread_tsd_size;
|
---|
| 223 | size = attr._a_stacksize;
|
---|
| 224 | stack = (void *)(attr._a_stackaddr & -16);
|
---|
| 225 | stack_limit = (void *)(attr._a_stackaddr - size);
|
---|
| 226 | /* Use application-provided stack for TLS only when
|
---|
| 227 | * it does not take more than ~12% or 2k of the
|
---|
| 228 | * application's stack space. */
|
---|
| 229 | if (need < size/8 && need < 2048) {
|
---|
| 230 | tsd = stack - __pthread_tsd_size;
|
---|
| 231 | stack = tsd - libc.tls_size;
|
---|
| 232 | memset(stack, 0, need);
|
---|
| 233 | } else {
|
---|
| 234 | size = ROUND(need);
|
---|
| 235 | guard = 0;
|
---|
| 236 | }
|
---|
| 237 | } else {
|
---|
| 238 | guard = ROUND(attr._a_guardsize);
|
---|
| 239 | size = guard + ROUND(attr._a_stacksize
|
---|
| 240 | + libc.tls_size + __pthread_tsd_size);
|
---|
| 241 | }
|
---|
| 242 |
|
---|
| 243 | if (!tsd) {
|
---|
| 244 | if (guard) {
|
---|
| 245 | map = __mmap(0, size, PROT_NONE, MAP_PRIVATE|MAP_ANON, -1, 0);
|
---|
| 246 | if (map == MAP_FAILED) goto fail;
|
---|
| 247 | if (__mprotect(map+guard, size-guard, PROT_READ|PROT_WRITE)
|
---|
| 248 | && errno != ENOSYS) {
|
---|
| 249 | __munmap(map, size);
|
---|
| 250 | goto fail;
|
---|
| 251 | }
|
---|
| 252 | } else {
|
---|
| 253 | map = __mmap(0, size, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANON, -1, 0);
|
---|
| 254 | if (map == MAP_FAILED) goto fail;
|
---|
| 255 | }
|
---|
| 256 | tsd = map + size - __pthread_tsd_size;
|
---|
| 257 | if (!stack) {
|
---|
| 258 | stack = tsd - libc.tls_size;
|
---|
| 259 | stack_limit = map + guard;
|
---|
| 260 | }
|
---|
| 261 | }
|
---|
| 262 |
|
---|
| 263 | new = __copy_tls(tsd - libc.tls_size);
|
---|
| 264 | new->map_base = map;
|
---|
| 265 | new->map_size = size;
|
---|
| 266 | new->stack = stack;
|
---|
| 267 | new->stack_size = stack - stack_limit;
|
---|
| 268 | new->start = entry;
|
---|
| 269 | new->start_arg = arg;
|
---|
| 270 | new->self = new;
|
---|
| 271 | new->tsd = (void *)tsd;
|
---|
| 272 | new->locale = &libc.global_locale;
|
---|
| 273 | if (attr._a_detach) {
|
---|
| 274 | new->detached = 1;
|
---|
| 275 | flags -= CLONE_CHILD_CLEARTID;
|
---|
| 276 | }
|
---|
| 277 | if (attr._a_sched) {
|
---|
| 278 | do_sched = new->startlock[0] = 1;
|
---|
| 279 | __block_app_sigs(new->sigmask);
|
---|
| 280 | }
|
---|
| 281 | new->robust_list.head = &new->robust_list.head;
|
---|
| 282 | new->unblock_cancel = self->cancel;
|
---|
| 283 | new->CANARY = self->CANARY;
|
---|
| 284 |
|
---|
| 285 | a_inc(&libc.threads_minus_1);
|
---|
| 286 | ret = __clone((c11 ? start_c11 : start), stack, flags, new, &new->tid, TP_ADJ(new), &new->tid);
|
---|
| 287 |
|
---|
| 288 | __release_ptc();
|
---|
| 289 |
|
---|
| 290 | if (do_sched) {
|
---|
| 291 | __restore_sigs(new->sigmask);
|
---|
| 292 | }
|
---|
| 293 |
|
---|
| 294 | if (ret < 0) {
|
---|
| 295 | a_dec(&libc.threads_minus_1);
|
---|
| 296 | if (map) __munmap(map, size);
|
---|
| 297 | return EAGAIN;
|
---|
| 298 | }
|
---|
| 299 |
|
---|
| 300 | if (do_sched) {
|
---|
| 301 | ret = __syscall(SYS_sched_setscheduler, new->tid,
|
---|
| 302 | attr._a_policy, &attr._a_prio);
|
---|
| 303 | a_store(new->startlock, ret<0 ? 2 : 3);
|
---|
| 304 | __wake(new->startlock, 1, 1);
|
---|
| 305 | if (ret < 0) return -ret;
|
---|
| 306 | }
|
---|
| 307 |
|
---|
| 308 | *res = new;
|
---|
| 309 | return 0;
|
---|
| 310 | fail:
|
---|
| 311 | __release_ptc();
|
---|
| 312 | return EAGAIN;
|
---|
| 313 | }
|
---|
| 314 |
|
---|
| 315 | weak_alias(__pthread_exit, pthread_exit);
|
---|
| 316 | weak_alias(__pthread_create, pthread_create);
|
---|