From 0f73cacad0091d80c56321dfad2ef278d0cdaeae Mon Sep 17 00:00:00 2001 From: Dmitrii Kuvaiskii Date: Sun, 24 Nov 2019 03:14:32 -0800 Subject: [PATCH] [LibOS,Pal] Correctly emulate CLONE_CHILD_CLEARTID When child thread exits, it wakes up its parent if CLONE_CHILD_CLEARTID was set during clone() call. Previously, this was done by the child thread itself as part of its own clean-up in release_clear_child_id(). But this child thread is still alive at this point and uses some resources, most notably the stack (that might have been provided by the parent) and the SGX TCS slot. Upon waking up, the parent might decide to free that stack (as Pthreads do) or re-use the TCS slot, causing data races. This commit introduces a correct emulation of CLONE_CHILD_CLEARTID: - A new argument `PAL_PTR clear_child_tid` is added to DkThreadCreate(); it points to memory that is erased on child exit to notify parent. - At PAL layer, when thread finally exits, it sets *clear_child_tid = -1; this signals to LibOS layer that the thread stopped using resources. - At LibOS layer, Async Helper thread is set up to wait for the signal from PAL; it is now the responsibility of Async Helper thread to call release_clear_child_id() to wake up the parent thread. - Async Helper thread waits for clear_child_tid == -1 and then sets it to 0 and wakes up all waiting parents via del_futex_waiter_wakeup(). Note that for Linux-SGX PAL, clear_child_tid is set to -1 not immediately but as part of handle_thread_reset, otherwise the TCS slot could be still occupied when LibOS wakes up parents. This commit also fixes all regression tests to use the new signature of DkThreadCreate() and increases the number of SGX threads slightly (to accommodate the newly used Async Helper thread). --- Documentation/oldwiki/PAL-Host-ABI.md | 3 +- LibOS/shim/include/shim_internal.h | 7 ++-- LibOS/shim/src/bookkeep/shim_thread.c | 2 +- LibOS/shim/src/ipc/shim_ipc_helper.c | 3 +- LibOS/shim/src/shim_async.c | 35 +++++++++++++++---- LibOS/shim/src/sys/shim_clone.c | 4 +-- LibOS/shim/src/sys/shim_exit.c | 7 ++-- LibOS/shim/src/sys/shim_futex.c | 35 ++++++++++++++----- LibOS/shim/test/regression/futex.c | 2 +- .../test/regression/futex.manifest.template | 2 +- LibOS/shim/test/regression/manifest.template | 2 ++ Pal/regression/Event.c | 2 +- Pal/regression/Thread.c | 2 +- Pal/regression/Thread2.c | 6 ++-- Pal/src/db_threading.c | 5 +-- Pal/src/host/FreeBSD/db_threading.c | 5 ++- Pal/src/host/Linux-SGX/db_threading.c | 28 ++++++++++----- Pal/src/host/Linux-SGX/enclave_ecalls.c | 7 ++-- Pal/src/host/Linux-SGX/enclave_entry.S | 9 +++++ Pal/src/host/Linux-SGX/generated-offsets.c | 1 + Pal/src/host/Linux-SGX/sgx_tls.h | 1 + Pal/src/host/Linux/db_threading.c | 23 +++++++----- Pal/src/host/Linux/pal_linux.h | 7 ++-- Pal/src/host/Skeleton/db_threading.c | 3 +- Pal/src/pal.h | 2 +- Pal/src/pal_internal.h | 4 +-- Pal/test/Event.c | 2 +- Pal/test/Fork.c | 2 +- Pal/test/Select.c | 2 +- Pal/test/Thread.c | 4 +-- Pal/test/Wait.c | 4 +-- Pal/test/Yield.c | 2 +- 32 files changed, 148 insertions(+), 75 deletions(-) diff --git a/Documentation/oldwiki/PAL-Host-ABI.md b/Documentation/oldwiki/PAL-Host-ABI.md index a84d3ba2..01f04d31 100644 --- a/Documentation/oldwiki/PAL-Host-ABI.md +++ b/Documentation/oldwiki/PAL-Host-ABI.md @@ -407,10 +407,11 @@ This API changes the name of an opened stream. #### DkThreadCreate - PAL_HANDLE DkThreadCreate(PAL_PTR addr, PAL_PTR param); + PAL_HANDLE DkThreadCreate(PAL_PTR addr, PAL_PTR param, PAL_PTR clear_child_tid); This API creates a thread in the current process. `addr` is the address of an entry point of execution for the new thread. `param` is the pointer argument that is passed to the new thread. +`clear_child_tid` is the pointer to memory that is erased on child exit to notify parent. #### DkThreadDelayExecution diff --git a/LibOS/shim/include/shim_internal.h b/LibOS/shim/include/shim_internal.h index e92ad724..fc6e8393 100644 --- a/LibOS/shim/include/shim_internal.h +++ b/LibOS/shim/include/shim_internal.h @@ -456,10 +456,9 @@ static inline void enable_locking (void) lock_enabled = true; } -static inline PAL_HANDLE thread_create (void * func, void * arg) -{ +static inline PAL_HANDLE thread_create(void* func, void* arg, void* clear_child_tid) { assert(lock_enabled); - return DkThreadCreate(func, arg); + return DkThreadCreate(func, arg, clear_child_tid); } static inline int64_t __disable_preempt (shim_tcb_t * tcb) @@ -797,6 +796,8 @@ void set_rlimit_cur(int resource, uint64_t rlim); int object_wait_with_retry(PAL_HANDLE handle); +void release_clear_child_id(IDTYPE caller, void* clear_child_tid_ptr); + #ifdef __x86_64__ #define __SWITCH_STACK(stack_top, func, arg) \ do { \ diff --git a/LibOS/shim/src/bookkeep/shim_thread.c b/LibOS/shim/src/bookkeep/shim_thread.c index a4908717..742cde93 100644 --- a/LibOS/shim/src/bookkeep/shim_thread.c +++ b/LibOS/shim/src/bookkeep/shim_thread.c @@ -759,7 +759,7 @@ BEGIN_RS_FUNC(running_thread) NUM_SIGS); if (cur_thread) { - PAL_HANDLE handle = DkThreadCreate(resume_wrapper, thread); + PAL_HANDLE handle = DkThreadCreate(resume_wrapper, thread, thread->clear_child_tid); if (!thread) return -PAL_ERRNO; diff --git a/LibOS/shim/src/ipc/shim_ipc_helper.c b/LibOS/shim/src/ipc/shim_ipc_helper.c index 745d8a4b..9f254254 100644 --- a/LibOS/shim/src/ipc/shim_ipc_helper.c +++ b/LibOS/shim/src/ipc/shim_ipc_helper.c @@ -780,6 +780,7 @@ noreturn static void shim_ipc_helper(void* dummy) { free(object_list); free(palhandle_list); + __disable_preempt(self->shim_tcb); put_thread(self); debug("IPC helper thread terminated\n"); @@ -828,7 +829,7 @@ static int create_ipc_helper(void) { ipc_helper_thread = new; ipc_helper_state = HELPER_ALIVE; - PAL_HANDLE handle = thread_create(shim_ipc_helper_prepare, new); + PAL_HANDLE handle = thread_create(shim_ipc_helper_prepare, new, /*clear_child_tid=*/NULL); if (!handle) { int ret = -PAL_ERRNO; /* put_thread() may overwrite errno */ diff --git a/LibOS/shim/src/shim_async.c b/LibOS/shim/src/shim_async.c index da352ab6..691e970a 100644 --- a/LibOS/shim/src/shim_async.c +++ b/LibOS/shim/src/shim_async.c @@ -38,6 +38,7 @@ struct async_event { void * arg; PAL_HANDLE object; /* handle (async IO) to wait on */ uint64_t expire_time; /* alarm/timer to wait on */ + bool todelete; }; DEFINE_LISTP(async_event); static LISTP_TYPE(async_event) async_list; @@ -82,10 +83,11 @@ int64_t install_async_event(PAL_HANDLE object, uint64_t time, event->caller = get_cur_tid(); event->object = object; event->expire_time = time ? now + time : 0; + event->todelete = false; lock(&async_helper_lock); - if (!object) { + if (callback != &release_clear_child_id && !object) { /* This is alarm() or setitimer() emulation, treat both according to * alarm() syscall semantics: cancel any pending alarm/timer. */ struct async_event * tmp, * n; @@ -203,9 +205,18 @@ static void shim_async_helper(void * arg) { struct async_event * tmp, * n; LISTP_FOR_EACH_ENTRY_SAFE(tmp, n, &async_list, list) { - /* First check if this event was triggered; note that IO events - * stay in the list whereas alarms/timers are fired only once. */ - if (polled && tmp->object == polled) { + /* First check if this event was triggered; there are three types: + * 1. Exited child: trigger callback and remove from the list; + * 2. IO events: trigger callback and keep in the list; + * 3. alarms/timers: trigger callback and remove from the list. */ + if (tmp->callback == &release_clear_child_id) { + debug("Child exited, notifying parents if any\n"); + tmp->todelete = true; + unlock(&async_helper_lock); + release_clear_child_id(tmp->caller, tmp->arg); + lock(&async_helper_lock); + continue; + } else if (polled && tmp->object == polled) { debug("Async IO event triggered at %lu\n", now); unlock(&async_helper_lock); tmp->callback(tmp->caller, tmp->arg); @@ -213,14 +224,16 @@ static void shim_async_helper(void * arg) { } else if (tmp->expire_time && tmp->expire_time <= now) { debug("Async alarm/timer triggered at %lu (expired at %lu)\n", now, tmp->expire_time); - LISTP_DEL(tmp, &async_list, list); + tmp->todelete = true; unlock(&async_helper_lock); tmp->callback(tmp->caller, tmp->arg); - free(tmp); lock(&async_helper_lock); continue; } + if (tmp->todelete) + continue; + /* Now re-add this IO event to the list or re-add this timer */ if (tmp->object) { if (object_num == object_list_size) { @@ -243,6 +256,13 @@ static void shim_async_helper(void * arg) { } } + LISTP_FOR_EACH_ENTRY_SAFE(tmp, n, &async_list, list) { + if (tmp->todelete) { + LISTP_DEL(tmp, &async_list, list); + free(tmp); + } + } + uint64_t sleep_time; if (next_expire_time) { sleep_time = next_expire_time - now; @@ -269,6 +289,7 @@ static void shim_async_helper(void * arg) { polled = DkObjectsWaitAny(object_num + 1, object_list, sleep_time); } + __disable_preempt(self->shim_tcb); put_thread(self); debug("Async helper thread terminated\n"); free(object_list); @@ -288,7 +309,7 @@ static int create_async_helper(void) { async_helper_thread = new; async_helper_state = HELPER_ALIVE; - PAL_HANDLE handle = thread_create(shim_async_helper, new); + PAL_HANDLE handle = thread_create(shim_async_helper, new, /*clear_child_tid=*/NULL); if (!handle) { async_helper_thread = NULL; diff --git a/LibOS/shim/src/sys/shim_clone.c b/LibOS/shim/src/sys/shim_clone.c index 27d54a57..8b9c54c5 100644 --- a/LibOS/shim/src/sys/shim_clone.c +++ b/LibOS/shim/src/sys/shim_clone.c @@ -288,7 +288,7 @@ int shim_do_clone (int flags, void * user_stack_addr, int * parent_tidptr, if (flags & CLONE_CHILD_CLEARTID) /* Implemented in shim_futex.c: release_clear_child_id */ - thread->clear_child_tid = parent_tidptr; + thread->clear_child_tid = child_tidptr; if (flags & CLONE_SETTLS) { if (!tls) { @@ -402,7 +402,7 @@ int shim_do_clone (int flags, void * user_stack_addr, int * parent_tidptr, // returns .The parent comes back here - however, the child is Happily // running the function we gave to DkThreadCreate. PAL_HANDLE pal_handle = thread_create(clone_implementation_wrapper, - &new_args); + &new_args, thread->clear_child_tid); if (!pal_handle) { ret = -PAL_ERRNO; put_thread(new_args.thread); diff --git a/LibOS/shim/src/sys/shim_exit.c b/LibOS/shim/src/sys/shim_exit.c index e75519d5..30c0e15b 100644 --- a/LibOS/shim/src/sys/shim_exit.c +++ b/LibOS/shim/src/sys/shim_exit.c @@ -40,8 +40,6 @@ void release_robust_list (struct robust_list_head * head); -void release_clear_child_id (int * clear_child_tid); - int thread_exit(struct shim_thread * self, bool send_ipc) { bool sent_exit_msg = false; @@ -122,8 +120,9 @@ int thread_exit(struct shim_thread * self, bool send_ipc) if (robust_list) release_robust_list(robust_list); - if (self->clear_child_tid) - release_clear_child_id (self->clear_child_tid); + /* ask Async Helper thread to wake up parent when this child thread finally exits */ + if (parent && self->in_vm && self->clear_child_tid) + install_async_event(NULL, 0, &release_clear_child_id, self->clear_child_tid); DkEventSet(self->exit_event); return 0; diff --git a/LibOS/shim/src/sys/shim_futex.c b/LibOS/shim/src/sys/shim_futex.c index 9205ae19..71779cff 100644 --- a/LibOS/shim/src/sys/shim_futex.c +++ b/LibOS/shim/src/sys/shim_futex.c @@ -427,40 +427,57 @@ void release_robust_list(struct robust_list_head* head) { } } -void release_clear_child_id(int* clear_child_tid) { - debug("clear child tid at %p\n", clear_child_tid); +/* Function is called by Async Helper thread to wait on *clear_child_tid to be changed + * to -1 (PAL does it when child thread finally exits). Since it is a callback to Async + * Helper thread, it must follow the `void (*callback) (IDTYPE caller, void * arg)` + * function signature even though we don't use caller. */ +void release_clear_child_id(IDTYPE caller, void* clear_child_tid_ptr) { + __UNUSED(caller); + int* clear_child_tid = (int*) clear_child_tid_ptr; + if (!clear_child_tid) + return; + + /* wait on clear_child_tid to become -1; this signifies that PAL layer exited child thread */ + while (__atomic_load_n(clear_child_tid, __ATOMIC_ACQUIRE) != -1) { + __asm__ volatile ("pause"); + } + + /* child thread exited, now parent can wake up; note that if PAL layer would set it to 0, + * parent thread could spuriously wake up, notice 0, and continue its execution without + * waiting for this function to succeed first */ *clear_child_tid = 0; + /* at this point, child thread finally exited, can wake up parents if any */ create_lock_runtime(&futex_list_lock); struct shim_futex_handle* tmp; struct shim_futex_handle* futex = NULL; - lock(&futex_list_lock); + lock(&futex_list_lock); LISTP_FOR_EACH_ENTRY(tmp, &futex_list, list) { if (tmp->uaddr == (void*)clear_child_tid) { futex = tmp; break; } } - unlock(&futex_list_lock); - if (!futex) + if (!futex) { + /* no parent threads waiting on this child to exit */ return; + } + debug("release futex at %p\n", clear_child_tid); struct futex_waiter* waiter; struct futex_waiter* wtmp; struct shim_handle* hdl = container_of(futex, struct shim_handle, info.futex); + get_handle(hdl); lock(&hdl->lock); - - debug("release futex at %p\n", clear_child_tid); - *clear_child_tid = 0; LISTP_FOR_EACH_ENTRY_SAFE(waiter, wtmp, &futex->waiters, list) { + /* wake up every parent waiting on this child */ del_futex_waiter_wakeup(waiter, futex); } - unlock(&hdl->lock); put_handle(hdl); } diff --git a/LibOS/shim/test/regression/futex.c b/LibOS/shim/test/regression/futex.c index da045334..aa930563 100644 --- a/LibOS/shim/test/regression/futex.c +++ b/LibOS/shim/test/regression/futex.c @@ -20,7 +20,7 @@ static int myfutex = 0; static int futex(int* uaddr, int futex_op, int val, const struct timespec* timeout, int* uaddr2, int val3) { - return syscall(SYS_futex, uaddr, futex_op, val, timeout, uaddr, val3); + return syscall(SYS_futex, uaddr, futex_op, val, timeout, uaddr2, val3); } void* thread_function(void* argument) { diff --git a/LibOS/shim/test/regression/futex.manifest.template b/LibOS/shim/test/regression/futex.manifest.template index 9c2cb7d3..9d8c5a1d 100644 --- a/LibOS/shim/test/regression/futex.manifest.template +++ b/LibOS/shim/test/regression/futex.manifest.template @@ -19,4 +19,4 @@ net.rules.2 = 0.0.0.0:0-65535:127.0.0.1:8000 sgx.trusted_files.ld = file:../../../../Runtime/ld-linux-x86-64.so.2 sgx.trusted_files.libc = file:../../../../Runtime/libc.so.6 sgx.trusted_files.libpthread = file:../../../../Runtime/libpthread.so.0 -sgx.thread_num = 4 +sgx.thread_num = 6 diff --git a/LibOS/shim/test/regression/manifest.template b/LibOS/shim/test/regression/manifest.template index adf27594..61af0af6 100644 --- a/LibOS/shim/test/regression/manifest.template +++ b/LibOS/shim/test/regression/manifest.template @@ -37,4 +37,6 @@ sgx.trusted_children.victim = file:exec_victim.sig sgx.allow_file_creation = 1 +sgx.thread_num = 6 + sgx.allowed_files.tmp_dir = file:tmp/ diff --git a/Pal/regression/Event.c b/Pal/regression/Event.c index 3244ba0d..41159723 100644 --- a/Pal/regression/Event.c +++ b/Pal/regression/Event.c @@ -41,7 +41,7 @@ int main() { return 1; } - PAL_HANDLE thread2 = DkThreadCreate(thread2_run, NULL); + PAL_HANDLE thread2 = DkThreadCreate(thread2_run, NULL, /*clear_child_tid=*/NULL); if (thread2 == NULL) { pal_printf("DkThreadCreate failed\n"); return 1; diff --git a/Pal/regression/Thread.c b/Pal/regression/Thread.c index b1ddca8a..be1b2ab6 100644 --- a/Pal/regression/Thread.c +++ b/Pal/regression/Thread.c @@ -40,7 +40,7 @@ int main(int argc, const char** argv, const char** envp) { __asm__ volatile("mov %%fs:0, %0" : "=r"(ptr1)::"memory"); pal_printf("Private Message (FS Segment) 1: %s\n", ptr1); - PAL_HANDLE thread1 = DkThreadCreate(callback1, "Hello World"); + PAL_HANDLE thread1 = DkThreadCreate(callback1, "Hello World", /*clear_child_tid=*/NULL); if (thread1) { pal_printf("Child Thread Created\n"); diff --git a/Pal/regression/Thread2.c b/Pal/regression/Thread2.c index 74d075f5..34b3be38 100644 --- a/Pal/regression/Thread2.c +++ b/Pal/regression/Thread2.c @@ -51,7 +51,7 @@ int thread4_run(void* args) { int main() { pal_printf("Thread 1 (main) started.\n"); - PAL_HANDLE thread2 = DkThreadCreate(thread2_run, NULL); + PAL_HANDLE thread2 = DkThreadCreate(thread2_run, NULL, /*clear_child_tid=*/NULL); if (!thread2) { pal_printf("DkThreadCreate failed for thread 2.\n"); return 1; @@ -65,7 +65,7 @@ int main() { pal_printf("Thread 2 ok.\n"); } - PAL_HANDLE thread3 = DkThreadCreate(thread3_run, NULL); + PAL_HANDLE thread3 = DkThreadCreate(thread3_run, NULL, /*clear_child_tid=*/NULL); if (!thread3) { pal_printf("DkThreadCreate failed for thread 3.\n"); return 1; @@ -77,7 +77,7 @@ int main() { pal_printf("Thread 3 ok.\n"); } - PAL_HANDLE thread4 = DkThreadCreate(thread4_run, NULL); + PAL_HANDLE thread4 = DkThreadCreate(thread4_run, NULL, /*clear_child_tid=*/NULL); if (!thread4) { pal_printf("DkThreadCreate failed for thread 4.\n"); return 1; diff --git a/Pal/src/db_threading.c b/Pal/src/db_threading.c index 0ff2183e..ac159ab0 100644 --- a/Pal/src/db_threading.c +++ b/Pal/src/db_threading.c @@ -30,11 +30,12 @@ /* PAL call DkThreadCreate: create a thread inside the current process */ PAL_HANDLE -DkThreadCreate(PAL_PTR addr, PAL_PTR param) { +DkThreadCreate(PAL_PTR addr, PAL_PTR param, PAL_PTR clear_child_tid) { ENTER_PAL_CALL(DkThreadCreate); PAL_HANDLE handle = NULL; - int ret = _DkThreadCreate(&handle, (int (*)(void*))addr, (const void*)param); + int ret = _DkThreadCreate(&handle, (int (*)(void*))addr, (const void*)param, + (const void*)clear_child_tid); if (ret < 0) { _DkRaiseFailure(-ret); diff --git a/Pal/src/host/FreeBSD/db_threading.c b/Pal/src/host/FreeBSD/db_threading.c index 20627894..78f23b3f 100644 --- a/Pal/src/host/FreeBSD/db_threading.c +++ b/Pal/src/host/FreeBSD/db_threading.c @@ -42,9 +42,8 @@ /* _DkThreadCreate for internal use. Create an internal thread inside the current process. The arguments callback and param specify the starting function and parameters */ -int _DkThreadCreate (PAL_HANDLE * handle, int (*callback) (void *), - const void * param) -{ +int _DkThreadCreate(PAL_HANDLE* handle, int (*callback) (void*), + const void* param, const void* clear_child_tid) { void * child_stack = NULL; if (_DkVirtualMemoryAlloc(&child_stack, THREAD_STACK_SIZE, 0, diff --git a/Pal/src/host/Linux-SGX/db_threading.c b/Pal/src/host/Linux-SGX/db_threading.c index 5064162e..1fde013d 100644 --- a/Pal/src/host/Linux-SGX/db_threading.c +++ b/Pal/src/host/Linux-SGX/db_threading.c @@ -44,8 +44,9 @@ DEFINE_LISTP(pal_handle_thread); static LISTP_TYPE(pal_handle_thread) thread_list = LISTP_INIT; struct thread_param { - int (*callback) (void *); - const void * param; + int (*callback) (void*); + const void* param; + const void* clear_child_tid; }; extern void * enclave_base; @@ -82,12 +83,17 @@ void pal_start_thread (void) struct thread_param * thread_param = (struct thread_param *) new_thread->param; - int (*callback) (void *) = thread_param->callback; - const void * param = thread_param->param; + int (*callback) (void*) = thread_param->callback; + const void* param = thread_param->param; + const void* clear_child_tid = thread_param->clear_child_tid; + free(thread_param); new_thread->param = NULL; + SET_ENCLAVE_TLS(thread, new_thread); SET_ENCLAVE_TLS(ready_for_exceptions, 1UL); + SET_ENCLAVE_TLS(clear_child_tid, clear_child_tid); + callback((void *) param); _DkThreadExit(); } @@ -95,9 +101,8 @@ void pal_start_thread (void) /* _DkThreadCreate for internal use. Create an internal thread inside the current process. The arguments callback and param specify the starting function and parameters */ -int _DkThreadCreate (PAL_HANDLE * handle, int (*callback) (void *), - const void * param) -{ +int _DkThreadCreate(PAL_HANDLE* handle, int (*callback) (void*), + const void* param, const void* clear_child_tid) { PAL_HANDLE new_thread = malloc(HANDLE_SIZE(thread)); SET_HANDLE_TYPE(new_thread, thread); /* @@ -108,8 +113,9 @@ int _DkThreadCreate (PAL_HANDLE * handle, int (*callback) (void *), new_thread->thread.tcs = NULL; INIT_LIST_HEAD(&new_thread->thread, list); struct thread_param * thread_param = malloc(sizeof(struct thread_param)); - thread_param->callback = callback; - thread_param->param = param; + thread_param->callback = callback; + thread_param->param = param; + thread_param->clear_child_tid = clear_child_tid; new_thread->thread.param = (void *) thread_param; _DkInternalLock(&thread_list_lock); @@ -142,6 +148,10 @@ noreturn void _DkThreadExit (void) { struct pal_handle_thread* exiting_thread = GET_ENCLAVE_TLS(thread); + /* thread is ready to exit, must inform LibOS by erasing clear_child_tid; + * note that we don't do it now (because this thread still occupies SGX + * TCS slot) but during handle_thread_reset in assembly code */ + /* main thread is not part of the thread_list */ if(exiting_thread != &pal_control.first_thread->thread) { _DkInternalLock(&thread_list_lock); diff --git a/Pal/src/host/Linux-SGX/enclave_ecalls.c b/Pal/src/host/Linux-SGX/enclave_ecalls.c index beed4ac6..0f4dab63 100644 --- a/Pal/src/host/Linux-SGX/enclave_ecalls.c +++ b/Pal/src/host/Linux-SGX/enclave_ecalls.c @@ -50,9 +50,10 @@ void handle_ecall (long ecall_index, void * ecall_args, void * exit_target, enclave_top = enclave_base_addr + GET_ENCLAVE_TLS(enclave_size); } - SET_ENCLAVE_TLS(exit_target, exit_target); - SET_ENCLAVE_TLS(ustack_top, untrusted_stack); - SET_ENCLAVE_TLS(ustack, untrusted_stack); + SET_ENCLAVE_TLS(exit_target, exit_target); + SET_ENCLAVE_TLS(ustack_top, untrusted_stack); + SET_ENCLAVE_TLS(ustack, untrusted_stack); + SET_ENCLAVE_TLS(clear_child_tid, NULL); if (atomic_cmpxchg(&enclave_start_called, 0, 1) == 0) { // ENCLAVE_START not yet called, so only valid ecall is ENCLAVE_START. diff --git a/Pal/src/host/Linux-SGX/enclave_entry.S b/Pal/src/host/Linux-SGX/enclave_entry.S index 0aa9fa27..171ce19e 100644 --- a/Pal/src/host/Linux-SGX/enclave_entry.S +++ b/Pal/src/host/Linux-SGX/enclave_entry.S @@ -117,6 +117,15 @@ enclave_entry: .Lhandle_thread_reset: movq $0, %gs:SGX_READY_FOR_EXCEPTIONS + # At this point, the thread has completely exited from the point of view + # of LibOS. We can now set *clear_child_tid to -1, which will trigger + # async helper thread in LibOS, who will wake up parent thread if any. + cmpq $0, %gs:SGX_CLEAR_CHILD_TID + je 1f + movq %gs:SGX_CLEAR_CHILD_TID, %rbx + movl $-1, (%rbx) + +1: # Signals are impossible at this point: benign untrusted runtime blocks # all signals (see sgx_ocall_exit()), and even if malicious one doesn't # block them, signals are ignored due to SGX_READY_FOR_EXCEPTIONS = 0. diff --git a/Pal/src/host/Linux-SGX/generated-offsets.c b/Pal/src/host/Linux-SGX/generated-offsets.c index c0b8040e..c1dd2aea 100644 --- a/Pal/src/host/Linux-SGX/generated-offsets.c +++ b/Pal/src/host/Linux-SGX/generated-offsets.c @@ -91,6 +91,7 @@ void dummy(void) OFFSET(SGX_HEAP_MAX, enclave_tls, heap_max); OFFSET(SGX_EXEC_ADDR, enclave_tls, exec_addr); OFFSET(SGX_EXEC_SIZE, enclave_tls, exec_size); + OFFSET(SGX_CLEAR_CHILD_TID, enclave_tls, clear_child_tid); /* struct pal_tcb_linux aka PAL_TCB_LINUX */ OFFSET(PAL_TCB_LINUX_TCS, pal_tcb_linux, tcs); diff --git a/Pal/src/host/Linux-SGX/sgx_tls.h b/Pal/src/host/Linux-SGX/sgx_tls.h index ac643982..ea4d096f 100644 --- a/Pal/src/host/Linux-SGX/sgx_tls.h +++ b/Pal/src/host/Linux-SGX/sgx_tls.h @@ -33,6 +33,7 @@ struct enclave_tls { void* heap_max; void* exec_addr; uint64_t exec_size; + int* clear_child_tid; }; }; diff --git a/Pal/src/host/Linux/db_threading.c b/Pal/src/host/Linux/db_threading.c index 566a5d98..591c86a8 100644 --- a/Pal/src/host/Linux/db_threading.c +++ b/Pal/src/host/Linux/db_threading.c @@ -80,9 +80,8 @@ int pal_thread_init (void * tcbptr) /* _DkThreadCreate for internal use. Create an internal thread inside the current process. The arguments callback and param specify the starting function and parameters */ -int _DkThreadCreate (PAL_HANDLE * handle, int (*callback) (void *), - const void * param) -{ +int _DkThreadCreate(PAL_HANDLE* handle, int (*callback) (void*), + const void* param, const void* clear_child_tid) { int ret = 0; PAL_HANDLE hdl = NULL; void * stack = malloc(THREAD_STACK_SIZE + ALT_STACK_SIZE); @@ -119,11 +118,12 @@ int _DkThreadCreate (PAL_HANDLE * handle, int (*callback) (void *), // Initialize TCB at the top of the alternative stack. PAL_TCB_LINUX * tcb = child_stack + ALT_STACK_SIZE - sizeof(PAL_TCB_LINUX); - tcb->common.self = &tcb->common; - tcb->handle = hdl; - tcb->alt_stack = child_stack; // Stack bottom - tcb->callback = callback; - tcb->param = (void *) param; + tcb->common.self = &tcb->common; + tcb->handle = hdl; + tcb->alt_stack = child_stack; // Stack bottom + tcb->callback = callback; + tcb->param = (void*) param; + tcb->clear_child_tid = (void*) clear_child_tid; /* align child_stack to 16 */ child_stack = ALIGN_DOWN_PTR(child_stack, 16); @@ -190,6 +190,13 @@ noreturn void _DkThreadExit (void) PAL_HANDLE handle = tcb->handle; block_async_signals(true); + + if (tcb->clear_child_tid) { + /* thread is ready to exit, must inform LibOS by setting *clear_child_tid to -1; + * async helper thread in LibOS is waiting on this to wake up parent */ + *tcb->clear_child_tid = -1; + } + if (tcb->alt_stack) { stack_t ss; ss.ss_sp = NULL; diff --git a/Pal/src/host/Linux/pal_linux.h b/Pal/src/host/Linux/pal_linux.h index 90ed5409..c6b5be89 100644 --- a/Pal/src/host/Linux/pal_linux.h +++ b/Pal/src/host/Linux/pal_linux.h @@ -194,9 +194,10 @@ typedef struct pal_tcb_linux { int pending_event; LISTP_TYPE(event_queue) pending_queue; PAL_HANDLE handle; - void * alt_stack; - int (*callback) (void *); - void * param; + void* alt_stack; + int (*callback) (void*); + void* param; + int* clear_child_tid; }; } PAL_TCB_LINUX; diff --git a/Pal/src/host/Skeleton/db_threading.c b/Pal/src/host/Skeleton/db_threading.c index 30be6f00..2267c022 100644 --- a/Pal/src/host/Skeleton/db_threading.c +++ b/Pal/src/host/Skeleton/db_threading.c @@ -30,7 +30,8 @@ /* _DkThreadCreate for internal use. Create an internal thread inside the current process. The arguments callback and param specify the starting function and parameters */ -int _DkThreadCreate(PAL_HANDLE* handle, int (*callback)(void*), const void* param) { +int _DkThreadCreate(PAL_HANDLE* handle, int (*callback)(void*), const void* param, + const void* clear_child_tid) { return -PAL_ERROR_NOTIMPLEMENTED; } diff --git a/Pal/src/pal.h b/Pal/src/pal.h index 986989d1..b330c546 100644 --- a/Pal/src/pal.h +++ b/Pal/src/pal.h @@ -410,7 +410,7 @@ DkStreamChangeName (PAL_HANDLE handle, PAL_STR uri); #define PAL_THREAD_MASK 0 PAL_HANDLE -DkThreadCreate (PAL_PTR addr, PAL_PTR param); +DkThreadCreate (PAL_PTR addr, PAL_PTR param, PAL_PTR clear_child_tid); // assuming duration to be in microseconds PAL_NUM diff --git a/Pal/src/pal_internal.h b/Pal/src/pal_internal.h index f9cb8b40..3820ff1e 100644 --- a/Pal/src/pal_internal.h +++ b/Pal/src/pal_internal.h @@ -292,8 +292,8 @@ int _DkReceiveHandle(PAL_HANDLE hdl, PAL_HANDLE * cargo); PAL_HANDLE _DkBroadcastStreamOpen (void); /* DkProcess and DkThread calls */ -int _DkThreadCreate (PAL_HANDLE * handle, int (*callback) (void *), - const void * param); +int _DkThreadCreate(PAL_HANDLE* handle, int (*callback) (void*), + const void* param, const void* clear_child_tid); noreturn void _DkThreadExit (void); int _DkThreadDelayExecution (unsigned long * duration); void _DkThreadYieldExecution (void); diff --git a/Pal/test/Event.c b/Pal/test/Event.c index 0df35bd0..caac9a9b 100644 --- a/Pal/test/Event.c +++ b/Pal/test/Event.c @@ -33,7 +33,7 @@ int main(int argc, char** argv) { return -1; } - thd1 = DkThreadCreate(&thread_1, 0); + thd1 = DkThreadCreate(&thread_1, 0, /*clear_child_tid=*/NULL); if (thd1 == NULL) { pal_printf("DkThreadCreate failed\n"); diff --git a/Pal/test/Fork.c b/Pal/test/Fork.c index 22befb3c..a81fa31f 100644 --- a/Pal/test/Fork.c +++ b/Pal/test/Fork.c @@ -15,7 +15,7 @@ PAL_HANDLE _fork(void* args) { if (args == NULL) { struct stack_frame cur_frame = *frame; pal_printf("return address is %p\n", cur_frame.ret); - return DkThreadCreate(&_fork, &cur_frame); + return DkThreadCreate(&_fork, &cur_frame, /*clear_child_tid=*/NULL); } else { struct stack_frame* las_frame = (struct stack_frame*)args; pal_printf("(in child) return address is %p\n", las_frame->ret); diff --git a/Pal/test/Select.c b/Pal/test/Select.c index 8abe8ade..bc636249 100644 --- a/Pal/test/Select.c +++ b/Pal/test/Select.c @@ -26,7 +26,7 @@ int main() { handles[2] = DkStreamOpen("pipe:", PAL_ACCESS_RDWR, 0, 0, 0); wakeup = handles[2]; - PAL_HANDLE thd = DkThreadCreate(&thread, NULL); + PAL_HANDLE thd = DkThreadCreate(&thread, NULL, /*clear_child_tid=*/NULL); if (thd == NULL) { pal_printf("DkThreadCreate failed\n"); diff --git a/Pal/test/Thread.c b/Pal/test/Thread.c index c4f82d19..24ed0939 100644 --- a/Pal/test/Thread.c +++ b/Pal/test/Thread.c @@ -27,14 +27,14 @@ int main() { PAL_HANDLE thd1, thd2; - thd1 = DkThreadCreate(&thread_1, NULL); + thd1 = DkThreadCreate(&thread_1, NULL, /*clear_child_tid=*/NULL); if (thd1 == NULL) { pal_printf("DkThreadCreate failed\n"); return -1; } - thd2 = DkThreadCreate(&thread_2, NULL); + thd2 = DkThreadCreate(&thread_2, NULL, /*clear_child_tid=*/NULL); if (thd2 == NULL) { pal_printf("DkThreadCreate failed\n"); diff --git a/Pal/test/Wait.c b/Pal/test/Wait.c index 26643626..bdbc6627 100644 --- a/Pal/test/Wait.c +++ b/Pal/test/Wait.c @@ -33,14 +33,14 @@ int main() { event1 = DkNotificationEventCreate(0); event2 = DkNotificationEventCreate(0); - thd1 = DkThreadCreate(&thread_1, 0); + thd1 = DkThreadCreate(&thread_1, 0, /*clear_child_tid=*/NULL); if (thd1 == NULL) { pal_printf("DkThreadCreate failed\n"); return -1; } - thd2 = DkThreadCreate(&thread_2, 0); + thd2 = DkThreadCreate(&thread_2, 0, /*clear_child_tid=*/NULL); if (thd2 == NULL) { pal_printf("DkThreadCreate failed\n"); diff --git a/Pal/test/Yield.c b/Pal/test/Yield.c index fcf5f6d8..b81819ef 100644 --- a/Pal/test/Yield.c +++ b/Pal/test/Yield.c @@ -22,7 +22,7 @@ int main(void) { pal_printf("Enter Parent Thread\n"); parent_thread = pal_control.first_thread; - child_thread = DkThreadCreate(&child, NULL); + child_thread = DkThreadCreate(&child, NULL, /*clear_child_tid=*/NULL); if (child_thread == NULL) { pal_printf("DkThreadCreate failed\n");