diff --git a/Documentation/oldwiki/PAL-Host-ABI.md b/Documentation/oldwiki/PAL-Host-ABI.md index a84d3ba2..01f04d31 100644 --- a/Documentation/oldwiki/PAL-Host-ABI.md +++ b/Documentation/oldwiki/PAL-Host-ABI.md @@ -407,10 +407,11 @@ This API changes the name of an opened stream. #### DkThreadCreate - PAL_HANDLE DkThreadCreate(PAL_PTR addr, PAL_PTR param); + PAL_HANDLE DkThreadCreate(PAL_PTR addr, PAL_PTR param, PAL_PTR clear_child_tid); This API creates a thread in the current process. `addr` is the address of an entry point of execution for the new thread. `param` is the pointer argument that is passed to the new thread. +`clear_child_tid` is the pointer to memory that is erased on child exit to notify parent. #### DkThreadDelayExecution diff --git a/LibOS/shim/include/shim_internal.h b/LibOS/shim/include/shim_internal.h index e92ad724..fc6e8393 100644 --- a/LibOS/shim/include/shim_internal.h +++ b/LibOS/shim/include/shim_internal.h @@ -456,10 +456,9 @@ static inline void enable_locking (void) lock_enabled = true; } -static inline PAL_HANDLE thread_create (void * func, void * arg) -{ +static inline PAL_HANDLE thread_create(void* func, void* arg, void* clear_child_tid) { assert(lock_enabled); - return DkThreadCreate(func, arg); + return DkThreadCreate(func, arg, clear_child_tid); } static inline int64_t __disable_preempt (shim_tcb_t * tcb) @@ -797,6 +796,8 @@ void set_rlimit_cur(int resource, uint64_t rlim); int object_wait_with_retry(PAL_HANDLE handle); +void release_clear_child_id(IDTYPE caller, void* clear_child_tid_ptr); + #ifdef __x86_64__ #define __SWITCH_STACK(stack_top, func, arg) \ do { \ diff --git a/LibOS/shim/src/bookkeep/shim_thread.c b/LibOS/shim/src/bookkeep/shim_thread.c index a4908717..742cde93 100644 --- a/LibOS/shim/src/bookkeep/shim_thread.c +++ b/LibOS/shim/src/bookkeep/shim_thread.c @@ -759,7 +759,7 @@ BEGIN_RS_FUNC(running_thread) NUM_SIGS); if (cur_thread) { - PAL_HANDLE handle = DkThreadCreate(resume_wrapper, thread); + PAL_HANDLE handle = DkThreadCreate(resume_wrapper, thread, thread->clear_child_tid); if (!thread) return -PAL_ERRNO; diff --git a/LibOS/shim/src/ipc/shim_ipc_helper.c b/LibOS/shim/src/ipc/shim_ipc_helper.c index 745d8a4b..9f254254 100644 --- a/LibOS/shim/src/ipc/shim_ipc_helper.c +++ b/LibOS/shim/src/ipc/shim_ipc_helper.c @@ -780,6 +780,7 @@ noreturn static void shim_ipc_helper(void* dummy) { free(object_list); free(palhandle_list); + __disable_preempt(self->shim_tcb); put_thread(self); debug("IPC helper thread terminated\n"); @@ -828,7 +829,7 @@ static int create_ipc_helper(void) { ipc_helper_thread = new; ipc_helper_state = HELPER_ALIVE; - PAL_HANDLE handle = thread_create(shim_ipc_helper_prepare, new); + PAL_HANDLE handle = thread_create(shim_ipc_helper_prepare, new, /*clear_child_tid=*/NULL); if (!handle) { int ret = -PAL_ERRNO; /* put_thread() may overwrite errno */ diff --git a/LibOS/shim/src/shim_async.c b/LibOS/shim/src/shim_async.c index da352ab6..691e970a 100644 --- a/LibOS/shim/src/shim_async.c +++ b/LibOS/shim/src/shim_async.c @@ -38,6 +38,7 @@ struct async_event { void * arg; PAL_HANDLE object; /* handle (async IO) to wait on */ uint64_t expire_time; /* alarm/timer to wait on */ + bool todelete; }; DEFINE_LISTP(async_event); static LISTP_TYPE(async_event) async_list; @@ -82,10 +83,11 @@ int64_t install_async_event(PAL_HANDLE object, uint64_t time, event->caller = get_cur_tid(); event->object = object; event->expire_time = time ? now + time : 0; + event->todelete = false; lock(&async_helper_lock); - if (!object) { + if (callback != &release_clear_child_id && !object) { /* This is alarm() or setitimer() emulation, treat both according to * alarm() syscall semantics: cancel any pending alarm/timer. */ struct async_event * tmp, * n; @@ -203,9 +205,18 @@ static void shim_async_helper(void * arg) { struct async_event * tmp, * n; LISTP_FOR_EACH_ENTRY_SAFE(tmp, n, &async_list, list) { - /* First check if this event was triggered; note that IO events - * stay in the list whereas alarms/timers are fired only once. */ - if (polled && tmp->object == polled) { + /* First check if this event was triggered; there are three types: + * 1. Exited child: trigger callback and remove from the list; + * 2. IO events: trigger callback and keep in the list; + * 3. alarms/timers: trigger callback and remove from the list. */ + if (tmp->callback == &release_clear_child_id) { + debug("Child exited, notifying parents if any\n"); + tmp->todelete = true; + unlock(&async_helper_lock); + release_clear_child_id(tmp->caller, tmp->arg); + lock(&async_helper_lock); + continue; + } else if (polled && tmp->object == polled) { debug("Async IO event triggered at %lu\n", now); unlock(&async_helper_lock); tmp->callback(tmp->caller, tmp->arg); @@ -213,14 +224,16 @@ static void shim_async_helper(void * arg) { } else if (tmp->expire_time && tmp->expire_time <= now) { debug("Async alarm/timer triggered at %lu (expired at %lu)\n", now, tmp->expire_time); - LISTP_DEL(tmp, &async_list, list); + tmp->todelete = true; unlock(&async_helper_lock); tmp->callback(tmp->caller, tmp->arg); - free(tmp); lock(&async_helper_lock); continue; } + if (tmp->todelete) + continue; + /* Now re-add this IO event to the list or re-add this timer */ if (tmp->object) { if (object_num == object_list_size) { @@ -243,6 +256,13 @@ static void shim_async_helper(void * arg) { } } + LISTP_FOR_EACH_ENTRY_SAFE(tmp, n, &async_list, list) { + if (tmp->todelete) { + LISTP_DEL(tmp, &async_list, list); + free(tmp); + } + } + uint64_t sleep_time; if (next_expire_time) { sleep_time = next_expire_time - now; @@ -269,6 +289,7 @@ static void shim_async_helper(void * arg) { polled = DkObjectsWaitAny(object_num + 1, object_list, sleep_time); } + __disable_preempt(self->shim_tcb); put_thread(self); debug("Async helper thread terminated\n"); free(object_list); @@ -288,7 +309,7 @@ static int create_async_helper(void) { async_helper_thread = new; async_helper_state = HELPER_ALIVE; - PAL_HANDLE handle = thread_create(shim_async_helper, new); + PAL_HANDLE handle = thread_create(shim_async_helper, new, /*clear_child_tid=*/NULL); if (!handle) { async_helper_thread = NULL; diff --git a/LibOS/shim/src/sys/shim_clone.c b/LibOS/shim/src/sys/shim_clone.c index 27d54a57..8b9c54c5 100644 --- a/LibOS/shim/src/sys/shim_clone.c +++ b/LibOS/shim/src/sys/shim_clone.c @@ -288,7 +288,7 @@ int shim_do_clone (int flags, void * user_stack_addr, int * parent_tidptr, if (flags & CLONE_CHILD_CLEARTID) /* Implemented in shim_futex.c: release_clear_child_id */ - thread->clear_child_tid = parent_tidptr; + thread->clear_child_tid = child_tidptr; if (flags & CLONE_SETTLS) { if (!tls) { @@ -402,7 +402,7 @@ int shim_do_clone (int flags, void * user_stack_addr, int * parent_tidptr, // returns .The parent comes back here - however, the child is Happily // running the function we gave to DkThreadCreate. PAL_HANDLE pal_handle = thread_create(clone_implementation_wrapper, - &new_args); + &new_args, thread->clear_child_tid); if (!pal_handle) { ret = -PAL_ERRNO; put_thread(new_args.thread); diff --git a/LibOS/shim/src/sys/shim_exit.c b/LibOS/shim/src/sys/shim_exit.c index e75519d5..30c0e15b 100644 --- a/LibOS/shim/src/sys/shim_exit.c +++ b/LibOS/shim/src/sys/shim_exit.c @@ -40,8 +40,6 @@ void release_robust_list (struct robust_list_head * head); -void release_clear_child_id (int * clear_child_tid); - int thread_exit(struct shim_thread * self, bool send_ipc) { bool sent_exit_msg = false; @@ -122,8 +120,9 @@ int thread_exit(struct shim_thread * self, bool send_ipc) if (robust_list) release_robust_list(robust_list); - if (self->clear_child_tid) - release_clear_child_id (self->clear_child_tid); + /* ask Async Helper thread to wake up parent when this child thread finally exits */ + if (parent && self->in_vm && self->clear_child_tid) + install_async_event(NULL, 0, &release_clear_child_id, self->clear_child_tid); DkEventSet(self->exit_event); return 0; diff --git a/LibOS/shim/src/sys/shim_futex.c b/LibOS/shim/src/sys/shim_futex.c index 9205ae19..71779cff 100644 --- a/LibOS/shim/src/sys/shim_futex.c +++ b/LibOS/shim/src/sys/shim_futex.c @@ -427,40 +427,57 @@ void release_robust_list(struct robust_list_head* head) { } } -void release_clear_child_id(int* clear_child_tid) { - debug("clear child tid at %p\n", clear_child_tid); +/* Function is called by Async Helper thread to wait on *clear_child_tid to be changed + * to -1 (PAL does it when child thread finally exits). Since it is a callback to Async + * Helper thread, it must follow the `void (*callback) (IDTYPE caller, void * arg)` + * function signature even though we don't use caller. */ +void release_clear_child_id(IDTYPE caller, void* clear_child_tid_ptr) { + __UNUSED(caller); + int* clear_child_tid = (int*) clear_child_tid_ptr; + if (!clear_child_tid) + return; + + /* wait on clear_child_tid to become -1; this signifies that PAL layer exited child thread */ + while (__atomic_load_n(clear_child_tid, __ATOMIC_ACQUIRE) != -1) { + __asm__ volatile ("pause"); + } + + /* child thread exited, now parent can wake up; note that if PAL layer would set it to 0, + * parent thread could spuriously wake up, notice 0, and continue its execution without + * waiting for this function to succeed first */ *clear_child_tid = 0; + /* at this point, child thread finally exited, can wake up parents if any */ create_lock_runtime(&futex_list_lock); struct shim_futex_handle* tmp; struct shim_futex_handle* futex = NULL; - lock(&futex_list_lock); + lock(&futex_list_lock); LISTP_FOR_EACH_ENTRY(tmp, &futex_list, list) { if (tmp->uaddr == (void*)clear_child_tid) { futex = tmp; break; } } - unlock(&futex_list_lock); - if (!futex) + if (!futex) { + /* no parent threads waiting on this child to exit */ return; + } + debug("release futex at %p\n", clear_child_tid); struct futex_waiter* waiter; struct futex_waiter* wtmp; struct shim_handle* hdl = container_of(futex, struct shim_handle, info.futex); + get_handle(hdl); lock(&hdl->lock); - - debug("release futex at %p\n", clear_child_tid); - *clear_child_tid = 0; LISTP_FOR_EACH_ENTRY_SAFE(waiter, wtmp, &futex->waiters, list) { + /* wake up every parent waiting on this child */ del_futex_waiter_wakeup(waiter, futex); } - unlock(&hdl->lock); put_handle(hdl); } diff --git a/LibOS/shim/test/regression/futex.c b/LibOS/shim/test/regression/futex.c index da045334..aa930563 100644 --- a/LibOS/shim/test/regression/futex.c +++ b/LibOS/shim/test/regression/futex.c @@ -20,7 +20,7 @@ static int myfutex = 0; static int futex(int* uaddr, int futex_op, int val, const struct timespec* timeout, int* uaddr2, int val3) { - return syscall(SYS_futex, uaddr, futex_op, val, timeout, uaddr, val3); + return syscall(SYS_futex, uaddr, futex_op, val, timeout, uaddr2, val3); } void* thread_function(void* argument) { diff --git a/LibOS/shim/test/regression/futex.manifest.template b/LibOS/shim/test/regression/futex.manifest.template index 9c2cb7d3..9d8c5a1d 100644 --- a/LibOS/shim/test/regression/futex.manifest.template +++ b/LibOS/shim/test/regression/futex.manifest.template @@ -19,4 +19,4 @@ net.rules.2 = 0.0.0.0:0-65535:127.0.0.1:8000 sgx.trusted_files.ld = file:../../../../Runtime/ld-linux-x86-64.so.2 sgx.trusted_files.libc = file:../../../../Runtime/libc.so.6 sgx.trusted_files.libpthread = file:../../../../Runtime/libpthread.so.0 -sgx.thread_num = 4 +sgx.thread_num = 6 diff --git a/LibOS/shim/test/regression/manifest.template b/LibOS/shim/test/regression/manifest.template index adf27594..61af0af6 100644 --- a/LibOS/shim/test/regression/manifest.template +++ b/LibOS/shim/test/regression/manifest.template @@ -37,4 +37,6 @@ sgx.trusted_children.victim = file:exec_victim.sig sgx.allow_file_creation = 1 +sgx.thread_num = 6 + sgx.allowed_files.tmp_dir = file:tmp/ diff --git a/Pal/regression/Event.c b/Pal/regression/Event.c index 3244ba0d..41159723 100644 --- a/Pal/regression/Event.c +++ b/Pal/regression/Event.c @@ -41,7 +41,7 @@ int main() { return 1; } - PAL_HANDLE thread2 = DkThreadCreate(thread2_run, NULL); + PAL_HANDLE thread2 = DkThreadCreate(thread2_run, NULL, /*clear_child_tid=*/NULL); if (thread2 == NULL) { pal_printf("DkThreadCreate failed\n"); return 1; diff --git a/Pal/regression/Thread.c b/Pal/regression/Thread.c index b1ddca8a..be1b2ab6 100644 --- a/Pal/regression/Thread.c +++ b/Pal/regression/Thread.c @@ -40,7 +40,7 @@ int main(int argc, const char** argv, const char** envp) { __asm__ volatile("mov %%fs:0, %0" : "=r"(ptr1)::"memory"); pal_printf("Private Message (FS Segment) 1: %s\n", ptr1); - PAL_HANDLE thread1 = DkThreadCreate(callback1, "Hello World"); + PAL_HANDLE thread1 = DkThreadCreate(callback1, "Hello World", /*clear_child_tid=*/NULL); if (thread1) { pal_printf("Child Thread Created\n"); diff --git a/Pal/regression/Thread2.c b/Pal/regression/Thread2.c index 74d075f5..34b3be38 100644 --- a/Pal/regression/Thread2.c +++ b/Pal/regression/Thread2.c @@ -51,7 +51,7 @@ int thread4_run(void* args) { int main() { pal_printf("Thread 1 (main) started.\n"); - PAL_HANDLE thread2 = DkThreadCreate(thread2_run, NULL); + PAL_HANDLE thread2 = DkThreadCreate(thread2_run, NULL, /*clear_child_tid=*/NULL); if (!thread2) { pal_printf("DkThreadCreate failed for thread 2.\n"); return 1; @@ -65,7 +65,7 @@ int main() { pal_printf("Thread 2 ok.\n"); } - PAL_HANDLE thread3 = DkThreadCreate(thread3_run, NULL); + PAL_HANDLE thread3 = DkThreadCreate(thread3_run, NULL, /*clear_child_tid=*/NULL); if (!thread3) { pal_printf("DkThreadCreate failed for thread 3.\n"); return 1; @@ -77,7 +77,7 @@ int main() { pal_printf("Thread 3 ok.\n"); } - PAL_HANDLE thread4 = DkThreadCreate(thread4_run, NULL); + PAL_HANDLE thread4 = DkThreadCreate(thread4_run, NULL, /*clear_child_tid=*/NULL); if (!thread4) { pal_printf("DkThreadCreate failed for thread 4.\n"); return 1; diff --git a/Pal/src/db_threading.c b/Pal/src/db_threading.c index 0ff2183e..ac159ab0 100644 --- a/Pal/src/db_threading.c +++ b/Pal/src/db_threading.c @@ -30,11 +30,12 @@ /* PAL call DkThreadCreate: create a thread inside the current process */ PAL_HANDLE -DkThreadCreate(PAL_PTR addr, PAL_PTR param) { +DkThreadCreate(PAL_PTR addr, PAL_PTR param, PAL_PTR clear_child_tid) { ENTER_PAL_CALL(DkThreadCreate); PAL_HANDLE handle = NULL; - int ret = _DkThreadCreate(&handle, (int (*)(void*))addr, (const void*)param); + int ret = _DkThreadCreate(&handle, (int (*)(void*))addr, (const void*)param, + (const void*)clear_child_tid); if (ret < 0) { _DkRaiseFailure(-ret); diff --git a/Pal/src/host/FreeBSD/db_threading.c b/Pal/src/host/FreeBSD/db_threading.c index 20627894..78f23b3f 100644 --- a/Pal/src/host/FreeBSD/db_threading.c +++ b/Pal/src/host/FreeBSD/db_threading.c @@ -42,9 +42,8 @@ /* _DkThreadCreate for internal use. Create an internal thread inside the current process. The arguments callback and param specify the starting function and parameters */ -int _DkThreadCreate (PAL_HANDLE * handle, int (*callback) (void *), - const void * param) -{ +int _DkThreadCreate(PAL_HANDLE* handle, int (*callback) (void*), + const void* param, const void* clear_child_tid) { void * child_stack = NULL; if (_DkVirtualMemoryAlloc(&child_stack, THREAD_STACK_SIZE, 0, diff --git a/Pal/src/host/Linux-SGX/db_threading.c b/Pal/src/host/Linux-SGX/db_threading.c index 5064162e..1fde013d 100644 --- a/Pal/src/host/Linux-SGX/db_threading.c +++ b/Pal/src/host/Linux-SGX/db_threading.c @@ -44,8 +44,9 @@ DEFINE_LISTP(pal_handle_thread); static LISTP_TYPE(pal_handle_thread) thread_list = LISTP_INIT; struct thread_param { - int (*callback) (void *); - const void * param; + int (*callback) (void*); + const void* param; + const void* clear_child_tid; }; extern void * enclave_base; @@ -82,12 +83,17 @@ void pal_start_thread (void) struct thread_param * thread_param = (struct thread_param *) new_thread->param; - int (*callback) (void *) = thread_param->callback; - const void * param = thread_param->param; + int (*callback) (void*) = thread_param->callback; + const void* param = thread_param->param; + const void* clear_child_tid = thread_param->clear_child_tid; + free(thread_param); new_thread->param = NULL; + SET_ENCLAVE_TLS(thread, new_thread); SET_ENCLAVE_TLS(ready_for_exceptions, 1UL); + SET_ENCLAVE_TLS(clear_child_tid, clear_child_tid); + callback((void *) param); _DkThreadExit(); } @@ -95,9 +101,8 @@ void pal_start_thread (void) /* _DkThreadCreate for internal use. Create an internal thread inside the current process. The arguments callback and param specify the starting function and parameters */ -int _DkThreadCreate (PAL_HANDLE * handle, int (*callback) (void *), - const void * param) -{ +int _DkThreadCreate(PAL_HANDLE* handle, int (*callback) (void*), + const void* param, const void* clear_child_tid) { PAL_HANDLE new_thread = malloc(HANDLE_SIZE(thread)); SET_HANDLE_TYPE(new_thread, thread); /* @@ -108,8 +113,9 @@ int _DkThreadCreate (PAL_HANDLE * handle, int (*callback) (void *), new_thread->thread.tcs = NULL; INIT_LIST_HEAD(&new_thread->thread, list); struct thread_param * thread_param = malloc(sizeof(struct thread_param)); - thread_param->callback = callback; - thread_param->param = param; + thread_param->callback = callback; + thread_param->param = param; + thread_param->clear_child_tid = clear_child_tid; new_thread->thread.param = (void *) thread_param; _DkInternalLock(&thread_list_lock); @@ -142,6 +148,10 @@ noreturn void _DkThreadExit (void) { struct pal_handle_thread* exiting_thread = GET_ENCLAVE_TLS(thread); + /* thread is ready to exit, must inform LibOS by erasing clear_child_tid; + * note that we don't do it now (because this thread still occupies SGX + * TCS slot) but during handle_thread_reset in assembly code */ + /* main thread is not part of the thread_list */ if(exiting_thread != &pal_control.first_thread->thread) { _DkInternalLock(&thread_list_lock); diff --git a/Pal/src/host/Linux-SGX/enclave_ecalls.c b/Pal/src/host/Linux-SGX/enclave_ecalls.c index beed4ac6..0f4dab63 100644 --- a/Pal/src/host/Linux-SGX/enclave_ecalls.c +++ b/Pal/src/host/Linux-SGX/enclave_ecalls.c @@ -50,9 +50,10 @@ void handle_ecall (long ecall_index, void * ecall_args, void * exit_target, enclave_top = enclave_base_addr + GET_ENCLAVE_TLS(enclave_size); } - SET_ENCLAVE_TLS(exit_target, exit_target); - SET_ENCLAVE_TLS(ustack_top, untrusted_stack); - SET_ENCLAVE_TLS(ustack, untrusted_stack); + SET_ENCLAVE_TLS(exit_target, exit_target); + SET_ENCLAVE_TLS(ustack_top, untrusted_stack); + SET_ENCLAVE_TLS(ustack, untrusted_stack); + SET_ENCLAVE_TLS(clear_child_tid, NULL); if (atomic_cmpxchg(&enclave_start_called, 0, 1) == 0) { // ENCLAVE_START not yet called, so only valid ecall is ENCLAVE_START. diff --git a/Pal/src/host/Linux-SGX/enclave_entry.S b/Pal/src/host/Linux-SGX/enclave_entry.S index 0aa9fa27..171ce19e 100644 --- a/Pal/src/host/Linux-SGX/enclave_entry.S +++ b/Pal/src/host/Linux-SGX/enclave_entry.S @@ -117,6 +117,15 @@ enclave_entry: .Lhandle_thread_reset: movq $0, %gs:SGX_READY_FOR_EXCEPTIONS + # At this point, the thread has completely exited from the point of view + # of LibOS. We can now set *clear_child_tid to -1, which will trigger + # async helper thread in LibOS, who will wake up parent thread if any. + cmpq $0, %gs:SGX_CLEAR_CHILD_TID + je 1f + movq %gs:SGX_CLEAR_CHILD_TID, %rbx + movl $-1, (%rbx) + +1: # Signals are impossible at this point: benign untrusted runtime blocks # all signals (see sgx_ocall_exit()), and even if malicious one doesn't # block them, signals are ignored due to SGX_READY_FOR_EXCEPTIONS = 0. diff --git a/Pal/src/host/Linux-SGX/generated-offsets.c b/Pal/src/host/Linux-SGX/generated-offsets.c index c0b8040e..c1dd2aea 100644 --- a/Pal/src/host/Linux-SGX/generated-offsets.c +++ b/Pal/src/host/Linux-SGX/generated-offsets.c @@ -91,6 +91,7 @@ void dummy(void) OFFSET(SGX_HEAP_MAX, enclave_tls, heap_max); OFFSET(SGX_EXEC_ADDR, enclave_tls, exec_addr); OFFSET(SGX_EXEC_SIZE, enclave_tls, exec_size); + OFFSET(SGX_CLEAR_CHILD_TID, enclave_tls, clear_child_tid); /* struct pal_tcb_linux aka PAL_TCB_LINUX */ OFFSET(PAL_TCB_LINUX_TCS, pal_tcb_linux, tcs); diff --git a/Pal/src/host/Linux-SGX/sgx_tls.h b/Pal/src/host/Linux-SGX/sgx_tls.h index ac643982..ea4d096f 100644 --- a/Pal/src/host/Linux-SGX/sgx_tls.h +++ b/Pal/src/host/Linux-SGX/sgx_tls.h @@ -33,6 +33,7 @@ struct enclave_tls { void* heap_max; void* exec_addr; uint64_t exec_size; + int* clear_child_tid; }; }; diff --git a/Pal/src/host/Linux/db_threading.c b/Pal/src/host/Linux/db_threading.c index 566a5d98..591c86a8 100644 --- a/Pal/src/host/Linux/db_threading.c +++ b/Pal/src/host/Linux/db_threading.c @@ -80,9 +80,8 @@ int pal_thread_init (void * tcbptr) /* _DkThreadCreate for internal use. Create an internal thread inside the current process. The arguments callback and param specify the starting function and parameters */ -int _DkThreadCreate (PAL_HANDLE * handle, int (*callback) (void *), - const void * param) -{ +int _DkThreadCreate(PAL_HANDLE* handle, int (*callback) (void*), + const void* param, const void* clear_child_tid) { int ret = 0; PAL_HANDLE hdl = NULL; void * stack = malloc(THREAD_STACK_SIZE + ALT_STACK_SIZE); @@ -119,11 +118,12 @@ int _DkThreadCreate (PAL_HANDLE * handle, int (*callback) (void *), // Initialize TCB at the top of the alternative stack. PAL_TCB_LINUX * tcb = child_stack + ALT_STACK_SIZE - sizeof(PAL_TCB_LINUX); - tcb->common.self = &tcb->common; - tcb->handle = hdl; - tcb->alt_stack = child_stack; // Stack bottom - tcb->callback = callback; - tcb->param = (void *) param; + tcb->common.self = &tcb->common; + tcb->handle = hdl; + tcb->alt_stack = child_stack; // Stack bottom + tcb->callback = callback; + tcb->param = (void*) param; + tcb->clear_child_tid = (void*) clear_child_tid; /* align child_stack to 16 */ child_stack = ALIGN_DOWN_PTR(child_stack, 16); @@ -190,6 +190,13 @@ noreturn void _DkThreadExit (void) PAL_HANDLE handle = tcb->handle; block_async_signals(true); + + if (tcb->clear_child_tid) { + /* thread is ready to exit, must inform LibOS by setting *clear_child_tid to -1; + * async helper thread in LibOS is waiting on this to wake up parent */ + *tcb->clear_child_tid = -1; + } + if (tcb->alt_stack) { stack_t ss; ss.ss_sp = NULL; diff --git a/Pal/src/host/Linux/pal_linux.h b/Pal/src/host/Linux/pal_linux.h index 90ed5409..c6b5be89 100644 --- a/Pal/src/host/Linux/pal_linux.h +++ b/Pal/src/host/Linux/pal_linux.h @@ -194,9 +194,10 @@ typedef struct pal_tcb_linux { int pending_event; LISTP_TYPE(event_queue) pending_queue; PAL_HANDLE handle; - void * alt_stack; - int (*callback) (void *); - void * param; + void* alt_stack; + int (*callback) (void*); + void* param; + int* clear_child_tid; }; } PAL_TCB_LINUX; diff --git a/Pal/src/host/Skeleton/db_threading.c b/Pal/src/host/Skeleton/db_threading.c index 30be6f00..2267c022 100644 --- a/Pal/src/host/Skeleton/db_threading.c +++ b/Pal/src/host/Skeleton/db_threading.c @@ -30,7 +30,8 @@ /* _DkThreadCreate for internal use. Create an internal thread inside the current process. The arguments callback and param specify the starting function and parameters */ -int _DkThreadCreate(PAL_HANDLE* handle, int (*callback)(void*), const void* param) { +int _DkThreadCreate(PAL_HANDLE* handle, int (*callback)(void*), const void* param, + const void* clear_child_tid) { return -PAL_ERROR_NOTIMPLEMENTED; } diff --git a/Pal/src/pal.h b/Pal/src/pal.h index 986989d1..b330c546 100644 --- a/Pal/src/pal.h +++ b/Pal/src/pal.h @@ -410,7 +410,7 @@ DkStreamChangeName (PAL_HANDLE handle, PAL_STR uri); #define PAL_THREAD_MASK 0 PAL_HANDLE -DkThreadCreate (PAL_PTR addr, PAL_PTR param); +DkThreadCreate (PAL_PTR addr, PAL_PTR param, PAL_PTR clear_child_tid); // assuming duration to be in microseconds PAL_NUM diff --git a/Pal/src/pal_internal.h b/Pal/src/pal_internal.h index f9cb8b40..3820ff1e 100644 --- a/Pal/src/pal_internal.h +++ b/Pal/src/pal_internal.h @@ -292,8 +292,8 @@ int _DkReceiveHandle(PAL_HANDLE hdl, PAL_HANDLE * cargo); PAL_HANDLE _DkBroadcastStreamOpen (void); /* DkProcess and DkThread calls */ -int _DkThreadCreate (PAL_HANDLE * handle, int (*callback) (void *), - const void * param); +int _DkThreadCreate(PAL_HANDLE* handle, int (*callback) (void*), + const void* param, const void* clear_child_tid); noreturn void _DkThreadExit (void); int _DkThreadDelayExecution (unsigned long * duration); void _DkThreadYieldExecution (void); diff --git a/Pal/test/Event.c b/Pal/test/Event.c index 0df35bd0..caac9a9b 100644 --- a/Pal/test/Event.c +++ b/Pal/test/Event.c @@ -33,7 +33,7 @@ int main(int argc, char** argv) { return -1; } - thd1 = DkThreadCreate(&thread_1, 0); + thd1 = DkThreadCreate(&thread_1, 0, /*clear_child_tid=*/NULL); if (thd1 == NULL) { pal_printf("DkThreadCreate failed\n"); diff --git a/Pal/test/Fork.c b/Pal/test/Fork.c index 22befb3c..a81fa31f 100644 --- a/Pal/test/Fork.c +++ b/Pal/test/Fork.c @@ -15,7 +15,7 @@ PAL_HANDLE _fork(void* args) { if (args == NULL) { struct stack_frame cur_frame = *frame; pal_printf("return address is %p\n", cur_frame.ret); - return DkThreadCreate(&_fork, &cur_frame); + return DkThreadCreate(&_fork, &cur_frame, /*clear_child_tid=*/NULL); } else { struct stack_frame* las_frame = (struct stack_frame*)args; pal_printf("(in child) return address is %p\n", las_frame->ret); diff --git a/Pal/test/Select.c b/Pal/test/Select.c index 8abe8ade..bc636249 100644 --- a/Pal/test/Select.c +++ b/Pal/test/Select.c @@ -26,7 +26,7 @@ int main() { handles[2] = DkStreamOpen("pipe:", PAL_ACCESS_RDWR, 0, 0, 0); wakeup = handles[2]; - PAL_HANDLE thd = DkThreadCreate(&thread, NULL); + PAL_HANDLE thd = DkThreadCreate(&thread, NULL, /*clear_child_tid=*/NULL); if (thd == NULL) { pal_printf("DkThreadCreate failed\n"); diff --git a/Pal/test/Thread.c b/Pal/test/Thread.c index c4f82d19..24ed0939 100644 --- a/Pal/test/Thread.c +++ b/Pal/test/Thread.c @@ -27,14 +27,14 @@ int main() { PAL_HANDLE thd1, thd2; - thd1 = DkThreadCreate(&thread_1, NULL); + thd1 = DkThreadCreate(&thread_1, NULL, /*clear_child_tid=*/NULL); if (thd1 == NULL) { pal_printf("DkThreadCreate failed\n"); return -1; } - thd2 = DkThreadCreate(&thread_2, NULL); + thd2 = DkThreadCreate(&thread_2, NULL, /*clear_child_tid=*/NULL); if (thd2 == NULL) { pal_printf("DkThreadCreate failed\n"); diff --git a/Pal/test/Wait.c b/Pal/test/Wait.c index 26643626..bdbc6627 100644 --- a/Pal/test/Wait.c +++ b/Pal/test/Wait.c @@ -33,14 +33,14 @@ int main() { event1 = DkNotificationEventCreate(0); event2 = DkNotificationEventCreate(0); - thd1 = DkThreadCreate(&thread_1, 0); + thd1 = DkThreadCreate(&thread_1, 0, /*clear_child_tid=*/NULL); if (thd1 == NULL) { pal_printf("DkThreadCreate failed\n"); return -1; } - thd2 = DkThreadCreate(&thread_2, 0); + thd2 = DkThreadCreate(&thread_2, 0, /*clear_child_tid=*/NULL); if (thd2 == NULL) { pal_printf("DkThreadCreate failed\n"); diff --git a/Pal/test/Yield.c b/Pal/test/Yield.c index fcf5f6d8..b81819ef 100644 --- a/Pal/test/Yield.c +++ b/Pal/test/Yield.c @@ -22,7 +22,7 @@ int main(void) { pal_printf("Enter Parent Thread\n"); parent_thread = pal_control.first_thread; - child_thread = DkThreadCreate(&child, NULL); + child_thread = DkThreadCreate(&child, NULL, /*clear_child_tid=*/NULL); if (child_thread == NULL) { pal_printf("DkThreadCreate failed\n");