From 3d662b49cd79db2e086ca8294fb730da4f90c900 Mon Sep 17 00:00:00 2001 From: borysp Date: Mon, 2 Dec 2019 19:05:10 +0100 Subject: [PATCH] [LibOS] Add futexes tests --- LibOS/shim/test/regression/Makefile | 4 +- LibOS/shim/test/regression/futex-timeout.c | 21 -- .../regression/{futex.c => futex_bitset.c} | 50 +++-- ...emplate => futex_bitset.manifest.template} | 2 +- LibOS/shim/test/regression/futex_requeue.c | 141 ++++++++++++++ .../futex_requeue.manifest.template | 17 ++ LibOS/shim/test/regression/futex_timeout.c | 51 +++++ LibOS/shim/test/regression/futex_wake_op.c | 182 ++++++++++++++++++ .../futex_wake_op.manifest.template | 17 ++ LibOS/shim/test/regression/test_libos.py | 16 +- 10 files changed, 459 insertions(+), 42 deletions(-) delete mode 100644 LibOS/shim/test/regression/futex-timeout.c rename LibOS/shim/test/regression/{futex.c => futex_bitset.c} (66%) rename LibOS/shim/test/regression/{futex.manifest.template => futex_bitset.manifest.template} (97%) create mode 100644 LibOS/shim/test/regression/futex_requeue.c create mode 100644 LibOS/shim/test/regression/futex_requeue.manifest.template create mode 100644 LibOS/shim/test/regression/futex_timeout.c create mode 100644 LibOS/shim/test/regression/futex_wake_op.c create mode 100644 LibOS/shim/test/regression/futex_wake_op.manifest.template diff --git a/LibOS/shim/test/regression/Makefile b/LibOS/shim/test/regression/Makefile index 68758b34..16e805f1 100644 --- a/LibOS/shim/test/regression/Makefile +++ b/LibOS/shim/test/regression/Makefile @@ -23,7 +23,9 @@ CFLAGS-multi_pthread = -pthread CFLAGS-exit_group = -pthread CFLAGS-abort_multithread = -pthread CFLAGS-eventfd = -pthread -CFLAGS-futex = -pthread +CFLAGS-futex_bitset = -pthread +CFLAGS-futex_requeue = -pthread +CFLAGS-futex_wake_op = -pthread CFLAGS-spinlock += -I$(PALDIR)/../lib -pthread CFLAGS-sigprocmask += -pthread diff --git a/LibOS/shim/test/regression/futex-timeout.c b/LibOS/shim/test/regression/futex-timeout.c deleted file mode 100644 index 29131e7e..00000000 --- a/LibOS/shim/test/regression/futex-timeout.c +++ /dev/null @@ -1,21 +0,0 @@ -#include -#include -#include -#include -#include -#include - -int main(int argc, const char** argv) { - int myfutex = 0; - int ret; - - struct timespec t = {.tv_sec = 1, .tv_nsec = 0}; - - puts("invoke futex syscall with 1-second timeout"); - ret = syscall(SYS_futex, &myfutex, FUTEX_WAIT, 0, &t, NULL, 0); - if (ret == -1 && errno == ETIMEDOUT) { - puts("futex correctly timed out"); - } - - return 0; -} diff --git a/LibOS/shim/test/regression/futex.c b/LibOS/shim/test/regression/futex_bitset.c similarity index 66% rename from LibOS/shim/test/regression/futex.c rename to LibOS/shim/test/regression/futex_bitset.c index aa930563..281727cc 100644 --- a/LibOS/shim/test/regression/futex.c +++ b/LibOS/shim/test/regression/futex_bitset.c @@ -1,21 +1,19 @@ #define _GNU_SOURCE #include #include +#include #include -#include +#include #include #include #include +#include #include #include #include #include -#include -#include -// 64kB stack -#define FIBER_STACK (1024 * 64) -#define THREADS 2 +#define THREADS 8 static int myfutex = 0; static int futex(int* uaddr, int futex_op, int val, const struct timespec* timeout, int* uaddr2, @@ -25,18 +23,22 @@ static int futex(int* uaddr, int futex_op, int val, const struct timespec* timeo void* thread_function(void* argument) { int* ptr = (int*)argument; - int rv; + long rv; // Sleep on the futex rv = futex(&myfutex, FUTEX_WAIT_BITSET, 0, NULL, NULL, *ptr); - assert(rv == 0); - // printf("child thread %d awakened\n", getpid()); - return NULL; + + return (void*)rv; } int main(int argc, const char** argv) { pthread_t thread[THREADS]; - static int varx[THREADS]; + int varx[THREADS]; + + setbuf(stdout, NULL); + setbuf(stderr, NULL); + + static_assert(THREADS < sizeof(int) * 8 - 1, "Left shift in the loop below would overflow!"); for (int i = 0; i < THREADS; i++) { varx[i] = (1 << i); @@ -45,7 +47,7 @@ int main(int argc, const char** argv) { if (ret) { errno = ret; perror("pthread_create"); - _exit(2); + return 1; } } @@ -67,15 +69,31 @@ int main(int argc, const char** argv) { sleep(1); } } while (rv == 0); - printf("FUTEX_WAKE_BITSET i = %d rv = %d\n", i, rv); - assert(rv == 1); + printf("FUTEX_WAKE_BITSET i = %d rv = %d (expected: 1)\n", i, rv); + if (rv != 1) { + return 1; + } // Wait for the child thread to exit - int ret = pthread_join(thread[i], NULL); + intptr_t retval = 0; + int ret = pthread_join(thread[i], (void**)&retval); if (ret) { errno = ret; perror("pthread_join"); - _exit(3); + return 1; + } + if (retval != 0) { + printf("Thread %d returned %zd (%s)\n", i, retval, strerror(retval)); + return 1; + } + + if (i != 0) { + errno = 0; + ret = pthread_tryjoin_np(thread[0], (void**)&retval); + if (ret != EBUSY) { + printf("Unexpectedly pthread_tryjoin_np returned: %d (%s)\n", ret, strerror(ret)); + return 1; + } } } diff --git a/LibOS/shim/test/regression/futex.manifest.template b/LibOS/shim/test/regression/futex_bitset.manifest.template similarity index 97% rename from LibOS/shim/test/regression/futex.manifest.template rename to LibOS/shim/test/regression/futex_bitset.manifest.template index 9d8c5a1d..f6912831 100644 --- a/LibOS/shim/test/regression/futex.manifest.template +++ b/LibOS/shim/test/regression/futex_bitset.manifest.template @@ -19,4 +19,4 @@ net.rules.2 = 0.0.0.0:0-65535:127.0.0.1:8000 sgx.trusted_files.ld = file:../../../../Runtime/ld-linux-x86-64.so.2 sgx.trusted_files.libc = file:../../../../Runtime/libc.so.6 sgx.trusted_files.libpthread = file:../../../../Runtime/libpthread.so.0 -sgx.thread_num = 6 +sgx.thread_num = 16 diff --git a/LibOS/shim/test/regression/futex_requeue.c b/LibOS/shim/test/regression/futex_requeue.c new file mode 100644 index 00000000..92062cb9 --- /dev/null +++ b/LibOS/shim/test/regression/futex_requeue.c @@ -0,0 +1,141 @@ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +static int futex(int* uaddr, int futex_op, int val, const struct timespec* timeout, int* uaddr2, int val3) { + return syscall(SYS_futex, uaddr, futex_op, val, timeout, uaddr2, val3); +} + +static int futex_wait(int* uaddr, int val, const struct timespec* timeout) { + return futex(uaddr, FUTEX_WAIT | FUTEX_PRIVATE_FLAG, val, timeout, NULL, 0); +} + +static int futex_wake(int* uaddr, int to_wake) { + return futex(uaddr, FUTEX_WAKE | FUTEX_PRIVATE_FLAG, to_wake, NULL, NULL, 0); +} + +static int futex_cmp_requeue(int* uaddr, int val, int to_wake, int* uaddr2, unsigned int max_requeue) { + return futex(uaddr, FUTEX_CMP_REQUEUE | FUTEX_PRIVATE_FLAG, to_wake, (struct timespec*)(unsigned long)max_requeue, uaddr2, val); +} + +static void fail(const char* msg, int x) { + printf("%s failed with %d (%s)\n", msg, x, strerror(x)); + exit(1); +} + +static void check(int x) { + if (x) { + fail("pthread", x); + } +} + +static void store(int* ptr, int val) { + __atomic_store_n(ptr, val, __ATOMIC_SEQ_CST); +} +static int load(int* ptr) { + return __atomic_load_n(ptr, __ATOMIC_SEQ_CST); +} + +static int futex1 = 0; +static int futex2 = 0; + +#define THREADS 9 +#define THREADS_WAKE 2 +#define THREADS_REQUEUE 3 + +static int thread_state[THREADS] = { 0 }; + +static void* thread_func(void* arg) { + unsigned long i = (unsigned long)arg; + + store(&thread_state[i], 1); + + int ret = futex_wait(&futex1, futex1, NULL); + if (ret) { + printf("futex_wait in thread %lu returned %d (%s)\n", i, ret, strerror(ret)); + // skip setting state below + return arg; + } + + store(&thread_state[i], 2); + return arg; +} + +int main(void) { + pthread_t th[THREADS]; + unsigned long i; + int ret; + + for (i = 0; i < THREADS; i++) { + check(pthread_create(&th[i], NULL, thread_func, (void*)i)); + } + + // wait for all threads + for (i = 0; i < THREADS; i++) { + while (load(&thread_state[i]) != 1) { + usleep(1000u); + } + } + // and let them sleep on futex + usleep(100000u); + + ret = futex_cmp_requeue(&futex1, futex1, THREADS_WAKE, &futex2, THREADS_REQUEUE); + if (ret < 0) { + fail("futex_cmp_requeue", errno); + } + if (ret != THREADS_WAKE + THREADS_REQUEUE) { + printf("futex_cmp_requeue returned %d instead of %d!\n", ret, THREADS_WAKE + THREADS_REQUEUE); + return 1; + } + + // let the woken thread(s) end + usleep(100000u); + + ret = 0; + for (i = 0; i < THREADS; i++) { + if (load(&thread_state[i]) == 2) { + ret++; + check(pthread_join(th[i], NULL)); + store(&thread_state[i], 3); + } + } + if (ret != THREADS_WAKE) { + printf("futex_cmp_requeue woke-up %d threads instead of %d!\n", ret, THREADS_WAKE); + return 1; + } + + ret = futex_wake(&futex1, INT_MAX); + if (ret < 0) { + fail("futex_wake(&futex1)", errno); + } + if (ret != (THREADS - THREADS_WAKE - THREADS_REQUEUE)) { + printf("futex_wake on futex1 woke-up %d threads instead of %d!\n", ret, THREADS - THREADS_WAKE - THREADS_REQUEUE); + return 1; + } + + ret = futex_wake(&futex2, INT_MAX); + if (ret < 0) { + fail("futex_wake(&futex2)", errno); + } + if (ret != THREADS_REQUEUE) { + printf("futex_wake on futex2 woke-up %d threads instead of %d!\n", ret, THREADS_REQUEUE); + return 1; + } + + for (i = 0; i < THREADS; i++) { + if (load(&thread_state[i]) != 3) { + check(pthread_join(th[i], NULL)); + } + } + + puts("Test successful!"); + return 0; +} diff --git a/LibOS/shim/test/regression/futex_requeue.manifest.template b/LibOS/shim/test/regression/futex_requeue.manifest.template new file mode 100644 index 00000000..8e21856e --- /dev/null +++ b/LibOS/shim/test/regression/futex_requeue.manifest.template @@ -0,0 +1,17 @@ +loader.preload = file:../../src/libsysdb.so +loader.env.LD_LIBRARY_PATH = /lib +loader.debug_type = none +loader.syscall_symbol = syscalldb + +fs.mount.lib.type = chroot +fs.mount.lib.path = /lib +fs.mount.lib.uri = file:../../../../Runtime + +fs.mount.bin.type = chroot +fs.mount.bin.path = /bin +fs.mount.bin.uri = file:/bin + +sgx.trusted_files.ld = file:../../../../Runtime/ld-linux-x86-64.so.2 +sgx.trusted_files.libc = file:../../../../Runtime/libc.so.6 +sgx.trusted_files.libpthread = file:../../../../Runtime/libpthread.so.0 +sgx.thread_num = 16 diff --git a/LibOS/shim/test/regression/futex_timeout.c b/LibOS/shim/test/regression/futex_timeout.c new file mode 100644 index 00000000..9f44c9ec --- /dev/null +++ b/LibOS/shim/test/regression/futex_timeout.c @@ -0,0 +1,51 @@ +#include +#include +#include +#include +#include +#include +#include + +#define SLEEP_SEC 1 + +int main(int argc, const char** argv) { + int myfutex = 0; + int ret; + int futex_errno = 0; + + struct timespec t = {.tv_sec = SLEEP_SEC, .tv_nsec = 0}; + struct timeval tv1 = { 0 }; + struct timeval tv2 = { 0 }; + + printf("invoke futex syscall with a %d second timeout\n", SLEEP_SEC); + if (gettimeofday(&tv1, NULL)) { + printf("Cannot get time 1: %m\n"); + return 1; + } + ret = syscall(SYS_futex, &myfutex, FUTEX_WAIT, 0, &t, NULL, 0); + futex_errno = errno; + if (gettimeofday(&tv2, NULL)) { + printf("Cannot get time 2: %m\n"); + return 1; + } + + if (ret != -1 || futex_errno != ETIMEDOUT) { + printf("futex syscall returned: %d with errno: %d (%s)\n", ret, futex_errno, strerror(futex_errno)); + return 1; + } + + long long diff = (tv2.tv_sec - tv1.tv_sec) * 1000000ll; + diff += tv2.tv_usec - tv1.tv_usec; + + if (diff < 0) { + printf("Just moved back in time (%lld), better call Ghostbusters!\n", diff); + return 1; + } + if (diff < 1000000ll * SLEEP_SEC) { + printf("Slept for %lld microseconds, which is less than %d seconds\n", diff, SLEEP_SEC); + return 1; + } + + puts("futex correctly timed out"); + return 0; +} diff --git a/LibOS/shim/test/regression/futex_wake_op.c b/LibOS/shim/test/regression/futex_wake_op.c new file mode 100644 index 00000000..58816a06 --- /dev/null +++ b/LibOS/shim/test/regression/futex_wake_op.c @@ -0,0 +1,182 @@ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +static int futex(int* uaddr, int futex_op, int val, const struct timespec* timeout, int* uaddr2, int val3) { + return syscall(SYS_futex, uaddr, futex_op, val, timeout, uaddr2, val3); +} + +static int futex_wait(int* uaddr, int val, const struct timespec* timeout) { + return futex(uaddr, FUTEX_WAIT | FUTEX_PRIVATE_FLAG, val, timeout, NULL, 0); +} + +static int futex_wake(int* uaddr, int to_wake) { + return futex(uaddr, FUTEX_WAKE | FUTEX_PRIVATE_FLAG, to_wake, NULL, NULL, 0); +} + +static int futex_wake_op(int* uaddr1, int to_wake1, int* uaddr2, int to_wake2, int op) { + return futex(uaddr1, FUTEX_WAKE_OP | FUTEX_PRIVATE_FLAG, to_wake1, (struct timespec*)(unsigned long)to_wake2, uaddr2, op); +} + +static void fail(const char* msg, int x) { + printf("%s failed with %d (%s)\n", msg, x, strerror(x)); + exit(1); +} + +static void check(int x) { + if (x) { + fail("pthread", x); + } +} + +static void store(int* ptr, int val) { + __atomic_store_n(ptr, val, __ATOMIC_SEQ_CST); +} +static int load(int* ptr) { + return __atomic_load_n(ptr, __ATOMIC_SEQ_CST); +} + +static int wakeop_arg_extend(int x) { + if (x >= 0x800) { + return 0xfffff000 | x; + } + return x; +} + +static int futex1 = 0; +static int futex2 = 0; + +#define THREADS1 4 +#define THREADS2 5 +#define THREADS_WAKE1 2 +#define THREADS_WAKE2 3 + +static int thread_state[THREADS1 + THREADS2] = { 0 }; + +static void* thread_func(void* arg) { + unsigned long i = (unsigned long)arg; + int ret = -1; + + store(&thread_state[i], 1); + + if (i < THREADS1) { + ret = futex_wait(&futex1, futex1, NULL); + } else { + ret = futex_wait(&futex2, futex2, NULL); + } + if (ret != 0) { + printf("futex_wait in thread %lu returned %d (%s)\n", i, ret, strerror(ret)); + // skip setting state below + return arg; + } + + store(&thread_state[i], 2); + return arg; +} + +int main(void) { + pthread_t th[THREADS1 + THREADS2]; + unsigned long i; + int ret; + int arg1 = 0x123846; + int arg2 = 0xc73; + int cmparg = 0x746; // so that arg1 >= cmparg + int op = FUTEX_OP(FUTEX_OP_XOR, arg2, FUTEX_OP_CMP_GE, cmparg); + + store(&futex2, arg1); + + for (i = 0; i < THREADS1 + THREADS2; i++) { + check(pthread_create(&th[i], NULL, thread_func, (void*)i)); + } + + // wait for all threads + for (i = 0; i < THREADS1 + THREADS2; i++) { + while (load(&thread_state[i]) != 1) { + usleep(1000u); + } + } + // and let them sleep on futex + usleep(100000u); + + ret = futex_wake_op(&futex1, THREADS_WAKE1, &futex2, THREADS_WAKE2, op); + + arg2 = wakeop_arg_extend(arg2); + + op = load(&futex2); + if (op != (arg1 ^ arg2)) { + printf("futex_wake_op did not set futex2 value correctly: current value: 0x%x, expected: 0x%x\n", op, arg1 ^ arg2); + return 1; + } + + if (ret < 0) { + fail("futex_wake_op", errno); + } + if (ret != THREADS_WAKE1 + THREADS_WAKE2) { + printf("futex_wake_op returned %d instead of %d!\n", ret, THREADS_WAKE1 + THREADS_WAKE2); + return 1; + } + + // let the woken thread(s) end + usleep(100000u); + + ret = 0; + for (i = 0; i < THREADS1; i++) { + if (load(&thread_state[i]) == 2) { + ret++; + check(pthread_join(th[i], NULL)); + store(&thread_state[i], 3); + } + } + if (ret != THREADS_WAKE1) { + printf("futex_wake_op woke-up %d threads on futex1 instead of %d!\n", ret, THREADS_WAKE1); + return 1; + } + + ret = 0; + for (i = THREADS1; i < THREADS1 + THREADS2; i++) { + if (load(&thread_state[i]) == 2) { + ret++; + check(pthread_join(th[i], NULL)); + store(&thread_state[i], 3); + } + } + if (ret != THREADS_WAKE2) { + printf("futex_wake_op woke-up %d threads on futex2 instead of %d!\n", ret, THREADS_WAKE2); + return 1; + } + + ret = futex_wake(&futex1, INT_MAX); + if (ret < 0) { + fail("futex_wake(&futex1)", errno); + } + if (ret != (THREADS1 - THREADS_WAKE1)) { + printf("futex_wake on futex1 woke-up %d threads instead of %d!\n", ret, THREADS1 - THREADS_WAKE1); + return 1; + } + + ret = futex_wake(&futex2, INT_MAX); + if (ret < 0) { + fail("futex_wake(&futex2)", errno); + } + if (ret != (THREADS2 - THREADS_WAKE2)) { + printf("futex_wake on futex2 woke-up %d threads instead of %d!\n", ret, THREADS2 - THREADS_WAKE2); + return 1; + } + + for (i = 0; i < THREADS1 + THREADS2; i++) { + if (load(&thread_state[i]) != 3) { + check(pthread_join(th[i], NULL)); + } + } + + puts("Test successful!"); + return 0; +} diff --git a/LibOS/shim/test/regression/futex_wake_op.manifest.template b/LibOS/shim/test/regression/futex_wake_op.manifest.template new file mode 100644 index 00000000..8e21856e --- /dev/null +++ b/LibOS/shim/test/regression/futex_wake_op.manifest.template @@ -0,0 +1,17 @@ +loader.preload = file:../../src/libsysdb.so +loader.env.LD_LIBRARY_PATH = /lib +loader.debug_type = none +loader.syscall_symbol = syscalldb + +fs.mount.lib.type = chroot +fs.mount.lib.path = /lib +fs.mount.lib.uri = file:../../../../Runtime + +fs.mount.bin.type = chroot +fs.mount.bin.path = /bin +fs.mount.bin.uri = file:/bin + +sgx.trusted_files.ld = file:../../../../Runtime/ld-linux-x86-64.so.2 +sgx.trusted_files.libc = file:../../../../Runtime/libc.so.6 +sgx.trusted_files.libpthread = file:../../../../Runtime/libpthread.so.0 +sgx.thread_num = 16 diff --git a/LibOS/shim/test/regression/test_libos.py b/LibOS/shim/test/regression/test_libos.py index c4dd9537..afff8e9f 100644 --- a/LibOS/shim/test/regression/test_libos.py +++ b/LibOS/shim/test/regression/test_libos.py @@ -263,18 +263,28 @@ class TC_30_Syscall(RegressionTestCase): # fopen corner cases self.assertIn('Successfully read from file: Hello World', stdout) - def test_040_futex_wake(self): - stdout, stderr = self.run_binary(['futex']) + def test_040_futex_bitset(self): + stdout, _ = self.run_binary(['futex_bitset']) # Futex Wake Test self.assertIn('Woke all kiddos', stdout) def test_041_futex_timeout(self): - stdout, stderr = self.run_binary(['futex-timeout']) + stdout, _ = self.run_binary(['futex_timeout']) # Futex Timeout Test self.assertIn('futex correctly timed out', stdout) + def test_042_futex_requeue(self): + stdout, _ = self.run_binary(['futex_requeue']) + + self.assertIn('Test successful!', stdout) + + def test_043_futex_wake_op(self): + stdout, _ = self.run_binary(['futex_wake_op']) + + self.assertIn('Test successful!', stdout) + def test_050_mmap(self): stdout, stderr = self.run_binary(['mmap-file'], timeout=60)