From dafdb1d47b800fa4496f5d5abc24809c54f543ce Mon Sep 17 00:00:00 2001
From: Isaku Yamahata <isaku.yamahata@gmail.com>
Date: Wed, 6 Nov 2019 12:22:06 -0800
Subject: [PATCH] [LibOS/regression] Multiple fixes for the futex test

- Loop on futex(WAKE) when no thread is awaken.
- Use pthread instead of raw clone() because clone() does not
  initialize Glibc TCB, thus any function call to Glibc results
  in SIGBUS.
---
 LibOS/shim/test/regression/Makefile           |  1 +
 LibOS/shim/test/regression/futex.c            | 69 +++++++------------
 .../test/regression/futex.manifest.template   |  1 +
 3 files changed, 28 insertions(+), 43 deletions(-)

diff --git a/LibOS/shim/test/regression/Makefile b/LibOS/shim/test/regression/Makefile
index a6a35a8c..8c660cb6 100644
--- a/LibOS/shim/test/regression/Makefile
+++ b/LibOS/shim/test/regression/Makefile
@@ -23,6 +23,7 @@ CFLAGS-multi_pthread = -pthread
 CFLAGS-exit_group = -pthread
 CFLAGS-abort_multithread = -pthread
 CFLAGS-eventfd = -pthread
+CFLAGS-futex = -pthread
 CFLAGS-spinlock += -I$(PALDIR)/../lib -pthread
 
 %: %.c
diff --git a/LibOS/shim/test/regression/futex.c b/LibOS/shim/test/regression/futex.c
index c98fd8c7..da045334 100644
--- a/LibOS/shim/test/regression/futex.c
+++ b/LibOS/shim/test/regression/futex.c
@@ -10,90 +10,73 @@
 #include <sys/types.h>
 #include <sys/wait.h>
 #include <unistd.h>
+#include <errno.h>
+#include <pthread.h>
 
 // 64kB stack
 #define FIBER_STACK (1024 * 64)
 #define THREADS     2
 static int myfutex = 0;
-struct atomic_int {
-    volatile int counter;
-};
-static struct atomic_int my_counter;
-
-static inline void atomic_inc(struct atomic_int* v) {
-    asm volatile("lock; incl %0" : "+m"(v->counter));
-}
 
 static int futex(int* uaddr, int futex_op, int val, const struct timespec* timeout, int* uaddr2,
                  int val3) {
     return syscall(SYS_futex, uaddr, futex_op, val, timeout, uaddr, val3);
 }
 
-int thread_function(void* argument) {
+void* thread_function(void* argument) {
     int* ptr = (int*)argument;
     int rv;
-    atomic_inc(&my_counter);
 
     // Sleep on the futex
     rv = futex(&myfutex, FUTEX_WAIT_BITSET, 0, NULL, NULL, *ptr);
     assert(rv == 0);
     // printf("child thread %d awakened\n", getpid());
-    return 0;
+    return NULL;
 }
 
 int main(int argc, const char** argv) {
-    void* stacks[THREADS];
-    pid_t pids[THREADS];
-    int varx[THREADS];
-    my_counter.counter = 0;
+    pthread_t thread[THREADS];
+    static int varx[THREADS];
 
     for (int i = 0; i < THREADS; i++) {
         varx[i] = (1 << i);
 
-        // Allocate the stack
-        stacks[i] = malloc(FIBER_STACK);
-        if (stacks[i] == 0) {
-            perror("malloc: could not allocate stack");
-            _exit(1);
-        }
-
-        // Call the clone system call to create the child thread
-        pids[i] = clone(&thread_function, (void*)stacks[i] + FIBER_STACK,
-                        CLONE_FS | CLONE_FILES | CLONE_SIGHAND | CLONE_VM | CLONE_THREAD, &varx[i]);
-
-        // printf("clone() creates new thread %d\n", pids[i]);
-
-        if (pids[i] == -1) {
-            perror("clone");
+        int ret = pthread_create(&thread[i], NULL, &thread_function, &varx[i]);
+        if (ret) {
+            errno = ret;
+            perror("pthread_create");
             _exit(2);
         }
     }
 
-    // Make sure the threads are sleeping
-    do {
-        sleep(1);
-    } while (my_counter.counter != THREADS);
-
     printf("Waking up kiddos\n");
     /* Wake in reverse order */
     for (int i = THREADS - 1; i >= 0; i--) {
-        pid_t pid;
         int rv;
         int var = (1 << i);
 
         // Wake up the thread
-        rv = futex(&myfutex, FUTEX_WAKE_BITSET, 1, NULL, NULL, var);
+        do {
+            rv = futex(&myfutex, FUTEX_WAKE_BITSET, 1, NULL, NULL, var);
+            if (rv == 0) {
+                // the thread of thread_function() may not reach
+                // futex(FUTEX_WAIT_BITSET) yet.
+                // Wait for the thread to sleep and try again.
+                // Since synchronization primitive, futex, is being tested,
+                // futex can't be used here. resort to use sleep.
+                sleep(1);
+            }
+        } while (rv == 0);
+        printf("FUTEX_WAKE_BITSET i = %d rv = %d\n", i, rv);
         assert(rv == 1);
 
         // Wait for the child thread to exit
-        pid = waitpid(pids[i], NULL, __WALL);
-        if (pid == -1) {
-            perror("waitpid");
+        int ret = pthread_join(thread[i], NULL);
+        if (ret) {
+            errno = ret;
+            perror("pthread_join");
             _exit(3);
         }
-
-        // Free the stack
-        free(stacks[i]);
     }
 
     printf("Woke all kiddos\n");
diff --git a/LibOS/shim/test/regression/futex.manifest.template b/LibOS/shim/test/regression/futex.manifest.template
index ec9a0b9c..9c2cb7d3 100644
--- a/LibOS/shim/test/regression/futex.manifest.template
+++ b/LibOS/shim/test/regression/futex.manifest.template
@@ -18,4 +18,5 @@ net.rules.2 = 0.0.0.0:0-65535:127.0.0.1:8000
 
 sgx.trusted_files.ld = file:../../../../Runtime/ld-linux-x86-64.so.2
 sgx.trusted_files.libc = file:../../../../Runtime/libc.so.6
+sgx.trusted_files.libpthread = file:../../../../Runtime/libpthread.so.0
 sgx.thread_num = 4