[Pal/Linux-SGX] Correctly emulate RDTSC and RDTSCP instructions

Under SGX PAL, if the CPU doesn't support RDTSC/RDTSCP inside SGX
enclave, Graphene uses trap-and-emulate on these instructions.
Previously, Graphene only emulated RDTSC and not RDTSCP. Moreover,
Graphene emulated RDTSC by simply returning zeros, which could lead
to faults in applications not expecting a zero value. This commit
emulates (imprecisely) both RDTSC and RDTSCP via gettime() syscall.
New LibOS test is added (SGX-only).

LibOS/shim/test/regression/.gitignore

@@ -57,6 +57,7 @@
 /proc_cpuinfo
 /proc_path
 /pselect
+/rdtsc
 /readdir
 /sched
 /select

LibOS/shim/test/regression/Makefile

@@ -1,3 +1,5 @@
+include ../../../../Scripts/Makefile.configs
+
 c_executables = \
 	abort \
 	abort_multithread \
@@ -69,6 +71,11 @@ c_executables = \
 	unix \
 	vfork_and_exec
 
+ifeq ($(ARCH),x86_64)
+c_executables += \
+	rdtsc
+endif
+
 cxx_executables = bootstrap_c++
 
 manifests = \
@@ -120,7 +127,6 @@ extra_rules = \
 	-e 's:\$$(RA_CLIENT_SPID):$(if $(RA_CLIENT_SPID),$(RA_CLIENT_SPID),AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA):g' \
 	-e 's:\$$(RA_CLIENT_LINKABLE):$(if $(RA_CLIENT_LINKABLE),$(RA_CLIENT_LINKABLE),0):g'
 
-include ../../../../Scripts/Makefile.configs
 include ../../../../Scripts/Makefile.manifest
 include ../../../../Scripts/Makefile.Test
 

LibOS/shim/test/regression/rdtsc.c

@@ -0,0 +1,49 @@
+#include <stdint.h>
+#include <stdio.h>
+#include <time.h>
+#include <unistd.h>
+
+static uint64_t rdtsc_start(void) {
+    uint32_t low, high;
+    __asm__ volatile(
+            "mfence; lfence;" /* memory barriers to sync RDTSC */
+            "rdtsc;"
+            : "=d"(high), "=a"(low) : : "memory");
+    return (low | ((uint64_t)high) << 32);
+}
+
+static uint64_t rdtsc_end(void) {
+    uint32_t low, high;
+    __asm__ volatile(
+            "rdtscp;"        /* use RDTSCP instead of RDTSC, just for testing */
+            "lfence;"        /* memory barrier to sync RDTSCP */
+            : "=d"(high), "=a"(low) : : "%rcx", "memory");
+    return (low | ((uint64_t)high) << 32);
+}
+
+int main(int argc, char** argv) {
+    int tries = 0;
+    uint64_t start = rdtsc_start();
+    while (1) {
+        /* sleep may return prematurely, make sure we sleep for 1s at least once */
+        unsigned int remaining = sleep(1);
+        if (!remaining)
+            break;
+        if (tries++ == 100) {
+            /* give up after several tries */
+            fprintf(stderr, "failed to sleep for 1 second!\n");
+            return 1;
+        }
+    }
+    uint64_t end = rdtsc_end();
+
+    if (end < start || end - start < 1000000) {
+        /* after a sleep of 1s, there must have been at least 1M ticks, otherwise smth wrong */
+        fprintf(stderr, "RDTSC difference doesn't correspond to sleep of 1 second!\n");
+        return 1;
+    }
+
+    printf("RDTSC: start = %lu, end = %lu, diff = %lu\n", start, end, end - start);
+    puts("TEST OK!");
+    return 0;
+}

LibOS/shim/test/regression/test_libos.py

@@ -7,6 +7,7 @@ import subprocess
 
 from regression import (
     HAS_SGX,
+    ON_X86,
     RegressionTestCase,
 )
 
@@ -643,3 +644,11 @@ class TC_90_CpuidSGX(RegressionTestCase):
     def test_000_cpuid(self):
         stdout, _ = self.run_binary(['cpuid'])
         self.assertIn('CPUID test passed.', stdout)
+
+# note that `rdtsc` also correctly runs on non-SGX PAL, but non-SGX CPU may not have rdtscp
+@unittest.skipUnless(HAS_SGX,
+    'This test is only meaningful on SGX PAL because only SGX emulates RDTSC/RDTSCP.')
+class TC_91_RdtscSGX(RegressionTestCase):
+    def test_000_rdtsc(self):
+        stdout, _ = self.run_binary(['rdtsc'])
+        self.assertIn('TEST OK', stdout)

Pal/src/host/Linux-SGX/db_exception.c

@@ -109,6 +109,19 @@ static void save_pal_context(PAL_CONTEXT* ctx, sgx_cpu_context_t* uc,
     }
 }
 
+static void emulate_rdtsc_and_print_warning(sgx_cpu_context_t* uc) {
+    static int first = 0;
+    if (__atomic_exchange_n(&first, 1, __ATOMIC_RELAXED) == 0) {
+        SGX_DBG(DBG_E, "WARNING: all RDTSC/RDTSCP instructions are emulated (imprecisely) via "
+                       "gettime() syscall.\n");
+    }
+
+    /* FIXME: Ideally, we would like to scale microseconds back to RDTSC clock cycles */
+    uint64_t usec = _DkSystemTimeQuery();
+    uc->rdx = (uint32_t)(usec >> 32);
+    uc->rax = (uint32_t)usec;
+}
+
 /* return value: true if #UD was handled and execution can be continued without propagating #UD;
  *               false if #UD was not handled and exception needs to be raised up to LibOS/app */
 static bool handle_ud(sgx_cpu_context_t* uc) {
@@ -126,9 +139,14 @@ static bool handle_ud(sgx_cpu_context_t* uc) {
         }
     } else if (instr[0] == 0x0f && instr[1] == 0x31) {
         /* rdtsc */
+        emulate_rdtsc_and_print_warning(uc);
         uc->rip += 2;
-        uc->rdx = 0;
-        uc->rax = 0;
+        return true;
+    }  else if (instr[0] == 0x0f && instr[1] == 0x01 && instr[2] == 0xf9) {
+        /* rdtscp */
+        emulate_rdtsc_and_print_warning(uc);
+        uc->rip += 3;
+        uc->rcx = 0; /* dummy IA32_TSC_AUX; Linux encodes it as (numa_id << 12) | cpu_id */
         return true;
     } else if (instr[0] == 0x0f && instr[1] == 0x05) {
         /* syscall: LibOS may know how to handle this */

Pal/src/host/Linux-SGX/db_misc.c

@@ -33,6 +33,8 @@ unsigned long _DkSystemTimeQuery(void) {
     int ret = ocall_gettime(&microsec);
     if (ret)
         return -PAL_ERROR_DENIED;
+
+    /* TODO: result comes from the untrusted host, introduce some schielding */
     return microsec;
 }