[LibOS,Pal] Add MSG_DONTWAIT and SCM_RIGHTS for recvmsg/sendmsg

This commit improves the emulation of recvfrom/sendfrom,
recvmsg/sendmsg, and recvmmsg/sendmmsg system calls. In particular,
MSG_DONTWAIT flag is allowed though not really emulated (benign in
most cases). Also, it is possible now to send/receive FDs via
SCM_RIGHTS on a UNIX domain socket (only send/recv of pipes and UNIX
domain sockets is currently supported). Corresponding LibOS test
is added.

LibOS/shim/include/shim_table.h

@@ -490,11 +490,11 @@ int shim_do_epoll_create1(int flags);
 int shim_do_pipe2(int* fildes, int flags);
 int shim_do_mknod(const char *pathname, mode_t mode, dev_t dev);
 int shim_do_mknodat(int dirfd, const char *pathname, mode_t mode, dev_t dev);
-ssize_t shim_do_recvmmsg(int sockfd, struct mmsghdr* msg, unsigned int vlen, int flags,
-                         struct __kernel_timespec* timeout);
+int shim_do_recvmmsg(int sockfd, struct mmsghdr* msg, unsigned int vlen, int flags,
+                     struct __kernel_timespec* timeout);
 int shim_do_prlimit64(pid_t pid, int resource, const struct __kernel_rlimit64* new_rlim,
                       struct __kernel_rlimit64* old_rlim);
-ssize_t shim_do_sendmmsg(int sockfd, struct mmsghdr* msg, unsigned int vlen, int flags);
+int shim_do_sendmmsg(int sockfd, struct mmsghdr* msg, unsigned int vlen, int flags);
 int shim_do_eventfd2(unsigned int count, int flags);
 int shim_do_eventfd(unsigned int count);
 
@@ -816,11 +816,11 @@ int shim_pwritev(unsigned long fd, const struct iovec* vec, unsigned long vlen,
 int shim_rt_tgsigqueueinfo(pid_t tgid, pid_t pid, int sig, siginfo_t* uinfo);
 int shim_perf_event_open(struct perf_event_attr* attr_uptr, pid_t pid, int cpu, int group_fd,
                          int flags);
-ssize_t shim_recvmmsg(int sockfd, struct mmsghdr* msg, unsigned int vlen, int flags,
-                      struct __kernel_timespec* timeout);
+int shim_recvmmsg(int sockfd, struct mmsghdr* msg, unsigned int vlen, int flags,
+                  struct __kernel_timespec* timeout);
 int shim_prlimit64(pid_t pid, int resource, const struct __kernel_rlimit64* new_rlim,
                    struct __kernel_rlimit64* old_rlim);
-ssize_t shim_sendmmsg(int sockfd, struct mmsghdr* msg, unsigned int vlen, int flags);
+int shim_sendmmsg(int sockfd, struct mmsghdr* msg, unsigned int vlen, int flags);
 
 /* libos call wrappers */
 int shim_msgpersist(int msqid, int cmd);

LibOS/shim/include/shim_types.h

@@ -333,10 +333,12 @@ struct __kernel_ustat
 /* bits/socket.h */
 enum
 {
-    MSG_OOB  = 0x01, /* Process out-of-band data. */
-    MSG_PEEK = 0x02, /* Peek at incoming messages. */
+    MSG_OOB      = 0x01, /* Process out-of-band data. */
+    MSG_PEEK     = 0x02, /* Peek at incoming messages. */
+    MSG_DONTWAIT = 0x40, /* Nonblocking IO.  */
 #define MSG_OOB MSG_OOB
 #define MSG_PEEK MSG_PEEK
+#define MSG_DONTWAIT MSG_DONTWAIT
 };
 
 struct msghdr {
@@ -358,6 +360,17 @@ struct mmsghdr {
     unsigned int msg_len;   /* Number of received bytes for the entry.  */
 };
 
+/* Structure used for storage of ancillary data object information. */
+struct cmsghdr {
+    size_t cmsg_len;
+    int cmsg_level;
+    int cmsg_type;
+};
+
+#ifndef SCM_RIGHTS
+#define SCM_RIGHTS 1
+#endif
+
 /* POSIX.1g specifies this type name for the `sa_family' member.  */
 typedef unsigned short int sa_family_t;
 

LibOS/shim/include/shim_utils.h

@@ -211,6 +211,7 @@ int create_pipe(char* name, char* uri, size_t size, PAL_HANDLE* hdl, struct shim
 int create_dir(const char* prefix, char* path, size_t size, struct shim_handle** hdl);
 int create_file(const char* prefix, char* path, size_t size, struct shim_handle** hdl);
 int create_handle(const char* prefix, char* path, size_t size, PAL_HANDLE* hdl, unsigned int* id);
+int bind_dummy_socket_to_pal_handle(PAL_HANDLE pal_hdl, struct shim_handle** out_hdl);
 
 /* Asynchronous event support */
 int init_async(void);

LibOS/shim/src/shim_syscalls.c

@@ -1008,7 +1008,7 @@ SHIM_SYSCALL_RETURN_ENOSYS(rt_tgsigqueueinfo, 4, int, pid_t, tgid, pid_t, pid, i
 SHIM_SYSCALL_RETURN_ENOSYS(perf_event_open, 5, int, struct perf_event_attr*, attr_uptr, pid_t, pid,
                            int, cpu, int, group_fd, int, flags)
 
-DEFINE_SHIM_SYSCALL(recvmmsg, 5, shim_do_recvmmsg, ssize_t, int, fd, struct mmsghdr*, msg,
+DEFINE_SHIM_SYSCALL(recvmmsg, 5, shim_do_recvmmsg, int, int, fd, struct mmsghdr*, msg,
                     unsigned int, vlen, int, flags, struct __kernel_timespec*, timeout)
 
 SHIM_SYSCALL_RETURN_ENOSYS(fanotify_init, 2, int, int, flags, int, event_f_flags)
@@ -1029,7 +1029,7 @@ SHIM_SYSCALL_RETURN_ENOSYS(clock_adjtime, 2, int, clockid_t, which_clock, struct
 
 SHIM_SYSCALL_RETURN_ENOSYS(syncfs, 1, int, int, fd)
 
-DEFINE_SHIM_SYSCALL(sendmmsg, 4, shim_do_sendmmsg, ssize_t, int, fd, struct mmsghdr*, msg,
+DEFINE_SHIM_SYSCALL(sendmmsg, 4, shim_do_sendmmsg, int, int, fd, struct mmsghdr*, msg,
                     unsigned int, vlen, int, flags)
 
 SHIM_SYSCALL_RETURN_ENOSYS(setns, 2, int, int, fd, int, nstype)

LibOS/shim/src/sys/shim_pipe.c

@@ -33,6 +33,43 @@
 #include "shim_types.h"
 #include "shim_utils.h"
 
+/* TODO: create a "dummy" shim handle as in socketpair() for SCM_RIGHTS emulation;
+ *       remove this once Graphene has proper serialization of shim_handles */
+int bind_dummy_socket_to_pal_handle(PAL_HANDLE pal_hdl, struct shim_handle** out_hdl) {
+    switch (PAL_GET_TYPE(pal_hdl)) {
+        case pal_type_pipe:
+        case pal_type_pipesrv:
+        case pal_type_pipecli:
+            /* only underlying PAL pipes are allowed */
+            break;
+        default:
+            DkObjectClose(pal_hdl);
+            return -EINVAL;
+    }
+
+    struct shim_handle* hdl = get_new_handle();
+    if (!hdl) {
+        DkObjectClose(pal_hdl);
+        return -ENOMEM;
+    }
+
+    struct shim_sock_handle* sock = &hdl->info.sock;
+
+    hdl->type = TYPE_SOCK;
+    set_handle_fs(hdl, &socket_builtin_fs);
+    hdl->pal_handle  = pal_hdl;
+    hdl->flags       = O_RDWR;
+    hdl->acc_mode    = MAY_READ | MAY_WRITE;
+    sock->domain     = AF_UNIX;
+    sock->sock_type  = SOCK_STREAM;
+    sock->protocol   = 0;
+    sock->sock_state = SOCK_CONNECTED;
+    /* we don't set URI/name: they don't matter for already opened pipes/sockets */
+
+    *out_hdl = hdl;
+    return 0;
+}
+
 static int create_pipes(PAL_HANDLE* srv, PAL_HANDLE* cli, int flags, char* name,
                         struct shim_qstr* qstr) {
     int ret = 0;

LibOS/shim/src/sys/shim_socket.c

@@ -59,6 +59,10 @@
 
 #define AF_UNSPEC 0
 
+/* macros for emulating send/recv of ancillary data */
+#define HANDLES_SCM_RIGHTS_MAX 32
+#define HANDLES_SCM_RIGHTS_HDR "GRAPHENE_SCM_RIGHTS:"
+
 static int rebase_on_lo __attribute_migratable = -1;
 
 static size_t minimal_addrlen(int domain) {
@@ -1016,10 +1020,14 @@ int shim_do_accept4(int fd, struct sockaddr* addr, int* addrlen, int flags) {
 }
 
 static ssize_t do_sendmsg(int fd, struct iovec* bufs, int nbufs, int flags,
-                          const struct sockaddr* addr, int addrlen) {
+                          const struct sockaddr* addr, int addrlen,
+                          void* msg_control, size_t msg_controllen) {
     // Issue #752 - https://github.com/oscarlab/graphene/issues/752
     __UNUSED(flags);
 
+    struct shim_handle* hdls_to_send[HANDLES_SCM_RIGHTS_MAX];
+    size_t hdls_to_send_cnt = 0;
+
     struct shim_handle* hdl = get_fd_handle(fd, NULL, NULL);
     if (!hdl)
         return -EBADF;
@@ -1042,12 +1050,51 @@ static ssize_t do_sendmsg(int fd, struct iovec* bufs, int nbufs, int flags,
             goto out;
     }
 
+    if (msg_control && test_user_memory(msg_control, msg_controllen, /*write=*/false))
+        goto out;
+
     lock(&hdl->lock);
 
     PAL_HANDLE pal_hdl = hdl->pal_handle;
     char* uri          = NULL;
 
-    /* Data gram sock need not be conneted or bound at all */
+    if (msg_control) {
+        /* prepare ancillary data to be sent (we support only SCM_RIGHTS on UNIX sockets) */
+        if (sock->domain != AF_UNIX) {
+            ret = -EINVAL;
+            goto out_locked;
+        }
+
+        struct cmsghdr* cmsg = (struct cmsghdr*)msg_control;
+        while ((char*)cmsg < (char*)msg_control + msg_controllen) {
+            ssize_t cmsg_data_len = cmsg->cmsg_len - sizeof(*cmsg);
+            if (cmsg_data_len <= 0 || cmsg->cmsg_level != SOL_SOCKET
+                    || cmsg->cmsg_type != SCM_RIGHTS) {
+                ret = -EINVAL;
+                goto out_locked;
+            }
+
+            int* fds_to_send       = (int*)((char*)cmsg + sizeof(*cmsg));
+            size_t fds_to_send_cnt = cmsg_data_len / sizeof(*fds_to_send);
+
+            for (size_t i = 0; i < fds_to_send_cnt; i++) {
+                if (hdls_to_send_cnt == HANDLES_SCM_RIGHTS_MAX) {
+                    ret = -EINVAL;
+                    goto out_locked;
+                }
+
+                struct shim_handle* hdl_to_send = get_fd_handle(fds_to_send[i], NULL, NULL);
+                if (!hdl_to_send) {
+                    ret = -EBADF;
+                    goto out_locked;
+                }
+                hdls_to_send[hdls_to_send_cnt++] = hdl_to_send;
+            }
+
+            cmsg = (struct cmsghdr*)((char*)cmsg + ALIGN_UP(cmsg->cmsg_len, sizeof(size_t)));
+        }
+    }
+
     if (sock->sock_type == SOCK_STREAM && sock->sock_state != SOCK_CONNECTED &&
         sock->sock_state != SOCK_BOUNDCONNECTED && sock->sock_state != SOCK_ACCEPTED) {
         ret = -ENOTCONN;
@@ -1109,6 +1156,32 @@ static ssize_t do_sendmsg(int fd, struct iovec* bufs, int nbufs, int flags,
         debug("next packet send to %s\n", uri);
     }
 
+    /* send ancillary data: first header with number of handles to send, then each handle */
+    if (hdls_to_send_cnt) {
+        char hdls_to_send_hdr[sizeof(HANDLES_SCM_RIGHTS_HDR) + sizeof(hdls_to_send_cnt)];
+        memcpy(hdls_to_send_hdr, HANDLES_SCM_RIGHTS_HDR, sizeof(HANDLES_SCM_RIGHTS_HDR));
+        memcpy(hdls_to_send_hdr + sizeof(HANDLES_SCM_RIGHTS_HDR), &hdls_to_send_cnt,
+               sizeof(hdls_to_send_cnt));
+
+        PAL_NUM pal_ret = DkStreamWrite(pal_hdl, /*offset=*/0, sizeof(hdls_to_send_hdr),
+                                        hdls_to_send_hdr, uri);
+        if (pal_ret == PAL_STREAM_ERROR) {
+            ret = (PAL_NATIVE_ERRNO == PAL_ERROR_STREAMEXIST) ? -ECONNABORTED : -PAL_ERRNO;
+            lock(&hdl->lock);
+            goto out_locked;
+        }
+        assert(pal_ret == sizeof(hdls_to_send_hdr));  /* always true for UNIX sockets */
+
+        for (size_t i = 0; i < hdls_to_send_cnt; i++) {
+            if (!DkSendHandle(pal_hdl, hdls_to_send[i]->pal_handle)) {
+                ret = -PAL_ERRNO;
+                lock(&hdl->lock);
+                goto out_locked;
+            }
+        }
+    }
+
+    /* send regular data */
     int bytes = 0;
     ret       = 0;
 
@@ -1137,6 +1210,8 @@ out_locked:
 
     unlock(&hdl->lock);
 out:
+    for (size_t i = 0; i < hdls_to_send_cnt; i++)
+        put_handle(hdls_to_send[i]);
     put_handle(hdl);
     return ret;
 }
@@ -1147,28 +1222,28 @@ ssize_t shim_do_sendto(int sockfd, const void* buf, size_t len, int flags,
     iovbuf.iov_base = (void*)buf;
     iovbuf.iov_len  = len;
 
-    return do_sendmsg(sockfd, &iovbuf, 1, flags, addr, addrlen);
+    return do_sendmsg(sockfd, &iovbuf, 1, flags, addr, addrlen, NULL, 0);
 }
 
 ssize_t shim_do_sendmsg(int sockfd, struct msghdr* msg, int flags) {
-    return do_sendmsg(sockfd, msg->msg_iov, msg->msg_iovlen, flags, msg->msg_name,
-                      msg->msg_namelen);
-}
-
-ssize_t shim_do_sendmmsg(int sockfd, struct mmsghdr* msg, unsigned int vlen, int flags) {
-    if (test_user_memory(msg, vlen, /*write=*/true))
+    if (test_user_memory(msg, sizeof(*msg), /*write=*/false))
         return -EFAULT;
 
-    ssize_t total = 0;
-    for (size_t i = 0; i * sizeof(struct mmsghdr) < vlen; i++) {
-        struct msghdr* m = &msg[i].msg_hdr;
+    return do_sendmsg(sockfd, msg->msg_iov, msg->msg_iovlen, flags, msg->msg_name,
+                      msg->msg_namelen, msg->msg_control, msg->msg_controllen);
+}
 
-        ssize_t bytes =
-            do_sendmsg(sockfd, m->msg_iov, m->msg_iovlen, flags, m->msg_name, m->msg_namelen);
+int shim_do_sendmmsg(int sockfd, struct mmsghdr* msgvec, unsigned int vlen, int flags) {
+    if (test_user_memory(msgvec, vlen * sizeof(*msgvec), /*write=*/true))
+        return -EFAULT;
+
+    int total = 0;
+    for (size_t i = 0; i < vlen; i++) {
+        ssize_t bytes = shim_do_sendmsg(sockfd, &msgvec[i].msg_hdr, flags);
         if (bytes < 0)
             return total > 0 ? total : bytes;
 
-        msg[i].msg_len = bytes;
+        msgvec[i].msg_len = bytes;
         total++;
     }
 
@@ -1176,7 +1251,13 @@ ssize_t shim_do_sendmmsg(int sockfd, struct mmsghdr* msg, unsigned int vlen, int
 }
 
 static ssize_t do_recvmsg(int fd, struct iovec* bufs, size_t nbufs, int flags,
-                          struct sockaddr* addr, int* addrlen) {
+                          struct sockaddr* addr, int* addrlen,
+                          void* msg_control, size_t* msg_controllen) {
+    size_t hdls_to_recv_cnt = 0;
+
+    struct shim_handle* received_hdls[HANDLES_SCM_RIGHTS_MAX];
+    size_t received_hdls_cnt = 0;
+
     struct shim_handle* hdl = get_fd_handle(fd, NULL, NULL);
     if (!hdl)
         return -EBADF;
@@ -1220,12 +1301,24 @@ static ssize_t do_recvmsg(int fd, struct iovec* bufs, size_t nbufs, int flags,
         expected_size += bufs[i].iov_len;
     }
 
-    if (flags & ~MSG_PEEK) {
-        debug("recvmsg()/recvmmsg()/recvfrom(): unknown flag (only MSG_PEEK is supported).\n");
+    if (msg_control && msg_controllen && test_user_memory(msg_control, *msg_controllen, /*write=*/true))
+        goto out;
+
+    if (flags & ~(MSG_PEEK | MSG_DONTWAIT)) {
+        debug("recvmsg()/recvmmsg()/recvfrom(): unknown flag (only MSG_PEEK and MSG_DONTWAIT are"
+              " supported).\n");
         ret = -EOPNOTSUPP;
         goto out;
     }
 
+    if (flags & MSG_DONTWAIT) {
+        if (!(hdl->flags & O_NONBLOCK)) {
+            debug("Warning: MSG_DONTWAIT on blocking socket is ignored, may lead to a read that"
+                  " unexpectedly blocks.\n");
+        }
+        flags &= ~MSG_DONTWAIT;
+    }
+
     lock(&hdl->lock);
     peek_buffer        = sock->peek_buffer;
     sock->peek_buffer  = NULL;
@@ -1255,6 +1348,103 @@ static ssize_t do_recvmsg(int fd, struct iovec* bufs, size_t nbufs, int flags,
 
     unlock(&hdl->lock);
 
+    if (msg_control) {
+        assert(msg_controllen);
+
+        if (sock->domain != AF_UNIX || flags & MSG_PEEK || peek_buffer) {
+            /* TODO: unclear what to do if we have both ancillary data and peeks */
+            ret = -EINVAL;
+            lock(&hdl->lock);
+            goto out_locked;
+        }
+
+        /* assume that if user supplied msg_control, then we expect ancillary data;
+         * we will return partial read if data is just normal */
+        char hdls_to_recv_hdr[sizeof(HANDLES_SCM_RIGHTS_HDR) + sizeof(hdls_to_recv_cnt)];
+
+        PAL_NUM pal_ret = DkStreamRead(pal_hdl, /*offset=*/0, sizeof(hdls_to_recv_hdr),
+                                       hdls_to_recv_hdr, uri, uri ? SOCK_URI_SIZE : 0);
+        if (pal_ret == PAL_STREAM_ERROR) {
+            ret = (PAL_NATIVE_ERRNO == PAL_ERROR_STREAMNOTEXIST) ? -ECONNABORTED : -PAL_ERRNO;
+            lock(&hdl->lock);
+            goto out_locked;
+        }
+
+        if (pal_ret < sizeof(hdls_to_recv_hdr) ||
+                strcmp_static(hdls_to_recv_hdr, HANDLES_SCM_RIGHTS_HDR)) {
+            /* looks like we have a "normal" message, try to return whatever was read in bufs */
+            *msg_controllen = 0;
+            ret = -EINVAL;
+
+            if (nbufs && pal_ret < bufs[0].iov_len) {
+                memcpy(bufs[0].iov_base, hdls_to_recv_hdr, pal_ret);
+                ret = pal_ret;
+            }
+
+            if (ret >= 0 && addr) {
+                unix_copy_addr(addr, sock->addr.un.dentry);
+                *addrlen = sizeof(struct sockaddr_un);
+            }
+
+            lock(&hdl->lock);
+            goto out_locked;
+        }
+
+        memcpy(&hdls_to_recv_cnt, hdls_to_recv_hdr + sizeof(HANDLES_SCM_RIGHTS_HDR),
+               sizeof(hdls_to_recv_cnt));
+        if (hdls_to_recv_cnt > HANDLES_SCM_RIGHTS_MAX) {
+            ret = -EINVAL;
+            lock(&hdl->lock);
+            goto out_locked;
+        }
+
+        size_t cmsg_len = sizeof(struct cmsghdr) + hdls_to_recv_cnt * sizeof(int);
+        cmsg_len        = ALIGN_UP(cmsg_len, sizeof(size_t));
+        if (*msg_controllen < cmsg_len) {
+            ret = -ENOMEM;
+            lock(&hdl->lock);
+            goto out_locked;
+        }
+
+        struct cmsghdr* cmsg = (struct cmsghdr*)msg_control;
+        cmsg->cmsg_len   = cmsg_len;
+        cmsg->cmsg_level = SOL_SOCKET;
+        cmsg->cmsg_type  = SCM_RIGHTS;
+
+        int* cmsg_vfds = (int*)((char*)msg_control + sizeof(*cmsg));
+
+        for (size_t i = 0; i < hdls_to_recv_cnt; i++) {
+            PAL_HANDLE received_pal_hdl = DkReceiveHandle(pal_hdl);
+            if (!received_pal_hdl) {
+                ret = -PAL_ERRNO;
+                lock(&hdl->lock);
+                goto out_locked;
+            }
+
+            /* FIXME: currently only allow to receive pipes and UNIX domain sockets */
+            struct shim_handle* received_hdl = NULL;
+            ret = bind_dummy_socket_to_pal_handle(received_pal_hdl, &received_hdl);
+            if (ret < 0) {
+                lock(&hdl->lock);
+                goto out_locked;
+            }
+
+            assert(received_hdl);
+            received_hdls[received_hdls_cnt++] = received_hdl;
+
+            int vfd = set_new_fd_handle(received_hdl, 0, NULL);
+            if (vfd < 0) {
+                ret = vfd;
+                lock(&hdl->lock);
+                goto out_locked;
+            }
+
+            cmsg_vfds[i] = vfd;
+        }
+
+        *msg_controllen = cmsg_len;
+    }
+
     if (flags & MSG_PEEK) {
         if (!peek_buffer) {
             /* create new peek buffer with expected read size */
@@ -1398,8 +1588,12 @@ static ssize_t do_recvmsg(int fd, struct iovec* bufs, size_t nbufs, int flags,
     goto out;
 
 out_locked:
-    if (ret < 0)
+    if (ret < 0) {
+        for (size_t i = 0; i < received_hdls_cnt; i++)
+            put_handle(received_hdls[i]);
         sock->error = -ret;
+    }
+
     unlock(&hdl->lock);
     free(peek_buffer);
 out:
@@ -1413,36 +1607,34 @@ ssize_t shim_do_recvfrom(int sockfd, void* buf, size_t len, int flags, struct so
     iovbuf.iov_base = (void*)buf;
     iovbuf.iov_len  = len;
 
-    return do_recvmsg(sockfd, &iovbuf, 1, flags, addr, addrlen);
+    return do_recvmsg(sockfd, &iovbuf, 1, flags, addr, addrlen, NULL, NULL);
 }
 
 ssize_t shim_do_recvmsg(int sockfd, struct msghdr* msg, int flags) {
+    if (test_user_memory(msg, sizeof(*msg), /*write=*/true))
+        return -EFAULT;
+
     return do_recvmsg(sockfd, msg->msg_iov, msg->msg_iovlen, flags, msg->msg_name,
-                      &msg->msg_namelen);
+                      &msg->msg_namelen, msg->msg_control, &msg->msg_controllen);
 }
 
-ssize_t shim_do_recvmmsg(int sockfd, struct mmsghdr* msg, unsigned int vlen, int flags,
-                         struct __kernel_timespec* timeout) {
-    if (test_user_memory(msg, vlen, /*write=*/true))
+int shim_do_recvmmsg(int sockfd, struct mmsghdr* msgvec, unsigned int vlen, int flags,
+                     struct __kernel_timespec* timeout) {
+    if (test_user_memory(msgvec, vlen * sizeof(*msgvec), /*write=*/true))
         return -EFAULT;
 
-    ssize_t total = 0;
-    // Issue # 753 - https://github.com/oscarlab/graphene/issues/753
-    /* TODO(donporter): timeout properly. For now, explicitly return an error. */
     if (timeout) {
         debug("recvmmsg(): timeout parameter unsupported.\n");
         return -EOPNOTSUPP;
     }
 
-    for (size_t i = 0; i * sizeof(struct mmsghdr) < vlen; i++) {
-        struct msghdr* m = &msg[i].msg_hdr;
-
-        ssize_t bytes =
-            do_recvmsg(sockfd, m->msg_iov, m->msg_iovlen, flags, m->msg_name, &m->msg_namelen);
+    int total = 0;
+    for (size_t i = 0; i < vlen; i++) {
+        ssize_t bytes = shim_do_recvmsg(sockfd, &msgvec[i].msg_hdr, flags);
         if (bytes < 0)
-            return total > 0 ? total : bytes;
+            return total > 0 ? total : (int)bytes;
 
-        msg[i].msg_len = bytes;
+        msgvec[i].msg_len = bytes;
         total++;
     }
 

LibOS/shim/test/ltp/ltp.cfg

@@ -2309,9 +2309,11 @@ skip = yes
 [sendfile09_64]
 skip = yes
 
+# requires root and tries to execute system() to bring up loop back device
 [sendmsg01]
 skip = yes
 
+# requires root and is a bug reproducer for obscure selinux_socket_unix_may_send()
 [sendmsg02]
 skip = yes
 

LibOS/shim/test/regression/.gitignore

@@ -55,6 +55,7 @@
 /pselect
 /readdir
 /sched
+/scm_rights
 /select
 /shared_object
 /sigaltstack

LibOS/shim/test/regression/Makefile

@@ -47,6 +47,7 @@ c_executables = \
 	pselect \
 	readdir \
 	sched \
+	scm_rights \
 	select \
 	shared_object \
 	sigaltstack \

LibOS/shim/test/regression/scm_rights.c

@@ -0,0 +1,289 @@
+#define _GNU_SOURCE
+#include <errno.h>
+#include <fcntl.h>
+#include <sched.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/socket.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <sys/un.h>
+#include <sys/wait.h>
+#include <unistd.h>
+
+#define UNIX_SOCKET_NAME "dummy_unix_socket"
+
+#define VLEN 5
+#define BUFSIZE 100
+
+#define STR_ONE   "one"
+#define STR_TWO   "two"
+#define STR_THREE "three"
+#define STR_HELLO "hello world"
+
+/* ancillary data buffer, wrapped in a union in order to ensure it is suitably aligned */
+union {
+    char buf[CMSG_SPACE(sizeof(int)) * 2];  /* want to send two cmsg */
+    struct cmsghdr align;
+} cmsghdr_union;
+
+/* pipe to be transmitted from parent process to child via SCM_RIGHTS */
+int pipefds[2] = {-1, -1};
+
+int server(void) {
+    int ret;
+
+    ret = pipe(pipefds);
+    if (ret < 0) {
+        perror("[parent] pipe error");
+        exit(1);
+    }
+
+    int listen_fd = socket(AF_UNIX, SOCK_STREAM, 0);
+    if (listen_fd < 0) {
+        perror("[parent] socket error");
+        exit(1);
+    }
+
+    struct sockaddr_un address;
+    address.sun_family = AF_UNIX;
+    strncpy(address.sun_path, UNIX_SOCKET_NAME, sizeof(address.sun_path));
+
+    ret = bind(listen_fd, (struct sockaddr*)&address, sizeof(address));
+    if (ret < 0) {
+        close(listen_fd);
+        perror("[parent] bind error");
+        exit(1);
+    }
+
+    ret = listen(listen_fd, 3);
+    if (ret < 0) {
+        close(listen_fd);
+        perror("[parent] listen error");
+        exit(1);
+    }
+
+    socklen_t addrlen = sizeof(address);
+
+    int fd = accept(listen_fd, (struct sockaddr*)&address, &addrlen);
+    if (fd < 0) {
+        close(listen_fd);
+        perror("[parent] accept error");
+        exit(1);
+    }
+
+    if (close(listen_fd) < 0) {
+        perror("[parent] close error");
+        exit(1);
+    }
+
+    puts("[parent] The client is connected...");
+
+    struct mmsghdr msg[2] = {0};
+    struct iovec msg1[2]  = {0};
+    struct iovec msg2     = {0};
+
+	msg1[0].iov_base = STR_ONE;
+	msg1[0].iov_len = strlen(STR_ONE);
+	msg1[1].iov_base = STR_TWO;
+	msg1[1].iov_len = strlen(STR_TWO);
+
+	msg2.iov_base = STR_THREE;
+	msg2.iov_len = strlen(STR_THREE);
+
+	msg[0].msg_hdr.msg_iov = msg1;
+	msg[0].msg_hdr.msg_iovlen = 2;
+
+	msg[1].msg_hdr.msg_iov = &msg2;
+	msg[1].msg_hdr.msg_iovlen = 1;
+
+    /* send two ends of the pipe as ancillary data in two cmsg's (just for fun) */
+    msg[0].msg_hdr.msg_control = cmsghdr_union.buf;
+    msg[0].msg_hdr.msg_controllen = sizeof(cmsghdr_union.buf);
+
+    struct cmsghdr* cmsg;
+	cmsg = CMSG_FIRSTHDR(&msg[0].msg_hdr);
+	cmsg->cmsg_level = SOL_SOCKET;
+	cmsg->cmsg_type  = SCM_RIGHTS;
+	cmsg->cmsg_len   = CMSG_LEN(sizeof(int));
+	memcpy(CMSG_DATA(cmsg), &pipefds[0], sizeof(int));
+
+    cmsg = CMSG_NXTHDR(&msg[0].msg_hdr, cmsg);
+	cmsg->cmsg_level = SOL_SOCKET;
+	cmsg->cmsg_type  = SCM_RIGHTS;
+	cmsg->cmsg_len   = CMSG_LEN(sizeof(int));
+	memcpy(CMSG_DATA(cmsg), &pipefds[1], sizeof(int));
+
+	ret = sendmmsg(fd, msg, 2, /*flags=*/0);
+	if (ret < 0) {
+        close(fd);
+        perror("[parent] sendmmsg error\n");
+        exit(1);
+    }
+
+    if (msg[0].msg_len != strlen(STR_ONE) + strlen(STR_TWO) ||
+            msg[1].msg_len != strlen(STR_THREE)) {
+        close(fd);
+        fprintf(stderr, "[parent] sendmmsg error: not all messages were sent\n");
+        exit(1);
+    }
+
+    printf("[parent] %d messages sent\n", ret);
+
+    if (close(fd) < 0) {
+        perror("[parent] close error");
+        exit(1);
+    }
+
+    return 0;
+}
+
+int client(void) {
+    int ret;
+
+    int fd = socket(AF_UNIX, SOCK_STREAM, 0);
+    if (fd < 0) {
+        perror("[child] socket error");
+        exit(1);
+    }
+
+    struct sockaddr_un address;
+    address.sun_family = AF_UNIX;
+    strncpy(address.sun_path, UNIX_SOCKET_NAME, sizeof(address.sun_path));
+
+    ret = -1;
+    while (ret < 0) {
+        /* wait until client is ready for read */
+        errno = 0;
+        ret = connect(fd, (struct sockaddr*)&address, sizeof(address));
+        if (ret < 0 && errno != ENOENT && errno != ECONNREFUSED) {
+            close(fd);
+            perror("[child] connect error\n");
+            exit(1);
+        }
+        sched_yield();
+    }
+
+    puts("[child] Connected to the server, receiving...");
+
+	struct mmsghdr msgs[VLEN] = {0};
+	struct iovec iovecs[VLEN] = {0};
+	char bufs[VLEN][BUFSIZE + 1] = {0};
+
+	for (int i = 0; i < VLEN; i++) {
+		iovecs[i].iov_base         = bufs[i];
+		iovecs[i].iov_len          = BUFSIZE;
+		msgs[i].msg_hdr.msg_iov    = &iovecs[i];
+		msgs[i].msg_hdr.msg_iovlen = 1;
+	}
+
+    /* receive two ends of the pipe as ancillary data in cmsg (must come in the first msg) */
+    msgs[0].msg_hdr.msg_control = cmsghdr_union.buf;
+    msgs[0].msg_hdr.msg_controllen = sizeof(cmsghdr_union.buf);
+
+    ret = -1;
+    while (ret < 0) {
+        errno = 0;
+        ret = recvmmsg(fd, msgs, VLEN, MSG_DONTWAIT, /*timeout=*/NULL);
+        if (ret < 0 && errno != EAGAIN && errno != EWOULDBLOCK) {
+            close(fd);
+            perror("[child] recvmmsg error");
+            exit(1);
+        }
+        sched_yield();
+    }
+
+	printf("[child] %d messages received\n", ret);
+	for (int i = 0; i < ret; i++) {
+        if (!msgs[i].msg_len)
+            continue;
+
+		bufs[i][msgs[i].msg_len] = 0;
+		printf("[child] message %d: %s\n", i + 1, bufs[i]);
+
+        if (!msgs[i].msg_hdr.msg_control || !msgs[i].msg_hdr.msg_controllen)
+            continue;
+
+        struct cmsghdr* cmsg;
+        for (cmsg = CMSG_FIRSTHDR(&msgs[i].msg_hdr); cmsg != NULL;
+                cmsg = CMSG_NXTHDR(&msgs[i].msg_hdr, cmsg)) {
+            if (cmsg->cmsg_level != SOL_SOCKET || cmsg->cmsg_type != SCM_RIGHTS) {
+                close(fd);
+                fprintf(stderr, "[child] recvmmsg error: unexpected ancillary data\n");
+                exit(1);
+            }
+            int* received_fds = (int*)CMSG_DATA(cmsg);
+            pipefds[0] = received_fds[0];
+            pipefds[1] = received_fds[1];
+            break;
+        }
+	}
+
+    if (pipefds[0] == -1 || pipefds[1] == -1) {
+        close(fd);
+        fprintf(stderr, "[child] recvmmsg error: received incorrect pipefds as ancillary data\n");
+        exit(1);
+    }
+
+    /* test received pipe */
+    ssize_t bytes;
+    bytes = write(pipefds[1], STR_HELLO, sizeof(STR_HELLO));
+    if (bytes < 0) {
+        close(fd);
+        perror("[child] write error\n");
+        exit(1);
+    }
+
+    char buffer[128];
+    bytes = 0;
+    while (bytes <= 0) {
+        errno = 0;
+        bytes = read(pipefds[0], &buffer, sizeof(buffer));
+        if (bytes < 0 && errno != EAGAIN && errno != EWOULDBLOCK) {
+            close(fd);
+            perror("[child] read error");
+            exit(1);
+        }
+        sched_yield();
+    }
+
+    buffer[sizeof(buffer) - 1] = '\0';
+    if (bytes < sizeof(buffer))
+        buffer[bytes] = '\0';
+
+    printf("[child] read on received pipe: %s\n", buffer);
+
+    if (close(fd) < 0) {
+        perror("[child] close error");
+        exit(1);
+    }
+
+    return 0;
+}
+
+int main(int argc, char** argv) {
+    int pid = fork();
+    if (pid < 0) {
+        perror("fork error");
+        return 1;
+    }
+
+    if (pid == 0)
+        return client();
+
+    server();
+
+    pid = wait(NULL); /* wait for child termination, just for sanity */
+    if (pid < 0) {
+        perror("[parent] wait error");
+        return 1;
+    }
+
+    if (unlink(UNIX_SOCKET_NAME) < 0) {
+        perror("[parent] unlink error");
+        return 1;
+    }
+
+    return 0;
+}

LibOS/shim/test/regression/test_libos.py

@@ -505,7 +505,7 @@ class TC_80_Socket(RegressionTestCase):
         self.assertIn('read on FIFO: Hello from write end of FIFO!', stdout)
 
     def test_100_socket_unix(self):
-        stdout, _ = self.run_binary(['unix'])
+        stdout, _ = self.run_binary(['unix'], timeout=60)
         self.assertIn('Data: This is packet 0', stdout)
         self.assertIn('Data: This is packet 1', stdout)
         self.assertIn('Data: This is packet 2', stdout)
@@ -517,6 +517,13 @@ class TC_80_Socket(RegressionTestCase):
         self.assertIn('Data: This is packet 8', stdout)
         self.assertIn('Data: This is packet 9', stdout)
 
+    def test_101_scm_rights(self):
+        stdout, _ = self.run_binary(['scm_rights'], timeout=60)
+        self.assertIn('one', stdout)
+        self.assertIn('two', stdout)
+        self.assertIn('three', stdout)
+        self.assertIn('read on received pipe: hello world', stdout)
+
     def test_200_socket_udp(self):
         stdout, _ = self.run_binary(['udp'], timeout=50)
         self.assertIn('Data: This is packet 0', stdout)

Pal/src/host/Linux-SGX/db_streams.c

@@ -244,6 +244,24 @@ static int handle_deserialize(PAL_HANDLE* handle, const void* data, size_t size,
     return 0;
 }
 
+static int get_secure_fd_from_handle(PAL_HANDLE hdl, int* fd, void** ssl_ctx) {
+    assert(hdl && fd && ssl_ctx);
+    switch (PAL_GET_TYPE(hdl)) {
+        case pal_type_pipe:
+        case pal_type_pipesrv:
+        case pal_type_pipecli:
+            *fd      = hdl->pipe.fd;
+            *ssl_ctx = hdl->pipe.ssl_ctx;
+            break;
+        case pal_type_process:
+            *fd      = hdl->process.stream;
+            *ssl_ctx = hdl->process.ssl_ctx;
+            break;
+        default:
+            return -1;
+    }
+    return 0;
+}
 /*!
  * \brief Send `cargo` handle to a process identified via `hdl` handle.
  *
@@ -254,8 +272,11 @@ static int handle_deserialize(PAL_HANDLE* handle, const void* data, size_t size,
  * \return           0 on success, negative PAL error code otherwise.
  */
 int _DkSendHandle(PAL_HANDLE hdl, PAL_HANDLE cargo) {
-    if (!IS_HANDLE_TYPE(hdl, process))
+    int fd        = -1;
+    void* ssl_ctx = NULL;
+    if (get_secure_fd_from_handle(hdl, &fd, &ssl_ctx) < 0)
         return -PAL_ERROR_BADHANDLE;
+    assert(fd >= 0);
 
     /* serialize cargo handle into a blob hdl_data */
     void* hdl_data = NULL;
@@ -265,7 +286,6 @@ int _DkSendHandle(PAL_HANDLE hdl, PAL_HANDLE cargo) {
 
     ssize_t ret;
     struct hdl_header hdl_hdr = {.fds = 0, .data_size = hdl_data_size};
-    int fd = hdl->process.stream;
 
     /* apply bitmask of FDs-to-transfer to hdl_hdr.fds and populate `fds` with these FDs */
     int fds[MAX_FDS];
@@ -301,8 +321,8 @@ int _DkSendHandle(PAL_HANDLE hdl, PAL_HANDLE cargo) {
     }
 
     /* finally send the serialized cargo as payload (possibly encrypted) */
-    if (hdl->process.ssl_ctx) {
-        ret = _DkStreamSecureWrite(hdl->process.ssl_ctx, (uint8_t*)hdl_data, hdl_hdr.data_size);
+    if (ssl_ctx) {
+        ret = _DkStreamSecureWrite(ssl_ctx, (uint8_t*)hdl_data, hdl_hdr.data_size);
     } else {
         ret = ocall_write(fd, hdl_data, hdl_hdr.data_size);
         ret = IS_ERR(ret) ? unix_to_pal_error(ERRNO(ret)) : ret;
@@ -322,12 +342,14 @@ int _DkSendHandle(PAL_HANDLE hdl, PAL_HANDLE cargo) {
  * \return           0 on success, negative PAL error code otherwise.
  */
 int _DkReceiveHandle(PAL_HANDLE hdl, PAL_HANDLE* cargo) {
-    if (!IS_HANDLE_TYPE(hdl, process))
+    int fd        = -1;
+    void* ssl_ctx = NULL;
+    if (get_secure_fd_from_handle(hdl, &fd, &ssl_ctx) < 0)
         return -PAL_ERROR_BADHANDLE;
+    assert(fd >= 0);
 
     ssize_t ret;
     struct hdl_header hdl_hdr;
-    int fd = hdl->process.stream;
 
     /* first receive hdl_hdr so that we know how many FDs were transferred + how large is cargo */
     ret = ocall_recv(fd, &hdl_hdr, sizeof(hdl_hdr), NULL, NULL, NULL, NULL);
@@ -361,8 +383,8 @@ int _DkReceiveHandle(PAL_HANDLE hdl, PAL_HANDLE* cargo) {
     /* finally receive the serialized cargo as payload (possibly encrypted) */
     char hdl_data[hdl_hdr.data_size];
 
-    if (hdl->process.ssl_ctx) {
-        ret = _DkStreamSecureRead(hdl->process.ssl_ctx, (uint8_t*)hdl_data, hdl_hdr.data_size);
+    if (ssl_ctx) {
+        ret = _DkStreamSecureRead(ssl_ctx, (uint8_t*)hdl_data, hdl_hdr.data_size);
     } else {
         ret = ocall_read(fd, hdl_data, hdl_hdr.data_size);
         ret = IS_ERR(ret) ? unix_to_pal_error(ERRNO(ret)) : ret;

Pal/src/host/Linux/db_streams.c

@@ -222,6 +222,23 @@ int handle_deserialize(PAL_HANDLE* handle, const void* data, int size) {
     return 0;
 }
 
+static int get_fd_from_handle(PAL_HANDLE hdl, int* fd) {
+    assert(hdl && fd);
+    switch (PAL_GET_TYPE(hdl)) {
+        case pal_type_pipe:
+        case pal_type_pipesrv:
+        case pal_type_pipecli:
+            *fd = hdl->pipe.fd;
+            break;
+        case pal_type_process:
+            *fd = hdl->process.stream;
+            break;
+        default:
+            return -1;
+    }
+    return 0;
+}
+
 /*!
  * \brief Send `cargo` handle to a process identified via `hdl` handle.
  *
@@ -230,8 +247,10 @@ int handle_deserialize(PAL_HANDLE* handle, const void* data, int size) {
  * \return           0 on success, negative PAL error code otherwise.
  */
 int _DkSendHandle(PAL_HANDLE hdl, PAL_HANDLE cargo) {
-    if (!IS_HANDLE_TYPE(hdl, process))
+    int fd = -1;
+    if (get_fd_from_handle(hdl, &fd) < 0)
         return -PAL_ERROR_BADHANDLE;
+    assert(fd >= 0);
 
     /* serialize cargo handle into a blob hdl_data */
     void* hdl_data = NULL;
@@ -241,7 +260,6 @@ int _DkSendHandle(PAL_HANDLE hdl, PAL_HANDLE cargo) {
 
     ssize_t ret;
     struct hdl_header hdl_hdr = {.fds = 0, .data_size = hdl_data_size};
-    int fd = hdl->process.stream;
 
     /* apply bitmask of FDs-to-transfer to hdl_hdr.fds and populate `fds` with these FDs */
     int fds[MAX_FDS];
@@ -304,12 +322,13 @@ int _DkSendHandle(PAL_HANDLE hdl, PAL_HANDLE cargo) {
  * \return           0 on success, negative PAL error code otherwise.
  */
 int _DkReceiveHandle(PAL_HANDLE hdl, PAL_HANDLE* cargo) {
-    if (!IS_HANDLE_TYPE(hdl, process))
+    int fd = -1;
+    if (get_fd_from_handle(hdl, &fd) < 0)
         return -PAL_ERROR_BADHANDLE;
+    assert(fd >= 0);
 
     ssize_t ret;
     struct hdl_header hdl_hdr;
-    int fd = hdl->process.stream;
 
     /* first receive hdl_hdr so that we know how many FDs were transferred + how large is cargo */
     struct msghdr message_hdr = {0};