From 46d21eccafddb035492693cfdb8ed2fbcabe1840 Mon Sep 17 00:00:00 2001 From: Dmitrii Kuvaiskii Date: Fri, 15 May 2020 23:36:11 +0000 Subject: [PATCH] [LibOS,Pal] Add MSG_DONTWAIT and SCM_RIGHTS for recvmsg/sendmsg This commit improves the emulation of recvfrom/sendfrom, recvmsg/sendmsg, and recvmmsg/sendmmsg system calls. In particular, MSG_DONTWAIT flag is allowed though not really emulated (benign in most cases). Also, it is possible now to send/receive FDs via SCM_RIGHTS on a UNIX domain socket (only send/recv of pipes and UNIX domain sockets is currently supported). Corresponding LibOS test is added. --- LibOS/shim/include/shim_table.h | 12 +- LibOS/shim/include/shim_types.h | 17 +- LibOS/shim/include/shim_utils.h | 1 + LibOS/shim/src/shim_syscalls.c | 4 +- LibOS/shim/src/sys/shim_pipe.c | 37 +++ LibOS/shim/src/sys/shim_socket.c | 260 +++++++++++++++++--- LibOS/shim/test/ltp/ltp.cfg | 2 + LibOS/shim/test/regression/.gitignore | 1 + LibOS/shim/test/regression/Makefile | 1 + LibOS/shim/test/regression/scm_rights.c | 289 +++++++++++++++++++++++ LibOS/shim/test/regression/test_libos.py | 9 +- Pal/src/host/Linux-SGX/db_streams.c | 38 ++- Pal/src/host/Linux/db_streams.c | 27 ++- 13 files changed, 641 insertions(+), 57 deletions(-) create mode 100644 LibOS/shim/test/regression/scm_rights.c diff --git a/LibOS/shim/include/shim_table.h b/LibOS/shim/include/shim_table.h index bd188fff..8753f2e4 100644 --- a/LibOS/shim/include/shim_table.h +++ b/LibOS/shim/include/shim_table.h @@ -490,11 +490,11 @@ int shim_do_epoll_create1(int flags); int shim_do_pipe2(int* fildes, int flags); int shim_do_mknod(const char *pathname, mode_t mode, dev_t dev); int shim_do_mknodat(int dirfd, const char *pathname, mode_t mode, dev_t dev); -ssize_t shim_do_recvmmsg(int sockfd, struct mmsghdr* msg, unsigned int vlen, int flags, - struct __kernel_timespec* timeout); +int shim_do_recvmmsg(int sockfd, struct mmsghdr* msg, unsigned int vlen, int flags, + struct __kernel_timespec* timeout); int shim_do_prlimit64(pid_t pid, int resource, const struct __kernel_rlimit64* new_rlim, struct __kernel_rlimit64* old_rlim); -ssize_t shim_do_sendmmsg(int sockfd, struct mmsghdr* msg, unsigned int vlen, int flags); +int shim_do_sendmmsg(int sockfd, struct mmsghdr* msg, unsigned int vlen, int flags); int shim_do_eventfd2(unsigned int count, int flags); int shim_do_eventfd(unsigned int count); @@ -816,11 +816,11 @@ int shim_pwritev(unsigned long fd, const struct iovec* vec, unsigned long vlen, int shim_rt_tgsigqueueinfo(pid_t tgid, pid_t pid, int sig, siginfo_t* uinfo); int shim_perf_event_open(struct perf_event_attr* attr_uptr, pid_t pid, int cpu, int group_fd, int flags); -ssize_t shim_recvmmsg(int sockfd, struct mmsghdr* msg, unsigned int vlen, int flags, - struct __kernel_timespec* timeout); +int shim_recvmmsg(int sockfd, struct mmsghdr* msg, unsigned int vlen, int flags, + struct __kernel_timespec* timeout); int shim_prlimit64(pid_t pid, int resource, const struct __kernel_rlimit64* new_rlim, struct __kernel_rlimit64* old_rlim); -ssize_t shim_sendmmsg(int sockfd, struct mmsghdr* msg, unsigned int vlen, int flags); +int shim_sendmmsg(int sockfd, struct mmsghdr* msg, unsigned int vlen, int flags); /* libos call wrappers */ int shim_msgpersist(int msqid, int cmd); diff --git a/LibOS/shim/include/shim_types.h b/LibOS/shim/include/shim_types.h index 988b9710..dcdba795 100644 --- a/LibOS/shim/include/shim_types.h +++ b/LibOS/shim/include/shim_types.h @@ -333,10 +333,12 @@ struct __kernel_ustat /* bits/socket.h */ enum { - MSG_OOB = 0x01, /* Process out-of-band data. */ - MSG_PEEK = 0x02, /* Peek at incoming messages. */ + MSG_OOB = 0x01, /* Process out-of-band data. */ + MSG_PEEK = 0x02, /* Peek at incoming messages. */ + MSG_DONTWAIT = 0x40, /* Nonblocking IO. */ #define MSG_OOB MSG_OOB #define MSG_PEEK MSG_PEEK +#define MSG_DONTWAIT MSG_DONTWAIT }; struct msghdr { @@ -358,6 +360,17 @@ struct mmsghdr { unsigned int msg_len; /* Number of received bytes for the entry. */ }; +/* Structure used for storage of ancillary data object information. */ +struct cmsghdr { + size_t cmsg_len; + int cmsg_level; + int cmsg_type; +}; + +#ifndef SCM_RIGHTS +#define SCM_RIGHTS 1 +#endif + /* POSIX.1g specifies this type name for the `sa_family' member. */ typedef unsigned short int sa_family_t; diff --git a/LibOS/shim/include/shim_utils.h b/LibOS/shim/include/shim_utils.h index 7e607365..085b065b 100644 --- a/LibOS/shim/include/shim_utils.h +++ b/LibOS/shim/include/shim_utils.h @@ -211,6 +211,7 @@ int create_pipe(char* name, char* uri, size_t size, PAL_HANDLE* hdl, struct shim int create_dir(const char* prefix, char* path, size_t size, struct shim_handle** hdl); int create_file(const char* prefix, char* path, size_t size, struct shim_handle** hdl); int create_handle(const char* prefix, char* path, size_t size, PAL_HANDLE* hdl, unsigned int* id); +int bind_dummy_socket_to_pal_handle(PAL_HANDLE pal_hdl, struct shim_handle** out_hdl); /* Asynchronous event support */ int init_async(void); diff --git a/LibOS/shim/src/shim_syscalls.c b/LibOS/shim/src/shim_syscalls.c index cde7c325..57218d9f 100644 --- a/LibOS/shim/src/shim_syscalls.c +++ b/LibOS/shim/src/shim_syscalls.c @@ -1008,7 +1008,7 @@ SHIM_SYSCALL_RETURN_ENOSYS(rt_tgsigqueueinfo, 4, int, pid_t, tgid, pid_t, pid, i SHIM_SYSCALL_RETURN_ENOSYS(perf_event_open, 5, int, struct perf_event_attr*, attr_uptr, pid_t, pid, int, cpu, int, group_fd, int, flags) -DEFINE_SHIM_SYSCALL(recvmmsg, 5, shim_do_recvmmsg, ssize_t, int, fd, struct mmsghdr*, msg, +DEFINE_SHIM_SYSCALL(recvmmsg, 5, shim_do_recvmmsg, int, int, fd, struct mmsghdr*, msg, unsigned int, vlen, int, flags, struct __kernel_timespec*, timeout) SHIM_SYSCALL_RETURN_ENOSYS(fanotify_init, 2, int, int, flags, int, event_f_flags) @@ -1029,7 +1029,7 @@ SHIM_SYSCALL_RETURN_ENOSYS(clock_adjtime, 2, int, clockid_t, which_clock, struct SHIM_SYSCALL_RETURN_ENOSYS(syncfs, 1, int, int, fd) -DEFINE_SHIM_SYSCALL(sendmmsg, 4, shim_do_sendmmsg, ssize_t, int, fd, struct mmsghdr*, msg, +DEFINE_SHIM_SYSCALL(sendmmsg, 4, shim_do_sendmmsg, int, int, fd, struct mmsghdr*, msg, unsigned int, vlen, int, flags) SHIM_SYSCALL_RETURN_ENOSYS(setns, 2, int, int, fd, int, nstype) diff --git a/LibOS/shim/src/sys/shim_pipe.c b/LibOS/shim/src/sys/shim_pipe.c index 4f1a53de..d3837198 100644 --- a/LibOS/shim/src/sys/shim_pipe.c +++ b/LibOS/shim/src/sys/shim_pipe.c @@ -33,6 +33,43 @@ #include "shim_types.h" #include "shim_utils.h" +/* TODO: create a "dummy" shim handle as in socketpair() for SCM_RIGHTS emulation; + * remove this once Graphene has proper serialization of shim_handles */ +int bind_dummy_socket_to_pal_handle(PAL_HANDLE pal_hdl, struct shim_handle** out_hdl) { + switch (PAL_GET_TYPE(pal_hdl)) { + case pal_type_pipe: + case pal_type_pipesrv: + case pal_type_pipecli: + /* only underlying PAL pipes are allowed */ + break; + default: + DkObjectClose(pal_hdl); + return -EINVAL; + } + + struct shim_handle* hdl = get_new_handle(); + if (!hdl) { + DkObjectClose(pal_hdl); + return -ENOMEM; + } + + struct shim_sock_handle* sock = &hdl->info.sock; + + hdl->type = TYPE_SOCK; + set_handle_fs(hdl, &socket_builtin_fs); + hdl->pal_handle = pal_hdl; + hdl->flags = O_RDWR; + hdl->acc_mode = MAY_READ | MAY_WRITE; + sock->domain = AF_UNIX; + sock->sock_type = SOCK_STREAM; + sock->protocol = 0; + sock->sock_state = SOCK_CONNECTED; + /* we don't set URI/name: they don't matter for already opened pipes/sockets */ + + *out_hdl = hdl; + return 0; +} + static int create_pipes(PAL_HANDLE* srv, PAL_HANDLE* cli, int flags, char* name, struct shim_qstr* qstr) { int ret = 0; diff --git a/LibOS/shim/src/sys/shim_socket.c b/LibOS/shim/src/sys/shim_socket.c index 047d3260..0831c8e2 100644 --- a/LibOS/shim/src/sys/shim_socket.c +++ b/LibOS/shim/src/sys/shim_socket.c @@ -59,6 +59,10 @@ #define AF_UNSPEC 0 +/* macros for emulating send/recv of ancillary data */ +#define HANDLES_SCM_RIGHTS_MAX 32 +#define HANDLES_SCM_RIGHTS_HDR "GRAPHENE_SCM_RIGHTS:" + static int rebase_on_lo __attribute_migratable = -1; static size_t minimal_addrlen(int domain) { @@ -1016,10 +1020,14 @@ int shim_do_accept4(int fd, struct sockaddr* addr, int* addrlen, int flags) { } static ssize_t do_sendmsg(int fd, struct iovec* bufs, int nbufs, int flags, - const struct sockaddr* addr, int addrlen) { + const struct sockaddr* addr, int addrlen, + void* msg_control, size_t msg_controllen) { // Issue #752 - https://github.com/oscarlab/graphene/issues/752 __UNUSED(flags); + struct shim_handle* hdls_to_send[HANDLES_SCM_RIGHTS_MAX]; + size_t hdls_to_send_cnt = 0; + struct shim_handle* hdl = get_fd_handle(fd, NULL, NULL); if (!hdl) return -EBADF; @@ -1042,12 +1050,51 @@ static ssize_t do_sendmsg(int fd, struct iovec* bufs, int nbufs, int flags, goto out; } + if (msg_control && test_user_memory(msg_control, msg_controllen, /*write=*/false)) + goto out; + lock(&hdl->lock); PAL_HANDLE pal_hdl = hdl->pal_handle; char* uri = NULL; - /* Data gram sock need not be conneted or bound at all */ + if (msg_control) { + /* prepare ancillary data to be sent (we support only SCM_RIGHTS on UNIX sockets) */ + if (sock->domain != AF_UNIX) { + ret = -EINVAL; + goto out_locked; + } + + struct cmsghdr* cmsg = (struct cmsghdr*)msg_control; + while ((char*)cmsg < (char*)msg_control + msg_controllen) { + ssize_t cmsg_data_len = cmsg->cmsg_len - sizeof(*cmsg); + if (cmsg_data_len <= 0 || cmsg->cmsg_level != SOL_SOCKET + || cmsg->cmsg_type != SCM_RIGHTS) { + ret = -EINVAL; + goto out_locked; + } + + int* fds_to_send = (int*)((char*)cmsg + sizeof(*cmsg)); + size_t fds_to_send_cnt = cmsg_data_len / sizeof(*fds_to_send); + + for (size_t i = 0; i < fds_to_send_cnt; i++) { + if (hdls_to_send_cnt == HANDLES_SCM_RIGHTS_MAX) { + ret = -EINVAL; + goto out_locked; + } + + struct shim_handle* hdl_to_send = get_fd_handle(fds_to_send[i], NULL, NULL); + if (!hdl_to_send) { + ret = -EBADF; + goto out_locked; + } + hdls_to_send[hdls_to_send_cnt++] = hdl_to_send; + } + + cmsg = (struct cmsghdr*)((char*)cmsg + ALIGN_UP(cmsg->cmsg_len, sizeof(size_t))); + } + } + if (sock->sock_type == SOCK_STREAM && sock->sock_state != SOCK_CONNECTED && sock->sock_state != SOCK_BOUNDCONNECTED && sock->sock_state != SOCK_ACCEPTED) { ret = -ENOTCONN; @@ -1109,6 +1156,32 @@ static ssize_t do_sendmsg(int fd, struct iovec* bufs, int nbufs, int flags, debug("next packet send to %s\n", uri); } + /* send ancillary data: first header with number of handles to send, then each handle */ + if (hdls_to_send_cnt) { + char hdls_to_send_hdr[sizeof(HANDLES_SCM_RIGHTS_HDR) + sizeof(hdls_to_send_cnt)]; + memcpy(hdls_to_send_hdr, HANDLES_SCM_RIGHTS_HDR, sizeof(HANDLES_SCM_RIGHTS_HDR)); + memcpy(hdls_to_send_hdr + sizeof(HANDLES_SCM_RIGHTS_HDR), &hdls_to_send_cnt, + sizeof(hdls_to_send_cnt)); + + PAL_NUM pal_ret = DkStreamWrite(pal_hdl, /*offset=*/0, sizeof(hdls_to_send_hdr), + hdls_to_send_hdr, uri); + if (pal_ret == PAL_STREAM_ERROR) { + ret = (PAL_NATIVE_ERRNO == PAL_ERROR_STREAMEXIST) ? -ECONNABORTED : -PAL_ERRNO; + lock(&hdl->lock); + goto out_locked; + } + assert(pal_ret == sizeof(hdls_to_send_hdr)); /* always true for UNIX sockets */ + + for (size_t i = 0; i < hdls_to_send_cnt; i++) { + if (!DkSendHandle(pal_hdl, hdls_to_send[i]->pal_handle)) { + ret = -PAL_ERRNO; + lock(&hdl->lock); + goto out_locked; + } + } + } + + /* send regular data */ int bytes = 0; ret = 0; @@ -1137,6 +1210,8 @@ out_locked: unlock(&hdl->lock); out: + for (size_t i = 0; i < hdls_to_send_cnt; i++) + put_handle(hdls_to_send[i]); put_handle(hdl); return ret; } @@ -1147,28 +1222,28 @@ ssize_t shim_do_sendto(int sockfd, const void* buf, size_t len, int flags, iovbuf.iov_base = (void*)buf; iovbuf.iov_len = len; - return do_sendmsg(sockfd, &iovbuf, 1, flags, addr, addrlen); + return do_sendmsg(sockfd, &iovbuf, 1, flags, addr, addrlen, NULL, 0); } ssize_t shim_do_sendmsg(int sockfd, struct msghdr* msg, int flags) { - return do_sendmsg(sockfd, msg->msg_iov, msg->msg_iovlen, flags, msg->msg_name, - msg->msg_namelen); -} - -ssize_t shim_do_sendmmsg(int sockfd, struct mmsghdr* msg, unsigned int vlen, int flags) { - if (test_user_memory(msg, vlen, /*write=*/true)) + if (test_user_memory(msg, sizeof(*msg), /*write=*/false)) return -EFAULT; - ssize_t total = 0; - for (size_t i = 0; i * sizeof(struct mmsghdr) < vlen; i++) { - struct msghdr* m = &msg[i].msg_hdr; + return do_sendmsg(sockfd, msg->msg_iov, msg->msg_iovlen, flags, msg->msg_name, + msg->msg_namelen, msg->msg_control, msg->msg_controllen); +} - ssize_t bytes = - do_sendmsg(sockfd, m->msg_iov, m->msg_iovlen, flags, m->msg_name, m->msg_namelen); +int shim_do_sendmmsg(int sockfd, struct mmsghdr* msgvec, unsigned int vlen, int flags) { + if (test_user_memory(msgvec, vlen * sizeof(*msgvec), /*write=*/true)) + return -EFAULT; + + int total = 0; + for (size_t i = 0; i < vlen; i++) { + ssize_t bytes = shim_do_sendmsg(sockfd, &msgvec[i].msg_hdr, flags); if (bytes < 0) return total > 0 ? total : bytes; - msg[i].msg_len = bytes; + msgvec[i].msg_len = bytes; total++; } @@ -1176,7 +1251,13 @@ ssize_t shim_do_sendmmsg(int sockfd, struct mmsghdr* msg, unsigned int vlen, int } static ssize_t do_recvmsg(int fd, struct iovec* bufs, size_t nbufs, int flags, - struct sockaddr* addr, int* addrlen) { + struct sockaddr* addr, int* addrlen, + void* msg_control, size_t* msg_controllen) { + size_t hdls_to_recv_cnt = 0; + + struct shim_handle* received_hdls[HANDLES_SCM_RIGHTS_MAX]; + size_t received_hdls_cnt = 0; + struct shim_handle* hdl = get_fd_handle(fd, NULL, NULL); if (!hdl) return -EBADF; @@ -1220,12 +1301,24 @@ static ssize_t do_recvmsg(int fd, struct iovec* bufs, size_t nbufs, int flags, expected_size += bufs[i].iov_len; } - if (flags & ~MSG_PEEK) { - debug("recvmsg()/recvmmsg()/recvfrom(): unknown flag (only MSG_PEEK is supported).\n"); + if (msg_control && msg_controllen && test_user_memory(msg_control, *msg_controllen, /*write=*/true)) + goto out; + + if (flags & ~(MSG_PEEK | MSG_DONTWAIT)) { + debug("recvmsg()/recvmmsg()/recvfrom(): unknown flag (only MSG_PEEK and MSG_DONTWAIT are" + " supported).\n"); ret = -EOPNOTSUPP; goto out; } + if (flags & MSG_DONTWAIT) { + if (!(hdl->flags & O_NONBLOCK)) { + debug("Warning: MSG_DONTWAIT on blocking socket is ignored, may lead to a read that" + " unexpectedly blocks.\n"); + } + flags &= ~MSG_DONTWAIT; + } + lock(&hdl->lock); peek_buffer = sock->peek_buffer; sock->peek_buffer = NULL; @@ -1255,6 +1348,103 @@ static ssize_t do_recvmsg(int fd, struct iovec* bufs, size_t nbufs, int flags, unlock(&hdl->lock); + if (msg_control) { + assert(msg_controllen); + + if (sock->domain != AF_UNIX || flags & MSG_PEEK || peek_buffer) { + /* TODO: unclear what to do if we have both ancillary data and peeks */ + ret = -EINVAL; + lock(&hdl->lock); + goto out_locked; + } + + /* assume that if user supplied msg_control, then we expect ancillary data; + * we will return partial read if data is just normal */ + char hdls_to_recv_hdr[sizeof(HANDLES_SCM_RIGHTS_HDR) + sizeof(hdls_to_recv_cnt)]; + + PAL_NUM pal_ret = DkStreamRead(pal_hdl, /*offset=*/0, sizeof(hdls_to_recv_hdr), + hdls_to_recv_hdr, uri, uri ? SOCK_URI_SIZE : 0); + if (pal_ret == PAL_STREAM_ERROR) { + ret = (PAL_NATIVE_ERRNO == PAL_ERROR_STREAMNOTEXIST) ? -ECONNABORTED : -PAL_ERRNO; + lock(&hdl->lock); + goto out_locked; + } + + if (pal_ret < sizeof(hdls_to_recv_hdr) || + strcmp_static(hdls_to_recv_hdr, HANDLES_SCM_RIGHTS_HDR)) { + /* looks like we have a "normal" message, try to return whatever was read in bufs */ + *msg_controllen = 0; + ret = -EINVAL; + + if (nbufs && pal_ret < bufs[0].iov_len) { + memcpy(bufs[0].iov_base, hdls_to_recv_hdr, pal_ret); + ret = pal_ret; + } + + if (ret >= 0 && addr) { + unix_copy_addr(addr, sock->addr.un.dentry); + *addrlen = sizeof(struct sockaddr_un); + } + + lock(&hdl->lock); + goto out_locked; + } + + memcpy(&hdls_to_recv_cnt, hdls_to_recv_hdr + sizeof(HANDLES_SCM_RIGHTS_HDR), + sizeof(hdls_to_recv_cnt)); + if (hdls_to_recv_cnt > HANDLES_SCM_RIGHTS_MAX) { + ret = -EINVAL; + lock(&hdl->lock); + goto out_locked; + } + + size_t cmsg_len = sizeof(struct cmsghdr) + hdls_to_recv_cnt * sizeof(int); + cmsg_len = ALIGN_UP(cmsg_len, sizeof(size_t)); + if (*msg_controllen < cmsg_len) { + ret = -ENOMEM; + lock(&hdl->lock); + goto out_locked; + } + + struct cmsghdr* cmsg = (struct cmsghdr*)msg_control; + cmsg->cmsg_len = cmsg_len; + cmsg->cmsg_level = SOL_SOCKET; + cmsg->cmsg_type = SCM_RIGHTS; + + int* cmsg_vfds = (int*)((char*)msg_control + sizeof(*cmsg)); + + for (size_t i = 0; i < hdls_to_recv_cnt; i++) { + PAL_HANDLE received_pal_hdl = DkReceiveHandle(pal_hdl); + if (!received_pal_hdl) { + ret = -PAL_ERRNO; + lock(&hdl->lock); + goto out_locked; + } + + /* FIXME: currently only allow to receive pipes and UNIX domain sockets */ + struct shim_handle* received_hdl = NULL; + ret = bind_dummy_socket_to_pal_handle(received_pal_hdl, &received_hdl); + if (ret < 0) { + lock(&hdl->lock); + goto out_locked; + } + + assert(received_hdl); + received_hdls[received_hdls_cnt++] = received_hdl; + + int vfd = set_new_fd_handle(received_hdl, 0, NULL); + if (vfd < 0) { + ret = vfd; + lock(&hdl->lock); + goto out_locked; + } + + cmsg_vfds[i] = vfd; + } + + *msg_controllen = cmsg_len; + } + if (flags & MSG_PEEK) { if (!peek_buffer) { /* create new peek buffer with expected read size */ @@ -1398,8 +1588,12 @@ static ssize_t do_recvmsg(int fd, struct iovec* bufs, size_t nbufs, int flags, goto out; out_locked: - if (ret < 0) + if (ret < 0) { + for (size_t i = 0; i < received_hdls_cnt; i++) + put_handle(received_hdls[i]); sock->error = -ret; + } + unlock(&hdl->lock); free(peek_buffer); out: @@ -1413,36 +1607,34 @@ ssize_t shim_do_recvfrom(int sockfd, void* buf, size_t len, int flags, struct so iovbuf.iov_base = (void*)buf; iovbuf.iov_len = len; - return do_recvmsg(sockfd, &iovbuf, 1, flags, addr, addrlen); + return do_recvmsg(sockfd, &iovbuf, 1, flags, addr, addrlen, NULL, NULL); } ssize_t shim_do_recvmsg(int sockfd, struct msghdr* msg, int flags) { + if (test_user_memory(msg, sizeof(*msg), /*write=*/true)) + return -EFAULT; + return do_recvmsg(sockfd, msg->msg_iov, msg->msg_iovlen, flags, msg->msg_name, - &msg->msg_namelen); + &msg->msg_namelen, msg->msg_control, &msg->msg_controllen); } -ssize_t shim_do_recvmmsg(int sockfd, struct mmsghdr* msg, unsigned int vlen, int flags, - struct __kernel_timespec* timeout) { - if (test_user_memory(msg, vlen, /*write=*/true)) +int shim_do_recvmmsg(int sockfd, struct mmsghdr* msgvec, unsigned int vlen, int flags, + struct __kernel_timespec* timeout) { + if (test_user_memory(msgvec, vlen * sizeof(*msgvec), /*write=*/true)) return -EFAULT; - ssize_t total = 0; - // Issue # 753 - https://github.com/oscarlab/graphene/issues/753 - /* TODO(donporter): timeout properly. For now, explicitly return an error. */ if (timeout) { debug("recvmmsg(): timeout parameter unsupported.\n"); return -EOPNOTSUPP; } - for (size_t i = 0; i * sizeof(struct mmsghdr) < vlen; i++) { - struct msghdr* m = &msg[i].msg_hdr; - - ssize_t bytes = - do_recvmsg(sockfd, m->msg_iov, m->msg_iovlen, flags, m->msg_name, &m->msg_namelen); + int total = 0; + for (size_t i = 0; i < vlen; i++) { + ssize_t bytes = shim_do_recvmsg(sockfd, &msgvec[i].msg_hdr, flags); if (bytes < 0) - return total > 0 ? total : bytes; + return total > 0 ? total : (int)bytes; - msg[i].msg_len = bytes; + msgvec[i].msg_len = bytes; total++; } diff --git a/LibOS/shim/test/ltp/ltp.cfg b/LibOS/shim/test/ltp/ltp.cfg index ed875812..c291c874 100644 --- a/LibOS/shim/test/ltp/ltp.cfg +++ b/LibOS/shim/test/ltp/ltp.cfg @@ -2309,9 +2309,11 @@ skip = yes [sendfile09_64] skip = yes +# requires root and tries to execute system() to bring up loop back device [sendmsg01] skip = yes +# requires root and is a bug reproducer for obscure selinux_socket_unix_may_send() [sendmsg02] skip = yes diff --git a/LibOS/shim/test/regression/.gitignore b/LibOS/shim/test/regression/.gitignore index 96826e4d..d6e50735 100644 --- a/LibOS/shim/test/regression/.gitignore +++ b/LibOS/shim/test/regression/.gitignore @@ -55,6 +55,7 @@ /pselect /readdir /sched +/scm_rights /select /shared_object /sigaltstack diff --git a/LibOS/shim/test/regression/Makefile b/LibOS/shim/test/regression/Makefile index aa2e6382..8dc0d0b6 100644 --- a/LibOS/shim/test/regression/Makefile +++ b/LibOS/shim/test/regression/Makefile @@ -47,6 +47,7 @@ c_executables = \ pselect \ readdir \ sched \ + scm_rights \ select \ shared_object \ sigaltstack \ diff --git a/LibOS/shim/test/regression/scm_rights.c b/LibOS/shim/test/regression/scm_rights.c new file mode 100644 index 00000000..e6fd2bda --- /dev/null +++ b/LibOS/shim/test/regression/scm_rights.c @@ -0,0 +1,289 @@ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define UNIX_SOCKET_NAME "dummy_unix_socket" + +#define VLEN 5 +#define BUFSIZE 100 + +#define STR_ONE "one" +#define STR_TWO "two" +#define STR_THREE "three" +#define STR_HELLO "hello world" + +/* ancillary data buffer, wrapped in a union in order to ensure it is suitably aligned */ +union { + char buf[CMSG_SPACE(sizeof(int)) * 2]; /* want to send two cmsg */ + struct cmsghdr align; +} cmsghdr_union; + +/* pipe to be transmitted from parent process to child via SCM_RIGHTS */ +int pipefds[2] = {-1, -1}; + +int server(void) { + int ret; + + ret = pipe(pipefds); + if (ret < 0) { + perror("[parent] pipe error"); + exit(1); + } + + int listen_fd = socket(AF_UNIX, SOCK_STREAM, 0); + if (listen_fd < 0) { + perror("[parent] socket error"); + exit(1); + } + + struct sockaddr_un address; + address.sun_family = AF_UNIX; + strncpy(address.sun_path, UNIX_SOCKET_NAME, sizeof(address.sun_path)); + + ret = bind(listen_fd, (struct sockaddr*)&address, sizeof(address)); + if (ret < 0) { + close(listen_fd); + perror("[parent] bind error"); + exit(1); + } + + ret = listen(listen_fd, 3); + if (ret < 0) { + close(listen_fd); + perror("[parent] listen error"); + exit(1); + } + + socklen_t addrlen = sizeof(address); + + int fd = accept(listen_fd, (struct sockaddr*)&address, &addrlen); + if (fd < 0) { + close(listen_fd); + perror("[parent] accept error"); + exit(1); + } + + if (close(listen_fd) < 0) { + perror("[parent] close error"); + exit(1); + } + + puts("[parent] The client is connected..."); + + struct mmsghdr msg[2] = {0}; + struct iovec msg1[2] = {0}; + struct iovec msg2 = {0}; + + msg1[0].iov_base = STR_ONE; + msg1[0].iov_len = strlen(STR_ONE); + msg1[1].iov_base = STR_TWO; + msg1[1].iov_len = strlen(STR_TWO); + + msg2.iov_base = STR_THREE; + msg2.iov_len = strlen(STR_THREE); + + msg[0].msg_hdr.msg_iov = msg1; + msg[0].msg_hdr.msg_iovlen = 2; + + msg[1].msg_hdr.msg_iov = &msg2; + msg[1].msg_hdr.msg_iovlen = 1; + + /* send two ends of the pipe as ancillary data in two cmsg's (just for fun) */ + msg[0].msg_hdr.msg_control = cmsghdr_union.buf; + msg[0].msg_hdr.msg_controllen = sizeof(cmsghdr_union.buf); + + struct cmsghdr* cmsg; + cmsg = CMSG_FIRSTHDR(&msg[0].msg_hdr); + cmsg->cmsg_level = SOL_SOCKET; + cmsg->cmsg_type = SCM_RIGHTS; + cmsg->cmsg_len = CMSG_LEN(sizeof(int)); + memcpy(CMSG_DATA(cmsg), &pipefds[0], sizeof(int)); + + cmsg = CMSG_NXTHDR(&msg[0].msg_hdr, cmsg); + cmsg->cmsg_level = SOL_SOCKET; + cmsg->cmsg_type = SCM_RIGHTS; + cmsg->cmsg_len = CMSG_LEN(sizeof(int)); + memcpy(CMSG_DATA(cmsg), &pipefds[1], sizeof(int)); + + ret = sendmmsg(fd, msg, 2, /*flags=*/0); + if (ret < 0) { + close(fd); + perror("[parent] sendmmsg error\n"); + exit(1); + } + + if (msg[0].msg_len != strlen(STR_ONE) + strlen(STR_TWO) || + msg[1].msg_len != strlen(STR_THREE)) { + close(fd); + fprintf(stderr, "[parent] sendmmsg error: not all messages were sent\n"); + exit(1); + } + + printf("[parent] %d messages sent\n", ret); + + if (close(fd) < 0) { + perror("[parent] close error"); + exit(1); + } + + return 0; +} + +int client(void) { + int ret; + + int fd = socket(AF_UNIX, SOCK_STREAM, 0); + if (fd < 0) { + perror("[child] socket error"); + exit(1); + } + + struct sockaddr_un address; + address.sun_family = AF_UNIX; + strncpy(address.sun_path, UNIX_SOCKET_NAME, sizeof(address.sun_path)); + + ret = -1; + while (ret < 0) { + /* wait until client is ready for read */ + errno = 0; + ret = connect(fd, (struct sockaddr*)&address, sizeof(address)); + if (ret < 0 && errno != ENOENT && errno != ECONNREFUSED) { + close(fd); + perror("[child] connect error\n"); + exit(1); + } + sched_yield(); + } + + puts("[child] Connected to the server, receiving..."); + + struct mmsghdr msgs[VLEN] = {0}; + struct iovec iovecs[VLEN] = {0}; + char bufs[VLEN][BUFSIZE + 1] = {0}; + + for (int i = 0; i < VLEN; i++) { + iovecs[i].iov_base = bufs[i]; + iovecs[i].iov_len = BUFSIZE; + msgs[i].msg_hdr.msg_iov = &iovecs[i]; + msgs[i].msg_hdr.msg_iovlen = 1; + } + + /* receive two ends of the pipe as ancillary data in cmsg (must come in the first msg) */ + msgs[0].msg_hdr.msg_control = cmsghdr_union.buf; + msgs[0].msg_hdr.msg_controllen = sizeof(cmsghdr_union.buf); + + ret = -1; + while (ret < 0) { + errno = 0; + ret = recvmmsg(fd, msgs, VLEN, MSG_DONTWAIT, /*timeout=*/NULL); + if (ret < 0 && errno != EAGAIN && errno != EWOULDBLOCK) { + close(fd); + perror("[child] recvmmsg error"); + exit(1); + } + sched_yield(); + } + + printf("[child] %d messages received\n", ret); + for (int i = 0; i < ret; i++) { + if (!msgs[i].msg_len) + continue; + + bufs[i][msgs[i].msg_len] = 0; + printf("[child] message %d: %s\n", i + 1, bufs[i]); + + if (!msgs[i].msg_hdr.msg_control || !msgs[i].msg_hdr.msg_controllen) + continue; + + struct cmsghdr* cmsg; + for (cmsg = CMSG_FIRSTHDR(&msgs[i].msg_hdr); cmsg != NULL; + cmsg = CMSG_NXTHDR(&msgs[i].msg_hdr, cmsg)) { + if (cmsg->cmsg_level != SOL_SOCKET || cmsg->cmsg_type != SCM_RIGHTS) { + close(fd); + fprintf(stderr, "[child] recvmmsg error: unexpected ancillary data\n"); + exit(1); + } + int* received_fds = (int*)CMSG_DATA(cmsg); + pipefds[0] = received_fds[0]; + pipefds[1] = received_fds[1]; + break; + } + } + + if (pipefds[0] == -1 || pipefds[1] == -1) { + close(fd); + fprintf(stderr, "[child] recvmmsg error: received incorrect pipefds as ancillary data\n"); + exit(1); + } + + /* test received pipe */ + ssize_t bytes; + bytes = write(pipefds[1], STR_HELLO, sizeof(STR_HELLO)); + if (bytes < 0) { + close(fd); + perror("[child] write error\n"); + exit(1); + } + + char buffer[128]; + bytes = 0; + while (bytes <= 0) { + errno = 0; + bytes = read(pipefds[0], &buffer, sizeof(buffer)); + if (bytes < 0 && errno != EAGAIN && errno != EWOULDBLOCK) { + close(fd); + perror("[child] read error"); + exit(1); + } + sched_yield(); + } + + buffer[sizeof(buffer) - 1] = '\0'; + if (bytes < sizeof(buffer)) + buffer[bytes] = '\0'; + + printf("[child] read on received pipe: %s\n", buffer); + + if (close(fd) < 0) { + perror("[child] close error"); + exit(1); + } + + return 0; +} + +int main(int argc, char** argv) { + int pid = fork(); + if (pid < 0) { + perror("fork error"); + return 1; + } + + if (pid == 0) + return client(); + + server(); + + pid = wait(NULL); /* wait for child termination, just for sanity */ + if (pid < 0) { + perror("[parent] wait error"); + return 1; + } + + if (unlink(UNIX_SOCKET_NAME) < 0) { + perror("[parent] unlink error"); + return 1; + } + + return 0; +} diff --git a/LibOS/shim/test/regression/test_libos.py b/LibOS/shim/test/regression/test_libos.py index 4fbe4433..fb0bf745 100644 --- a/LibOS/shim/test/regression/test_libos.py +++ b/LibOS/shim/test/regression/test_libos.py @@ -505,7 +505,7 @@ class TC_80_Socket(RegressionTestCase): self.assertIn('read on FIFO: Hello from write end of FIFO!', stdout) def test_100_socket_unix(self): - stdout, _ = self.run_binary(['unix']) + stdout, _ = self.run_binary(['unix'], timeout=60) self.assertIn('Data: This is packet 0', stdout) self.assertIn('Data: This is packet 1', stdout) self.assertIn('Data: This is packet 2', stdout) @@ -517,6 +517,13 @@ class TC_80_Socket(RegressionTestCase): self.assertIn('Data: This is packet 8', stdout) self.assertIn('Data: This is packet 9', stdout) + def test_101_scm_rights(self): + stdout, _ = self.run_binary(['scm_rights'], timeout=60) + self.assertIn('one', stdout) + self.assertIn('two', stdout) + self.assertIn('three', stdout) + self.assertIn('read on received pipe: hello world', stdout) + def test_200_socket_udp(self): stdout, _ = self.run_binary(['udp'], timeout=50) self.assertIn('Data: This is packet 0', stdout) diff --git a/Pal/src/host/Linux-SGX/db_streams.c b/Pal/src/host/Linux-SGX/db_streams.c index 3a9fd43e..90ee7961 100644 --- a/Pal/src/host/Linux-SGX/db_streams.c +++ b/Pal/src/host/Linux-SGX/db_streams.c @@ -244,6 +244,24 @@ static int handle_deserialize(PAL_HANDLE* handle, const void* data, size_t size, return 0; } +static int get_secure_fd_from_handle(PAL_HANDLE hdl, int* fd, void** ssl_ctx) { + assert(hdl && fd && ssl_ctx); + switch (PAL_GET_TYPE(hdl)) { + case pal_type_pipe: + case pal_type_pipesrv: + case pal_type_pipecli: + *fd = hdl->pipe.fd; + *ssl_ctx = hdl->pipe.ssl_ctx; + break; + case pal_type_process: + *fd = hdl->process.stream; + *ssl_ctx = hdl->process.ssl_ctx; + break; + default: + return -1; + } + return 0; +} /*! * \brief Send `cargo` handle to a process identified via `hdl` handle. * @@ -254,8 +272,11 @@ static int handle_deserialize(PAL_HANDLE* handle, const void* data, size_t size, * \return 0 on success, negative PAL error code otherwise. */ int _DkSendHandle(PAL_HANDLE hdl, PAL_HANDLE cargo) { - if (!IS_HANDLE_TYPE(hdl, process)) + int fd = -1; + void* ssl_ctx = NULL; + if (get_secure_fd_from_handle(hdl, &fd, &ssl_ctx) < 0) return -PAL_ERROR_BADHANDLE; + assert(fd >= 0); /* serialize cargo handle into a blob hdl_data */ void* hdl_data = NULL; @@ -265,7 +286,6 @@ int _DkSendHandle(PAL_HANDLE hdl, PAL_HANDLE cargo) { ssize_t ret; struct hdl_header hdl_hdr = {.fds = 0, .data_size = hdl_data_size}; - int fd = hdl->process.stream; /* apply bitmask of FDs-to-transfer to hdl_hdr.fds and populate `fds` with these FDs */ int fds[MAX_FDS]; @@ -301,8 +321,8 @@ int _DkSendHandle(PAL_HANDLE hdl, PAL_HANDLE cargo) { } /* finally send the serialized cargo as payload (possibly encrypted) */ - if (hdl->process.ssl_ctx) { - ret = _DkStreamSecureWrite(hdl->process.ssl_ctx, (uint8_t*)hdl_data, hdl_hdr.data_size); + if (ssl_ctx) { + ret = _DkStreamSecureWrite(ssl_ctx, (uint8_t*)hdl_data, hdl_hdr.data_size); } else { ret = ocall_write(fd, hdl_data, hdl_hdr.data_size); ret = IS_ERR(ret) ? unix_to_pal_error(ERRNO(ret)) : ret; @@ -322,12 +342,14 @@ int _DkSendHandle(PAL_HANDLE hdl, PAL_HANDLE cargo) { * \return 0 on success, negative PAL error code otherwise. */ int _DkReceiveHandle(PAL_HANDLE hdl, PAL_HANDLE* cargo) { - if (!IS_HANDLE_TYPE(hdl, process)) + int fd = -1; + void* ssl_ctx = NULL; + if (get_secure_fd_from_handle(hdl, &fd, &ssl_ctx) < 0) return -PAL_ERROR_BADHANDLE; + assert(fd >= 0); ssize_t ret; struct hdl_header hdl_hdr; - int fd = hdl->process.stream; /* first receive hdl_hdr so that we know how many FDs were transferred + how large is cargo */ ret = ocall_recv(fd, &hdl_hdr, sizeof(hdl_hdr), NULL, NULL, NULL, NULL); @@ -361,8 +383,8 @@ int _DkReceiveHandle(PAL_HANDLE hdl, PAL_HANDLE* cargo) { /* finally receive the serialized cargo as payload (possibly encrypted) */ char hdl_data[hdl_hdr.data_size]; - if (hdl->process.ssl_ctx) { - ret = _DkStreamSecureRead(hdl->process.ssl_ctx, (uint8_t*)hdl_data, hdl_hdr.data_size); + if (ssl_ctx) { + ret = _DkStreamSecureRead(ssl_ctx, (uint8_t*)hdl_data, hdl_hdr.data_size); } else { ret = ocall_read(fd, hdl_data, hdl_hdr.data_size); ret = IS_ERR(ret) ? unix_to_pal_error(ERRNO(ret)) : ret; diff --git a/Pal/src/host/Linux/db_streams.c b/Pal/src/host/Linux/db_streams.c index 3990e55a..ae22ada0 100644 --- a/Pal/src/host/Linux/db_streams.c +++ b/Pal/src/host/Linux/db_streams.c @@ -222,6 +222,23 @@ int handle_deserialize(PAL_HANDLE* handle, const void* data, int size) { return 0; } +static int get_fd_from_handle(PAL_HANDLE hdl, int* fd) { + assert(hdl && fd); + switch (PAL_GET_TYPE(hdl)) { + case pal_type_pipe: + case pal_type_pipesrv: + case pal_type_pipecli: + *fd = hdl->pipe.fd; + break; + case pal_type_process: + *fd = hdl->process.stream; + break; + default: + return -1; + } + return 0; +} + /*! * \brief Send `cargo` handle to a process identified via `hdl` handle. * @@ -230,8 +247,10 @@ int handle_deserialize(PAL_HANDLE* handle, const void* data, int size) { * \return 0 on success, negative PAL error code otherwise. */ int _DkSendHandle(PAL_HANDLE hdl, PAL_HANDLE cargo) { - if (!IS_HANDLE_TYPE(hdl, process)) + int fd = -1; + if (get_fd_from_handle(hdl, &fd) < 0) return -PAL_ERROR_BADHANDLE; + assert(fd >= 0); /* serialize cargo handle into a blob hdl_data */ void* hdl_data = NULL; @@ -241,7 +260,6 @@ int _DkSendHandle(PAL_HANDLE hdl, PAL_HANDLE cargo) { ssize_t ret; struct hdl_header hdl_hdr = {.fds = 0, .data_size = hdl_data_size}; - int fd = hdl->process.stream; /* apply bitmask of FDs-to-transfer to hdl_hdr.fds and populate `fds` with these FDs */ int fds[MAX_FDS]; @@ -304,12 +322,13 @@ int _DkSendHandle(PAL_HANDLE hdl, PAL_HANDLE cargo) { * \return 0 on success, negative PAL error code otherwise. */ int _DkReceiveHandle(PAL_HANDLE hdl, PAL_HANDLE* cargo) { - if (!IS_HANDLE_TYPE(hdl, process)) + int fd = -1; + if (get_fd_from_handle(hdl, &fd) < 0) return -PAL_ERROR_BADHANDLE; + assert(fd >= 0); ssize_t ret; struct hdl_header hdl_hdr; - int fd = hdl->process.stream; /* first receive hdl_hdr so that we know how many FDs were transferred + how large is cargo */ struct msghdr message_hdr = {0};