Commit 0608c69c authored by John Fastabend's avatar John Fastabend Committed by Daniel Borkmann

bpf: sk_msg, sock{map|hash} redirect through ULP

A sockmap program that redirects through a kTLS ULP enabled socket
will not work correctly because the ULP layer is skipped. This
fixes the behavior to call through the ULP layer on redirect to
ensure any operations required on the data stream at the ULP layer
continue to be applied.

To do this we add an internal flag MSG_SENDPAGE_NOPOLICY to avoid
calling the BPF layer on a redirected message. This is
required to avoid calling the BPF layer multiple times (possibly
recursively) which is not the current/expected behavior without
ULPs. In the future we may add a redirect flag if users _do_
want the policy applied again but this would need to work for both
ULP and non-ULP sockets and be opt-in to avoid breaking existing
programs.

Also to avoid polluting the flag space with an internal flag we
reuse the flag space overlapping MSG_SENDPAGE_NOPOLICY with
MSG_WAITFORONE. Here WAITFORONE is specific to recv path and
SENDPAGE_NOPOLICY is only used for sendpage hooks. The last thing
to verify is user space API is masked correctly to ensure the flag
can not be set by user. (Note this needs to be true regardless
because we have internal flags already in-use that user space
should not be able to set). But for completeness we have two UAPI
paths into sendpage, sendfile and splice.

In the sendfile case the function do_sendfile() zero's flags,

./fs/read_write.c:
 static ssize_t do_sendfile(int out_fd, int in_fd, loff_t *ppos,
		   	    size_t count, loff_t max)
 {
   ...
   fl = 0;
#if 0
   /*
    * We need to debate whether we can enable this or not. The
    * man page documents EAGAIN return for the output at least,
    * and the application is arguably buggy if it doesn't expect
    * EAGAIN on a non-blocking file descriptor.
    */
    if (in.file->f_flags & O_NONBLOCK)
	fl = SPLICE_F_NONBLOCK;
#endif
    file_start_write(out.file);
    retval = do_splice_direct(in.file, &pos, out.file, &out_pos, count, fl);
 }

In the splice case the pipe_to_sendpage "actor" is used which
masks flags with SPLICE_F_MORE.

./fs/splice.c:
 static int pipe_to_sendpage(struct pipe_inode_info *pipe,
			    struct pipe_buffer *buf, struct splice_desc *sd)
 {
   ...
   more = (sd->flags & SPLICE_F_MORE) ? MSG_MORE : 0;
   ...
 }

Confirming what we expect that internal flags  are in fact internal
to socket side.

Fixes: d3b18ad3 ("tls: add bpf support to sk_msg handling")
Signed-off-by: default avatarJohn Fastabend <john.fastabend@gmail.com>
Signed-off-by: default avatarDaniel Borkmann <daniel@iogearbox.net>
parent a136678c
...@@ -286,6 +286,7 @@ struct ucred { ...@@ -286,6 +286,7 @@ struct ucred {
#define MSG_NOSIGNAL 0x4000 /* Do not generate SIGPIPE */ #define MSG_NOSIGNAL 0x4000 /* Do not generate SIGPIPE */
#define MSG_MORE 0x8000 /* Sender will send more */ #define MSG_MORE 0x8000 /* Sender will send more */
#define MSG_WAITFORONE 0x10000 /* recvmmsg(): block until 1+ packets avail */ #define MSG_WAITFORONE 0x10000 /* recvmmsg(): block until 1+ packets avail */
#define MSG_SENDPAGE_NOPOLICY 0x10000 /* sendpage() internal : do no apply policy */
#define MSG_SENDPAGE_NOTLAST 0x20000 /* sendpage() internal : not the last page */ #define MSG_SENDPAGE_NOTLAST 0x20000 /* sendpage() internal : not the last page */
#define MSG_BATCH 0x40000 /* sendmmsg(): more messages coming */ #define MSG_BATCH 0x40000 /* sendmmsg(): more messages coming */
#define MSG_EOF MSG_FIN #define MSG_EOF MSG_FIN
......
...@@ -454,6 +454,15 @@ tls_offload_ctx_tx(const struct tls_context *tls_ctx) ...@@ -454,6 +454,15 @@ tls_offload_ctx_tx(const struct tls_context *tls_ctx)
return (struct tls_offload_context_tx *)tls_ctx->priv_ctx_tx; return (struct tls_offload_context_tx *)tls_ctx->priv_ctx_tx;
} }
static inline bool tls_sw_has_ctx_tx(const struct sock *sk)
{
struct tls_context *ctx = tls_get_ctx(sk);
if (!ctx)
return false;
return !!tls_sw_ctx_tx(ctx);
}
static inline struct tls_offload_context_rx * static inline struct tls_offload_context_rx *
tls_offload_ctx_rx(const struct tls_context *tls_ctx) tls_offload_ctx_rx(const struct tls_context *tls_ctx)
{ {
......
...@@ -8,6 +8,7 @@ ...@@ -8,6 +8,7 @@
#include <linux/wait.h> #include <linux/wait.h>
#include <net/inet_common.h> #include <net/inet_common.h>
#include <net/tls.h>
static bool tcp_bpf_stream_read(const struct sock *sk) static bool tcp_bpf_stream_read(const struct sock *sk)
{ {
...@@ -218,6 +219,8 @@ static int tcp_bpf_push(struct sock *sk, struct sk_msg *msg, u32 apply_bytes, ...@@ -218,6 +219,8 @@ static int tcp_bpf_push(struct sock *sk, struct sk_msg *msg, u32 apply_bytes,
u32 off; u32 off;
while (1) { while (1) {
bool has_tx_ulp;
sge = sk_msg_elem(msg, msg->sg.start); sge = sk_msg_elem(msg, msg->sg.start);
size = (apply && apply_bytes < sge->length) ? size = (apply && apply_bytes < sge->length) ?
apply_bytes : sge->length; apply_bytes : sge->length;
...@@ -226,7 +229,15 @@ static int tcp_bpf_push(struct sock *sk, struct sk_msg *msg, u32 apply_bytes, ...@@ -226,7 +229,15 @@ static int tcp_bpf_push(struct sock *sk, struct sk_msg *msg, u32 apply_bytes,
tcp_rate_check_app_limited(sk); tcp_rate_check_app_limited(sk);
retry: retry:
ret = do_tcp_sendpages(sk, page, off, size, flags); has_tx_ulp = tls_sw_has_ctx_tx(sk);
if (has_tx_ulp) {
flags |= MSG_SENDPAGE_NOPOLICY;
ret = kernel_sendpage_locked(sk,
page, off, size, flags);
} else {
ret = do_tcp_sendpages(sk, page, off, size, flags);
}
if (ret <= 0) if (ret <= 0)
return ret; return ret;
if (apply) if (apply)
......
...@@ -686,12 +686,13 @@ static int bpf_exec_tx_verdict(struct sk_msg *msg, struct sock *sk, ...@@ -686,12 +686,13 @@ static int bpf_exec_tx_verdict(struct sk_msg *msg, struct sock *sk,
struct sk_psock *psock; struct sk_psock *psock;
struct sock *sk_redir; struct sock *sk_redir;
struct tls_rec *rec; struct tls_rec *rec;
bool enospc, policy;
int err = 0, send; int err = 0, send;
u32 delta = 0; u32 delta = 0;
bool enospc;
policy = !(flags & MSG_SENDPAGE_NOPOLICY);
psock = sk_psock_get(sk); psock = sk_psock_get(sk);
if (!psock) if (!psock || !policy)
return tls_push_record(sk, flags, record_type); return tls_push_record(sk, flags, record_type);
more_data: more_data:
enospc = sk_msg_full(msg); enospc = sk_msg_full(msg);
...@@ -1017,8 +1018,8 @@ int tls_sw_sendmsg(struct sock *sk, struct msghdr *msg, size_t size) ...@@ -1017,8 +1018,8 @@ int tls_sw_sendmsg(struct sock *sk, struct msghdr *msg, size_t size)
return copied ? copied : ret; return copied ? copied : ret;
} }
int tls_sw_sendpage(struct sock *sk, struct page *page, int tls_sw_do_sendpage(struct sock *sk, struct page *page,
int offset, size_t size, int flags) int offset, size_t size, int flags)
{ {
long timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT); long timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT);
struct tls_context *tls_ctx = tls_get_ctx(sk); struct tls_context *tls_ctx = tls_get_ctx(sk);
...@@ -1033,15 +1034,7 @@ int tls_sw_sendpage(struct sock *sk, struct page *page, ...@@ -1033,15 +1034,7 @@ int tls_sw_sendpage(struct sock *sk, struct page *page,
int ret = 0; int ret = 0;
bool eor; bool eor;
if (flags & ~(MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL |
MSG_SENDPAGE_NOTLAST))
return -ENOTSUPP;
/* No MSG_EOR from splice, only look at MSG_MORE */
eor = !(flags & (MSG_MORE | MSG_SENDPAGE_NOTLAST)); eor = !(flags & (MSG_MORE | MSG_SENDPAGE_NOTLAST));
lock_sock(sk);
sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk); sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk);
/* Wait till there is any pending write on socket */ /* Wait till there is any pending write on socket */
...@@ -1145,10 +1138,34 @@ int tls_sw_sendpage(struct sock *sk, struct page *page, ...@@ -1145,10 +1138,34 @@ int tls_sw_sendpage(struct sock *sk, struct page *page,
} }
sendpage_end: sendpage_end:
ret = sk_stream_error(sk, flags, ret); ret = sk_stream_error(sk, flags, ret);
release_sock(sk);
return copied ? copied : ret; return copied ? copied : ret;
} }
int tls_sw_sendpage_locked(struct sock *sk, struct page *page,
int offset, size_t size, int flags)
{
if (flags & ~(MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL |
MSG_SENDPAGE_NOTLAST | MSG_SENDPAGE_NOPOLICY))
return -ENOTSUPP;
return tls_sw_do_sendpage(sk, page, offset, size, flags);
}
int tls_sw_sendpage(struct sock *sk, struct page *page,
int offset, size_t size, int flags)
{
int ret;
if (flags & ~(MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL |
MSG_SENDPAGE_NOTLAST | MSG_SENDPAGE_NOPOLICY))
return -ENOTSUPP;
lock_sock(sk);
ret = tls_sw_do_sendpage(sk, page, offset, size, flags);
release_sock(sk);
return ret;
}
static struct sk_buff *tls_wait_data(struct sock *sk, struct sk_psock *psock, static struct sk_buff *tls_wait_data(struct sock *sk, struct sk_psock *psock,
int flags, long timeo, int *err) int flags, long timeo, int *err)
{ {
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment