From 4cbc325ed6b4dce4910be06d9d6940a8b919c59b Mon Sep 17 00:00:00 2001 From: Jakub Kicinski Date: Thu, 14 Jul 2022 22:22:25 -0700 Subject: tls: rx: allow only one reader at a time recvmsg() in TLS gets data from the skb list (rx_list) or fresh skbs we read from TCP via strparser. The former holds skbs which were already decrypted for peek or decrypted and partially consumed. tls_wait_data() only notices appearance of fresh skbs coming out of TCP (or psock). It is possible, if there is a concurrent call to peek() and recv() that the peek() will move the data from input to rx_list without recv() noticing. recv() will then read data out of order or never wake up. This is not a practical use case/concern, but it makes the self tests less reliable. This patch solves the problem by allowing only one reader in. Because having multiple processes calling read()/peek() is not normal avoid adding a lock and try to fast-path the single reader case. Signed-off-by: Jakub Kicinski Signed-off-by: David S. Miller --- include/net/tls.h | 3 +++ 1 file changed, 3 insertions(+) (limited to 'include') diff --git a/include/net/tls.h b/include/net/tls.h index 8742e13bc362..e8935cfe0cd6 100644 --- a/include/net/tls.h +++ b/include/net/tls.h @@ -116,11 +116,14 @@ struct tls_sw_context_rx { void (*saved_data_ready)(struct sock *sk); struct sk_buff *recv_pkt; + u8 reader_present; u8 async_capable:1; u8 zc_capable:1; + u8 reader_contended:1; atomic_t decrypt_pending; /* protect crypto_wait with decrypt_pending*/ spinlock_t decrypt_compl_lock; + struct wait_queue_head wq; }; struct tls_record_info { -- cgit v1.2.3 From 53d57999fe02785040bc53e2f12efc881f13ae17 Mon Sep 17 00:00:00 2001 From: Jakub Kicinski Date: Thu, 14 Jul 2022 22:22:28 -0700 Subject: tls: rx: remove the message decrypted tracking We no longer allow a decrypted skb to remain linked to ctx->recv_pkt. Anything on the list is decrypted, anything on ctx->recv_pkt needs to be decrypted. Signed-off-by: Jakub Kicinski Signed-off-by: David S. Miller --- include/net/strparser.h | 1 - net/tls/tls_sw.c | 10 ---------- 2 files changed, 11 deletions(-) (limited to 'include') diff --git a/include/net/strparser.h b/include/net/strparser.h index 88900b05443e..41e2ce9e9e10 100644 --- a/include/net/strparser.h +++ b/include/net/strparser.h @@ -72,7 +72,6 @@ struct sk_skb_cb { /* strp users' data follows */ struct tls_msg { u8 control; - u8 decrypted; } tls; /* temp_reg is a temporary register used for bpf_convert_data_end_access * when dst_reg == src_reg. diff --git a/net/tls/tls_sw.c b/net/tls/tls_sw.c index f5f06d1ba024..49cfaa8119c6 100644 --- a/net/tls/tls_sw.c +++ b/net/tls/tls_sw.c @@ -1563,21 +1563,13 @@ static int decrypt_skb_update(struct sock *sk, struct sk_buff *skb, struct tls_context *tls_ctx = tls_get_ctx(sk); struct tls_prot_info *prot = &tls_ctx->prot_info; struct strp_msg *rxm = strp_msg(skb); - struct tls_msg *tlm = tls_msg(skb); int pad, err; - if (tlm->decrypted) { - darg->zc = false; - darg->async = false; - return 0; - } - if (tls_ctx->rx_conf == TLS_HW) { err = tls_device_decrypted(sk, tls_ctx, skb, rxm); if (err < 0) return err; if (err > 0) { - tlm->decrypted = 1; darg->zc = false; darg->async = false; goto decrypt_done; @@ -1610,7 +1602,6 @@ decrypt_done: rxm->full_len -= pad; rxm->offset += prot->prepend_size; rxm->full_len -= prot->overhead_size; - tlm->decrypted = 1; decrypt_next: tls_advance_record_sn(sk, prot, &tls_ctx->rx); @@ -2130,7 +2121,6 @@ static int tls_read_size(struct strparser *strp, struct sk_buff *skb) if (ret < 0) goto read_failure; - tlm->decrypted = 0; tlm->control = header[0]; data_len = ((header[4] & 0xFF) | (header[3] << 8)); -- cgit v1.2.3 From c618db2afe7c31d13ca8cf05b60f17165fbdc282 Mon Sep 17 00:00:00 2001 From: Jakub Kicinski Date: Thu, 14 Jul 2022 22:22:33 -0700 Subject: tls: rx: async: hold onto the input skb Async crypto currently benefits from the fact that we decrypt in place. When we allow input and output to be different skbs we will have to hang onto the input while we move to the next record. Clone the inputs and keep them on a list. Signed-off-by: Jakub Kicinski Signed-off-by: David S. Miller --- include/net/tls.h | 1 + net/tls/Makefile | 2 +- net/tls/tls.h | 3 +++ net/tls/tls_strp.c | 17 +++++++++++++++++ net/tls/tls_sw.c | 26 +++++++++++++++++--------- 5 files changed, 39 insertions(+), 10 deletions(-) create mode 100644 net/tls/tls_strp.c (limited to 'include') diff --git a/include/net/tls.h b/include/net/tls.h index e8935cfe0cd6..181c496b01b8 100644 --- a/include/net/tls.h +++ b/include/net/tls.h @@ -123,6 +123,7 @@ struct tls_sw_context_rx { atomic_t decrypt_pending; /* protect crypto_wait with decrypt_pending*/ spinlock_t decrypt_compl_lock; + struct sk_buff_head async_hold; struct wait_queue_head wq; }; diff --git a/net/tls/Makefile b/net/tls/Makefile index f1ffbfe8968d..e41c800489ac 100644 --- a/net/tls/Makefile +++ b/net/tls/Makefile @@ -7,7 +7,7 @@ CFLAGS_trace.o := -I$(src) obj-$(CONFIG_TLS) += tls.o -tls-y := tls_main.o tls_sw.o tls_proc.o trace.o +tls-y := tls_main.o tls_sw.o tls_proc.o trace.o tls_strp.o tls-$(CONFIG_TLS_TOE) += tls_toe.o tls-$(CONFIG_TLS_DEVICE) += tls_device.o tls_device_fallback.o diff --git a/net/tls/tls.h b/net/tls/tls.h index 44522b221717..c818dc68955d 100644 --- a/net/tls/tls.h +++ b/net/tls/tls.h @@ -124,6 +124,9 @@ int tls_sw_fallback_init(struct sock *sk, struct tls_offload_context_tx *offload_ctx, struct tls_crypto_info *crypto_info); +int tls_strp_msg_hold(struct sock *sk, struct sk_buff *skb, + struct sk_buff_head *dst); + static inline struct tls_msg *tls_msg(struct sk_buff *skb) { struct sk_skb_cb *scb = (struct sk_skb_cb *)skb->cb; diff --git a/net/tls/tls_strp.c b/net/tls/tls_strp.c new file mode 100644 index 000000000000..9ccab79a6e1e --- /dev/null +++ b/net/tls/tls_strp.c @@ -0,0 +1,17 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include + +#include "tls.h" + +int tls_strp_msg_hold(struct sock *sk, struct sk_buff *skb, + struct sk_buff_head *dst) +{ + struct sk_buff *clone; + + clone = skb_clone(skb, sk->sk_allocation); + if (!clone) + return -ENOMEM; + __skb_queue_tail(dst, clone); + return 0; +} diff --git a/net/tls/tls_sw.c b/net/tls/tls_sw.c index 09fe2cfff51a..f767501e178d 100644 --- a/net/tls/tls_sw.c +++ b/net/tls/tls_sw.c @@ -1535,8 +1535,13 @@ fallback_to_reg_recv: goto exit_free_pages; darg->skb = tls_strp_msg(ctx); - if (darg->async) - return 0; + + if (unlikely(darg->async)) { + err = tls_strp_msg_hold(sk, skb, &ctx->async_hold); + if (err) + __skb_queue_tail(&ctx->async_hold, darg->skb); + return err; + } if (prot->tail_size) darg->tail = dctx->tail; @@ -1998,14 +2003,16 @@ recv_end: reinit_completion(&ctx->async_wait.completion); pending = atomic_read(&ctx->decrypt_pending); spin_unlock_bh(&ctx->decrypt_compl_lock); - if (pending) { + ret = 0; + if (pending) ret = crypto_wait_req(-EINPROGRESS, &ctx->async_wait); - if (ret) { - if (err >= 0 || err == -EINPROGRESS) - err = ret; - decrypted = 0; - goto end; - } + __skb_queue_purge(&ctx->async_hold); + + if (ret) { + if (err >= 0 || err == -EINPROGRESS) + err = ret; + decrypted = 0; + goto end; } /* Drain records from the rx_list & copy if required */ @@ -2440,6 +2447,7 @@ int tls_set_sw_offload(struct sock *sk, struct tls_context *ctx, int tx) crypto_info = &ctx->crypto_recv.info; cctx = &ctx->rx; skb_queue_head_init(&sw_ctx_rx->rx_list); + skb_queue_head_init(&sw_ctx_rx->async_hold); aead = &sw_ctx_rx->aead_recv; } -- cgit v1.2.3