nagai/kernel
3
0
Fork 0
forked from mirrors/linux
Code Pull requests Activity

net/tls: Split conf to rx + tx

Browse source 
In TLS inline crypto, we can have one direction in software
and another in hardware. Thus, we split the TLS configuration to separate
structures for receive and transmit.

Signed-off-by: Boris Pismenny <borisp@mellanox.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
This commit is contained in:
Boris Pismenny 2018-04-30 10:16:15 +03:00 committed by David S. Miller
parent 2342a8512a
commit f66de3ee2c
3 changed files with 158 additions and 124 deletions
repo.diff.show_diff_stats Download patch file Download diff file Expand all files Collapse all files

51
include/net/tls.h
View file

@ -83,21 +83,10 @@ struct tls_device {
void (*unhash)(struct tls_device *device, struct sock *sk); void (*unhash)(struct tls_device *device, struct sock *sk);
}; };
struct tls_sw_context { struct tls_sw_context_tx {
struct crypto_aead *aead_send; struct crypto_aead *aead_send;
struct crypto_aead *aead_recv;
struct crypto_wait async_wait; struct crypto_wait async_wait;
/* Receive context */
struct strparser strp;
void (*saved_data_ready)(struct sock *sk);
unsigned int (*sk_poll)(struct file *file, struct socket *sock,
struct poll_table_struct *wait);
struct sk_buff *recv_pkt;
u8 control;
bool decrypted;
/* Sending context */
char aad_space[TLS_AAD_SPACE_SIZE]; char aad_space[TLS_AAD_SPACE_SIZE];
unsigned int sg_plaintext_size; unsigned int sg_plaintext_size;
@ -114,6 +103,19 @@ struct tls_sw_context {
struct scatterlist sg_aead_out[2]; struct scatterlist sg_aead_out[2];
}; };
struct tls_sw_context_rx {
struct crypto_aead *aead_recv;
struct crypto_wait async_wait;
struct strparser strp;
void (*saved_data_ready)(struct sock *sk);
unsigned int (*sk_poll)(struct file *file, struct socket *sock,
struct poll_table_struct *wait);
struct sk_buff *recv_pkt;
u8 control;
bool decrypted;
};
enum { enum {
TLS_PENDING_CLOSED_RECORD TLS_PENDING_CLOSED_RECORD
}; };
@ -138,9 +140,15 @@ struct tls_context {
struct tls12_crypto_info_aes_gcm_128 crypto_recv_aes_gcm_128; struct tls12_crypto_info_aes_gcm_128 crypto_recv_aes_gcm_128;
}; };
void *priv_ctx; struct list_head list;
struct net_device *netdev;
refcount_t refcount;
u8 conf:3; void *priv_ctx_tx;
void *priv_ctx_rx;
u8 tx_conf:3;
u8 rx_conf:3;
struct cipher_context tx; struct cipher_context tx;
struct cipher_context rx; struct cipher_context rx;
@ -177,7 +185,8 @@ int tls_sw_sendmsg(struct sock *sk, struct msghdr *msg, size_t size);
int tls_sw_sendpage(struct sock *sk, struct page *page, int tls_sw_sendpage(struct sock *sk, struct page *page,
int offset, size_t size, int flags); int offset, size_t size, int flags);
void tls_sw_close(struct sock *sk, long timeout); void tls_sw_close(struct sock *sk, long timeout);
void tls_sw_free_resources(struct sock *sk); void tls_sw_free_resources_tx(struct sock *sk);
void tls_sw_free_resources_rx(struct sock *sk);
int tls_sw_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int tls_sw_recvmsg(struct sock *sk, struct msghdr *msg, size_t len,
int nonblock, int flags, int *addr_len); int nonblock, int flags, int *addr_len);
unsigned int tls_sw_poll(struct file *file, struct socket *sock, unsigned int tls_sw_poll(struct file *file, struct socket *sock,
@ -297,16 +306,22 @@ static inline struct tls_context *tls_get_ctx(const struct sock *sk)
return icsk->icsk_ulp_data; return icsk->icsk_ulp_data;
} }
static inline struct tls_sw_context *tls_sw_ctx( static inline struct tls_sw_context_rx *tls_sw_ctx_rx(
const struct tls_context *tls_ctx) const struct tls_context *tls_ctx)
{ {
return (struct tls_sw_context *)tls_ctx->priv_ctx; return (struct tls_sw_context_rx *)tls_ctx->priv_ctx_rx;
}
static inline struct tls_sw_context_tx *tls_sw_ctx_tx(
const struct tls_context *tls_ctx)
{
return (struct tls_sw_context_tx *)tls_ctx->priv_ctx_tx;
} }
static inline struct tls_offload_context *tls_offload_ctx( static inline struct tls_offload_context *tls_offload_ctx(
const struct tls_context *tls_ctx) const struct tls_context *tls_ctx)
{ {
return (struct tls_offload_context *)tls_ctx->priv_ctx; return (struct tls_offload_context *)tls_ctx->priv_ctx_tx;
} }
int tls_proccess_cmsg(struct sock *sk, struct msghdr *msg, int tls_proccess_cmsg(struct sock *sk, struct msghdr *msg,

89
net/tls/tls_main.c
View file

@ -51,12 +51,9 @@ enum {
TLSV6, TLSV6,
TLS_NUM_PROTS, TLS_NUM_PROTS,
}; };
enum { enum {
TLS_BASE, TLS_BASE,
TLS_SW_TX, TLS_SW,
TLS_SW_RX,
TLS_SW_RXTX,
TLS_HW_RECORD, TLS_HW_RECORD,
TLS_NUM_CONFIG, TLS_NUM_CONFIG,
}; };
@ -65,14 +62,14 @@ static struct proto *saved_tcpv6_prot;
static DEFINE_MUTEX(tcpv6_prot_mutex); static DEFINE_MUTEX(tcpv6_prot_mutex);
static LIST_HEAD(device_list); static LIST_HEAD(device_list);
static DEFINE_MUTEX(device_mutex); static DEFINE_MUTEX(device_mutex);
static struct proto tls_prots[TLS_NUM_PROTS][TLS_NUM_CONFIG]; static struct proto tls_prots[TLS_NUM_PROTS][TLS_NUM_CONFIG][TLS_NUM_CONFIG];
static struct proto_ops tls_sw_proto_ops; static struct proto_ops tls_sw_proto_ops;
static inline void update_sk_prot(struct sock *sk, struct tls_context *ctx) static void update_sk_prot(struct sock *sk, struct tls_context *ctx)
{ {
int ip_ver = sk->sk_family == AF_INET6 ? TLSV6 : TLSV4; int ip_ver = sk->sk_family == AF_INET6 ? TLSV6 : TLSV4;
sk->sk_prot = &tls_prots[ip_ver][ctx->conf]; sk->sk_prot = &tls_prots[ip_ver][ctx->tx_conf][ctx->rx_conf];
} }
int wait_on_pending_writer(struct sock *sk, long *timeo) int wait_on_pending_writer(struct sock *sk, long *timeo)
@ -245,10 +242,10 @@ static void tls_sk_proto_close(struct sock *sk, long timeout)
lock_sock(sk); lock_sock(sk);
sk_proto_close = ctx->sk_proto_close; sk_proto_close = ctx->sk_proto_close;
if (ctx->conf == TLS_HW_RECORD) if (ctx->tx_conf == TLS_HW_RECORD && ctx->rx_conf == TLS_HW_RECORD)
goto skip_tx_cleanup; goto skip_tx_cleanup;
if (ctx->conf == TLS_BASE) { if (ctx->tx_conf == TLS_BASE && ctx->rx_conf == TLS_BASE) {
kfree(ctx); kfree(ctx);
ctx = NULL; ctx = NULL;
goto skip_tx_cleanup; goto skip_tx_cleanup;
@ -270,15 +267,17 @@ static void tls_sk_proto_close(struct sock *sk, long timeout)
} }
} }
/* We need these for tls_sw_fallback handling of other packets */
if (ctx->tx_conf == TLS_SW) {
kfree(ctx->tx.rec_seq); kfree(ctx->tx.rec_seq);
kfree(ctx->tx.iv); kfree(ctx->tx.iv);
tls_sw_free_resources_tx(sk);
}
if (ctx->rx_conf == TLS_SW) {
kfree(ctx->rx.rec_seq); kfree(ctx->rx.rec_seq);
kfree(ctx->rx.iv); kfree(ctx->rx.iv);
tls_sw_free_resources_rx(sk);
if (ctx->conf == TLS_SW_TX ||
ctx->conf == TLS_SW_RX ||
ctx->conf == TLS_SW_RXTX) {
tls_sw_free_resources(sk);
} }
skip_tx_cleanup: skip_tx_cleanup:
@ -287,7 +286,8 @@ static void tls_sk_proto_close(struct sock *sk, long timeout)
/* free ctx for TLS_HW_RECORD, used by tcp_set_state /* free ctx for TLS_HW_RECORD, used by tcp_set_state
* for sk->sk_prot->unhash [tls_hw_unhash] * for sk->sk_prot->unhash [tls_hw_unhash]
*/ */
if (ctx && ctx->conf == TLS_HW_RECORD) if (ctx && ctx->tx_conf == TLS_HW_RECORD &&
ctx->rx_conf == TLS_HW_RECORD)
kfree(ctx); kfree(ctx);
} }
@ -441,25 +441,21 @@ static int do_tls_setsockopt_conf(struct sock *sk, char __user *optval,
goto err_crypto_info; goto err_crypto_info;
} }
/* currently SW is default, we will have ethtool in future */
if (tx) { if (tx) {
rc = tls_set_sw_offload(sk, ctx, 1); rc = tls_set_sw_offload(sk, ctx, 1);
if (ctx->conf == TLS_SW_RX) conf = TLS_SW;
conf = TLS_SW_RXTX;
else
conf = TLS_SW_TX;
} else { } else {
rc = tls_set_sw_offload(sk, ctx, 0); rc = tls_set_sw_offload(sk, ctx, 0);
if (ctx->conf == TLS_SW_TX) conf = TLS_SW;
conf = TLS_SW_RXTX;
else
conf = TLS_SW_RX;
} }
if (rc) if (rc)
goto err_crypto_info; goto err_crypto_info;
ctx->conf = conf; if (tx)
ctx->tx_conf = conf;
else
ctx->rx_conf = conf;
update_sk_prot(sk, ctx); update_sk_prot(sk, ctx);
if (tx) { if (tx) {
ctx->sk_write_space = sk->sk_write_space; ctx->sk_write_space = sk->sk_write_space;
@ -535,7 +531,8 @@ static int tls_hw_prot(struct sock *sk)
ctx->hash = sk->sk_prot->hash; ctx->hash = sk->sk_prot->hash;
ctx->unhash = sk->sk_prot->unhash; ctx->unhash = sk->sk_prot->unhash;
ctx->sk_proto_close = sk->sk_prot->close; ctx->sk_proto_close = sk->sk_prot->close;
ctx->conf = TLS_HW_RECORD; ctx->rx_conf = TLS_HW_RECORD;
ctx->tx_conf = TLS_HW_RECORD;
update_sk_prot(sk, ctx); update_sk_prot(sk, ctx);
rc = 1; rc = 1;
break; break;
@ -579,29 +576,30 @@ static int tls_hw_hash(struct sock *sk)
return err; return err;
} }
static void build_protos(struct proto *prot, struct proto *base) static void build_protos(struct proto prot[TLS_NUM_CONFIG][TLS_NUM_CONFIG],
struct proto *base)
{ {
prot[TLS_BASE] = *base; prot[TLS_BASE][TLS_BASE] = *base;
prot[TLS_BASE].setsockopt = tls_setsockopt; prot[TLS_BASE][TLS_BASE].setsockopt = tls_setsockopt;
prot[TLS_BASE].getsockopt = tls_getsockopt; prot[TLS_BASE][TLS_BASE].getsockopt = tls_getsockopt;
prot[TLS_BASE].close = tls_sk_proto_close; prot[TLS_BASE][TLS_BASE].close = tls_sk_proto_close;
prot[TLS_SW_TX] = prot[TLS_BASE]; prot[TLS_SW][TLS_BASE] = prot[TLS_BASE][TLS_BASE];
prot[TLS_SW_TX].sendmsg = tls_sw_sendmsg; prot[TLS_SW][TLS_BASE].sendmsg = tls_sw_sendmsg;
prot[TLS_SW_TX].sendpage = tls_sw_sendpage; prot[TLS_SW][TLS_BASE].sendpage = tls_sw_sendpage;
prot[TLS_SW_RX] = prot[TLS_BASE]; prot[TLS_BASE][TLS_SW] = prot[TLS_BASE][TLS_BASE];
prot[TLS_SW_RX].recvmsg = tls_sw_recvmsg; prot[TLS_BASE][TLS_SW].recvmsg = tls_sw_recvmsg;
prot[TLS_SW_RX].close = tls_sk_proto_close; prot[TLS_BASE][TLS_SW].close = tls_sk_proto_close;
prot[TLS_SW_RXTX] = prot[TLS_SW_TX]; prot[TLS_SW][TLS_SW] = prot[TLS_SW][TLS_BASE];
prot[TLS_SW_RXTX].recvmsg = tls_sw_recvmsg; prot[TLS_SW][TLS_SW].recvmsg = tls_sw_recvmsg;
prot[TLS_SW_RXTX].close = tls_sk_proto_close; prot[TLS_SW][TLS_SW].close = tls_sk_proto_close;
prot[TLS_HW_RECORD] = *base; prot[TLS_HW_RECORD][TLS_HW_RECORD] = *base;
prot[TLS_HW_RECORD].hash = tls_hw_hash; prot[TLS_HW_RECORD][TLS_HW_RECORD].hash = tls_hw_hash;
prot[TLS_HW_RECORD].unhash = tls_hw_unhash; prot[TLS_HW_RECORD][TLS_HW_RECORD].unhash = tls_hw_unhash;
prot[TLS_HW_RECORD].close = tls_sk_proto_close; prot[TLS_HW_RECORD][TLS_HW_RECORD].close = tls_sk_proto_close;
} }
static int tls_init(struct sock *sk) static int tls_init(struct sock *sk)
@ -643,7 +641,8 @@ static int tls_init(struct sock *sk)
mutex_unlock(&tcpv6_prot_mutex); mutex_unlock(&tcpv6_prot_mutex);
} }
ctx->conf = TLS_BASE; ctx->tx_conf = TLS_BASE;
ctx->rx_conf = TLS_BASE;
update_sk_prot(sk, ctx); update_sk_prot(sk, ctx);
out: out:
return rc; return rc;

136
net/tls/tls_sw.c
View file

@ -52,7 +52,7 @@ static int tls_do_decryption(struct sock *sk,
gfp_t flags) gfp_t flags)
{ {
struct tls_context *tls_ctx = tls_get_ctx(sk); struct tls_context *tls_ctx = tls_get_ctx(sk);
struct tls_sw_context *ctx = tls_sw_ctx(tls_ctx); struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
struct strp_msg *rxm = strp_msg(skb); struct strp_msg *rxm = strp_msg(skb);
struct aead_request *aead_req; struct aead_request *aead_req;
@ -122,7 +122,7 @@ static void trim_sg(struct sock *sk, struct scatterlist *sg,
static void trim_both_sgl(struct sock *sk, int target_size) static void trim_both_sgl(struct sock *sk, int target_size)
{ {
struct tls_context *tls_ctx = tls_get_ctx(sk); struct tls_context *tls_ctx = tls_get_ctx(sk);
struct tls_sw_context *ctx = tls_sw_ctx(tls_ctx); struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
trim_sg(sk, ctx->sg_plaintext_data, trim_sg(sk, ctx->sg_plaintext_data,
&ctx->sg_plaintext_num_elem, &ctx->sg_plaintext_num_elem,
@ -141,7 +141,7 @@ static void trim_both_sgl(struct sock *sk, int target_size)
static int alloc_encrypted_sg(struct sock *sk, int len) static int alloc_encrypted_sg(struct sock *sk, int len)
{ {
struct tls_context *tls_ctx = tls_get_ctx(sk); struct tls_context *tls_ctx = tls_get_ctx(sk);
struct tls_sw_context *ctx = tls_sw_ctx(tls_ctx); struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
int rc = 0; int rc = 0;
rc = sk_alloc_sg(sk, len, rc = sk_alloc_sg(sk, len,
@ -155,7 +155,7 @@ static int alloc_encrypted_sg(struct sock *sk, int len)
static int alloc_plaintext_sg(struct sock *sk, int len) static int alloc_plaintext_sg(struct sock *sk, int len)
{ {
struct tls_context *tls_ctx = tls_get_ctx(sk); struct tls_context *tls_ctx = tls_get_ctx(sk);
struct tls_sw_context *ctx = tls_sw_ctx(tls_ctx); struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
int rc = 0; int rc = 0;
rc = sk_alloc_sg(sk, len, ctx->sg_plaintext_data, 0, rc = sk_alloc_sg(sk, len, ctx->sg_plaintext_data, 0,
@ -181,7 +181,7 @@ static void free_sg(struct sock *sk, struct scatterlist *sg,
static void tls_free_both_sg(struct sock *sk) static void tls_free_both_sg(struct sock *sk)
{ {
struct tls_context *tls_ctx = tls_get_ctx(sk); struct tls_context *tls_ctx = tls_get_ctx(sk);
struct tls_sw_context *ctx = tls_sw_ctx(tls_ctx); struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
free_sg(sk, ctx->sg_encrypted_data, &ctx->sg_encrypted_num_elem, free_sg(sk, ctx->sg_encrypted_data, &ctx->sg_encrypted_num_elem,
&ctx->sg_encrypted_size); &ctx->sg_encrypted_size);
@ -191,7 +191,7 @@ static void tls_free_both_sg(struct sock *sk)
} }
static int tls_do_encryption(struct tls_context *tls_ctx, static int tls_do_encryption(struct tls_context *tls_ctx,
struct tls_sw_context *ctx, size_t data_len, struct tls_sw_context_tx *ctx, size_t data_len,
gfp_t flags) gfp_t flags)
{ {
unsigned int req_size = sizeof(struct aead_request) + unsigned int req_size = sizeof(struct aead_request) +
@ -227,7 +227,7 @@ static int tls_push_record(struct sock *sk, int flags,
unsigned char record_type) unsigned char record_type)
{ {
struct tls_context *tls_ctx = tls_get_ctx(sk); struct tls_context *tls_ctx = tls_get_ctx(sk);
struct tls_sw_context *ctx = tls_sw_ctx(tls_ctx); struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
int rc; int rc;
sg_mark_end(ctx->sg_plaintext_data + ctx->sg_plaintext_num_elem - 1); sg_mark_end(ctx->sg_plaintext_data + ctx->sg_plaintext_num_elem - 1);
@ -339,7 +339,7 @@ static int memcopy_from_iter(struct sock *sk, struct iov_iter *from,
int bytes) int bytes)
{ {
struct tls_context *tls_ctx = tls_get_ctx(sk); struct tls_context *tls_ctx = tls_get_ctx(sk);
struct tls_sw_context *ctx = tls_sw_ctx(tls_ctx); struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
struct scatterlist *sg = ctx->sg_plaintext_data; struct scatterlist *sg = ctx->sg_plaintext_data;
int copy, i, rc = 0; int copy, i, rc = 0;
@ -367,7 +367,7 @@ static int memcopy_from_iter(struct sock *sk, struct iov_iter *from,
int tls_sw_sendmsg(struct sock *sk, struct msghdr *msg, size_t size) int tls_sw_sendmsg(struct sock *sk, struct msghdr *msg, size_t size)
{ {
struct tls_context *tls_ctx = tls_get_ctx(sk); struct tls_context *tls_ctx = tls_get_ctx(sk);
struct tls_sw_context *ctx = tls_sw_ctx(tls_ctx); struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
int ret = 0; int ret = 0;
int required_size; int required_size;
long timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT); long timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT);
@ -522,7 +522,7 @@ int tls_sw_sendpage(struct sock *sk, struct page *page,
int offset, size_t size, int flags) int offset, size_t size, int flags)
{ {
struct tls_context *tls_ctx = tls_get_ctx(sk); struct tls_context *tls_ctx = tls_get_ctx(sk);
struct tls_sw_context *ctx = tls_sw_ctx(tls_ctx); struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
int ret = 0; int ret = 0;
long timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT); long timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT);
bool eor; bool eor;
@ -636,7 +636,7 @@ static struct sk_buff *tls_wait_data(struct sock *sk, int flags,
long timeo, int *err) long timeo, int *err)
{ {
struct tls_context *tls_ctx = tls_get_ctx(sk); struct tls_context *tls_ctx = tls_get_ctx(sk);
struct tls_sw_context *ctx = tls_sw_ctx(tls_ctx); struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
struct sk_buff *skb; struct sk_buff *skb;
DEFINE_WAIT_FUNC(wait, woken_wake_function); DEFINE_WAIT_FUNC(wait, woken_wake_function);
@ -674,7 +674,7 @@ static int decrypt_skb(struct sock *sk, struct sk_buff *skb,
struct scatterlist *sgout) struct scatterlist *sgout)
{ {
struct tls_context *tls_ctx = tls_get_ctx(sk); struct tls_context *tls_ctx = tls_get_ctx(sk);
struct tls_sw_context *ctx = tls_sw_ctx(tls_ctx); struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
char iv[TLS_CIPHER_AES_GCM_128_SALT_SIZE + MAX_IV_SIZE]; char iv[TLS_CIPHER_AES_GCM_128_SALT_SIZE + MAX_IV_SIZE];
struct scatterlist sgin_arr[MAX_SKB_FRAGS + 2]; struct scatterlist sgin_arr[MAX_SKB_FRAGS + 2];
struct scatterlist *sgin = &sgin_arr[0]; struct scatterlist *sgin = &sgin_arr[0];
@ -723,7 +723,7 @@ static bool tls_sw_advance_skb(struct sock *sk, struct sk_buff *skb,
unsigned int len) unsigned int len)
{ {
struct tls_context *tls_ctx = tls_get_ctx(sk); struct tls_context *tls_ctx = tls_get_ctx(sk);
struct tls_sw_context *ctx = tls_sw_ctx(tls_ctx); struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
struct strp_msg *rxm = strp_msg(skb); struct strp_msg *rxm = strp_msg(skb);
if (len < rxm->full_len) { if (len < rxm->full_len) {
@ -749,7 +749,7 @@ int tls_sw_recvmsg(struct sock *sk,
int *addr_len) int *addr_len)
{ {
struct tls_context *tls_ctx = tls_get_ctx(sk); struct tls_context *tls_ctx = tls_get_ctx(sk);
struct tls_sw_context *ctx = tls_sw_ctx(tls_ctx); struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
unsigned char control; unsigned char control;
struct strp_msg *rxm; struct strp_msg *rxm;
struct sk_buff *skb; struct sk_buff *skb;
@ -869,7 +869,7 @@ ssize_t tls_sw_splice_read(struct socket *sock, loff_t *ppos,
size_t len, unsigned int flags) size_t len, unsigned int flags)
{ {
struct tls_context *tls_ctx = tls_get_ctx(sock->sk); struct tls_context *tls_ctx = tls_get_ctx(sock->sk);
struct tls_sw_context *ctx = tls_sw_ctx(tls_ctx); struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
struct strp_msg *rxm = NULL; struct strp_msg *rxm = NULL;
struct sock *sk = sock->sk; struct sock *sk = sock->sk;
struct sk_buff *skb; struct sk_buff *skb;
@ -922,7 +922,7 @@ unsigned int tls_sw_poll(struct file *file, struct socket *sock,
unsigned int ret; unsigned int ret;
struct sock *sk = sock->sk; struct sock *sk = sock->sk;
struct tls_context *tls_ctx = tls_get_ctx(sk); struct tls_context *tls_ctx = tls_get_ctx(sk);
struct tls_sw_context *ctx = tls_sw_ctx(tls_ctx); struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
/* Grab POLLOUT and POLLHUP from the underlying socket */ /* Grab POLLOUT and POLLHUP from the underlying socket */
ret = ctx->sk_poll(file, sock, wait); ret = ctx->sk_poll(file, sock, wait);
@ -938,7 +938,7 @@ unsigned int tls_sw_poll(struct file *file, struct socket *sock,
static int tls_read_size(struct strparser *strp, struct sk_buff *skb) static int tls_read_size(struct strparser *strp, struct sk_buff *skb)
{ {
struct tls_context *tls_ctx = tls_get_ctx(strp->sk); struct tls_context *tls_ctx = tls_get_ctx(strp->sk);
struct tls_sw_context *ctx = tls_sw_ctx(tls_ctx); struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
char header[tls_ctx->rx.prepend_size]; char header[tls_ctx->rx.prepend_size];
struct strp_msg *rxm = strp_msg(skb); struct strp_msg *rxm = strp_msg(skb);
size_t cipher_overhead; size_t cipher_overhead;
@ -987,7 +987,7 @@ static int tls_read_size(struct strparser *strp, struct sk_buff *skb)
static void tls_queue(struct strparser *strp, struct sk_buff *skb) static void tls_queue(struct strparser *strp, struct sk_buff *skb)
{ {
struct tls_context *tls_ctx = tls_get_ctx(strp->sk); struct tls_context *tls_ctx = tls_get_ctx(strp->sk);
struct tls_sw_context *ctx = tls_sw_ctx(tls_ctx); struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
struct strp_msg *rxm; struct strp_msg *rxm;
rxm = strp_msg(skb); rxm = strp_msg(skb);
@ -1003,18 +1003,28 @@ static void tls_queue(struct strparser *strp, struct sk_buff *skb)
static void tls_data_ready(struct sock *sk) static void tls_data_ready(struct sock *sk)
{ {
struct tls_context *tls_ctx = tls_get_ctx(sk); struct tls_context *tls_ctx = tls_get_ctx(sk);
struct tls_sw_context *ctx = tls_sw_ctx(tls_ctx); struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
strp_data_ready(&ctx->strp); strp_data_ready(&ctx->strp);
} }
void tls_sw_free_resources(struct sock *sk) void tls_sw_free_resources_tx(struct sock *sk)
{ {
struct tls_context *tls_ctx = tls_get_ctx(sk); struct tls_context *tls_ctx = tls_get_ctx(sk);
struct tls_sw_context *ctx = tls_sw_ctx(tls_ctx); struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
if (ctx->aead_send) if (ctx->aead_send)
crypto_free_aead(ctx->aead_send); crypto_free_aead(ctx->aead_send);
tls_free_both_sg(sk);
kfree(ctx);
}
void tls_sw_free_resources_rx(struct sock *sk)
{
struct tls_context *tls_ctx = tls_get_ctx(sk);
struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
if (ctx->aead_recv) { if (ctx->aead_recv) {
if (ctx->recv_pkt) { if (ctx->recv_pkt) {
kfree_skb(ctx->recv_pkt); kfree_skb(ctx->recv_pkt);
@ -1030,10 +1040,7 @@ void tls_sw_free_resources(struct sock *sk)
lock_sock(sk); lock_sock(sk);
} }
tls_free_both_sg(sk);
kfree(ctx); kfree(ctx);
kfree(tls_ctx);
} }
int tls_set_sw_offload(struct sock *sk, struct tls_context *ctx, int tx) int tls_set_sw_offload(struct sock *sk, struct tls_context *ctx, int tx)
@ -1041,7 +1048,8 @@ int tls_set_sw_offload(struct sock *sk, struct tls_context *ctx, int tx)
char keyval[TLS_CIPHER_AES_GCM_128_KEY_SIZE]; char keyval[TLS_CIPHER_AES_GCM_128_KEY_SIZE];
struct tls_crypto_info *crypto_info; struct tls_crypto_info *crypto_info;
struct tls12_crypto_info_aes_gcm_128 *gcm_128_info; struct tls12_crypto_info_aes_gcm_128 *gcm_128_info;
struct tls_sw_context *sw_ctx; struct tls_sw_context_tx *sw_ctx_tx = NULL;
struct tls_sw_context_rx *sw_ctx_rx = NULL;
struct cipher_context *cctx; struct cipher_context *cctx;
struct crypto_aead **aead; struct crypto_aead **aead;
struct strp_callbacks cb; struct strp_callbacks cb;
@ -1054,27 +1062,32 @@ int tls_set_sw_offload(struct sock *sk, struct tls_context *ctx, int tx)
goto out; goto out;
} }
if (!ctx->priv_ctx) { if (tx) {
sw_ctx = kzalloc(sizeof(*sw_ctx), GFP_KERNEL); sw_ctx_tx = kzalloc(sizeof(*sw_ctx_tx), GFP_KERNEL);
if (!sw_ctx) { if (!sw_ctx_tx) {
rc = -ENOMEM; rc = -ENOMEM;
goto out; goto out;
} }
crypto_init_wait(&sw_ctx->async_wait); crypto_init_wait(&sw_ctx_tx->async_wait);
ctx->priv_ctx_tx = sw_ctx_tx;
} else { } else {
sw_ctx = ctx->priv_ctx; sw_ctx_rx = kzalloc(sizeof(*sw_ctx_rx), GFP_KERNEL);
if (!sw_ctx_rx) {
rc = -ENOMEM;
goto out;
}
crypto_init_wait(&sw_ctx_rx->async_wait);
ctx->priv_ctx_rx = sw_ctx_rx;
} }
ctx->priv_ctx = (struct tls_offload_context *)sw_ctx;
if (tx) { if (tx) {
crypto_info = &ctx->crypto_send; crypto_info = &ctx->crypto_send;
cctx = &ctx->tx; cctx = &ctx->tx;
aead = &sw_ctx->aead_send; aead = &sw_ctx_tx->aead_send;
} else { } else {
crypto_info = &ctx->crypto_recv; crypto_info = &ctx->crypto_recv;
cctx = &ctx->rx; cctx = &ctx->rx;
aead = &sw_ctx->aead_recv; aead = &sw_ctx_rx->aead_recv;
} }
switch (crypto_info->cipher_type) { switch (crypto_info->cipher_type) {
@ -1121,22 +1134,24 @@ int tls_set_sw_offload(struct sock *sk, struct tls_context *ctx, int tx)
} }
memcpy(cctx->rec_seq, rec_seq, rec_seq_size); memcpy(cctx->rec_seq, rec_seq, rec_seq_size);
if (tx) { if (sw_ctx_tx) {
sg_init_table(sw_ctx->sg_encrypted_data, sg_init_table(sw_ctx_tx->sg_encrypted_data,
ARRAY_SIZE(sw_ctx->sg_encrypted_data)); ARRAY_SIZE(sw_ctx_tx->sg_encrypted_data));
sg_init_table(sw_ctx->sg_plaintext_data, sg_init_table(sw_ctx_tx->sg_plaintext_data,
ARRAY_SIZE(sw_ctx->sg_plaintext_data)); ARRAY_SIZE(sw_ctx_tx->sg_plaintext_data));
sg_init_table(sw_ctx->sg_aead_in, 2); sg_init_table(sw_ctx_tx->sg_aead_in, 2);
sg_set_buf(&sw_ctx->sg_aead_in[0], sw_ctx->aad_space, sg_set_buf(&sw_ctx_tx->sg_aead_in[0], sw_ctx_tx->aad_space,
sizeof(sw_ctx->aad_space)); sizeof(sw_ctx_tx->aad_space));
sg_unmark_end(&sw_ctx->sg_aead_in[1]); sg_unmark_end(&sw_ctx_tx->sg_aead_in[1]);
sg_chain(sw_ctx->sg_aead_in, 2, sw_ctx->sg_plaintext_data); sg_chain(sw_ctx_tx->sg_aead_in, 2,
sg_init_table(sw_ctx->sg_aead_out, 2); sw_ctx_tx->sg_plaintext_data);
sg_set_buf(&sw_ctx->sg_aead_out[0], sw_ctx->aad_space, sg_init_table(sw_ctx_tx->sg_aead_out, 2);
sizeof(sw_ctx->aad_space)); sg_set_buf(&sw_ctx_tx->sg_aead_out[0], sw_ctx_tx->aad_space,
sg_unmark_end(&sw_ctx->sg_aead_out[1]); sizeof(sw_ctx_tx->aad_space));
sg_chain(sw_ctx->sg_aead_out, 2, sw_ctx->sg_encrypted_data); sg_unmark_end(&sw_ctx_tx->sg_aead_out[1]);
sg_chain(sw_ctx_tx->sg_aead_out, 2,
sw_ctx_tx->sg_encrypted_data);
} }
if (!*aead) { if (!*aead) {
@ -1161,22 +1176,22 @@ int tls_set_sw_offload(struct sock *sk, struct tls_context *ctx, int tx)
if (rc) if (rc)
goto free_aead; goto free_aead;
if (!tx) { if (sw_ctx_rx) {
/* Set up strparser */ /* Set up strparser */
memset(&cb, 0, sizeof(cb)); memset(&cb, 0, sizeof(cb));
cb.rcv_msg = tls_queue; cb.rcv_msg = tls_queue;
cb.parse_msg = tls_read_size; cb.parse_msg = tls_read_size;
strp_init(&sw_ctx->strp, sk, &cb); strp_init(&sw_ctx_rx->strp, sk, &cb);
write_lock_bh(&sk->sk_callback_lock); write_lock_bh(&sk->sk_callback_lock);
sw_ctx->saved_data_ready = sk->sk_data_ready; sw_ctx_rx->saved_data_ready = sk->sk_data_ready;
sk->sk_data_ready = tls_data_ready; sk->sk_data_ready = tls_data_ready;
write_unlock_bh(&sk->sk_callback_lock); write_unlock_bh(&sk->sk_callback_lock);
sw_ctx->sk_poll = sk->sk_socket->ops->poll; sw_ctx_rx->sk_poll = sk->sk_socket->ops->poll;
strp_check_rcv(&sw_ctx->strp); strp_check_rcv(&sw_ctx_rx->strp);
} }
goto out; goto out;
@ -1188,11 +1203,16 @@ int tls_set_sw_offload(struct sock *sk, struct tls_context *ctx, int tx)
kfree(cctx->rec_seq); kfree(cctx->rec_seq);
cctx->rec_seq = NULL; cctx->rec_seq = NULL;
free_iv: free_iv:
kfree(ctx->tx.iv); kfree(cctx->iv);
ctx->tx.iv = NULL; cctx->iv = NULL;
free_priv: free_priv:
kfree(ctx->priv_ctx); if (tx) {
ctx->priv_ctx = NULL; kfree(ctx->priv_ctx_tx);
ctx->priv_ctx_tx = NULL;
} else {
kfree(ctx->priv_ctx_rx);
ctx->priv_ctx_rx = NULL;
}
out: out:
return rc; return rc;
} }