forked from mirrors/linux
		
	crypto: lib/poly1305 - Use block-only interface
Now that every architecture provides a block function, use that to implement the lib/poly1305 and remove the old per-arch code. Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
This commit is contained in:
		
							parent
							
								
									ceef731b0e
								
							
						
					
					
						commit
						10a6d72ea3
					
				
					 7 changed files with 32 additions and 358 deletions
				
			
		|  | @ -12,7 +12,6 @@ | ||||||
| #include <linux/jump_label.h> | #include <linux/jump_label.h> | ||||||
| #include <linux/kernel.h> | #include <linux/kernel.h> | ||||||
| #include <linux/module.h> | #include <linux/module.h> | ||||||
| #include <linux/string.h> |  | ||||||
| #include <linux/unaligned.h> | #include <linux/unaligned.h> | ||||||
| 
 | 
 | ||||||
| asmlinkage void poly1305_block_init_arch( | asmlinkage void poly1305_block_init_arch( | ||||||
|  | @ -35,17 +34,6 @@ void __weak poly1305_blocks_neon(struct poly1305_block_state *state, | ||||||
| 
 | 
 | ||||||
| static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon); | static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon); | ||||||
| 
 | 
 | ||||||
| void poly1305_init_arch(struct poly1305_desc_ctx *dctx, const u8 key[POLY1305_KEY_SIZE]) |  | ||||||
| { |  | ||||||
| 	dctx->s[0] = get_unaligned_le32(key + 16); |  | ||||||
| 	dctx->s[1] = get_unaligned_le32(key + 20); |  | ||||||
| 	dctx->s[2] = get_unaligned_le32(key + 24); |  | ||||||
| 	dctx->s[3] = get_unaligned_le32(key + 28); |  | ||||||
| 	dctx->buflen = 0; |  | ||||||
| 	poly1305_block_init_arch(&dctx->state, key); |  | ||||||
| } |  | ||||||
| EXPORT_SYMBOL(poly1305_init_arch); |  | ||||||
| 
 |  | ||||||
| void poly1305_blocks_arch(struct poly1305_block_state *state, const u8 *src, | void poly1305_blocks_arch(struct poly1305_block_state *state, const u8 *src, | ||||||
| 			  unsigned int len, u32 padbit) | 			  unsigned int len, u32 padbit) | ||||||
| { | { | ||||||
|  | @ -67,51 +55,6 @@ void poly1305_blocks_arch(struct poly1305_block_state *state, const u8 *src, | ||||||
| } | } | ||||||
| EXPORT_SYMBOL_GPL(poly1305_blocks_arch); | EXPORT_SYMBOL_GPL(poly1305_blocks_arch); | ||||||
| 
 | 
 | ||||||
| void poly1305_update_arch(struct poly1305_desc_ctx *dctx, const u8 *src, |  | ||||||
| 			  unsigned int nbytes) |  | ||||||
| { |  | ||||||
| 	if (unlikely(dctx->buflen)) { |  | ||||||
| 		u32 bytes = min(nbytes, POLY1305_BLOCK_SIZE - dctx->buflen); |  | ||||||
| 
 |  | ||||||
| 		memcpy(dctx->buf + dctx->buflen, src, bytes); |  | ||||||
| 		src += bytes; |  | ||||||
| 		nbytes -= bytes; |  | ||||||
| 		dctx->buflen += bytes; |  | ||||||
| 
 |  | ||||||
| 		if (dctx->buflen == POLY1305_BLOCK_SIZE) { |  | ||||||
| 			poly1305_blocks_arch(&dctx->state, dctx->buf, |  | ||||||
| 					     POLY1305_BLOCK_SIZE, 1); |  | ||||||
| 			dctx->buflen = 0; |  | ||||||
| 		} |  | ||||||
| 	} |  | ||||||
| 
 |  | ||||||
| 	if (likely(nbytes >= POLY1305_BLOCK_SIZE)) { |  | ||||||
| 		poly1305_blocks_arch(&dctx->state, src, nbytes, 1); |  | ||||||
| 		src += round_down(nbytes, POLY1305_BLOCK_SIZE); |  | ||||||
| 		nbytes %= POLY1305_BLOCK_SIZE; |  | ||||||
| 	} |  | ||||||
| 
 |  | ||||||
| 	if (unlikely(nbytes)) { |  | ||||||
| 		dctx->buflen = nbytes; |  | ||||||
| 		memcpy(dctx->buf, src, nbytes); |  | ||||||
| 	} |  | ||||||
| } |  | ||||||
| EXPORT_SYMBOL(poly1305_update_arch); |  | ||||||
| 
 |  | ||||||
| void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst) |  | ||||||
| { |  | ||||||
| 	if (unlikely(dctx->buflen)) { |  | ||||||
| 		dctx->buf[dctx->buflen++] = 1; |  | ||||||
| 		memset(dctx->buf + dctx->buflen, 0, |  | ||||||
| 		       POLY1305_BLOCK_SIZE - dctx->buflen); |  | ||||||
| 		poly1305_blocks_arch(&dctx->state, dctx->buf, POLY1305_BLOCK_SIZE, 0); |  | ||||||
| 	} |  | ||||||
| 
 |  | ||||||
| 	poly1305_emit_arch(&dctx->h, dst, dctx->s); |  | ||||||
| 	*dctx = (struct poly1305_desc_ctx){}; |  | ||||||
| } |  | ||||||
| EXPORT_SYMBOL(poly1305_final_arch); |  | ||||||
| 
 |  | ||||||
| bool poly1305_is_arch_optimized(void) | bool poly1305_is_arch_optimized(void) | ||||||
| { | { | ||||||
| 	/* We always can use at least the ARM scalar implementation. */ | 	/* We always can use at least the ARM scalar implementation. */ | ||||||
|  |  | ||||||
|  | @ -12,7 +12,6 @@ | ||||||
| #include <linux/jump_label.h> | #include <linux/jump_label.h> | ||||||
| #include <linux/kernel.h> | #include <linux/kernel.h> | ||||||
| #include <linux/module.h> | #include <linux/module.h> | ||||||
| #include <linux/string.h> |  | ||||||
| #include <linux/unaligned.h> | #include <linux/unaligned.h> | ||||||
| 
 | 
 | ||||||
| asmlinkage void poly1305_block_init_arch( | asmlinkage void poly1305_block_init_arch( | ||||||
|  | @ -30,17 +29,6 @@ EXPORT_SYMBOL_GPL(poly1305_emit_arch); | ||||||
| 
 | 
 | ||||||
| static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon); | static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon); | ||||||
| 
 | 
 | ||||||
| void poly1305_init_arch(struct poly1305_desc_ctx *dctx, const u8 key[POLY1305_KEY_SIZE]) |  | ||||||
| { |  | ||||||
| 	dctx->s[0] = get_unaligned_le32(key + 16); |  | ||||||
| 	dctx->s[1] = get_unaligned_le32(key + 20); |  | ||||||
| 	dctx->s[2] = get_unaligned_le32(key + 24); |  | ||||||
| 	dctx->s[3] = get_unaligned_le32(key + 28); |  | ||||||
| 	dctx->buflen = 0; |  | ||||||
| 	poly1305_block_init_arch(&dctx->state, key); |  | ||||||
| } |  | ||||||
| EXPORT_SYMBOL(poly1305_init_arch); |  | ||||||
| 
 |  | ||||||
| void poly1305_blocks_arch(struct poly1305_block_state *state, const u8 *src, | void poly1305_blocks_arch(struct poly1305_block_state *state, const u8 *src, | ||||||
| 			  unsigned int len, u32 padbit) | 			  unsigned int len, u32 padbit) | ||||||
| { | { | ||||||
|  | @ -61,52 +49,6 @@ void poly1305_blocks_arch(struct poly1305_block_state *state, const u8 *src, | ||||||
| } | } | ||||||
| EXPORT_SYMBOL_GPL(poly1305_blocks_arch); | EXPORT_SYMBOL_GPL(poly1305_blocks_arch); | ||||||
| 
 | 
 | ||||||
| void poly1305_update_arch(struct poly1305_desc_ctx *dctx, const u8 *src, |  | ||||||
| 			  unsigned int nbytes) |  | ||||||
| { |  | ||||||
| 	if (unlikely(dctx->buflen)) { |  | ||||||
| 		u32 bytes = min(nbytes, POLY1305_BLOCK_SIZE - dctx->buflen); |  | ||||||
| 
 |  | ||||||
| 		memcpy(dctx->buf + dctx->buflen, src, bytes); |  | ||||||
| 		src += bytes; |  | ||||||
| 		nbytes -= bytes; |  | ||||||
| 		dctx->buflen += bytes; |  | ||||||
| 
 |  | ||||||
| 		if (dctx->buflen == POLY1305_BLOCK_SIZE) { |  | ||||||
| 			poly1305_blocks_arch(&dctx->state, dctx->buf, |  | ||||||
| 					     POLY1305_BLOCK_SIZE, 1); |  | ||||||
| 			dctx->buflen = 0; |  | ||||||
| 		} |  | ||||||
| 	} |  | ||||||
| 
 |  | ||||||
| 	if (likely(nbytes >= POLY1305_BLOCK_SIZE)) { |  | ||||||
| 		poly1305_blocks_arch(&dctx->state, src, nbytes, 1); |  | ||||||
| 		src += round_down(nbytes, POLY1305_BLOCK_SIZE); |  | ||||||
| 		nbytes %= POLY1305_BLOCK_SIZE; |  | ||||||
| 	} |  | ||||||
| 
 |  | ||||||
| 	if (unlikely(nbytes)) { |  | ||||||
| 		dctx->buflen = nbytes; |  | ||||||
| 		memcpy(dctx->buf, src, nbytes); |  | ||||||
| 	} |  | ||||||
| } |  | ||||||
| EXPORT_SYMBOL(poly1305_update_arch); |  | ||||||
| 
 |  | ||||||
| void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst) |  | ||||||
| { |  | ||||||
| 	if (unlikely(dctx->buflen)) { |  | ||||||
| 		dctx->buf[dctx->buflen++] = 1; |  | ||||||
| 		memset(dctx->buf + dctx->buflen, 0, |  | ||||||
| 		       POLY1305_BLOCK_SIZE - dctx->buflen); |  | ||||||
| 		poly1305_blocks_arch(&dctx->state, dctx->buf, |  | ||||||
| 				     POLY1305_BLOCK_SIZE, 0); |  | ||||||
| 	} |  | ||||||
| 
 |  | ||||||
| 	poly1305_emit_arch(&dctx->h, dst, dctx->s); |  | ||||||
| 	memzero_explicit(dctx, sizeof(*dctx)); |  | ||||||
| } |  | ||||||
| EXPORT_SYMBOL(poly1305_final_arch); |  | ||||||
| 
 |  | ||||||
| bool poly1305_is_arch_optimized(void) | bool poly1305_is_arch_optimized(void) | ||||||
| { | { | ||||||
| 	/* We always can use at least the ARM64 scalar implementation. */ | 	/* We always can use at least the ARM64 scalar implementation. */ | ||||||
|  |  | ||||||
|  | @ -9,7 +9,6 @@ | ||||||
| #include <linux/cpufeature.h> | #include <linux/cpufeature.h> | ||||||
| #include <linux/kernel.h> | #include <linux/kernel.h> | ||||||
| #include <linux/module.h> | #include <linux/module.h> | ||||||
| #include <linux/string.h> |  | ||||||
| #include <linux/unaligned.h> | #include <linux/unaligned.h> | ||||||
| 
 | 
 | ||||||
| asmlinkage void poly1305_block_init_arch( | asmlinkage void poly1305_block_init_arch( | ||||||
|  | @ -24,65 +23,6 @@ asmlinkage void poly1305_emit_arch(const struct poly1305_state *state, | ||||||
| 				   const u32 nonce[4]); | 				   const u32 nonce[4]); | ||||||
| EXPORT_SYMBOL_GPL(poly1305_emit_arch); | EXPORT_SYMBOL_GPL(poly1305_emit_arch); | ||||||
| 
 | 
 | ||||||
| void poly1305_init_arch(struct poly1305_desc_ctx *dctx, const u8 key[POLY1305_KEY_SIZE]) |  | ||||||
| { |  | ||||||
| 	dctx->s[0] = get_unaligned_le32(key + 16); |  | ||||||
| 	dctx->s[1] = get_unaligned_le32(key + 20); |  | ||||||
| 	dctx->s[2] = get_unaligned_le32(key + 24); |  | ||||||
| 	dctx->s[3] = get_unaligned_le32(key + 28); |  | ||||||
| 	dctx->buflen = 0; |  | ||||||
| 	poly1305_block_init_arch(&dctx->state, key); |  | ||||||
| } |  | ||||||
| EXPORT_SYMBOL(poly1305_init_arch); |  | ||||||
| 
 |  | ||||||
| void poly1305_update_arch(struct poly1305_desc_ctx *dctx, const u8 *src, |  | ||||||
| 			  unsigned int nbytes) |  | ||||||
| { |  | ||||||
| 	if (unlikely(dctx->buflen)) { |  | ||||||
| 		u32 bytes = min(nbytes, POLY1305_BLOCK_SIZE - dctx->buflen); |  | ||||||
| 
 |  | ||||||
| 		memcpy(dctx->buf + dctx->buflen, src, bytes); |  | ||||||
| 		src += bytes; |  | ||||||
| 		nbytes -= bytes; |  | ||||||
| 		dctx->buflen += bytes; |  | ||||||
| 
 |  | ||||||
| 		if (dctx->buflen == POLY1305_BLOCK_SIZE) { |  | ||||||
| 			poly1305_blocks_arch(&dctx->state, dctx->buf, |  | ||||||
| 					     POLY1305_BLOCK_SIZE, 1); |  | ||||||
| 			dctx->buflen = 0; |  | ||||||
| 		} |  | ||||||
| 	} |  | ||||||
| 
 |  | ||||||
| 	if (likely(nbytes >= POLY1305_BLOCK_SIZE)) { |  | ||||||
| 		unsigned int len = round_down(nbytes, POLY1305_BLOCK_SIZE); |  | ||||||
| 
 |  | ||||||
| 		poly1305_blocks_arch(&dctx->state, src, len, 1); |  | ||||||
| 		src += len; |  | ||||||
| 		nbytes %= POLY1305_BLOCK_SIZE; |  | ||||||
| 	} |  | ||||||
| 
 |  | ||||||
| 	if (unlikely(nbytes)) { |  | ||||||
| 		dctx->buflen = nbytes; |  | ||||||
| 		memcpy(dctx->buf, src, nbytes); |  | ||||||
| 	} |  | ||||||
| } |  | ||||||
| EXPORT_SYMBOL(poly1305_update_arch); |  | ||||||
| 
 |  | ||||||
| void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst) |  | ||||||
| { |  | ||||||
| 	if (unlikely(dctx->buflen)) { |  | ||||||
| 		dctx->buf[dctx->buflen++] = 1; |  | ||||||
| 		memset(dctx->buf + dctx->buflen, 0, |  | ||||||
| 		       POLY1305_BLOCK_SIZE - dctx->buflen); |  | ||||||
| 		poly1305_blocks_arch(&dctx->state, dctx->buf, |  | ||||||
| 				     POLY1305_BLOCK_SIZE, 0); |  | ||||||
| 	} |  | ||||||
| 
 |  | ||||||
| 	poly1305_emit_arch(&dctx->h, dst, dctx->s); |  | ||||||
| 	*dctx = (struct poly1305_desc_ctx){}; |  | ||||||
| } |  | ||||||
| EXPORT_SYMBOL(poly1305_final_arch); |  | ||||||
| 
 |  | ||||||
| bool poly1305_is_arch_optimized(void) | bool poly1305_is_arch_optimized(void) | ||||||
| { | { | ||||||
| 	return true; | 	return true; | ||||||
|  |  | ||||||
|  | @ -10,7 +10,6 @@ | ||||||
| #include <linux/jump_label.h> | #include <linux/jump_label.h> | ||||||
| #include <linux/kernel.h> | #include <linux/kernel.h> | ||||||
| #include <linux/module.h> | #include <linux/module.h> | ||||||
| #include <linux/string.h> |  | ||||||
| #include <linux/unaligned.h> | #include <linux/unaligned.h> | ||||||
| 
 | 
 | ||||||
| asmlinkage void poly1305_p10le_4blocks(struct poly1305_block_state *state, const u8 *m, u32 mlen); | asmlinkage void poly1305_p10le_4blocks(struct poly1305_block_state *state, const u8 *m, u32 mlen); | ||||||
|  | @ -45,17 +44,6 @@ void poly1305_block_init_arch(struct poly1305_block_state *dctx, | ||||||
| } | } | ||||||
| EXPORT_SYMBOL_GPL(poly1305_block_init_arch); | EXPORT_SYMBOL_GPL(poly1305_block_init_arch); | ||||||
| 
 | 
 | ||||||
| void poly1305_init_arch(struct poly1305_desc_ctx *dctx, const u8 key[POLY1305_KEY_SIZE]) |  | ||||||
| { |  | ||||||
| 	dctx->s[0] = get_unaligned_le32(key + 16); |  | ||||||
| 	dctx->s[1] = get_unaligned_le32(key + 20); |  | ||||||
| 	dctx->s[2] = get_unaligned_le32(key + 24); |  | ||||||
| 	dctx->s[3] = get_unaligned_le32(key + 28); |  | ||||||
| 	dctx->buflen = 0; |  | ||||||
| 	poly1305_block_init_arch(&dctx->state, key); |  | ||||||
| } |  | ||||||
| EXPORT_SYMBOL(poly1305_init_arch); |  | ||||||
| 
 |  | ||||||
| void poly1305_blocks_arch(struct poly1305_block_state *state, const u8 *src, | void poly1305_blocks_arch(struct poly1305_block_state *state, const u8 *src, | ||||||
| 			  unsigned int len, u32 padbit) | 			  unsigned int len, u32 padbit) | ||||||
| { | { | ||||||
|  | @ -76,57 +64,6 @@ void poly1305_blocks_arch(struct poly1305_block_state *state, const u8 *src, | ||||||
| } | } | ||||||
| EXPORT_SYMBOL_GPL(poly1305_blocks_arch); | EXPORT_SYMBOL_GPL(poly1305_blocks_arch); | ||||||
| 
 | 
 | ||||||
| void poly1305_update_arch(struct poly1305_desc_ctx *dctx, |  | ||||||
| 			  const u8 *src, unsigned int srclen) |  | ||||||
| { |  | ||||||
| 	unsigned int bytes; |  | ||||||
| 
 |  | ||||||
| 	if (!static_key_enabled(&have_p10)) |  | ||||||
| 		return poly1305_update_generic(dctx, src, srclen); |  | ||||||
| 
 |  | ||||||
| 	if (unlikely(dctx->buflen)) { |  | ||||||
| 		bytes = min(srclen, POLY1305_BLOCK_SIZE - dctx->buflen); |  | ||||||
| 		memcpy(dctx->buf + dctx->buflen, src, bytes); |  | ||||||
| 		src += bytes; |  | ||||||
| 		srclen -= bytes; |  | ||||||
| 		dctx->buflen += bytes; |  | ||||||
| 		if (dctx->buflen < POLY1305_BLOCK_SIZE) |  | ||||||
| 			return; |  | ||||||
| 		poly1305_blocks_arch(&dctx->state, dctx->buf, |  | ||||||
| 				     POLY1305_BLOCK_SIZE, 1); |  | ||||||
| 		dctx->buflen = 0; |  | ||||||
| 	} |  | ||||||
| 
 |  | ||||||
| 	if (likely(srclen >= POLY1305_BLOCK_SIZE)) { |  | ||||||
| 		poly1305_blocks_arch(&dctx->state, src, srclen, 1); |  | ||||||
| 		src += srclen - (srclen % POLY1305_BLOCK_SIZE); |  | ||||||
| 		srclen %= POLY1305_BLOCK_SIZE; |  | ||||||
| 	} |  | ||||||
| 
 |  | ||||||
| 	if (unlikely(srclen)) { |  | ||||||
| 		dctx->buflen = srclen; |  | ||||||
| 		memcpy(dctx->buf, src, srclen); |  | ||||||
| 	} |  | ||||||
| } |  | ||||||
| EXPORT_SYMBOL(poly1305_update_arch); |  | ||||||
| 
 |  | ||||||
| void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst) |  | ||||||
| { |  | ||||||
| 	if (!static_key_enabled(&have_p10)) |  | ||||||
| 		return poly1305_final_generic(dctx, dst); |  | ||||||
| 
 |  | ||||||
| 	if (dctx->buflen) { |  | ||||||
| 		dctx->buf[dctx->buflen++] = 1; |  | ||||||
| 		memset(dctx->buf + dctx->buflen, 0, |  | ||||||
| 		       POLY1305_BLOCK_SIZE - dctx->buflen); |  | ||||||
| 		poly1305_blocks_arch(&dctx->state, dctx->buf, |  | ||||||
| 				     POLY1305_BLOCK_SIZE, 0); |  | ||||||
| 	} |  | ||||||
| 
 |  | ||||||
| 	poly1305_emit_arch(&dctx->h, dst, dctx->s); |  | ||||||
| } |  | ||||||
| EXPORT_SYMBOL(poly1305_final_arch); |  | ||||||
| 
 |  | ||||||
| bool poly1305_is_arch_optimized(void) | bool poly1305_is_arch_optimized(void) | ||||||
| { | { | ||||||
| 	return static_key_enabled(&have_p10); | 	return static_key_enabled(&have_p10); | ||||||
|  |  | ||||||
|  | @ -10,7 +10,6 @@ | ||||||
| #include <linux/kernel.h> | #include <linux/kernel.h> | ||||||
| #include <linux/module.h> | #include <linux/module.h> | ||||||
| #include <linux/sizes.h> | #include <linux/sizes.h> | ||||||
| #include <linux/string.h> |  | ||||||
| #include <linux/unaligned.h> | #include <linux/unaligned.h> | ||||||
| 
 | 
 | ||||||
| struct poly1305_arch_internal { | struct poly1305_arch_internal { | ||||||
|  | @ -96,65 +95,6 @@ void poly1305_emit_arch(const struct poly1305_state *ctx, | ||||||
| } | } | ||||||
| EXPORT_SYMBOL_GPL(poly1305_emit_arch); | EXPORT_SYMBOL_GPL(poly1305_emit_arch); | ||||||
| 
 | 
 | ||||||
| void poly1305_init_arch(struct poly1305_desc_ctx *dctx, const u8 key[POLY1305_KEY_SIZE]) |  | ||||||
| { |  | ||||||
| 	dctx->s[0] = get_unaligned_le32(&key[16]); |  | ||||||
| 	dctx->s[1] = get_unaligned_le32(&key[20]); |  | ||||||
| 	dctx->s[2] = get_unaligned_le32(&key[24]); |  | ||||||
| 	dctx->s[3] = get_unaligned_le32(&key[28]); |  | ||||||
| 	dctx->buflen = 0; |  | ||||||
| 	poly1305_block_init_arch(&dctx->state, key); |  | ||||||
| } |  | ||||||
| EXPORT_SYMBOL(poly1305_init_arch); |  | ||||||
| 
 |  | ||||||
| void poly1305_update_arch(struct poly1305_desc_ctx *dctx, const u8 *src, |  | ||||||
| 			  unsigned int srclen) |  | ||||||
| { |  | ||||||
| 	unsigned int bytes; |  | ||||||
| 
 |  | ||||||
| 	if (unlikely(dctx->buflen)) { |  | ||||||
| 		bytes = min(srclen, POLY1305_BLOCK_SIZE - dctx->buflen); |  | ||||||
| 		memcpy(dctx->buf + dctx->buflen, src, bytes); |  | ||||||
| 		src += bytes; |  | ||||||
| 		srclen -= bytes; |  | ||||||
| 		dctx->buflen += bytes; |  | ||||||
| 
 |  | ||||||
| 		if (dctx->buflen == POLY1305_BLOCK_SIZE) { |  | ||||||
| 			poly1305_blocks_arch(&dctx->state, dctx->buf, |  | ||||||
| 					     POLY1305_BLOCK_SIZE, 1); |  | ||||||
| 			dctx->buflen = 0; |  | ||||||
| 		} |  | ||||||
| 	} |  | ||||||
| 
 |  | ||||||
| 	if (likely(srclen >= POLY1305_BLOCK_SIZE)) { |  | ||||||
| 		bytes = round_down(srclen, POLY1305_BLOCK_SIZE); |  | ||||||
| 		poly1305_blocks_arch(&dctx->state, src, bytes, 1); |  | ||||||
| 		src += bytes; |  | ||||||
| 		srclen -= bytes; |  | ||||||
| 	} |  | ||||||
| 
 |  | ||||||
| 	if (unlikely(srclen)) { |  | ||||||
| 		dctx->buflen = srclen; |  | ||||||
| 		memcpy(dctx->buf, src, srclen); |  | ||||||
| 	} |  | ||||||
| } |  | ||||||
| EXPORT_SYMBOL(poly1305_update_arch); |  | ||||||
| 
 |  | ||||||
| void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst) |  | ||||||
| { |  | ||||||
| 	if (unlikely(dctx->buflen)) { |  | ||||||
| 		dctx->buf[dctx->buflen++] = 1; |  | ||||||
| 		memset(dctx->buf + dctx->buflen, 0, |  | ||||||
| 		       POLY1305_BLOCK_SIZE - dctx->buflen); |  | ||||||
| 		poly1305_blocks_arch(&dctx->state, dctx->buf, |  | ||||||
| 				     POLY1305_BLOCK_SIZE, 0); |  | ||||||
| 	} |  | ||||||
| 
 |  | ||||||
| 	poly1305_emit_arch(&dctx->h, dst, dctx->s); |  | ||||||
| 	memzero_explicit(dctx, sizeof(*dctx)); |  | ||||||
| } |  | ||||||
| EXPORT_SYMBOL(poly1305_final_arch); |  | ||||||
| 
 |  | ||||||
| bool poly1305_is_arch_optimized(void) | bool poly1305_is_arch_optimized(void) | ||||||
| { | { | ||||||
| 	return static_key_enabled(&poly1305_use_avx); | 	return static_key_enabled(&poly1305_use_avx); | ||||||
|  |  | ||||||
|  | @ -55,55 +55,14 @@ struct poly1305_desc_ctx { | ||||||
| 	unsigned int buflen; | 	unsigned int buflen; | ||||||
| 	/* finalize key */ | 	/* finalize key */ | ||||||
| 	u32 s[4]; | 	u32 s[4]; | ||||||
| 	union { |  | ||||||
| 		struct { |  | ||||||
| 			struct poly1305_state h; |  | ||||||
| 			union { |  | ||||||
| 				struct poly1305_key opaque_r[CONFIG_CRYPTO_LIB_POLY1305_RSIZE]; |  | ||||||
| 				struct poly1305_core_key core_r; |  | ||||||
| 			}; |  | ||||||
| 		}; |  | ||||||
| 	struct poly1305_block_state state; | 	struct poly1305_block_state state; | ||||||
| 	}; |  | ||||||
| }; | }; | ||||||
| 
 | 
 | ||||||
| void poly1305_init_arch(struct poly1305_desc_ctx *desc, | void poly1305_init(struct poly1305_desc_ctx *desc, | ||||||
| 		   const u8 key[POLY1305_KEY_SIZE]); | 		   const u8 key[POLY1305_KEY_SIZE]); | ||||||
| void poly1305_init_generic(struct poly1305_desc_ctx *desc, | void poly1305_update(struct poly1305_desc_ctx *desc, | ||||||
| 			   const u8 key[POLY1305_KEY_SIZE]); | 		     const u8 *src, unsigned int nbytes); | ||||||
| 
 | void poly1305_final(struct poly1305_desc_ctx *desc, u8 *digest); | ||||||
| static inline void poly1305_init(struct poly1305_desc_ctx *desc, const u8 *key) |  | ||||||
| { |  | ||||||
| 	if (IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_POLY1305)) |  | ||||||
| 		poly1305_init_arch(desc, key); |  | ||||||
| 	else |  | ||||||
| 		poly1305_init_generic(desc, key); |  | ||||||
| } |  | ||||||
| 
 |  | ||||||
| void poly1305_update_arch(struct poly1305_desc_ctx *desc, const u8 *src, |  | ||||||
| 			  unsigned int nbytes); |  | ||||||
| void poly1305_update_generic(struct poly1305_desc_ctx *desc, const u8 *src, |  | ||||||
| 			     unsigned int nbytes); |  | ||||||
| 
 |  | ||||||
| static inline void poly1305_update(struct poly1305_desc_ctx *desc, |  | ||||||
| 				   const u8 *src, unsigned int nbytes) |  | ||||||
| { |  | ||||||
| 	if (IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_POLY1305)) |  | ||||||
| 		poly1305_update_arch(desc, src, nbytes); |  | ||||||
| 	else |  | ||||||
| 		poly1305_update_generic(desc, src, nbytes); |  | ||||||
| } |  | ||||||
| 
 |  | ||||||
| void poly1305_final_arch(struct poly1305_desc_ctx *desc, u8 *digest); |  | ||||||
| void poly1305_final_generic(struct poly1305_desc_ctx *desc, u8 *digest); |  | ||||||
| 
 |  | ||||||
| static inline void poly1305_final(struct poly1305_desc_ctx *desc, u8 *digest) |  | ||||||
| { |  | ||||||
| 	if (IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_POLY1305)) |  | ||||||
| 		poly1305_final_arch(desc, digest); |  | ||||||
| 	else |  | ||||||
| 		poly1305_final_generic(desc, digest); |  | ||||||
| } |  | ||||||
| 
 | 
 | ||||||
| #if IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_POLY1305) | #if IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_POLY1305) | ||||||
| bool poly1305_is_arch_optimized(void); | bool poly1305_is_arch_optimized(void); | ||||||
|  |  | ||||||
|  | @ -22,7 +22,7 @@ void poly1305_block_init_generic(struct poly1305_block_state *desc, | ||||||
| } | } | ||||||
| EXPORT_SYMBOL_GPL(poly1305_block_init_generic); | EXPORT_SYMBOL_GPL(poly1305_block_init_generic); | ||||||
| 
 | 
 | ||||||
| void poly1305_init_generic(struct poly1305_desc_ctx *desc, | void poly1305_init(struct poly1305_desc_ctx *desc, | ||||||
| 		   const u8 key[POLY1305_KEY_SIZE]) | 		   const u8 key[POLY1305_KEY_SIZE]) | ||||||
| { | { | ||||||
| 	desc->s[0] = get_unaligned_le32(key + 16); | 	desc->s[0] = get_unaligned_le32(key + 16); | ||||||
|  | @ -30,39 +30,52 @@ void poly1305_init_generic(struct poly1305_desc_ctx *desc, | ||||||
| 	desc->s[2] = get_unaligned_le32(key + 24); | 	desc->s[2] = get_unaligned_le32(key + 24); | ||||||
| 	desc->s[3] = get_unaligned_le32(key + 28); | 	desc->s[3] = get_unaligned_le32(key + 28); | ||||||
| 	desc->buflen = 0; | 	desc->buflen = 0; | ||||||
|  | 	if (IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_POLY1305)) | ||||||
|  | 		poly1305_block_init_arch(&desc->state, key); | ||||||
|  | 	else | ||||||
| 		poly1305_block_init_generic(&desc->state, key); | 		poly1305_block_init_generic(&desc->state, key); | ||||||
| } | } | ||||||
| EXPORT_SYMBOL_GPL(poly1305_init_generic); | EXPORT_SYMBOL(poly1305_init); | ||||||
| 
 | 
 | ||||||
| static inline void poly1305_blocks(struct poly1305_block_state *state, | static inline void poly1305_blocks(struct poly1305_block_state *state, | ||||||
| 				   const u8 *src, unsigned int len) | 				   const u8 *src, unsigned int len) | ||||||
| { | { | ||||||
|  | 	if (IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_POLY1305)) | ||||||
|  | 		poly1305_blocks_arch(state, src, len, 1); | ||||||
|  | 	else | ||||||
| 		poly1305_blocks_generic(state, src, len, 1); | 		poly1305_blocks_generic(state, src, len, 1); | ||||||
| } | } | ||||||
| 
 | 
 | ||||||
| void poly1305_update_generic(struct poly1305_desc_ctx *desc, const u8 *src, | void poly1305_update(struct poly1305_desc_ctx *desc, | ||||||
| 			     unsigned int nbytes) | 		     const u8 *src, unsigned int nbytes) | ||||||
| { | { | ||||||
| 	desc->buflen = BLOCK_HASH_UPDATE(poly1305_blocks, &desc->state, | 	desc->buflen = BLOCK_HASH_UPDATE(poly1305_blocks, &desc->state, | ||||||
| 					 src, nbytes, POLY1305_BLOCK_SIZE, | 					 src, nbytes, POLY1305_BLOCK_SIZE, | ||||||
| 					 desc->buf, desc->buflen); | 					 desc->buf, desc->buflen); | ||||||
| } | } | ||||||
| EXPORT_SYMBOL_GPL(poly1305_update_generic); | EXPORT_SYMBOL(poly1305_update); | ||||||
| 
 | 
 | ||||||
| void poly1305_final_generic(struct poly1305_desc_ctx *desc, u8 *dst) | void poly1305_final(struct poly1305_desc_ctx *desc, u8 *dst) | ||||||
| { | { | ||||||
| 	if (unlikely(desc->buflen)) { | 	if (unlikely(desc->buflen)) { | ||||||
| 		desc->buf[desc->buflen++] = 1; | 		desc->buf[desc->buflen++] = 1; | ||||||
| 		memset(desc->buf + desc->buflen, 0, | 		memset(desc->buf + desc->buflen, 0, | ||||||
| 		       POLY1305_BLOCK_SIZE - desc->buflen); | 		       POLY1305_BLOCK_SIZE - desc->buflen); | ||||||
|  | 		if (IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_POLY1305)) | ||||||
|  | 			poly1305_blocks_arch(&desc->state, desc->buf, | ||||||
|  | 					     POLY1305_BLOCK_SIZE, 0); | ||||||
|  | 		else | ||||||
| 			poly1305_blocks_generic(&desc->state, desc->buf, | 			poly1305_blocks_generic(&desc->state, desc->buf, | ||||||
| 						POLY1305_BLOCK_SIZE, 0); | 						POLY1305_BLOCK_SIZE, 0); | ||||||
| 	} | 	} | ||||||
| 
 | 
 | ||||||
| 	poly1305_emit_generic(&desc->h, dst, desc->s); | 	if (IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_POLY1305)) | ||||||
|  | 		poly1305_emit_arch(&desc->state.h, dst, desc->s); | ||||||
|  | 	else | ||||||
|  | 		poly1305_emit_generic(&desc->state.h, dst, desc->s); | ||||||
| 	*desc = (struct poly1305_desc_ctx){}; | 	*desc = (struct poly1305_desc_ctx){}; | ||||||
| } | } | ||||||
| EXPORT_SYMBOL_GPL(poly1305_final_generic); | EXPORT_SYMBOL(poly1305_final); | ||||||
| 
 | 
 | ||||||
| MODULE_LICENSE("GPL"); | MODULE_LICENSE("GPL"); | ||||||
| MODULE_AUTHOR("Martin Willi <martin@strongswan.org>"); | MODULE_AUTHOR("Martin Willi <martin@strongswan.org>"); | ||||||
|  |  | ||||||
		Loading…
	
		Reference in a new issue
	
	 Herbert Xu
						Herbert Xu