From e5e13aeeb133191abe9a3fd8e91f6f038d085537 Mon Sep 17 00:00:00 2001 From: Jens Wiklander Date: Thu, 16 Nov 2017 12:44:37 +0100 Subject: [PATCH 01/12] core: arm64: add mov_imm assembly macro Implement a macro mov_imm that can be used to move an immediate constant into a 64-bit register, using between 2 and 4 movz/movk instructions (depending on the operand) Acked-by: Jerome Forissier Acked-by: Ard Biesheuvel Signed-off-by: Jens Wiklander --- core/arch/arm/include/arm64_macros.S | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/core/arch/arm/include/arm64_macros.S b/core/arch/arm/include/arm64_macros.S index 981a150b272..04d684c15df 100644 --- a/core/arch/arm/include/arm64_macros.S +++ b/core/arch/arm/include/arm64_macros.S @@ -123,3 +123,17 @@ ldp \r1, \r2, [sp], #16 .endm + .macro mov_imm _reg, _val + .if (((\_val) >> 31) == 0 || ((\_val) >> 31) == 0x1ffffffff) + movz \_reg, :abs_g1_s:\_val + .else + .if (((\_val) >> 47) == 0 || ((\_val) >> 47) == 0x1ffff) + movz \_reg, :abs_g2_s:\_val + .else + movz \_reg, :abs_g3:\_val + movk \_reg, :abs_g2_nc:\_val + .endif + movk \_reg, :abs_g1_nc:\_val + .endif + movk \_reg, :abs_g0_nc:\_val + .endm From b367de3677f921af0a80eb28a34fd2461185807f Mon Sep 17 00:00:00 2001 From: Jens Wiklander Date: Thu, 16 Nov 2017 12:44:38 +0100 Subject: [PATCH 02/12] core: io.h: add {get,put}_be{16,32,64}() Reviewed-by: Jerome Forissier Signed-off-by: Jens Wiklander --- core/arch/arm/mm/pager_aes_gcm.c | 16 +--------------- core/include/io.h | 31 +++++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 15 deletions(-) diff --git a/core/arch/arm/mm/pager_aes_gcm.c b/core/arch/arm/mm/pager_aes_gcm.c index e0ff286f51e..871c2005ffc 100644 --- a/core/arch/arm/mm/pager_aes_gcm.c +++ b/core/arch/arm/mm/pager_aes_gcm.c @@ -44,6 +44,7 @@ #include #include +#include #include "pager_private.h" #include #include @@ -59,21 +60,6 @@ #define BLOCK_ALIGNMENT sizeof(uint64_t) -static uint32_t get_be32(const void *a) -{ - return TEE_U32_FROM_BIG_ENDIAN(*(const uint32_t *)a); -} - -static void put_be32(void *a, uint32_t val) -{ - *(uint32_t *)a = TEE_U32_TO_BIG_ENDIAN(val); -} - -static void put_be64(void *a, uint64_t val) -{ - *(uint64_t *)a = TEE_U64_TO_BIG_ENDIAN(val); -} - static void aes_encrypt(symmetric_key *skey, const uint8_t *plain, uint8_t *crypt) { diff --git a/core/include/io.h b/core/include/io.h index 83b2efc8947..b2ece1c93d0 100644 --- a/core/include/io.h +++ b/core/include/io.h @@ -29,6 +29,7 @@ #include #include +#include static inline void write8(uint8_t val, vaddr_t addr) { @@ -75,4 +76,34 @@ static inline void io_mask32(vaddr_t addr, uint32_t val, uint32_t mask) write32((read32(addr) & ~mask) | (val & mask), addr); } +static inline uint64_t get_be64(const void *p) +{ + return TEE_U64_FROM_BIG_ENDIAN(*(const uint64_t *)p); +} + +static inline void put_be64(void *p, uint64_t val) +{ + *(uint64_t *)p = TEE_U64_TO_BIG_ENDIAN(val); +} + +static inline uint32_t get_be32(const void *p) +{ + return TEE_U32_FROM_BIG_ENDIAN(*(const uint32_t *)p); +} + +static inline void put_be32(void *p, uint32_t val) +{ + *(uint32_t *)p = TEE_U32_TO_BIG_ENDIAN(val); +} + +static inline uint16_t get_be16(const void *p) +{ + return TEE_U16_FROM_BIG_ENDIAN(*(const uint16_t *)p); +} + +static inline void put_be16(void *p, uint16_t val) +{ + *(uint16_t *)p = TEE_U16_TO_BIG_ENDIAN(val); +} + #endif /*IO_H*/ From fdd3aa8a6131497cafc487b5bcbc474b6cd358ec Mon Sep 17 00:00:00 2001 From: Jens Wiklander Date: Thu, 16 Nov 2017 12:44:40 +0100 Subject: [PATCH 03/12] core: import GHASH acceleration routines Imports assembly code for accelerated GHASH computations and provides an interface for using these low level functions. The assembly code relies on features now available in all ARM cores. No assembly code is enabled at all unless CFG_WITH_VFP = y. Code using the PMULL/PMULL2 instruction is enabled with CFG_HWSUPP_PMULL = y. The assembly code is written by Ard Biesheuvel and modified slightly here to fit better into OP-TEE. Acked-by: Jerome Forissier Signed-off-by: Jens Wiklander --- core/arch/arm/arm.mk | 2 +- core/arch/arm/crypto/ghash-ce-core_a32.S | 251 ++++++++++ core/arch/arm/crypto/ghash-ce-core_a64.S | 462 +++++++++++++++++++ core/arch/arm/crypto/sub.mk | 4 + core/arch/arm/include/crypto/ghash-ce-core.h | 31 ++ 5 files changed, 749 insertions(+), 1 deletion(-) create mode 100644 core/arch/arm/crypto/ghash-ce-core_a32.S create mode 100644 core/arch/arm/crypto/ghash-ce-core_a64.S create mode 100644 core/arch/arm/crypto/sub.mk create mode 100644 core/arch/arm/include/crypto/ghash-ce-core.h diff --git a/core/arch/arm/arm.mk b/core/arch/arm/arm.mk index 377971a9816..8b7efe09354 100644 --- a/core/arch/arm/arm.mk +++ b/core/arch/arm/arm.mk @@ -52,7 +52,7 @@ endif core-platform-cppflags += -I$(arch-dir)/include core-platform-subdirs += \ - $(addprefix $(arch-dir)/, kernel mm tee pta) $(platform-dir) + $(addprefix $(arch-dir)/, kernel crypto mm tee pta) $(platform-dir) ifneq ($(CFG_WITH_ARM_TRUSTED_FW),y) core-platform-subdirs += $(arch-dir)/sm diff --git a/core/arch/arm/crypto/ghash-ce-core_a32.S b/core/arch/arm/crypto/ghash-ce-core_a32.S new file mode 100644 index 00000000000..a644d9300f7 --- /dev/null +++ b/core/arch/arm/crypto/ghash-ce-core_a32.S @@ -0,0 +1,251 @@ +/* + * Accelerated GHASH implementation with ARMv8 PMULL instructions. + * + * Copyright (C) 2014 - 2017 Linaro Ltd. + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include + +#define ENTRY(func) \ + .global func ; \ + .type func , %function ; \ + func : + +#define ENDPROC(func) \ + .size func , .-func + +#define CPU_LE(x...) x + + SHASH .req q0 + T1 .req q1 + XL .req q2 + XM .req q3 + XH .req q4 + IN1 .req q4 + + SHASH_L .req d0 + SHASH_H .req d1 + T1_L .req d2 + T1_H .req d3 + XL_L .req d4 + XL_H .req d5 + XM_L .req d6 + XM_H .req d7 + XH_L .req d8 + + t0l .req d10 + t0h .req d11 + t1l .req d12 + t1h .req d13 + t2l .req d14 + t2h .req d15 + t3l .req d16 + t3h .req d17 + t4l .req d18 + t4h .req d19 + + t0q .req q5 + t1q .req q6 + t2q .req q7 + t3q .req q8 + t4q .req q9 + T2 .req q9 + + s1l .req d20 + s1h .req d21 + s2l .req d22 + s2h .req d23 + s3l .req d24 + s3h .req d25 + s4l .req d26 + s4h .req d27 + + MASK .req d28 + SHASH2_p8 .req d28 + + k16 .req d29 + k32 .req d30 + k48 .req d31 + SHASH2_p64 .req d31 + + .text + .fpu crypto-neon-fp-armv8 + + .macro __pmull_p64, rd, rn, rm, b1, b2, b3, b4 + vmull.p64 \rd, \rn, \rm + .endm + + /* + * This implementation of 64x64 -> 128 bit polynomial multiplication + * using vmull.p8 instructions (8x8 -> 16) is taken from the paper + * "Fast Software Polynomial Multiplication on ARM Processors Using + * the NEON Engine" by Danilo Camara, Conrado Gouvea, Julio Lopez and + * Ricardo Dahab (https://hal.inria.fr/hal-01506572) + * + * It has been slightly tweaked for in-order performance, and to allow + * 'rq' to overlap with 'ad' or 'bd'. + */ + .macro __pmull_p8, rq, ad, bd, b1=t4l, b2=t3l, b3=t4l, b4=t3l + vext.8 t0l, \ad, \ad, #1 @ A1 + .ifc \b1, t4l + vext.8 t4l, \bd, \bd, #1 @ B1 + .endif + vmull.p8 t0q, t0l, \bd @ F = A1*B + vext.8 t1l, \ad, \ad, #2 @ A2 + vmull.p8 t4q, \ad, \b1 @ E = A*B1 + .ifc \b2, t3l + vext.8 t3l, \bd, \bd, #2 @ B2 + .endif + vmull.p8 t1q, t1l, \bd @ H = A2*B + vext.8 t2l, \ad, \ad, #3 @ A3 + vmull.p8 t3q, \ad, \b2 @ G = A*B2 + veor t0q, t0q, t4q @ L = E + F + .ifc \b3, t4l + vext.8 t4l, \bd, \bd, #3 @ B3 + .endif + vmull.p8 t2q, t2l, \bd @ J = A3*B + veor t0l, t0l, t0h @ t0 = (L) (P0 + P1) << 8 + veor t1q, t1q, t3q @ M = G + H + .ifc \b4, t3l + vext.8 t3l, \bd, \bd, #4 @ B4 + .endif + vmull.p8 t4q, \ad, \b3 @ I = A*B3 + veor t1l, t1l, t1h @ t1 = (M) (P2 + P3) << 16 + vmull.p8 t3q, \ad, \b4 @ K = A*B4 + vand t0h, t0h, k48 + vand t1h, t1h, k32 + veor t2q, t2q, t4q @ N = I + J + veor t0l, t0l, t0h + veor t1l, t1l, t1h + veor t2l, t2l, t2h @ t2 = (N) (P4 + P5) << 24 + vand t2h, t2h, k16 + veor t3l, t3l, t3h @ t3 = (K) (P6 + P7) << 32 + vmov.i64 t3h, #0 + vext.8 t0q, t0q, t0q, #15 + veor t2l, t2l, t2h + vext.8 t1q, t1q, t1q, #14 + vmull.p8 \rq, \ad, \bd @ D = A*B + vext.8 t2q, t2q, t2q, #13 + vext.8 t3q, t3q, t3q, #12 + veor t0q, t0q, t1q + veor t2q, t2q, t3q + veor \rq, \rq, t0q + veor \rq, \rq, t2q + .endm + + // + // PMULL (64x64->128) based reduction for CPUs that can do + // it in a single instruction. + // + .macro __pmull_reduce_p64 + vmull.p64 T1, XL_L, MASK + + veor XH_L, XH_L, XM_H + vext.8 T1, T1, T1, #8 + veor XL_H, XL_H, XM_L + veor T1, T1, XL + + vmull.p64 XL, T1_H, MASK + .endm + + // + // Alternative reduction for CPUs that lack support for the + // 64x64->128 PMULL instruction + // + .macro __pmull_reduce_p8 + veor XL_H, XL_H, XM_L + veor XH_L, XH_L, XM_H + + vshl.i64 T1, XL, #57 + vshl.i64 T2, XL, #62 + veor T1, T1, T2 + vshl.i64 T2, XL, #63 + veor T1, T1, T2 + veor XL_H, XL_H, T1_L + veor XH_L, XH_L, T1_H + + vshr.u64 T1, XL, #1 + veor XH, XH, XL + veor XL, XL, T1 + vshr.u64 T1, T1, #6 + vshr.u64 XL, XL, #1 + .endm + + .macro ghash_update, pn + vld1.64 {XL}, [r1] + + /* do the head block first, if supplied */ + ldr ip, [sp] + teq ip, #0 + beq 0f + vld1.64 {T1}, [ip] + teq r0, #0 + b 1f + +0: vld1.64 {T1}, [r2]! + subs r0, r0, #1 + +1: /* multiply XL by SHASH in GF(2^128) */ +#ifndef CONFIG_CPU_BIG_ENDIAN + vrev64.8 T1, T1 +#endif + vext.8 IN1, T1, T1, #8 + veor T1_L, T1_L, XL_H + veor XL, XL, IN1 + + __pmull_\pn XH, XL_H, SHASH_H, s1h, s2h, s3h, s4h @ a1 * b1 + veor T1, T1, XL + __pmull_\pn XL, XL_L, SHASH_L, s1l, s2l, s3l, s4l @ a0 * b0 + __pmull_\pn XM, T1_L, SHASH2_\pn @ (a1+a0)(b1+b0) + + veor T1, XL, XH + veor XM, XM, T1 + + __pmull_reduce_\pn + + veor T1, T1, XH + veor XL, XL, T1 + + bne 0b + + vst1.64 {XL}, [r1] + bx lr + .endm + + /* + * void pmull_ghash_update(int blocks, u64 dg[], const char *src, + * struct ghash_key const *k, const char *head) + */ + .section .text.pmull_ghash_update_p64 +ENTRY(pmull_ghash_update_p64) + vld1.64 {SHASH}, [r3] + veor SHASH2_p64, SHASH_L, SHASH_H + + vmov.i8 MASK, #0xe1 + vshl.u64 MASK, MASK, #57 + + ghash_update p64 +ENDPROC(pmull_ghash_update_p64) + + .section .text.pmull_ghash_update_p8 +ENTRY(pmull_ghash_update_p8) + vld1.64 {SHASH}, [r3] + veor SHASH2_p8, SHASH_L, SHASH_H + + vext.8 s1l, SHASH_L, SHASH_L, #1 + vext.8 s2l, SHASH_L, SHASH_L, #2 + vext.8 s3l, SHASH_L, SHASH_L, #3 + vext.8 s4l, SHASH_L, SHASH_L, #4 + vext.8 s1h, SHASH_H, SHASH_H, #1 + vext.8 s2h, SHASH_H, SHASH_H, #2 + vext.8 s3h, SHASH_H, SHASH_H, #3 + vext.8 s4h, SHASH_H, SHASH_H, #4 + + vmov.i64 k16, #0xffff + vmov.i64 k32, #0xffffffff + vmov.i64 k48, #0xffffffffffff + + ghash_update p8 +ENDPROC(pmull_ghash_update_p8) diff --git a/core/arch/arm/crypto/ghash-ce-core_a64.S b/core/arch/arm/crypto/ghash-ce-core_a64.S new file mode 100644 index 00000000000..584c49f05c4 --- /dev/null +++ b/core/arch/arm/crypto/ghash-ce-core_a64.S @@ -0,0 +1,462 @@ +/* + * Accelerated GHASH implementation with ARMv8 PMULL instructions. + * + * Copyright (C) 2014 - 2017 Linaro Ltd. + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include + +#define ENTRY(func) \ + .global func ; \ + .type func , %function ; \ + func : + +#define ENDPROC(func) \ + .size func , .-func + +#define CPU_LE(x...) x + + SHASH .req v0 + SHASH2 .req v1 + T1 .req v2 + T2 .req v3 + MASK .req v4 + XL .req v5 + XM .req v6 + XH .req v7 + IN1 .req v7 + + k00_16 .req v8 + k32_48 .req v9 + + t3 .req v10 + t4 .req v11 + t5 .req v12 + t6 .req v13 + t7 .req v14 + t8 .req v15 + t9 .req v16 + + perm1 .req v17 + perm2 .req v18 + perm3 .req v19 + + sh1 .req v20 + sh2 .req v21 + sh3 .req v22 + sh4 .req v23 + + ss1 .req v24 + ss2 .req v25 + ss3 .req v26 + ss4 .req v27 + + .text + .arch armv8-a+crypto + + .macro __pmull_p64, rd, rn, rm + pmull \rd\().1q, \rn\().1d, \rm\().1d + .endm + + .macro __pmull2_p64, rd, rn, rm + pmull2 \rd\().1q, \rn\().2d, \rm\().2d + .endm + + .macro __pmull_p8, rq, ad, bd + ext t3.8b, \ad\().8b, \ad\().8b, #1 // A1 + ext t5.8b, \ad\().8b, \ad\().8b, #2 // A2 + ext t7.8b, \ad\().8b, \ad\().8b, #3 // A3 + + __pmull_p8_\bd \rq, \ad + .endm + + .macro __pmull2_p8, rq, ad, bd + tbl t3.16b, {\ad\().16b}, perm1.16b // A1 + tbl t5.16b, {\ad\().16b}, perm2.16b // A2 + tbl t7.16b, {\ad\().16b}, perm3.16b // A3 + + __pmull2_p8_\bd \rq, \ad + .endm + + .macro __pmull_p8_SHASH, rq, ad + __pmull_p8_tail \rq, \ad\().8b, SHASH.8b, 8b,, sh1, sh2, sh3, sh4 + .endm + + .macro __pmull_p8_SHASH2, rq, ad + __pmull_p8_tail \rq, \ad\().8b, SHASH2.8b, 8b,, ss1, ss2, ss3, ss4 + .endm + + .macro __pmull2_p8_SHASH, rq, ad + __pmull_p8_tail \rq, \ad\().16b, SHASH.16b, 16b, 2, sh1, sh2, sh3, sh4 + .endm + + .macro __pmull_p8_tail, rq, ad, bd, nb, t, b1, b2, b3, b4 + pmull\t t3.8h, t3.\nb, \bd // F = A1*B + pmull\t t4.8h, \ad, \b1\().\nb // E = A*B1 + pmull\t t5.8h, t5.\nb, \bd // H = A2*B + pmull\t t6.8h, \ad, \b2\().\nb // G = A*B2 + pmull\t t7.8h, t7.\nb, \bd // J = A3*B + pmull\t t8.8h, \ad, \b3\().\nb // I = A*B3 + pmull\t t9.8h, \ad, \b4\().\nb // K = A*B4 + pmull\t \rq\().8h, \ad, \bd // D = A*B + + eor t3.16b, t3.16b, t4.16b // L = E + F + eor t5.16b, t5.16b, t6.16b // M = G + H + eor t7.16b, t7.16b, t8.16b // N = I + J + + uzp1 t4.2d, t3.2d, t5.2d + uzp2 t3.2d, t3.2d, t5.2d + uzp1 t6.2d, t7.2d, t9.2d + uzp2 t7.2d, t7.2d, t9.2d + + // t3 = (L) (P0 + P1) << 8 + // t5 = (M) (P2 + P3) << 16 + eor t4.16b, t4.16b, t3.16b + and t3.16b, t3.16b, k32_48.16b + + // t7 = (N) (P4 + P5) << 24 + // t9 = (K) (P6 + P7) << 32 + eor t6.16b, t6.16b, t7.16b + and t7.16b, t7.16b, k00_16.16b + + eor t4.16b, t4.16b, t3.16b + eor t6.16b, t6.16b, t7.16b + + zip2 t5.2d, t4.2d, t3.2d + zip1 t3.2d, t4.2d, t3.2d + zip2 t9.2d, t6.2d, t7.2d + zip1 t7.2d, t6.2d, t7.2d + + ext t3.16b, t3.16b, t3.16b, #15 + ext t5.16b, t5.16b, t5.16b, #14 + ext t7.16b, t7.16b, t7.16b, #13 + ext t9.16b, t9.16b, t9.16b, #12 + + eor t3.16b, t3.16b, t5.16b + eor t7.16b, t7.16b, t9.16b + eor \rq\().16b, \rq\().16b, t3.16b + eor \rq\().16b, \rq\().16b, t7.16b + .endm + + .macro __pmull_pre_p64 + movi MASK.16b, #0xe1 + shl MASK.2d, MASK.2d, #57 + .endm + + .macro __pmull_pre_p8 + // k00_16 := 0x0000000000000000_000000000000ffff + // k32_48 := 0x00000000ffffffff_0000ffffffffffff + movi k32_48.2d, #0xffffffff + mov k32_48.h[2], k32_48.h[0] + ushr k00_16.2d, k32_48.2d, #32 + + // prepare the permutation vectors + mov_imm x5, 0x080f0e0d0c0b0a09 + movi T1.8b, #8 + dup perm1.2d, x5 + eor perm1.16b, perm1.16b, T1.16b + ushr perm2.2d, perm1.2d, #8 + ushr perm3.2d, perm1.2d, #16 + ushr T1.2d, perm1.2d, #24 + sli perm2.2d, perm1.2d, #56 + sli perm3.2d, perm1.2d, #48 + sli T1.2d, perm1.2d, #40 + + // precompute loop invariants + tbl sh1.16b, {SHASH.16b}, perm1.16b + tbl sh2.16b, {SHASH.16b}, perm2.16b + tbl sh3.16b, {SHASH.16b}, perm3.16b + tbl sh4.16b, {SHASH.16b}, T1.16b + ext ss1.8b, SHASH2.8b, SHASH2.8b, #1 + ext ss2.8b, SHASH2.8b, SHASH2.8b, #2 + ext ss3.8b, SHASH2.8b, SHASH2.8b, #3 + ext ss4.8b, SHASH2.8b, SHASH2.8b, #4 + .endm + + // + // PMULL (64x64->128) based reduction for CPUs that can do + // it in a single instruction. + // + .macro __pmull_reduce_p64 + pmull T2.1q, XL.1d, MASK.1d + eor XM.16b, XM.16b, T1.16b + + mov XH.d[0], XM.d[1] + mov XM.d[1], XL.d[0] + + eor XL.16b, XM.16b, T2.16b + ext T2.16b, XL.16b, XL.16b, #8 + pmull XL.1q, XL.1d, MASK.1d + .endm + + // + // Alternative reduction for CPUs that lack support for the + // 64x64->128 PMULL instruction + // + .macro __pmull_reduce_p8 + eor XM.16b, XM.16b, T1.16b + + mov XL.d[1], XM.d[0] + mov XH.d[0], XM.d[1] + + shl T1.2d, XL.2d, #57 + shl T2.2d, XL.2d, #62 + eor T2.16b, T2.16b, T1.16b + shl T1.2d, XL.2d, #63 + eor T2.16b, T2.16b, T1.16b + ext T1.16b, XL.16b, XH.16b, #8 + eor T2.16b, T2.16b, T1.16b + + mov XL.d[1], T2.d[0] + mov XH.d[0], T2.d[1] + + ushr T2.2d, XL.2d, #1 + eor XH.16b, XH.16b, XL.16b + eor XL.16b, XL.16b, T2.16b + ushr T2.2d, T2.2d, #6 + ushr XL.2d, XL.2d, #1 + .endm + + .macro __pmull_ghash, pn + ld1 {SHASH.2d}, [x3] + ld1 {XL.2d}, [x1] + ext SHASH2.16b, SHASH.16b, SHASH.16b, #8 + eor SHASH2.16b, SHASH2.16b, SHASH.16b + + __pmull_pre_\pn + + /* do the head block first, if supplied */ + cbz x4, 0f + ld1 {T1.2d}, [x4] + b 1f + +0: ld1 {T1.2d}, [x2], #16 + sub w0, w0, #1 + +1: /* multiply XL by SHASH in GF(2^128) */ +CPU_LE( rev64 T1.16b, T1.16b ) + + ext T2.16b, XL.16b, XL.16b, #8 + ext IN1.16b, T1.16b, T1.16b, #8 + eor T1.16b, T1.16b, T2.16b + eor XL.16b, XL.16b, IN1.16b + + __pmull2_\pn XH, XL, SHASH // a1 * b1 + eor T1.16b, T1.16b, XL.16b + __pmull_\pn XL, XL, SHASH // a0 * b0 + __pmull_\pn XM, T1, SHASH2 // (a1 + a0)(b1 + b0) + + eor T2.16b, XL.16b, XH.16b + ext T1.16b, XL.16b, XH.16b, #8 + eor XM.16b, XM.16b, T2.16b + + __pmull_reduce_\pn + + eor T2.16b, T2.16b, XH.16b + eor XL.16b, XL.16b, T2.16b + + cbnz w0, 0b + + st1 {XL.2d}, [x1] + ret + .endm + + /* + * void pmull_ghash_update(int blocks, u64 dg[], const char *src, + * struct ghash_key const *k, const char *head) + */ + .section .text.pmull_ghash_update_p64 +ENTRY(pmull_ghash_update_p64) + __pmull_ghash p64 +ENDPROC(pmull_ghash_update_p64) + + .section .text.pmull_ghash_update_p8 +ENTRY(pmull_ghash_update_p8) + __pmull_ghash p8 +ENDPROC(pmull_ghash_update_p8) + + KS .req v8 + CTR .req v9 + INP .req v10 + + .macro load_round_keys, rounds, rk + cmp \rounds, #12 + blo 2222f /* 128 bits */ + beq 1111f /* 192 bits */ + ld1 {v17.4s-v18.4s}, [\rk], #32 +1111: ld1 {v19.4s-v20.4s}, [\rk], #32 +2222: ld1 {v21.4s-v24.4s}, [\rk], #64 + ld1 {v25.4s-v28.4s}, [\rk], #64 + ld1 {v29.4s-v31.4s}, [\rk] + .endm + + .macro enc_round, state, key + aese \state\().16b, \key\().16b + aesmc \state\().16b, \state\().16b + .endm + + .macro enc_block, state, rounds + cmp \rounds, #12 + b.lo 2222f /* 128 bits */ + b.eq 1111f /* 192 bits */ + enc_round \state, v17 + enc_round \state, v18 +1111: enc_round \state, v19 + enc_round \state, v20 +2222: .irp key, v21, v22, v23, v24, v25, v26, v27, v28, v29 + enc_round \state, \key + .endr + aese \state\().16b, v30.16b + eor \state\().16b, \state\().16b, v31.16b + .endm + + .macro pmull_gcm_do_crypt, enc + ld1 {SHASH.2d}, [x4] + ld1 {XL.2d}, [x1] + ldr x8, [x5, #8] // load lower counter + + movi MASK.16b, #0xe1 + ext SHASH2.16b, SHASH.16b, SHASH.16b, #8 +CPU_LE( rev x8, x8 ) + shl MASK.2d, MASK.2d, #57 + eor SHASH2.16b, SHASH2.16b, SHASH.16b + + .if \enc == 1 + ld1 {KS.16b}, [x7] + .endif + +0: ld1 {CTR.8b}, [x5] // load upper counter + ld1 {INP.16b}, [x3], #16 + rev x9, x8 + add x8, x8, #1 + sub w0, w0, #1 + ins CTR.d[1], x9 // set lower counter + + .if \enc == 1 + eor INP.16b, INP.16b, KS.16b // encrypt input + st1 {INP.16b}, [x2], #16 + .endif + + rev64 T1.16b, INP.16b + + cmp w6, #12 + b.ge 2f // AES-192/256? + +1: enc_round CTR, v21 + + ext T2.16b, XL.16b, XL.16b, #8 + ext IN1.16b, T1.16b, T1.16b, #8 + + enc_round CTR, v22 + + eor T1.16b, T1.16b, T2.16b + eor XL.16b, XL.16b, IN1.16b + + enc_round CTR, v23 + + pmull2 XH.1q, SHASH.2d, XL.2d // a1 * b1 + eor T1.16b, T1.16b, XL.16b + + enc_round CTR, v24 + + pmull XL.1q, SHASH.1d, XL.1d // a0 * b0 + pmull XM.1q, SHASH2.1d, T1.1d // (a1 + a0)(b1 + b0) + + enc_round CTR, v25 + + ext T1.16b, XL.16b, XH.16b, #8 + eor T2.16b, XL.16b, XH.16b + eor XM.16b, XM.16b, T1.16b + + enc_round CTR, v26 + + eor XM.16b, XM.16b, T2.16b + pmull T2.1q, XL.1d, MASK.1d + + enc_round CTR, v27 + + mov XH.d[0], XM.d[1] + mov XM.d[1], XL.d[0] + + enc_round CTR, v28 + + eor XL.16b, XM.16b, T2.16b + + enc_round CTR, v29 + + ext T2.16b, XL.16b, XL.16b, #8 + + aese CTR.16b, v30.16b + + pmull XL.1q, XL.1d, MASK.1d + eor T2.16b, T2.16b, XH.16b + + eor KS.16b, CTR.16b, v31.16b + + eor XL.16b, XL.16b, T2.16b + + .if \enc == 0 + eor INP.16b, INP.16b, KS.16b + st1 {INP.16b}, [x2], #16 + .endif + + cbnz w0, 0b + +CPU_LE( rev x8, x8 ) + st1 {XL.2d}, [x1] + str x8, [x5, #8] // store lower counter + + .if \enc == 1 + st1 {KS.16b}, [x7] + .endif + + ret + +2: b.eq 3f // AES-192? + enc_round CTR, v17 + enc_round CTR, v18 +3: enc_round CTR, v19 + enc_round CTR, v20 + b 1b + .endm + + /* + * void pmull_gcm_encrypt(int blocks, u64 dg[], u8 dst[], + * const u8 src[], struct ghash_key const *k, + * u8 ctr[], int rounds, u8 ks[]) + */ + .section .text.pmull_gcm_encrypt +ENTRY(pmull_gcm_encrypt) + pmull_gcm_do_crypt 1 +ENDPROC(pmull_gcm_encrypt) + + /* + * void pmull_gcm_decrypt(int blocks, u64 dg[], u8 dst[], + * const u8 src[], struct ghash_key const *k, + * u8 ctr[], int rounds) + */ + .section .text.pmull_gcm_decrypt +ENTRY(pmull_gcm_decrypt) + pmull_gcm_do_crypt 0 +ENDPROC(pmull_gcm_decrypt) + + /* + * void pmull_gcm_encrypt_block(u8 dst[], u8 src[], int rounds) + */ + .section .text.pmull_gcm_encrypt_block +ENTRY(pmull_gcm_encrypt_block) + ld1 {v0.16b}, [x1] + enc_block v0, w2 + st1 {v0.16b}, [x0] + ret +ENDPROC(pmull_gcm_encrypt_block) + + /* + * pmull_gcm_load_round_keys(uint64_t rk[2], int rounds) + */ +ENTRY(pmull_gcm_load_round_keys) + load_round_keys w1, x0 + ret +ENDPROC(pmull_gcm_load_round_keys) diff --git a/core/arch/arm/crypto/sub.mk b/core/arch/arm/crypto/sub.mk new file mode 100644 index 00000000000..9a905a131bc --- /dev/null +++ b/core/arch/arm/crypto/sub.mk @@ -0,0 +1,4 @@ +ifeq ($(CFG_CRYPTO_WITH_CE),y) +srcs-$(CFG_ARM64_core) += ghash-ce-core_a64.S +srcs-$(CFG_ARM32_core) += ghash-ce-core_a32.S +endif diff --git a/core/arch/arm/include/crypto/ghash-ce-core.h b/core/arch/arm/include/crypto/ghash-ce-core.h new file mode 100644 index 00000000000..d9e8fc645d1 --- /dev/null +++ b/core/arch/arm/include/crypto/ghash-ce-core.h @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2017, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#ifndef __GHASH_CE_CORE_H +#define __GHASH_CE_CORE_H + +#include + +void pmull_ghash_update_p64(int blocks, uint64_t dg[2], const uint8_t *src, + const uint64_t k[2], const uint8_t *head); +void pmull_ghash_update_p8(int blocks, uint64_t dg[2], const uint8_t *src, + const uint64_t k[2], const uint8_t *head); + +void pmull_gcm_load_round_keys(uint64_t rk[30], int rounds); + +void pmull_gcm_encrypt(int blocks, uint64_t dg[2], uint8_t dst[], + const uint8_t src[], const uint64_t k[2], + const uint8_t ctr[], int rounds, uint8_t ks[]); + + +void pmull_gcm_decrypt(int blocks, uint64_t dg[2], uint8_t dst[], + const uint8_t src[], const uint64_t k[2], + const uint8_t ctr[], int rounds); + +void pmull_gcm_encrypt_block(uint8_t dst[], const uint8_t src[], int rounds); + +#endif /*__GHASH_CE_CORE_H*/ From 80427c4266ee57023705afd6f56a8c93f9cf31d1 Mon Sep 17 00:00:00 2001 From: Jens Wiklander Date: Thu, 16 Nov 2017 12:44:41 +0100 Subject: [PATCH 04/12] LTC: add GHASH acceleration Acked-by: Jerome Forissier Signed-off-by: Jens Wiklander --- .../src/encauth/gcm/gcm_mult_h_arm_ce.c | 50 +++++++++++++++++++ core/lib/libtomcrypt/src/encauth/gcm/sub.mk | 4 ++ 2 files changed, 54 insertions(+) create mode 100644 core/lib/libtomcrypt/src/encauth/gcm/gcm_mult_h_arm_ce.c diff --git a/core/lib/libtomcrypt/src/encauth/gcm/gcm_mult_h_arm_ce.c b/core/lib/libtomcrypt/src/encauth/gcm/gcm_mult_h_arm_ce.c new file mode 100644 index 00000000000..2c769ac5d73 --- /dev/null +++ b/core/lib/libtomcrypt/src/encauth/gcm/gcm_mult_h_arm_ce.c @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2017, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include +#include +#include +#include +#include + +/** + GCM multiply by H + @param gcm The GCM state which holds the H value + @param I The value to multiply H by + */ +void gcm_mult_h(gcm_state *gcm, unsigned char *I) +{ + struct tomcrypt_arm_neon_state state; + const uint8_t zeroes[TEE_AES_BLOCK_SIZE] = { 0 }; + uint64_t k[2]; + uint64_t a; + uint64_t b; + uint64_t dg[2]; + + b = get_be64(gcm->H); + a = get_be64(gcm->H + 8); + + k[0] = (a << 1) | (b >> 63); + k[1] = (b << 1) | (a >> 63); + if (b >> 63) + k[1] ^= 0xc200000000000000UL; + + dg[1] = get_be64(I); + dg[0] = get_be64(I + 8); + + tomcrypt_arm_neon_enable(&state); +#ifdef CFG_HWSUPP_PMULL + pmull_ghash_update_p64(1, dg, zeroes, k, NULL); +#else + pmull_ghash_update_p8(1, dg, zeroes, k, NULL); +#endif + tomcrypt_arm_neon_disable(&state); + + put_be64(I, dg[1]); + put_be64(I + 8, dg[0]); +} + diff --git a/core/lib/libtomcrypt/src/encauth/gcm/sub.mk b/core/lib/libtomcrypt/src/encauth/gcm/sub.mk index 62946f5e67c..8315f62d205 100644 --- a/core/lib/libtomcrypt/src/encauth/gcm/sub.mk +++ b/core/lib/libtomcrypt/src/encauth/gcm/sub.mk @@ -1,7 +1,11 @@ srcs-y += gcm_add_aad.c srcs-y += gcm_add_iv.c srcs-y += gcm_done.c +ifeq ($(CFG_CRYPTO_WITH_CE),y) +srcs-y += gcm_mult_h_arm_ce.c +else srcs-y += gcm_gf_mult.c +endif srcs-y += gcm_init.c srcs-y += gcm_memory.c srcs-y += gcm_mult_h.c From 993a8c6edb4f5774e9ad3d3d06735fc4710e2588 Mon Sep 17 00:00:00 2001 From: Jens Wiklander Date: Thu, 16 Nov 2017 12:44:42 +0100 Subject: [PATCH 05/12] hikey{,960}: CFG_HWSUPP_PMULL=y Takes full advantage of LTC GHASH acceleration by using the pmull instruction. Acked-by: Jerome Forissier Signed-off-by: Jens Wiklander --- core/arch/arm/plat-hikey/conf.mk | 1 + 1 file changed, 1 insertion(+) diff --git a/core/arch/arm/plat-hikey/conf.mk b/core/arch/arm/plat-hikey/conf.mk index 345a1ae1af5..f943bffa4ae 100644 --- a/core/arch/arm/plat-hikey/conf.mk +++ b/core/arch/arm/plat-hikey/conf.mk @@ -22,6 +22,7 @@ endif CFG_NUM_THREADS ?= 8 CFG_CRYPTO_WITH_CE ?= y +CFG_HWSUPP_PMULL ?= y CFG_WITH_STACK_CANARIES ?= y ifeq ($(PLATFORM_FLAVOR),hikey) From f73a51e8b094d81a98e68133492b7af553f105df Mon Sep 17 00:00:00 2001 From: Jens Wiklander Date: Thu, 16 Nov 2017 12:44:44 +0100 Subject: [PATCH 06/12] plat-vexpress: Juno: CFG_HWSUPP_PMULL=y Takes full advantage of LTC GHASH acceleration by using the pmull instruction. Signed-off-by: Jens Wiklander --- core/arch/arm/plat-vexpress/conf.mk | 1 + 1 file changed, 1 insertion(+) diff --git a/core/arch/arm/plat-vexpress/conf.mk b/core/arch/arm/plat-vexpress/conf.mk index 7a8ebd74c69..0d5956e90d1 100644 --- a/core/arch/arm/plat-vexpress/conf.mk +++ b/core/arch/arm/plat-vexpress/conf.mk @@ -50,6 +50,7 @@ CFG_WITH_STATS ?= y ifeq ($(PLATFORM_FLAVOR),juno) CFG_CRYPTO_WITH_CE ?= y +CFG_HWSUPP_PMULL ?= y endif ifeq ($(PLATFORM_FLAVOR),qemu_virt) From 053918a93f507fc842c9e90d559bb36aea63fdcd Mon Sep 17 00:00:00 2001 From: Jens Wiklander Date: Thu, 16 Nov 2017 12:44:45 +0100 Subject: [PATCH 07/12] core: add temporary CFG_LTC_CRYPTO_GCM Adds temporary CFG_LTC_CRYPTO_GCM which is a mirror of CFG_CRYPTO_GCM. Signed-off-by: Jens Wiklander --- core/core.mk | 1 + core/lib/libtomcrypt/include/tomcrypt_custom.h | 2 +- core/lib/libtomcrypt/src/encauth/sub.mk | 2 +- core/lib/libtomcrypt/src/tee_ltc_provider.c | 4 ++-- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/core/core.mk b/core/core.mk index 7eb90e3a9ef..1be2504f99a 100644 --- a/core/core.mk +++ b/core/core.mk @@ -15,6 +15,7 @@ PLATFORM_$(PLATFORM) := y PLATFORM_FLAVOR_$(PLATFORM_FLAVOR) := y $(call cfg-depends-all,CFG_PAGED_USER_TA,CFG_WITH_PAGER CFG_WITH_USER_TA) +CFG_LTC_CRYPTO_GCM = $(CFG_CRYPTO_GCM) # Setup compiler for this sub module COMPILER_$(sm) ?= $(COMPILER) diff --git a/core/lib/libtomcrypt/include/tomcrypt_custom.h b/core/lib/libtomcrypt/include/tomcrypt_custom.h index 223af26cf1b..16afd440c52 100644 --- a/core/lib/libtomcrypt/include/tomcrypt_custom.h +++ b/core/lib/libtomcrypt/include/tomcrypt_custom.h @@ -211,7 +211,7 @@ #ifdef CFG_CRYPTO_CCM #define LTC_CCM_MODE #endif -#ifdef CFG_CRYPTO_GCM +#ifdef CFG_LTC_CRYPTO_GCM #define LTC_GCM_MODE #endif diff --git a/core/lib/libtomcrypt/src/encauth/sub.mk b/core/lib/libtomcrypt/src/encauth/sub.mk index 14341220b48..52159cd3d44 100644 --- a/core/lib/libtomcrypt/src/encauth/sub.mk +++ b/core/lib/libtomcrypt/src/encauth/sub.mk @@ -1,2 +1,2 @@ subdirs-$(CFG_CRYPTO_CCM) += ccm -subdirs-$(CFG_CRYPTO_GCM) += gcm +subdirs-$(CFG_LTC_CRYPTO_GCM) += gcm diff --git a/core/lib/libtomcrypt/src/tee_ltc_provider.c b/core/lib/libtomcrypt/src/tee_ltc_provider.c index 36378fd0697..8ae90c634e4 100644 --- a/core/lib/libtomcrypt/src/tee_ltc_provider.c +++ b/core/lib/libtomcrypt/src/tee_ltc_provider.c @@ -2689,7 +2689,7 @@ void crypto_aes_ccm_final(void *ctx) } #endif /*CFG_CRYPTO_CCM*/ -#if defined(CFG_CRYPTO_GCM) +#if defined(CFG_LTC_CRYPTO_GCM) struct tee_gcm_state { gcm_state ctx; /* the gcm state as defined by LTC */ size_t tag_len; /* tag length */ @@ -2845,7 +2845,7 @@ void crypto_aes_gcm_final(void *ctx) gcm_reset(&gcm->ctx); } -#endif /*CFG_CRYPTO_GCM*/ +#endif /*CFG_LTC_CRYPTO_GCM*/ /****************************************************************************** * Pseudo Random Number Generator From f3ddbfdeb40e2716e10635dfa12c9301b6540f88 Mon Sep 17 00:00:00 2001 From: Jens Wiklander Date: Thu, 16 Nov 2017 12:44:47 +0100 Subject: [PATCH 08/12] core: crypto: add new AES-GCM implementation Adds a new AES-GCM implementation optimized for hardware acceleration. Signed-off-by: Jens Wiklander --- core/arch/arm/crypto/aes-gcm-ce.c | 121 +++++++++++ core/arch/arm/crypto/sub.mk | 1 + core/core.mk | 1 - core/crypto/aes-gcm-ghash.c | 79 ++++++++ core/crypto/aes-gcm-sw.c | 102 ++++++++++ core/crypto/aes-gcm.c | 320 ++++++++++++++++++++++++++++++ core/crypto/sub.mk | 3 + core/include/crypto/aes-gcm_ext.h | 56 ++++++ 8 files changed, 682 insertions(+), 1 deletion(-) create mode 100644 core/arch/arm/crypto/aes-gcm-ce.c create mode 100644 core/crypto/aes-gcm-ghash.c create mode 100644 core/crypto/aes-gcm-sw.c create mode 100644 core/crypto/aes-gcm.c create mode 100644 core/include/crypto/aes-gcm_ext.h diff --git a/core/arch/arm/crypto/aes-gcm-ce.c b/core/arch/arm/crypto/aes-gcm-ce.c new file mode 100644 index 00000000000..efb842eeb4b --- /dev/null +++ b/core/arch/arm/crypto/aes-gcm-ce.c @@ -0,0 +1,121 @@ +/* + * Copyright (c) 2017, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include +#include +#include +#include +#include +#include +#include + +TEE_Result crypto_aes_gcm_set_key(struct crypto_aes_gcm_ctx *ctx, + const void *key, size_t key_len) +{ + uint64_t k[2]; + uint64_t a; + uint64_t b; + + if (aes_setup(key, key_len, 0, &ctx->skey)) + return TEE_ERROR_BAD_PARAMETERS; + + crypto_aes_gcm_encrypt_block(ctx, ctx->ctr, ctx->hash_subkey); + + /* Store hash key in little endian and multiply by 'x' */ + b = get_be64(ctx->hash_subkey); + a = get_be64(ctx->hash_subkey + 8); + k[0] = (a << 1) | (b >> 63); + k[1] = (b << 1) | (a >> 63); + if (b >> 63) + k[1] ^= 0xc200000000000000UL; + + memcpy(ctx->hash_subkey, k, TEE_AES_BLOCK_SIZE); + return TEE_SUCCESS; +} + +static void get_dg(uint64_t dg[2], struct crypto_aes_gcm_ctx *ctx) +{ + dg[1] = get_be64(ctx->hash_state); + dg[0] = get_be64(ctx->hash_state + 8); +} + +static void put_dg(struct crypto_aes_gcm_ctx *ctx, uint64_t dg[2]) +{ + put_be64(ctx->hash_state, dg[1]); + put_be64(ctx->hash_state + 8, dg[0]); +} + +void crypto_aes_gcm_ghash_update(struct crypto_aes_gcm_ctx *ctx, + const void *head, const void *data, + size_t num_blocks) +{ + uint32_t vfp_state; + uint64_t dg[2]; + uint64_t *k; + + get_dg(dg, ctx); + + k = (void *)ctx->hash_subkey; + + vfp_state = thread_kernel_enable_vfp(); + +#ifdef CFG_HWSUPP_PMULL + pmull_ghash_update_p64(num_blocks, dg, data, k, head); +#else + pmull_ghash_update_p8(num_blocks, dg, data, k, head); +#endif + thread_kernel_disable_vfp(vfp_state); + + put_dg(ctx, dg); +} + +#ifdef ARM64 +void crypto_aes_gcm_encrypt_block(struct crypto_aes_gcm_ctx *ctx, + const void *src, void *dst) +{ + uint32_t vfp_state; + void *enc_key = ctx->skey.rijndael.eK; + size_t rounds = ctx->skey.rijndael.Nr; + + vfp_state = thread_kernel_enable_vfp(); + + pmull_gcm_load_round_keys(enc_key, rounds); + pmull_gcm_encrypt_block(dst, src, rounds); + + thread_kernel_disable_vfp(vfp_state); +} + +void crypto_aes_gcm_update_payload_block_aligned(struct crypto_aes_gcm_ctx *ctx, + TEE_OperationMode m, + const void *src, + size_t num_blocks, void *dst) +{ + uint32_t vfp_state; + uint64_t dg[2]; + uint64_t *k; + void *ctr = ctx->ctr; + void *enc_key = ctx->skey.rijndael.eK; + size_t rounds = ctx->skey.rijndael.Nr; + + get_dg(dg, ctx); + k = (void *)ctx->hash_subkey; + + vfp_state = thread_kernel_enable_vfp(); + + pmull_gcm_load_round_keys(enc_key, rounds); + + if (m == TEE_MODE_ENCRYPT) + pmull_gcm_encrypt(num_blocks, dg, dst, src, k, ctr, rounds, + ctx->buf_cryp); + else + pmull_gcm_decrypt(num_blocks, dg, dst, src, k, ctr, rounds); + + thread_kernel_disable_vfp(vfp_state); + + put_dg(ctx, dg); +} +#endif /*ARM64*/ diff --git a/core/arch/arm/crypto/sub.mk b/core/arch/arm/crypto/sub.mk index 9a905a131bc..fabfd54efdc 100644 --- a/core/arch/arm/crypto/sub.mk +++ b/core/arch/arm/crypto/sub.mk @@ -1,4 +1,5 @@ ifeq ($(CFG_CRYPTO_WITH_CE),y) srcs-$(CFG_ARM64_core) += ghash-ce-core_a64.S srcs-$(CFG_ARM32_core) += ghash-ce-core_a32.S +srcs-y += aes-gcm-ce.c endif diff --git a/core/core.mk b/core/core.mk index 1be2504f99a..7eb90e3a9ef 100644 --- a/core/core.mk +++ b/core/core.mk @@ -15,7 +15,6 @@ PLATFORM_$(PLATFORM) := y PLATFORM_FLAVOR_$(PLATFORM_FLAVOR) := y $(call cfg-depends-all,CFG_PAGED_USER_TA,CFG_WITH_PAGER CFG_WITH_USER_TA) -CFG_LTC_CRYPTO_GCM = $(CFG_CRYPTO_GCM) # Setup compiler for this sub module COMPILER_$(sm) ?= $(COMPILER) diff --git a/core/crypto/aes-gcm-ghash.c b/core/crypto/aes-gcm-ghash.c new file mode 100644 index 00000000000..e4ddfa5ad7b --- /dev/null +++ b/core/crypto/aes-gcm-ghash.c @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2010 Mike Belopuhov + * Copyright (c) 2017, Linaro Limited + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#include +#include +#include +#include +#include + +static void xor_block(void *dst, const void *src) +{ + uint64_t *d = dst; + const uint64_t *s = src; + + d[0] ^= s[0]; + d[1] ^= s[1]; +} + +/* + * gfmul() is based on ghash_gfmul() from + * https://github.com/openbsd/src/blob/master/sys/crypto/gmac.c + */ +static void gfmul(const uint64_t X[2], const uint64_t Y[2], uint64_t product[2]) +{ + uint64_t y[2]; + uint64_t z[2] = { 0 }; + const uint8_t *x = (const uint8_t *)X; + uint32_t mul; + size_t n; + + y[0] = TEE_U64_FROM_BIG_ENDIAN(Y[0]); + y[1] = TEE_U64_FROM_BIG_ENDIAN(Y[1]); + + for (n = 0; n < TEE_AES_BLOCK_SIZE * 8; n++) { + /* update Z */ + if (x[n >> 3] & (1 << (~n & 7))) + xor_block(z, y); + + /* update Y */ + mul = y[1] & 1; + y[1] = (y[0] << 63) | (y[1] >> 1); + y[0] = (y[0] >> 1) ^ (0xe100000000000000 * mul); + } + + product[0] = TEE_U64_TO_BIG_ENDIAN(z[0]); + product[1] = TEE_U64_TO_BIG_ENDIAN(z[1]); +} + +void __weak crypto_aes_gcm_ghash_update(struct crypto_aes_gcm_ctx *ctx, + const void *head, const void *data, + size_t num_blocks) +{ + const uint64_t *x = (const void *)data; + void *y = ctx->hash_state; + size_t n; + + if (head) { + xor_block(y, head); + gfmul((void *)ctx->hash_subkey, y, y); + } + for (n = 0; n < num_blocks; n++) { + xor_block(y, x + n * 2); + gfmul((void *)ctx->hash_subkey, y, y); + } +} diff --git a/core/crypto/aes-gcm-sw.c b/core/crypto/aes-gcm-sw.c new file mode 100644 index 00000000000..cd6f3af7459 --- /dev/null +++ b/core/crypto/aes-gcm-sw.c @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2017, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +/* + * gfmul() is based on ghash_gfmul from + * https://github.com/openbsd/src/blob/master/sys/crypto/gmac.c + * Which is: + * Copyright (c) 2010 Mike Belopuhov + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#include +#include +#include +#include +#include +#include + +static bool __maybe_unused ptr_is_block_aligned(const void *p) +{ + return !((vaddr_t)p & (TEE_AES_BLOCK_SIZE - 1)); +} + +static void xor_block(void *dst, const void *src) +{ + uint64_t *d = dst; + const uint64_t *s = src; + + d[0] ^= s[0]; + d[1] ^= s[1]; +} + +TEE_Result __weak crypto_aes_gcm_set_key(struct crypto_aes_gcm_ctx *ctx, + const void *key, size_t key_len) +{ + if (aes_setup(key, key_len, 0, &ctx->skey)) + return TEE_ERROR_BAD_PARAMETERS; + + if (aes_ecb_encrypt((void *)ctx->ctr, ctx->hash_subkey, &ctx->skey)) + panic(); + + return TEE_SUCCESS; +} + +void __weak +crypto_aes_gcm_update_payload_block_aligned(struct crypto_aes_gcm_ctx *ctx, + TEE_OperationMode m, + const void *src, size_t num_blocks, + void *dst) +{ + size_t n; + const uint8_t *s = src; + uint8_t *d = dst; + + assert(!ctx->buf_pos && num_blocks && + ptr_is_block_aligned(s) && ptr_is_block_aligned(d)); + + for (n = 0; n < num_blocks; n++) { + if (m == TEE_MODE_ENCRYPT) { + xor_block(ctx->buf_cryp, s); + crypto_aes_gcm_ghash_update(ctx, ctx->buf_cryp, + NULL, 0); + memcpy(d, ctx->buf_cryp, sizeof(ctx->buf_cryp)); + crypto_aes_gcm_encrypt_block(ctx, ctx->ctr, + ctx->buf_cryp); + crypto_aes_gcm_inc_ctr(ctx); + } else { + crypto_aes_gcm_encrypt_block(ctx, ctx->ctr, + ctx->buf_cryp); + + xor_block(ctx->buf_cryp, s); + crypto_aes_gcm_ghash_update(ctx, s, NULL, 0); + memcpy(d, ctx->buf_cryp, sizeof(ctx->buf_cryp)); + + crypto_aes_gcm_inc_ctr(ctx); + } + s += TEE_AES_BLOCK_SIZE; + d += TEE_AES_BLOCK_SIZE; + } +} + +void __weak crypto_aes_gcm_encrypt_block(struct crypto_aes_gcm_ctx *ctx, + const void *src, void *dst) +{ + if (aes_ecb_encrypt(src, dst, &ctx->skey)) + panic(); +} diff --git a/core/crypto/aes-gcm.c b/core/crypto/aes-gcm.c new file mode 100644 index 00000000000..93aa365fc59 --- /dev/null +++ b/core/crypto/aes-gcm.c @@ -0,0 +1,320 @@ +/* + * Copyright (c) 2017, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +size_t crypto_aes_gcm_get_ctx_size(void) +{ + return sizeof(struct crypto_aes_gcm_ctx); +} + +static void xor_buf(uint8_t *dst, const uint8_t *src, size_t len) +{ + size_t n; + + for (n = 0; n < len; n++) + dst[n] ^= src[n]; +} + +static bool ptr_is_block_aligned(const void *p) +{ + return !((vaddr_t)p & (TEE_AES_BLOCK_SIZE - 1)); +} + +static void ghash_update_pad_zero(struct crypto_aes_gcm_ctx *ctx, + const uint8_t *data, size_t len) +{ + size_t n = len / TEE_AES_BLOCK_SIZE; + uint64_t block[2]; + + if (n) { + if (ptr_is_block_aligned(data)) { + crypto_aes_gcm_ghash_update(ctx, NULL, data, n); + } else { + size_t m; + + for (m = 0; m < n; m++) { + + memcpy(block, data + m * sizeof(block), + sizeof(block)); + crypto_aes_gcm_ghash_update(ctx, NULL, + (void *)block, 1); + } + } + } + + if (len - n * TEE_AES_BLOCK_SIZE) { + memset(block, 0, sizeof(block)); + memcpy(block, data + n * TEE_AES_BLOCK_SIZE, + len - n * TEE_AES_BLOCK_SIZE); + crypto_aes_gcm_ghash_update(ctx, block, NULL, 0); + } +} + +static void ghash_update_lengths(struct crypto_aes_gcm_ctx *ctx, uint32_t l1, + uint32_t l2) +{ + uint64_t len_fields[2] = { + TEE_U64_TO_BIG_ENDIAN(l1 * 8), + TEE_U64_TO_BIG_ENDIAN(l2 * 8) + }; + + COMPILE_TIME_ASSERT(sizeof(len_fields) == TEE_AES_BLOCK_SIZE); + crypto_aes_gcm_ghash_update(ctx, (uint8_t *)len_fields, NULL, 0); +} + +TEE_Result crypto_aes_gcm_init(void *c, TEE_OperationMode mode, + const uint8_t *key, size_t key_len, + const uint8_t *nonce, size_t nonce_len, + size_t tag_len) +{ + struct crypto_aes_gcm_ctx *ctx = c; + TEE_Result res; + + COMPILE_TIME_ASSERT(sizeof(ctx->ctr) == TEE_AES_BLOCK_SIZE); + + if (tag_len > sizeof(ctx->buf_tag)) + return TEE_ERROR_BAD_PARAMETERS; + + memset(ctx, 0, sizeof(*ctx)); + + ctx->tag_len = tag_len; + res = crypto_aes_gcm_set_key(ctx, key, key_len); + if (res) + return res; + + if (nonce_len == (96 / 8)) { + memcpy(ctx->ctr, nonce, nonce_len); + crypto_aes_gcm_inc_ctr(ctx); + } else { + ghash_update_pad_zero(ctx, nonce, nonce_len); + ghash_update_lengths(ctx, 0, nonce_len); + + memcpy(ctx->ctr, ctx->hash_state, sizeof(ctx->ctr)); + memset(ctx->hash_state, 0, sizeof(ctx->hash_state)); + } + + crypto_aes_gcm_encrypt_block(ctx, ctx->ctr, ctx->buf_tag); + crypto_aes_gcm_inc_ctr(ctx); + if (mode == TEE_MODE_ENCRYPT) { + /* + * Encryption uses the pre-encrypted xor-buffer to encrypt + * while decryption encrypts the xor-buffer when needed + * instead. + * + * The reason for this is that the combined encryption and + * ghash implementation does both operations intertwined. + * In the decrypt case the xor-buffer is needed at the end + * of processing each block, while the encryption case + * needs xor-buffer before processing each block. + * + * In a pure software implementation we wouldn't have any + * use for this kind of optimization, but since this + * AES-GCM implementation is aimed at being combined with + * accelerated routines it's more convenient to always have + * this optimization activated. + */ + crypto_aes_gcm_encrypt_block(ctx, ctx->ctr, ctx->buf_cryp); + crypto_aes_gcm_inc_ctr(ctx); + } + + return TEE_SUCCESS; +} + +TEE_Result crypto_aes_gcm_update_aad(void *c, + const uint8_t *data, size_t len) +{ + struct crypto_aes_gcm_ctx *ctx = c; + const uint8_t *d = data; + size_t l = len; + const uint8_t *head = NULL; + size_t n; + + if (ctx->payload_bytes) + return TEE_ERROR_BAD_PARAMETERS; + + ctx->aad_bytes += len; + + while (l) { + if (ctx->buf_pos || !ptr_is_block_aligned(d) || + l < TEE_AES_BLOCK_SIZE) { + n = MIN(TEE_AES_BLOCK_SIZE - ctx->buf_pos, l); + memcpy(ctx->buf_hash + ctx->buf_pos, d, n); + ctx->buf_pos += n; + + if (ctx->buf_pos != TEE_AES_BLOCK_SIZE) + return TEE_SUCCESS; + + ctx->buf_pos = 0; + head = ctx->buf_hash; + d += n; + l -= n; + } + + if (ptr_is_block_aligned(d)) + n = l / TEE_AES_BLOCK_SIZE; + else + n = 0; + + crypto_aes_gcm_ghash_update(ctx, head, d, n); + l -= n * TEE_AES_BLOCK_SIZE; + d += n * TEE_AES_BLOCK_SIZE; + } + + return TEE_SUCCESS; +} + +TEE_Result crypto_aes_gcm_update_payload(void *c, TEE_OperationMode m, + const uint8_t *src, size_t len, + uint8_t *dst) +{ + struct crypto_aes_gcm_ctx *ctx = c; + size_t n; + const uint8_t *s = src; + uint8_t *d = dst; + size_t l = len; + + if (!ctx->payload_bytes && ctx->buf_pos) { + /* AAD part done, finish up the last bits. */ + memset(ctx->buf_hash + ctx->buf_pos, 0, + TEE_AES_BLOCK_SIZE - ctx->buf_pos); + crypto_aes_gcm_ghash_update(ctx, ctx->buf_hash, NULL, 0); + ctx->buf_pos = 0; + } + + ctx->payload_bytes += len; + + while (l) { + if (ctx->buf_pos || !ptr_is_block_aligned(s) || + !ptr_is_block_aligned(d) || l < TEE_AES_BLOCK_SIZE) { + n = MIN(TEE_AES_BLOCK_SIZE - ctx->buf_pos, l); + + if (!ctx->buf_pos && m == TEE_MODE_DECRYPT) { + crypto_aes_gcm_encrypt_block(ctx, ctx->ctr, + ctx->buf_cryp); + } + + xor_buf(ctx->buf_cryp + ctx->buf_pos, s, n); + memcpy(d, ctx->buf_cryp + ctx->buf_pos, n); + if (m == TEE_MODE_ENCRYPT) + memcpy(ctx->buf_hash + ctx->buf_pos, + ctx->buf_cryp + ctx->buf_pos, n); + else + memcpy(ctx->buf_hash + ctx->buf_pos, s, n); + + ctx->buf_pos += n; + + if (ctx->buf_pos != TEE_AES_BLOCK_SIZE) + return TEE_SUCCESS; + + crypto_aes_gcm_ghash_update(ctx, ctx->buf_hash, NULL, 0); + ctx->buf_pos = 0; + d += n; + s += n; + l -= n; + + if (m == TEE_MODE_ENCRYPT) + crypto_aes_gcm_encrypt_block(ctx, ctx->ctr, + ctx->buf_cryp); + crypto_aes_gcm_inc_ctr(ctx); + } else { + n = l / TEE_AES_BLOCK_SIZE; + crypto_aes_gcm_update_payload_block_aligned(ctx, m, s, + n, d); + s += n * TEE_AES_BLOCK_SIZE; + d += n * TEE_AES_BLOCK_SIZE; + l -= n * TEE_AES_BLOCK_SIZE; + } + } + + return TEE_SUCCESS; +} + +static TEE_Result operation_final(struct crypto_aes_gcm_ctx *ctx, + TEE_OperationMode m, const uint8_t *src, + size_t len, uint8_t *dst) +{ + TEE_Result res; + + res = crypto_aes_gcm_update_payload(ctx, m, src, len, dst); + if (res) + return res; + + if (ctx->buf_pos) { + memset(ctx->buf_hash + ctx->buf_pos, 0, + sizeof(ctx->buf_hash) - ctx->buf_pos); + crypto_aes_gcm_ghash_update(ctx, ctx->buf_hash, NULL, 0); + } + + ghash_update_lengths(ctx, ctx->aad_bytes, ctx->payload_bytes); + /* buf_tag was filled in with the first counter block aes_gcm_init() */ + xor_buf(ctx->buf_tag, ctx->hash_state, ctx->tag_len); + + return TEE_SUCCESS; +} + +TEE_Result crypto_aes_gcm_enc_final(void *c, const uint8_t *src, size_t len, + uint8_t *dst, uint8_t *tag, size_t *tag_len) +{ + struct crypto_aes_gcm_ctx *ctx = c; + TEE_Result res; + + if (*tag_len < ctx->tag_len) + return TEE_ERROR_SHORT_BUFFER; + + res = operation_final(ctx, TEE_MODE_ENCRYPT, src, len, dst); + if (res) + return res; + + memcpy(tag, ctx->buf_tag, ctx->tag_len); + *tag_len = ctx->tag_len; + + return TEE_SUCCESS; +} + +TEE_Result crypto_aes_gcm_dec_final(void *c, const uint8_t *src, size_t len, + uint8_t *dst, const uint8_t *tag, + size_t tag_len) +{ + struct crypto_aes_gcm_ctx *ctx = c; + TEE_Result res; + + if (tag_len != ctx->tag_len) + return TEE_ERROR_MAC_INVALID; + + res = operation_final(ctx, TEE_MODE_DECRYPT, src, len, dst); + if (res) + return res; + + if (buf_compare_ct(ctx->buf_tag, tag, tag_len)) + return TEE_ERROR_MAC_INVALID; + + return TEE_SUCCESS; +} + +void crypto_aes_gcm_inc_ctr(struct crypto_aes_gcm_ctx *ctx) +{ + uint64_t c; + + c = TEE_U64_FROM_BIG_ENDIAN(ctx->ctr[1]) + 1; + ctx->ctr[1] = TEE_U64_TO_BIG_ENDIAN(c); + if (!c) { + c = TEE_U64_FROM_BIG_ENDIAN(ctx->ctr[0]) + 1; + ctx->ctr[0] = TEE_U64_TO_BIG_ENDIAN(c); + } +} + +void crypto_aes_gcm_final(void *c __unused) +{ +} diff --git a/core/crypto/sub.mk b/core/crypto/sub.mk index 7f72ebd0775..99df944dd5b 100644 --- a/core/crypto/sub.mk +++ b/core/crypto/sub.mk @@ -1 +1,4 @@ srcs-y += crypto.c +srcs-y += aes-gcm.c +srcs-y += aes-gcm-sw.c +srcs-y += aes-gcm-ghash.c diff --git a/core/include/crypto/aes-gcm_ext.h b/core/include/crypto/aes-gcm_ext.h new file mode 100644 index 00000000000..d79773a55b4 --- /dev/null +++ b/core/include/crypto/aes-gcm_ext.h @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2017, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#ifndef __CRYPTO_AES_GCM_EXT_H +#define __CRYPTO_AES_GCM_EXT_H + +#include +#include +#include +#include + +struct crypto_aes_gcm_ctx { + uint64_t ctr[2]; + + uint8_t hash_subkey[TEE_AES_BLOCK_SIZE]; + uint8_t hash_state[TEE_AES_BLOCK_SIZE]; + + uint8_t buf_tag[TEE_AES_BLOCK_SIZE]; + uint8_t buf_hash[TEE_AES_BLOCK_SIZE]; + uint8_t buf_cryp[TEE_AES_BLOCK_SIZE]; + + symmetric_key skey; + + unsigned int tag_len; + unsigned int aad_bytes; + unsigned int payload_bytes; + unsigned int buf_pos; +}; + +void crypto_aes_gcm_inc_ctr(struct crypto_aes_gcm_ctx *ctx); + +/* + * Internal weak functions that can be overridden with hardware specific + * implementations. + */ +void crypto_aes_gcm_next_ctr(struct crypto_aes_gcm_ctx *ctx); + +void crypto_aes_gcm_encrypt_block(struct crypto_aes_gcm_ctx *ctx, + const void *src, void *dst); + +TEE_Result crypto_aes_gcm_set_key(struct crypto_aes_gcm_ctx *ctx, + const void *key, size_t key_len); + +void crypto_aes_gcm_ghash_update(struct crypto_aes_gcm_ctx *ctx, + const void *head, const void *data, + size_t num_blocks); + +void crypto_aes_gcm_update_payload_block_aligned(struct crypto_aes_gcm_ctx *ctx, + TEE_OperationMode m, + const void *src, + size_t num_blocks, void *dst); +#endif /*__CRYPTO_AES_GCM_EXT_H*/ From 56db8181e3a4c5eb6a26ef25fd1fbc9d1c5889be Mon Sep 17 00:00:00 2001 From: Jens Wiklander Date: Thu, 16 Nov 2017 12:44:48 +0100 Subject: [PATCH 09/12] core: ltc: remove redundant aes-gcm implementation Signed-off-by: Jens Wiklander --- .../lib/libtomcrypt/include/tomcrypt_custom.h | 3 - core/lib/libtomcrypt/src/encauth/sub.mk | 1 - core/lib/libtomcrypt/src/tee_ltc_provider.c | 161 ------------------ 3 files changed, 165 deletions(-) diff --git a/core/lib/libtomcrypt/include/tomcrypt_custom.h b/core/lib/libtomcrypt/include/tomcrypt_custom.h index 16afd440c52..ce5b3915acb 100644 --- a/core/lib/libtomcrypt/include/tomcrypt_custom.h +++ b/core/lib/libtomcrypt/include/tomcrypt_custom.h @@ -211,9 +211,6 @@ #ifdef CFG_CRYPTO_CCM #define LTC_CCM_MODE #endif -#ifdef CFG_LTC_CRYPTO_GCM - #define LTC_GCM_MODE -#endif #define LTC_NO_PK diff --git a/core/lib/libtomcrypt/src/encauth/sub.mk b/core/lib/libtomcrypt/src/encauth/sub.mk index 52159cd3d44..9ad603a4c37 100644 --- a/core/lib/libtomcrypt/src/encauth/sub.mk +++ b/core/lib/libtomcrypt/src/encauth/sub.mk @@ -1,2 +1 @@ subdirs-$(CFG_CRYPTO_CCM) += ccm -subdirs-$(CFG_LTC_CRYPTO_GCM) += gcm diff --git a/core/lib/libtomcrypt/src/tee_ltc_provider.c b/core/lib/libtomcrypt/src/tee_ltc_provider.c index 8ae90c634e4..ea4a2216681 100644 --- a/core/lib/libtomcrypt/src/tee_ltc_provider.c +++ b/core/lib/libtomcrypt/src/tee_ltc_provider.c @@ -27,7 +27,6 @@ #include #include -#include #include #include #include @@ -334,7 +333,6 @@ static TEE_Result tee_algo_to_ltc_cipherindex(uint32_t algo, case TEE_ALG_AES_CTS: case TEE_ALG_AES_XTS: case TEE_ALG_AES_CCM: - case TEE_ALG_AES_GCM: *ltc_cipherindex = find_cipher("aes"); break; #endif @@ -2523,7 +2521,6 @@ TEE_Result crypto_mac_final(void *ctx, uint32_t algo, uint8_t *digest, #define TEE_CCM_KEY_MAX_LENGTH 32 #define TEE_CCM_NONCE_MAX_LENGTH 13 #define TEE_CCM_TAG_MAX_LENGTH 16 -#define TEE_GCM_TAG_MAX_LENGTH 16 #if defined(CFG_CRYPTO_CCM) struct tee_ccm_state { @@ -2689,164 +2686,6 @@ void crypto_aes_ccm_final(void *ctx) } #endif /*CFG_CRYPTO_CCM*/ -#if defined(CFG_LTC_CRYPTO_GCM) -struct tee_gcm_state { - gcm_state ctx; /* the gcm state as defined by LTC */ - size_t tag_len; /* tag length */ -}; - -size_t crypto_aes_gcm_get_ctx_size(void) -{ - return sizeof(struct tee_gcm_state); -} - -TEE_Result crypto_aes_gcm_init(void *ctx, TEE_OperationMode mode __unused, - const uint8_t *key, size_t key_len, - const uint8_t *nonce, size_t nonce_len, - size_t tag_len) -{ - TEE_Result res; - int ltc_res; - int ltc_cipherindex; - struct tee_gcm_state *gcm = ctx; - - res = tee_algo_to_ltc_cipherindex(TEE_ALG_AES_GCM, <c_cipherindex); - if (res != TEE_SUCCESS) - return TEE_ERROR_NOT_SUPPORTED; - - /* reset the state */ - memset(gcm, 0, sizeof(struct tee_gcm_state)); - gcm->tag_len = tag_len; - - ltc_res = gcm_init(&gcm->ctx, ltc_cipherindex, key, key_len); - if (ltc_res != CRYPT_OK) - return TEE_ERROR_BAD_STATE; - - /* Add the IV */ - ltc_res = gcm_add_iv(&gcm->ctx, nonce, nonce_len); - if (ltc_res != CRYPT_OK) - return TEE_ERROR_BAD_STATE; - - return TEE_SUCCESS; -} - -TEE_Result crypto_aes_gcm_update_aad(void *ctx, const uint8_t *data, size_t len) -{ - struct tee_gcm_state *gcm = ctx; - int ltc_res; - - /* Add the AAD (note: aad can be NULL if aadlen == 0) */ - ltc_res = gcm_add_aad(&gcm->ctx, data, len); - if (ltc_res != CRYPT_OK) - return TEE_ERROR_BAD_STATE; - - return TEE_SUCCESS; -} - -TEE_Result crypto_aes_gcm_update_payload(void *ctx, TEE_OperationMode mode, - const uint8_t *src_data, - size_t len, uint8_t *dst_data) -{ - TEE_Result res; - int ltc_res, dir; - struct tee_gcm_state *gcm = ctx; - unsigned char *pt, *ct; /* the plain and the cipher text */ - - if (mode == TEE_MODE_ENCRYPT) { - pt = (unsigned char *)src_data; - ct = dst_data; - dir = GCM_ENCRYPT; - } else { - pt = dst_data; - ct = (unsigned char *)src_data; - dir = GCM_DECRYPT; - } - - /* aad is optional ==> add one without length */ - if (gcm->ctx.mode == LTC_GCM_MODE_IV) { - res = crypto_aes_gcm_update_aad(gcm, NULL, 0); - if (res != TEE_SUCCESS) - return res; - } - - /* process the data */ - ltc_res = gcm_process(&gcm->ctx, pt, len, ct, dir); - if (ltc_res != CRYPT_OK) - return TEE_ERROR_BAD_STATE; - - return TEE_SUCCESS; -} - -TEE_Result crypto_aes_gcm_enc_final(void *ctx, const uint8_t *src_data, - size_t len, uint8_t *dst_data, - uint8_t *dst_tag, size_t *dst_tag_len) -{ - TEE_Result res; - struct tee_gcm_state *gcm = ctx; - int ltc_res; - - /* Finalize the remaining buffer */ - res = crypto_aes_gcm_update_payload(ctx, TEE_MODE_ENCRYPT, src_data, - len, dst_data); - if (res != TEE_SUCCESS) - return res; - - /* Check the tag length */ - if (*dst_tag_len < gcm->tag_len) { - *dst_tag_len = gcm->tag_len; - return TEE_ERROR_SHORT_BUFFER; - } - *dst_tag_len = gcm->tag_len; - - /* Compute the tag */ - ltc_res = gcm_done(&gcm->ctx, dst_tag, (unsigned long *)dst_tag_len); - if (ltc_res != CRYPT_OK) - return TEE_ERROR_BAD_STATE; - - return TEE_SUCCESS; -} - -TEE_Result crypto_aes_gcm_dec_final(void *ctx, const uint8_t *src_data, - size_t len, uint8_t *dst_data, - const uint8_t *tag, size_t tag_len) -{ - TEE_Result res = TEE_ERROR_BAD_STATE; - struct tee_gcm_state *gcm = ctx; - int ltc_res; - uint8_t dst_tag[TEE_GCM_TAG_MAX_LENGTH]; - unsigned long ltc_tag_len = tag_len; - - if (tag_len == 0) - return TEE_ERROR_SHORT_BUFFER; - if (tag_len > TEE_GCM_TAG_MAX_LENGTH) - return TEE_ERROR_BAD_STATE; - - /* Process the last buffer, if any */ - res = crypto_aes_gcm_update_payload(ctx, TEE_MODE_DECRYPT, src_data, - len, dst_data); - if (res != TEE_SUCCESS) - return res; - - /* Finalize the authentication */ - ltc_res = gcm_done(&gcm->ctx, dst_tag, <c_tag_len); - if (ltc_res != CRYPT_OK) - return TEE_ERROR_BAD_STATE; - - if (buf_compare_ct(dst_tag, tag, tag_len) != 0) - res = TEE_ERROR_MAC_INVALID; - else - res = TEE_SUCCESS; - return res; -} - -void crypto_aes_gcm_final(void *ctx) -{ - struct tee_gcm_state *gcm = ctx; - - gcm_reset(&gcm->ctx); -} -#endif /*CFG_LTC_CRYPTO_GCM*/ - /****************************************************************************** * Pseudo Random Number Generator ******************************************************************************/ From a865014683c8385f70d27d97e5297745ca418b7b Mon Sep 17 00:00:00 2001 From: Jens Wiklander Date: Thu, 16 Nov 2017 12:44:50 +0100 Subject: [PATCH 10/12] core: arm: crypto: fix AES-GCM counter increase In pmull_gcm_encrypt() and pmull_gcm_decrypt() it was assumed that it's enough to only increase the least significant 64-bits of the counter fed to the block cipher. This can hold for 96-bit IVs, but not for IVs of any other length as the number stored in the least significant 64-bits of the counter can't be easily predicted. In this patch pmull_gcm_encrypt() and pmull_gcm_decrypt() are updated to increase the entire counter, at the same time is the interface changed to accept the counter in little endian format instead. Signed-off-by: Jens Wiklander --- core/arch/arm/crypto/aes-gcm-ce.c | 41 ++++++++++++-------- core/arch/arm/crypto/ghash-ce-core_a64.S | 36 +++++++++-------- core/arch/arm/include/crypto/ghash-ce-core.h | 4 +- 3 files changed, 46 insertions(+), 35 deletions(-) diff --git a/core/arch/arm/crypto/aes-gcm-ce.c b/core/arch/arm/crypto/aes-gcm-ce.c index efb842eeb4b..35851e48a8b 100644 --- a/core/arch/arm/crypto/aes-gcm-ce.c +++ b/core/arch/arm/crypto/aes-gcm-ce.c @@ -13,6 +13,22 @@ #include #include +static void get_be_block(void *dst, const void *src) +{ + uint64_t *d = dst; + + d[1] = get_be64(src); + d[0] = get_be64((const uint8_t *)src + 8); +} + +static void put_be_block(void *dst, const void *src) +{ + const uint64_t *s = src; + + put_be64(dst, s[1]); + put_be64((uint8_t *)dst + 8, s[0]); +} + TEE_Result crypto_aes_gcm_set_key(struct crypto_aes_gcm_ctx *ctx, const void *key, size_t key_len) { @@ -37,18 +53,6 @@ TEE_Result crypto_aes_gcm_set_key(struct crypto_aes_gcm_ctx *ctx, return TEE_SUCCESS; } -static void get_dg(uint64_t dg[2], struct crypto_aes_gcm_ctx *ctx) -{ - dg[1] = get_be64(ctx->hash_state); - dg[0] = get_be64(ctx->hash_state + 8); -} - -static void put_dg(struct crypto_aes_gcm_ctx *ctx, uint64_t dg[2]) -{ - put_be64(ctx->hash_state, dg[1]); - put_be64(ctx->hash_state + 8, dg[0]); -} - void crypto_aes_gcm_ghash_update(struct crypto_aes_gcm_ctx *ctx, const void *head, const void *data, size_t num_blocks) @@ -57,7 +61,7 @@ void crypto_aes_gcm_ghash_update(struct crypto_aes_gcm_ctx *ctx, uint64_t dg[2]; uint64_t *k; - get_dg(dg, ctx); + get_be_block(dg, ctx->hash_state); k = (void *)ctx->hash_subkey; @@ -70,7 +74,7 @@ void crypto_aes_gcm_ghash_update(struct crypto_aes_gcm_ctx *ctx, #endif thread_kernel_disable_vfp(vfp_state); - put_dg(ctx, dg); + put_be_block(ctx->hash_state, dg); } #ifdef ARM64 @@ -96,12 +100,14 @@ void crypto_aes_gcm_update_payload_block_aligned(struct crypto_aes_gcm_ctx *ctx, { uint32_t vfp_state; uint64_t dg[2]; + uint64_t ctr[2]; uint64_t *k; - void *ctr = ctx->ctr; void *enc_key = ctx->skey.rijndael.eK; size_t rounds = ctx->skey.rijndael.Nr; - get_dg(dg, ctx); + get_be_block(dg, ctx->hash_state); + get_be_block(ctr, ctx->ctr); + k = (void *)ctx->hash_subkey; vfp_state = thread_kernel_enable_vfp(); @@ -116,6 +122,7 @@ void crypto_aes_gcm_update_payload_block_aligned(struct crypto_aes_gcm_ctx *ctx, thread_kernel_disable_vfp(vfp_state); - put_dg(ctx, dg); + put_be_block(ctx->ctr, ctr); + put_be_block(ctx->hash_state, dg); } #endif /*ARM64*/ diff --git a/core/arch/arm/crypto/ghash-ce-core_a64.S b/core/arch/arm/crypto/ghash-ce-core_a64.S index 584c49f05c4..af40c556352 100644 --- a/core/arch/arm/crypto/ghash-ce-core_a64.S +++ b/core/arch/arm/crypto/ghash-ce-core_a64.S @@ -315,11 +315,11 @@ ENDPROC(pmull_ghash_update_p8) .macro pmull_gcm_do_crypt, enc ld1 {SHASH.2d}, [x4] ld1 {XL.2d}, [x1] - ldr x8, [x5, #8] // load lower counter + ldp x8, x9, [x5] // load counter movi MASK.16b, #0xe1 ext SHASH2.16b, SHASH.16b, SHASH.16b, #8 -CPU_LE( rev x8, x8 ) + shl MASK.2d, MASK.2d, #57 eor SHASH2.16b, SHASH2.16b, SHASH.16b @@ -327,12 +327,17 @@ CPU_LE( rev x8, x8 ) ld1 {KS.16b}, [x7] .endif -0: ld1 {CTR.8b}, [x5] // load upper counter - ld1 {INP.16b}, [x3], #16 - rev x9, x8 - add x8, x8, #1 +0: ins CTR.d[1], x8 // set counter + ins CTR.d[0], x9 +CPU_LE( rev64 CTR.16b, CTR.16b) + + add x8, x8, #1 // increase counter + cmp x8, #0 + b.ne 1f + add x9, x9, #1 + +1: ld1 {INP.16b}, [x3], #16 sub w0, w0, #1 - ins CTR.d[1], x9 // set lower counter .if \enc == 1 eor INP.16b, INP.16b, KS.16b // encrypt input @@ -342,9 +347,9 @@ CPU_LE( rev x8, x8 ) rev64 T1.16b, INP.16b cmp w6, #12 - b.ge 2f // AES-192/256? + b.ge 3f // AES-192/256? -1: enc_round CTR, v21 +2: enc_round CTR, v21 ext T2.16b, XL.16b, XL.16b, #8 ext IN1.16b, T1.16b, T1.16b, #8 @@ -404,9 +409,8 @@ CPU_LE( rev x8, x8 ) cbnz w0, 0b -CPU_LE( rev x8, x8 ) st1 {XL.2d}, [x1] - str x8, [x5, #8] // store lower counter + stp x8, x9, [x5] // store counter .if \enc == 1 st1 {KS.16b}, [x7] @@ -414,18 +418,18 @@ CPU_LE( rev x8, x8 ) ret -2: b.eq 3f // AES-192? +3: b.eq 4f // AES-192? enc_round CTR, v17 enc_round CTR, v18 -3: enc_round CTR, v19 +4: enc_round CTR, v19 enc_round CTR, v20 - b 1b + b 2b .endm /* * void pmull_gcm_encrypt(int blocks, u64 dg[], u8 dst[], * const u8 src[], struct ghash_key const *k, - * u8 ctr[], int rounds, u8 ks[]) + * u64 ctr[2], int rounds, u8 ks[]) */ .section .text.pmull_gcm_encrypt ENTRY(pmull_gcm_encrypt) @@ -435,7 +439,7 @@ ENDPROC(pmull_gcm_encrypt) /* * void pmull_gcm_decrypt(int blocks, u64 dg[], u8 dst[], * const u8 src[], struct ghash_key const *k, - * u8 ctr[], int rounds) + * u64 ctr[2], int rounds) */ .section .text.pmull_gcm_decrypt ENTRY(pmull_gcm_decrypt) diff --git a/core/arch/arm/include/crypto/ghash-ce-core.h b/core/arch/arm/include/crypto/ghash-ce-core.h index d9e8fc645d1..657910e4863 100644 --- a/core/arch/arm/include/crypto/ghash-ce-core.h +++ b/core/arch/arm/include/crypto/ghash-ce-core.h @@ -19,12 +19,12 @@ void pmull_gcm_load_round_keys(uint64_t rk[30], int rounds); void pmull_gcm_encrypt(int blocks, uint64_t dg[2], uint8_t dst[], const uint8_t src[], const uint64_t k[2], - const uint8_t ctr[], int rounds, uint8_t ks[]); + uint64_t ctr[], int rounds, uint8_t ks[]); void pmull_gcm_decrypt(int blocks, uint64_t dg[2], uint8_t dst[], const uint8_t src[], const uint64_t k[2], - const uint8_t ctr[], int rounds); + uint64_t ctr[], int rounds); void pmull_gcm_encrypt_block(uint8_t dst[], const uint8_t src[], int rounds); From b1538e99efe1b2798a2e7dadd89fabbd3e666975 Mon Sep 17 00:00:00 2001 From: Jens Wiklander Date: Thu, 16 Nov 2017 12:44:53 +0100 Subject: [PATCH 11/12] core: crypto: merge aes-gcm.h and aes-gcm_ext.h Merges aes-gcm.h and aes-gcm_ext.h and aligns types of function argument etc. Signed-off-by: Jens Wiklander --- core/arch/arm/crypto/aes-gcm-ce.c | 2 +- core/crypto/aes-gcm-ghash.c | 2 +- core/crypto/aes-gcm-sw.c | 2 +- core/crypto/aes-gcm.c | 53 ++++++++----------- core/include/crypto/aes-gcm.h | 84 ++++++++++++++++++++++++------- core/include/crypto/aes-gcm_ext.h | 56 --------------------- 6 files changed, 92 insertions(+), 107 deletions(-) delete mode 100644 core/include/crypto/aes-gcm_ext.h diff --git a/core/arch/arm/crypto/aes-gcm-ce.c b/core/arch/arm/crypto/aes-gcm-ce.c index 35851e48a8b..b4edb74fd55 100644 --- a/core/arch/arm/crypto/aes-gcm-ce.c +++ b/core/arch/arm/crypto/aes-gcm-ce.c @@ -5,7 +5,7 @@ * SPDX-License-Identifier: BSD-2-Clause */ -#include +#include #include #include #include diff --git a/core/crypto/aes-gcm-ghash.c b/core/crypto/aes-gcm-ghash.c index e4ddfa5ad7b..65a6cacca30 100644 --- a/core/crypto/aes-gcm-ghash.c +++ b/core/crypto/aes-gcm-ghash.c @@ -15,7 +15,7 @@ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ -#include +#include #include #include #include diff --git a/core/crypto/aes-gcm-sw.c b/core/crypto/aes-gcm-sw.c index cd6f3af7459..a239522b589 100644 --- a/core/crypto/aes-gcm-sw.c +++ b/core/crypto/aes-gcm-sw.c @@ -24,7 +24,7 @@ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ -#include +#include #include #include #include diff --git a/core/crypto/aes-gcm.c b/core/crypto/aes-gcm.c index 93aa365fc59..5055cd445bc 100644 --- a/core/crypto/aes-gcm.c +++ b/core/crypto/aes-gcm.c @@ -5,7 +5,7 @@ * SPDX-License-Identifier: BSD-2-Clause */ -#include +#include #include #include #include @@ -14,11 +14,6 @@ #include #include -size_t crypto_aes_gcm_get_ctx_size(void) -{ - return sizeof(struct crypto_aes_gcm_ctx); -} - static void xor_buf(uint8_t *dst, const uint8_t *src, size_t len) { size_t n; @@ -74,12 +69,11 @@ static void ghash_update_lengths(struct crypto_aes_gcm_ctx *ctx, uint32_t l1, crypto_aes_gcm_ghash_update(ctx, (uint8_t *)len_fields, NULL, 0); } -TEE_Result crypto_aes_gcm_init(void *c, TEE_OperationMode mode, - const uint8_t *key, size_t key_len, - const uint8_t *nonce, size_t nonce_len, - size_t tag_len) +TEE_Result crypto_aes_gcm_init(struct crypto_aes_gcm_ctx *ctx, + TEE_OperationMode mode, const void *key, + size_t key_len, const void *nonce, + size_t nonce_len, size_t tag_len) { - struct crypto_aes_gcm_ctx *ctx = c; TEE_Result res; COMPILE_TIME_ASSERT(sizeof(ctx->ctr) == TEE_AES_BLOCK_SIZE); @@ -132,10 +126,9 @@ TEE_Result crypto_aes_gcm_init(void *c, TEE_OperationMode mode, return TEE_SUCCESS; } -TEE_Result crypto_aes_gcm_update_aad(void *c, - const uint8_t *data, size_t len) +TEE_Result crypto_aes_gcm_update_aad(struct crypto_aes_gcm_ctx *ctx, + const void *data, size_t len) { - struct crypto_aes_gcm_ctx *ctx = c; const uint8_t *d = data; size_t l = len; const uint8_t *head = NULL; @@ -175,11 +168,10 @@ TEE_Result crypto_aes_gcm_update_aad(void *c, return TEE_SUCCESS; } -TEE_Result crypto_aes_gcm_update_payload(void *c, TEE_OperationMode m, - const uint8_t *src, size_t len, - uint8_t *dst) +TEE_Result crypto_aes_gcm_update_payload(struct crypto_aes_gcm_ctx *ctx, + TEE_OperationMode mode, + const void *src, size_t len, void *dst) { - struct crypto_aes_gcm_ctx *ctx = c; size_t n; const uint8_t *s = src; uint8_t *d = dst; @@ -200,14 +192,14 @@ TEE_Result crypto_aes_gcm_update_payload(void *c, TEE_OperationMode m, !ptr_is_block_aligned(d) || l < TEE_AES_BLOCK_SIZE) { n = MIN(TEE_AES_BLOCK_SIZE - ctx->buf_pos, l); - if (!ctx->buf_pos && m == TEE_MODE_DECRYPT) { + if (!ctx->buf_pos && mode == TEE_MODE_DECRYPT) { crypto_aes_gcm_encrypt_block(ctx, ctx->ctr, ctx->buf_cryp); } xor_buf(ctx->buf_cryp + ctx->buf_pos, s, n); memcpy(d, ctx->buf_cryp + ctx->buf_pos, n); - if (m == TEE_MODE_ENCRYPT) + if (mode == TEE_MODE_ENCRYPT) memcpy(ctx->buf_hash + ctx->buf_pos, ctx->buf_cryp + ctx->buf_pos, n); else @@ -224,14 +216,14 @@ TEE_Result crypto_aes_gcm_update_payload(void *c, TEE_OperationMode m, s += n; l -= n; - if (m == TEE_MODE_ENCRYPT) + if (mode == TEE_MODE_ENCRYPT) crypto_aes_gcm_encrypt_block(ctx, ctx->ctr, ctx->buf_cryp); crypto_aes_gcm_inc_ctr(ctx); } else { n = l / TEE_AES_BLOCK_SIZE; - crypto_aes_gcm_update_payload_block_aligned(ctx, m, s, - n, d); + crypto_aes_gcm_update_payload_block_aligned(ctx, mode, + s, n, d); s += n * TEE_AES_BLOCK_SIZE; d += n * TEE_AES_BLOCK_SIZE; l -= n * TEE_AES_BLOCK_SIZE; @@ -264,10 +256,10 @@ static TEE_Result operation_final(struct crypto_aes_gcm_ctx *ctx, return TEE_SUCCESS; } -TEE_Result crypto_aes_gcm_enc_final(void *c, const uint8_t *src, size_t len, - uint8_t *dst, uint8_t *tag, size_t *tag_len) +TEE_Result crypto_aes_gcm_enc_final(struct crypto_aes_gcm_ctx *ctx, + const void *src, size_t len, void *dst, + void *tag, size_t *tag_len) { - struct crypto_aes_gcm_ctx *ctx = c; TEE_Result res; if (*tag_len < ctx->tag_len) @@ -283,11 +275,10 @@ TEE_Result crypto_aes_gcm_enc_final(void *c, const uint8_t *src, size_t len, return TEE_SUCCESS; } -TEE_Result crypto_aes_gcm_dec_final(void *c, const uint8_t *src, size_t len, - uint8_t *dst, const uint8_t *tag, - size_t tag_len) +TEE_Result crypto_aes_gcm_dec_final(struct crypto_aes_gcm_ctx *ctx, + const void *src, size_t len, void *dst, + const void *tag, size_t tag_len) { - struct crypto_aes_gcm_ctx *ctx = c; TEE_Result res; if (tag_len != ctx->tag_len) @@ -315,6 +306,6 @@ void crypto_aes_gcm_inc_ctr(struct crypto_aes_gcm_ctx *ctx) } } -void crypto_aes_gcm_final(void *c __unused) +void crypto_aes_gcm_final(struct crypto_aes_gcm_ctx *ctx __unused) { } diff --git a/core/include/crypto/aes-gcm.h b/core/include/crypto/aes-gcm.h index 3b90eea519a..a2d71615ca3 100644 --- a/core/include/crypto/aes-gcm.h +++ b/core/include/crypto/aes-gcm.h @@ -9,23 +9,73 @@ #define __CRYPTO_AES_GCM_H #include +#include +#include +#include + +struct crypto_aes_gcm_ctx { + uint64_t ctr[2]; + + uint8_t hash_subkey[TEE_AES_BLOCK_SIZE]; + uint8_t hash_state[TEE_AES_BLOCK_SIZE]; + + uint8_t buf_tag[TEE_AES_BLOCK_SIZE]; + uint8_t buf_hash[TEE_AES_BLOCK_SIZE]; + uint8_t buf_cryp[TEE_AES_BLOCK_SIZE]; + + symmetric_key skey; + + unsigned int tag_len; + unsigned int aad_bytes; + unsigned int payload_bytes; + unsigned int buf_pos; +}; + +static inline size_t crypto_aes_gcm_get_ctx_size(void) +{ + return sizeof(struct crypto_aes_gcm_ctx); +} + +TEE_Result crypto_aes_gcm_init(struct crypto_aes_gcm_ctx *ctx, + TEE_OperationMode mode, const void *key, + size_t key_len, const void *nonce, + size_t nonce_len, size_t tag_len); +TEE_Result crypto_aes_gcm_update_aad(struct crypto_aes_gcm_ctx *ctx, + const void *data, size_t len); +TEE_Result crypto_aes_gcm_update_payload(struct crypto_aes_gcm_ctx *ctx, + TEE_OperationMode mode, + const void *src, size_t len, + void *dst); +TEE_Result crypto_aes_gcm_enc_final(struct crypto_aes_gcm_ctx *ctx, + const void *src, size_t len, void *dst, + void *tag, size_t *tag_len); +TEE_Result crypto_aes_gcm_dec_final(struct crypto_aes_gcm_ctx *ctx, + const void *src, size_t len, void *dst, + const void *tag, size_t tag_len); +void crypto_aes_gcm_final(struct crypto_aes_gcm_ctx *ctx); + +void crypto_aes_gcm_inc_ctr(struct crypto_aes_gcm_ctx *ctx); + +/* + * Internal weak functions that can be overridden with hardware specific + * implementations. + */ +void crypto_aes_gcm_next_ctr(struct crypto_aes_gcm_ctx *ctx); + +void crypto_aes_gcm_encrypt_block(struct crypto_aes_gcm_ctx *ctx, + const void *src, void *dst); + +TEE_Result crypto_aes_gcm_set_key(struct crypto_aes_gcm_ctx *ctx, + const void *key, size_t key_len); + +void crypto_aes_gcm_ghash_update(struct crypto_aes_gcm_ctx *ctx, + const void *head, const void *data, + size_t num_blocks); + +void crypto_aes_gcm_update_payload_block_aligned(struct crypto_aes_gcm_ctx *ctx, + TEE_OperationMode m, + const void *src, + size_t num_blocks, void *dst); -size_t crypto_aes_gcm_get_ctx_size(void); -TEE_Result crypto_aes_gcm_init(void *ctx, TEE_OperationMode mode, - const uint8_t *key, size_t key_len, - const uint8_t *nonce, size_t nonce_len, - size_t tag_len); -TEE_Result crypto_aes_gcm_update_aad(void *ctx, const uint8_t *data, - size_t len); -TEE_Result crypto_aes_gcm_update_payload(void *ctx, TEE_OperationMode mode, - const uint8_t *src_data, - size_t len, uint8_t *dst_data); -TEE_Result crypto_aes_gcm_enc_final(void *ctx, const uint8_t *src_data, - size_t len, uint8_t *dst_data, - uint8_t *dst_tag, size_t *dst_tag_len); -TEE_Result crypto_aes_gcm_dec_final(void *ctx, const uint8_t *src_data, - size_t len, uint8_t *dst_data, - const uint8_t *tag, size_t tag_len); -void crypto_aes_gcm_final(void *ctx); #endif /*__CRYPTO_AES_GCM_H*/ diff --git a/core/include/crypto/aes-gcm_ext.h b/core/include/crypto/aes-gcm_ext.h deleted file mode 100644 index d79773a55b4..00000000000 --- a/core/include/crypto/aes-gcm_ext.h +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright (c) 2017, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-2-Clause - */ - -#ifndef __CRYPTO_AES_GCM_EXT_H -#define __CRYPTO_AES_GCM_EXT_H - -#include -#include -#include -#include - -struct crypto_aes_gcm_ctx { - uint64_t ctr[2]; - - uint8_t hash_subkey[TEE_AES_BLOCK_SIZE]; - uint8_t hash_state[TEE_AES_BLOCK_SIZE]; - - uint8_t buf_tag[TEE_AES_BLOCK_SIZE]; - uint8_t buf_hash[TEE_AES_BLOCK_SIZE]; - uint8_t buf_cryp[TEE_AES_BLOCK_SIZE]; - - symmetric_key skey; - - unsigned int tag_len; - unsigned int aad_bytes; - unsigned int payload_bytes; - unsigned int buf_pos; -}; - -void crypto_aes_gcm_inc_ctr(struct crypto_aes_gcm_ctx *ctx); - -/* - * Internal weak functions that can be overridden with hardware specific - * implementations. - */ -void crypto_aes_gcm_next_ctr(struct crypto_aes_gcm_ctx *ctx); - -void crypto_aes_gcm_encrypt_block(struct crypto_aes_gcm_ctx *ctx, - const void *src, void *dst); - -TEE_Result crypto_aes_gcm_set_key(struct crypto_aes_gcm_ctx *ctx, - const void *key, size_t key_len); - -void crypto_aes_gcm_ghash_update(struct crypto_aes_gcm_ctx *ctx, - const void *head, const void *data, - size_t num_blocks); - -void crypto_aes_gcm_update_payload_block_aligned(struct crypto_aes_gcm_ctx *ctx, - TEE_OperationMode m, - const void *src, - size_t num_blocks, void *dst); -#endif /*__CRYPTO_AES_GCM_EXT_H*/ From 7d9fabe4f95d290ceab58b29c958dee849981a83 Mon Sep 17 00:00:00 2001 From: Jens Wiklander Date: Fri, 17 Nov 2017 09:19:23 +0100 Subject: [PATCH 12/12] [review] LTC: add GHASH acceleration Signed-off-by: Jens Wiklander --- core/lib/libtomcrypt/src/encauth/gcm/sub.mk | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/core/lib/libtomcrypt/src/encauth/gcm/sub.mk b/core/lib/libtomcrypt/src/encauth/gcm/sub.mk index 8315f62d205..3df1df51f8b 100644 --- a/core/lib/libtomcrypt/src/encauth/gcm/sub.mk +++ b/core/lib/libtomcrypt/src/encauth/gcm/sub.mk @@ -1,14 +1,14 @@ srcs-y += gcm_add_aad.c srcs-y += gcm_add_iv.c srcs-y += gcm_done.c -ifeq ($(CFG_CRYPTO_WITH_CE),y) -srcs-y += gcm_mult_h_arm_ce.c -else srcs-y += gcm_gf_mult.c -endif srcs-y += gcm_init.c srcs-y += gcm_memory.c +ifeq ($(CFG_CRYPTO_WITH_CE),y) +srcs-y += gcm_mult_h_arm_ce.c +else srcs-y += gcm_mult_h.c +endif srcs-y += gcm_process.c srcs-y += gcm_reset.c # srcs-y += gcm_test.c