core: crypto: add new AES-GCM implementation

Adds a new AES-GCM implementation optimized for hardware acceleration.

This implementation is enabled by default, to use the implementation in
libTomCrypt instead set CFG_CRYPTO_AES_GCM_FROM_CRYPTOLIB=y.

Tested-by: Jerome Forissier <jerome.forissier@linaro.org> (HiKey960)
Acked-by: Jerome Forissier <jerome.forissier@linaro.org>
Signed-off-by: Jens Wiklander <jens.wiklander@linaro.org>

core/arch/arm/crypto/aes-gcm-ce.c

@@ -0,0 +1,122 @@
+/*
+ * Copyright (c) 2017, Linaro Limited
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <crypto/internal_aes-gcm.h>
+#include <crypto/ghash-ce-core.h>
+#include <io.h>
+#include <kernel/panic.h>
+#include <kernel/thread.h>
+#include <tomcrypt.h>
+#include <types_ext.h>
+
+TEE_Result internal_aes_gcm_set_key(struct internal_aes_gcm_ctx *ctx,
+				    const void *key, size_t key_len)
+{
+	uint64_t k[2];
+	uint64_t a;
+	uint64_t b;
+
+	if (aes_setup(key, key_len, 0, &ctx->skey))
+		return TEE_ERROR_BAD_PARAMETERS;
+
+	internal_aes_gcm_encrypt_block(ctx, ctx->ctr, ctx->hash_subkey);
+
+	/* Store hash key in little endian and multiply by 'x' */
+	b = get_be64(ctx->hash_subkey);
+	a = get_be64(ctx->hash_subkey + 8);
+	k[0] = (a << 1) | (b >> 63);
+	k[1] = (b << 1) | (a >> 63);
+	if (b >> 63)
+		k[1] ^= 0xc200000000000000UL;
+
+	memcpy(ctx->hash_subkey, k, TEE_AES_BLOCK_SIZE);
+	return TEE_SUCCESS;
+}
+
+static void get_dg(uint64_t dg[2], struct internal_aes_gcm_ctx *ctx)
+{
+	dg[1] = get_be64(ctx->hash_state);
+	dg[0] = get_be64(ctx->hash_state + 8);
+}
+
+static void put_dg(struct internal_aes_gcm_ctx *ctx, uint64_t dg[2])
+{
+	put_be64(ctx->hash_state, dg[1]);
+	put_be64(ctx->hash_state + 8, dg[0]);
+}
+
+void internal_aes_gcm_ghash_update(struct internal_aes_gcm_ctx *ctx,
+				   const void *head, const void *data,
+				 size_t num_blocks)
+{
+	uint32_t vfp_state;
+	uint64_t dg[2];
+	uint64_t *k;
+
+	get_dg(dg, ctx);
+
+	k = (void *)ctx->hash_subkey;
+
+	vfp_state = thread_kernel_enable_vfp();
+
+#ifdef CFG_HWSUPP_PMULL
+	pmull_ghash_update_p64(num_blocks, dg, data, k, head);
+#else
+	pmull_ghash_update_p8(num_blocks, dg, data, k, head);
+#endif
+	thread_kernel_disable_vfp(vfp_state);
+
+	put_dg(ctx, dg);
+}
+
+#ifdef ARM64
+void internal_aes_gcm_encrypt_block(struct internal_aes_gcm_ctx *ctx,
+				    const void *src, void *dst)
+{
+	uint32_t vfp_state;
+	void *enc_key = ctx->skey.rijndael.eK;
+	size_t rounds = ctx->skey.rijndael.Nr;
+
+	vfp_state = thread_kernel_enable_vfp();
+
+	pmull_gcm_load_round_keys(enc_key, rounds);
+	pmull_gcm_encrypt_block(dst, src, rounds);
+
+	thread_kernel_disable_vfp(vfp_state);
+}
+
+void
+internal_aes_gcm_update_payload_block_aligned(struct internal_aes_gcm_ctx *ctx,
+					      TEE_OperationMode m,
+					      const void *src,
+					      size_t num_blocks, void *dst)
+{
+	uint32_t vfp_state;
+	uint64_t dg[2];
+	uint64_t *k;
+	void *ctr = ctx->ctr;
+	void *enc_key = ctx->skey.rijndael.eK;
+	size_t rounds = ctx->skey.rijndael.Nr;
+
+	get_dg(dg, ctx);
+	k = (void *)ctx->hash_subkey;
+
+	vfp_state = thread_kernel_enable_vfp();
+
+	pmull_gcm_load_round_keys(enc_key, rounds);
+
+	if (m == TEE_MODE_ENCRYPT)
+		pmull_gcm_encrypt(num_blocks, dg, dst, src, k, ctr, rounds,
+				  ctx->buf_cryp);
+	else
+		pmull_gcm_decrypt(num_blocks, dg, dst, src, k, ctr, rounds);
+
+	thread_kernel_disable_vfp(vfp_state);
+
+	put_dg(ctx, dg);
+}
+#endif /*ARM64*/

core/arch/arm/crypto/sub.mk

@@ -1,4 +1,5 @@
 ifeq ($(CFG_CRYPTO_WITH_CE),y)
 srcs-$(CFG_ARM64_core) += ghash-ce-core_a64.S
 srcs-$(CFG_ARM32_core) += ghash-ce-core_a32.S
+srcs-y += aes-gcm-ce.c
 endif

core/crypto.mk

@@ -36,7 +36,8 @@ CFG_CRYPTO_ECC ?= y
 # Authenticated encryption
 CFG_CRYPTO_CCM ?= y
 CFG_CRYPTO_GCM ?= y
-CFG_CRYPTO_AES_GCM_FROM_CRYPTOLIB = $(CFG_CRYPTO_GCM)
+# Default uses the OP-TEE internal AES-GCM implementation
+CFG_CRYPTO_AES_GCM_FROM_CRYPTOLIB ?= n
 
 endif
 

core/crypto/aes-gcm-ghash.c

@@ -0,0 +1,79 @@
+/*
+ * Copyright (c) 2010 Mike Belopuhov
+ * Copyright (c) 2017, Linaro Limited
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include <crypto/internal_aes-gcm.h>
+#include <kernel/panic.h>
+#include <string.h>
+#include <tee_api_types.h>
+#include <types_ext.h>
+
+static void xor_block(void *dst, const void *src)
+{
+	uint64_t *d = dst;
+	const uint64_t *s = src;
+
+	d[0] ^= s[0];
+	d[1] ^= s[1];
+}
+
+/*
+ * gfmul() is based on ghash_gfmul() from
+ * https://github.com/openbsd/src/blob/master/sys/crypto/gmac.c
+ */
+static void gfmul(const uint64_t X[2], const uint64_t Y[2], uint64_t product[2])
+{
+	uint64_t y[2];
+	uint64_t z[2] = { 0 };
+	const uint8_t *x = (const uint8_t *)X;
+	uint32_t mul;
+	size_t n;
+
+	y[0] = TEE_U64_FROM_BIG_ENDIAN(Y[0]);
+	y[1] = TEE_U64_FROM_BIG_ENDIAN(Y[1]);
+
+	for (n = 0; n < TEE_AES_BLOCK_SIZE * 8; n++) {
+		/* update Z */
+		if (x[n >> 3] & (1 << (~n & 7)))
+			xor_block(z, y);
+
+		/* update Y */
+		mul = y[1] & 1;
+		y[1] = (y[0] << 63) | (y[1] >> 1);
+		y[0] = (y[0] >> 1) ^ (0xe100000000000000 * mul);
+	}
+
+	product[0] = TEE_U64_TO_BIG_ENDIAN(z[0]);
+	product[1] = TEE_U64_TO_BIG_ENDIAN(z[1]);
+}
+
+void __weak internal_aes_gcm_ghash_update(struct internal_aes_gcm_ctx *ctx,
+					  const void *head, const void *data,
+					size_t num_blocks)
+{
+	const uint64_t *x = (const void *)data;
+	void *y = ctx->hash_state;
+	size_t n;
+
+	if (head) {
+		xor_block(y, head);
+		gfmul((void *)ctx->hash_subkey, y, y);
+	}
+	for (n = 0; n < num_blocks; n++) {
+		xor_block(y, x + n * 2);
+		gfmul((void *)ctx->hash_subkey, y, y);
+	}
+}

core/crypto/aes-gcm-sw.c

@@ -0,0 +1,102 @@
+/*
+ * Copyright (c) 2017, Linaro Limited
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+/*
+ * gfmul() is based on ghash_gfmul from
+ * https://github.com/openbsd/src/blob/master/sys/crypto/gmac.c
+ * Which is:
+ * Copyright (c) 2010 Mike Belopuhov
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include <crypto/internal_aes-gcm.h>
+#include <kernel/panic.h>
+#include <string.h>
+#include <tee_api_types.h>
+#include <tomcrypt.h>
+#include <types_ext.h>
+
+static bool __maybe_unused ptr_is_block_aligned(const void *p)
+{
+	return !((vaddr_t)p & (TEE_AES_BLOCK_SIZE - 1));
+}
+
+static void xor_block(void *dst, const void *src)
+{
+	uint64_t *d = dst;
+	const uint64_t *s = src;
+
+	d[0] ^= s[0];
+	d[1] ^= s[1];
+}
+
+TEE_Result __weak internal_aes_gcm_set_key(struct internal_aes_gcm_ctx *ctx,
+					   const void *key, size_t key_len)
+{
+	if (aes_setup(key, key_len, 0, &ctx->skey))
+		return TEE_ERROR_BAD_PARAMETERS;
+
+	if (aes_ecb_encrypt((void *)ctx->ctr, ctx->hash_subkey, &ctx->skey))
+		panic();
+
+	return TEE_SUCCESS;
+}
+
+void __weak
+internal_aes_gcm_update_payload_block_aligned(struct internal_aes_gcm_ctx *ctx,
+					      TEE_OperationMode m,
+					      const void *src,
+					      size_t num_blocks, void *dst)
+{
+	size_t n;
+	const uint8_t *s = src;
+	uint8_t *d = dst;
+
+	assert(!ctx->buf_pos && num_blocks &&
+	       ptr_is_block_aligned(s) && ptr_is_block_aligned(d));
+
+	for (n = 0; n < num_blocks; n++) {
+		if (m == TEE_MODE_ENCRYPT) {
+			xor_block(ctx->buf_cryp, s);
+			internal_aes_gcm_ghash_update(ctx, ctx->buf_cryp,
+						      NULL, 0);
+			memcpy(d, ctx->buf_cryp, sizeof(ctx->buf_cryp));
+			internal_aes_gcm_encrypt_block(ctx, ctx->ctr,
+						       ctx->buf_cryp);
+			internal_aes_gcm_inc_ctr(ctx);
+		} else {
+			internal_aes_gcm_encrypt_block(ctx, ctx->ctr,
+						       ctx->buf_cryp);
+
+			xor_block(ctx->buf_cryp, s);
+			internal_aes_gcm_ghash_update(ctx, s, NULL, 0);
+			memcpy(d, ctx->buf_cryp, sizeof(ctx->buf_cryp));
+
+			internal_aes_gcm_inc_ctr(ctx);
+		}
+		s += TEE_AES_BLOCK_SIZE;
+		d += TEE_AES_BLOCK_SIZE;
+	}
+}
+
+void __weak internal_aes_gcm_encrypt_block(struct internal_aes_gcm_ctx *ctx,
+					   const void *src, void *dst)
+{
+	if (aes_ecb_encrypt(src, dst, &ctx->skey))
+		panic();
+}