Dump call stack on TA panic

Adds support for dumping the call stack of a user-mode TA when it
panics. Stack unwinding happens in kernel mode by re-using
abort_print_error() in core/arch/arm/kernel/abort.c. Like for abort
dumps, the helper script scripts/symbolize.py may be used to obtain
source-level information.

This feature is enabled by default. Set CFG_UNWIND=n to disable it
(or CFG_TEE_CORE_DEBUG=n).

In libutee, the utee_panic() syscall wrapper is renamed __utee_panic()
and now takes an additional parameters: a stack pointer, in addition to
the panic code. utee_panic() is written in assembly and pushes some
registers onto the stack before calling __utee_panic(). When it is time
to return from syscall_panic(), tee_svc_sys_return_helper() uses the
stack pointer to get the information needed to unwind the TA stack.
A struct abort_info is created and abort_print_error() is called.

Signed-off-by: Jerome Forissier <jerome.forissier@linaro.org>
Reviewed-by: Jens Wiklander <jens.wiklander@linaro.org>
Tested-by: Jerome Forissier <jerome.forissier@linaro.org> (QEMU)
Tested-by: Jerome Forissier <jerome.forissier@linaro.org> (HiKey 32/64)
Tested-by: Jerome Forissier <jerome.forissier@linaro.org> (QEMUv8)

core/arch/arm/include/kernel/abort.h

@@ -31,6 +31,7 @@
 #define ABORT_TYPE_UNDEF	0
 #define ABORT_TYPE_PREFETCH	1
 #define ABORT_TYPE_DATA		2
+#define ABORT_TYPE_TA_PANIC	3 /* Dump stack on TA panic (not an abort) */
 
 #ifndef ASM
 

core/arch/arm/kernel/abort.c

@@ -341,8 +341,8 @@ static void __abort_print(struct abort_info *ai, bool stack_dump)
 		 */
 		paged_ta = true;
 #endif
-
-		__print_abort_info(ai, "User TA");
+		if (ai->abort_type != ABORT_TYPE_TA_PANIC)
+			__print_abort_info(ai, "User TA");
 		tee_ta_dump_current();
 	} else {
 		is_32bit = kernel_is32bit;

core/arch/arm/tee/arch_svc.c

@@ -27,9 +27,14 @@
 
 #include <arm.h>
 #include <assert.h>
+#include <kernel/abort.h>
 #include <kernel/misc.h>
+#include <kernel/panic.h>
+#include <kernel/tee_ta_manager.h>
 #include <kernel/thread.h>
 #include <kernel/trace_ta.h>
+#include <kernel/user_ta.h>
+#include <string.h>
 #include <tee/tee_svc.h>
 #include <tee/arch_svc.h>
 #include <tee/tee_svc_cryp.h>
@@ -237,14 +242,60 @@ void __weak tee_svc_handler(struct thread_svc_regs *regs)
 }
 
 #ifdef ARM32
+#ifdef CFG_UNWIND
+/* Get register values pushed onto the stack by utee_panic() */
+static void get_panic_regs_a32_ta(uint32_t *pushed, struct abort_info *ai)
+{
+	struct thread_abort_regs *ar = ai->regs;
+
+	ai->abort_type = ABORT_TYPE_TA_PANIC;
+
+	memset(ar, 0, sizeof(*ar));
+	ai->pc = pushed[0];
+	ar->r0 = pushed[1];
+	ar->r1 = pushed[2];
+	ar->r2 = pushed[3];
+	ar->r3 = pushed[4];
+	ar->r4 = pushed[5];
+	ar->r5 = pushed[6];
+	ar->r6 = pushed[7];
+	ar->r7 = pushed[8];
+	ar->r8 = pushed[9];
+	ar->r9 = pushed[10];
+	ar->r10 = pushed[11];
+	ar->r11 = pushed[12];
+	ar->usr_sp = (uint32_t)pushed;
+	ar->usr_lr = pushed[13];
+	ar->spsr = read_spsr();
+}
+
+static void print_panic_stack(struct thread_svc_regs *regs)
+{
+	struct abort_info ai;
+	struct thread_abort_regs ar;
+	uint32_t *pregs = (uint32_t *)regs->r1;
+
+	memset(&ai, 0, sizeof(ai));
+	ai.regs = &ar;
+	get_panic_regs_a32_ta(pregs, &ai);
+
+	abort_print_error(&ai);
+}
+#else /* CFG_UNWIND */
+static void print_panic_stack(struct thread_svc_regs *regs __unused)
+{
+}
+#endif
+
 uint32_t tee_svc_sys_return_helper(uint32_t ret, bool panic,
 			uint32_t panic_code, struct thread_svc_regs *regs)
 {
 	if (panic) {
-		TAMSG("TA panicked with code 0x%x usr_sp 0x%x usr_lr 0x%x",
-		      panic_code, read_mode_sp(CPSR_MODE_SYS),
-		      read_mode_lr(CPSR_MODE_SYS));
+		TAMSG_RAW("");
+		TAMSG_RAW("TA panicked with code 0x%" PRIx32, panic_code);
+		print_panic_stack(regs);
 	}
+
 	regs->r1 = panic;
 	regs->r2 = panic_code;
 	regs->lr = (uintptr_t)thread_unwind_user_mode;
@@ -253,13 +304,81 @@ uint32_t tee_svc_sys_return_helper(uint32_t ret, bool panic,
 }
 #endif /*ARM32*/
 #ifdef ARM64
+#ifdef CFG_UNWIND
+/* Get register values pushed onto the stack by utee_panic() (32-bit TA) */
+static void get_panic_regs_a32_ta(uint32_t *pushed, struct abort_info *ai)
+{
+	struct thread_abort_regs *ar = ai->regs;
+
+	ai->abort_type = ABORT_TYPE_TA_PANIC;
+
+	memset(ar, 0, sizeof(*ar));
+	ai->pc = pushed[0];
+	ar->x0 = pushed[1];
+	ar->x1 = pushed[2];
+	ar->x2 = pushed[3];
+	ar->x3 = pushed[4];
+	ar->x4 = pushed[5];
+	ar->x5 = pushed[6];
+	ar->x6 = pushed[7];
+	ar->x7 = pushed[8];
+	ar->x8 = pushed[9];
+	ar->x9 = pushed[10];
+	ar->x10 = pushed[11];
+	ar->x11 = pushed[12];
+	ar->x13 = (uint64_t)pushed;
+	ar->x14 = pushed[13];
+	ar->spsr = (SPSR_MODE_RW_32 << SPSR_MODE_RW_SHIFT);
+}
+
+/* Get register values pushed onto the stack by utee_panic() (64-bit TA) */
+static void get_panic_regs_a64_ta(uint64_t *pushed, struct abort_info *ai)
+{
+	struct thread_abort_regs *ar = ai->regs;
+
+	ai->abort_type = ABORT_TYPE_TA_PANIC;
+
+	memset(ar, 0, sizeof(*ar));
+	ar->x29 = pushed[0];
+	ar->elr = pushed[1];
+	ar->spsr = (SPSR_64_MODE_EL0 << SPSR_64_MODE_EL_SHIFT);
+}
+
+static void print_panic_stack(struct thread_svc_regs *regs)
+{
+	struct tee_ta_session *s;
+	struct user_ta_ctx *utc;
+	struct abort_info ai;
+	struct thread_abort_regs ar;
+
+	memset(&ai, 0, sizeof(ai));
+	ai.regs = &ar;
+
+	if (tee_ta_get_current_session(&s) != TEE_SUCCESS)
+		panic();
+	utc = to_user_ta_ctx(s->ctx);
+	if (utc->is_32bit)
+		get_panic_regs_a32_ta((uint32_t *)regs->x1, &ai);
+	else
+		get_panic_regs_a64_ta((uint64_t *)regs->x1, &ai);
+
+	abort_print_error(&ai);
+}
+#else /* CFG_UNWIND */
+static void print_panic_stack(struct thread_svc_regs *regs __unused)
+{
+}
+#endif /* CFG_UNWIND */
+
 uint32_t tee_svc_sys_return_helper(uint32_t ret, bool panic,
 			uint32_t panic_code, struct thread_svc_regs *regs)
 {
 	if (panic) {
-		TAMSG("TA panicked with code 0x%x usr_sp 0x%" PRIx64 " usr_lr 0x%" PRIx64,
-			panic_code, regs->x13, regs->x14);
+		TAMSG_RAW("");
+		TAMSG_RAW("TA panicked with code 0x%" PRIx32, panic_code);
+		print_panic_stack(regs);
 	}
+
 	regs->x1 = panic;
 	regs->x2 = panic_code;
 	regs->elr = (uintptr_t)thread_unwind_user_mode;

lib/libutee/abort.c

@@ -34,4 +34,7 @@ void abort(void)
 {
 	printf("Abort!\n");
 	utee_panic(0);
+	/* Not reached */
+	while (1)
+		;
 }

lib/libutee/arch/arm/utee_syscalls_a32.S

@@ -64,4 +64,17 @@ UNWIND( .fnend)
 	END_FUNC \name
         .endm
 
+	FUNC utee_panic, :
+UNWIND( .fnstart)
+	push	{r0-r11, lr}
+UNWIND(	.save	{r0-r11, lr})
+	mov	lr, pc
+	push	{lr}
+UNWIND(	.save	{lr})
+	mov	r1, sp
+	bl	__utee_panic
+	/* Not reached */
+UNWIND( .fnend)
+	END_FUNC utee_panic
+
 #include "utee_syscalls_asm.S"

lib/libutee/arch/arm/utee_syscalls_a64.S

@@ -42,5 +42,12 @@
         END_FUNC \name
         .endm
 
+	FUNC utee_panic, :
+	stp	x29, x30, [sp, #-16]!
+	mov	x1, sp
+	bl	__utee_panic
+	/* Not reached */
+	END_FUNC utee_panic
+
 #include "utee_syscalls_asm.S"
 

lib/libutee/arch/arm/utee_syscalls_asm.S

@@ -31,7 +31,7 @@
 
         UTEE_SYSCALL utee_log, TEE_SCN_LOG, 2
 
-        UTEE_SYSCALL utee_panic, TEE_SCN_PANIC, 1
+        UTEE_SYSCALL __utee_panic, TEE_SCN_PANIC, 2
 
         UTEE_SYSCALL utee_get_property, TEE_SCN_GET_PROPERTY, 7
 

lib/libutee/assert.c

@@ -44,4 +44,7 @@ void _assert_log(const char *expr __maybe_unused,
 void __noreturn _assert_break(void)
 {
 	utee_panic(TEE_ERROR_GENERIC);
+	/* Not reached */
+	while (1)
+		;
 }

lib/libutee/include/tee_api.h

@@ -33,9 +33,7 @@
 #include <compiler.h>
 #include <tee_api_defines.h>
 #include <tee_api_types.h>
-#if defined(CFG_TEE_PANIC_DEBUG)
 #include <trace.h>
-#endif
 
 /* Property access functions */
 
@@ -75,14 +73,7 @@ TEE_Result TEE_GetNextProperty(TEE_PropSetHandle enumerator);
 
 /* System API - Misc */
 
-void __TEE_Panic(TEE_Result panicCode);
 void TEE_Panic(TEE_Result panicCode);
-#if defined(CFG_TEE_PANIC_DEBUG)
-#define TEE_Panic(c) do { \
-		EMSG("Panic 0x%x", (c)); \
-		__TEE_Panic(c); \
-	} while (0)
-#endif
 
 /* System API - Internal Client API */
 

lib/libutee/include/utee_syscalls.h

@@ -55,7 +55,8 @@ void utee_return(unsigned long ret) __noreturn;
 
 void utee_log(const void *buf, size_t len);
 
-void utee_panic(unsigned long code) __noreturn;
+/* This is not __noreturn because AArch32 stack unwinding fails otherwise */
+void utee_panic(unsigned long code);
 
 /* prop_set is TEE_PROPSET_xxx*/
 TEE_Result utee_get_property(unsigned long prop_set, unsigned long index,

lib/libutee/tee_api_panic.c

@@ -27,17 +27,9 @@
 #include <tee_api.h>
 #include <utee_syscalls.h>
 
-#undef TEE_Panic
-
 /* System API - Misc */
 
-void __noreturn __TEE_Panic(TEE_Result panicCode)
+void TEE_Panic(TEE_Result panicCode)
 {
 	utee_panic(panicCode);
 }
-
-void __noreturn TEE_Panic(TEE_Result panicCode)
-{
-	__TEE_Panic(panicCode);
-}
-

mk/config.mk

@@ -61,11 +61,6 @@ CFG_TEE_TA_LOG_LEVEL ?= 1
 # CFG_TEE_TA_LOG_LEVEL. Otherwise, they are not output at all
 CFG_TEE_CORE_TA_TRACE ?= y
 
-# Define TEE_Panic as a macro to help debugging panics caused by calls to
-# TEE_Panic. This flag can have a different value when later compiling the
-# TA
-CFG_TEE_PANIC_DEBUG ?= y
-
 # If 1, enable debug features in TA memory allocation.
 # Debug features include check of buffer overflow, statistics, mark/check heap
 # feature.

ta/mk/ta_dev_kit.mk

@@ -53,11 +53,6 @@ cflags$(sm)    := $($(sm)-platform-cflags) $(CFLAGS_$(sm))
 CFG_TEE_TA_LOG_LEVEL ?= 2
 cppflags$(sm) += -DTRACE_LEVEL=$(CFG_TEE_TA_LOG_LEVEL)
 
-# CFG_TEE_PANIC_DEBUG is used in tee_api.h
-ifeq ($(CFG_TEE_PANIC_DEBUG),y)
-cppflags$(sm) += -DCFG_TEE_PANIC_DEBUG=1
-endif
-
 cppflags$(sm) += -I. -I$(ta-dev-kit-dir)/include
 
 libdirs += $(ta-dev-kit-dir)/lib