[libc] Add stack-checking alloca for GCC/IA16

elks/fs/exec.c

@@ -494,7 +494,7 @@ static int execve_aout(struct inode *inode, struct file *filp, char *sptr, size_
     /* set data/stack limits and copy argc/argv */
     currentp->t_enddata = (size_t)mh.dseg + (size_t)mh.bseg + base_data;
     currentp->t_endseg = len;
-    currentp->t_minstack = stack;
+    currentp->t_regs.dx = currentp->t_minstack = stack;
 
 #ifdef CONFIG_EXEC_LOW_STACK
     currentp->t_begstack = ((base_data   /* Just above the top of stack */

libc/crt0.S

@@ -36,6 +36,7 @@ _start:
 
 // C runtime startup
 // Stack is empty and immediately followed by argc, argv and envp
+// DX is stack size
 
 	pop %cx		// argc
 	mov %sp,%bx	// argv [0]
@@ -51,7 +52,7 @@ _start:
 	push %cx        // argc
 
 // ...Code fragments from .preinit & .preinit.* sections will go here...
-//    NOTE: these assume ax = envp, bx = argv, cx = argc & may clobber dx
+//    NOTE: these assume ax = envp, bx = argv, cx = argc, dx = stacksize & may clobber si
 
 // ...Then code fragments from .init & .init.* sections will go here...
 //    NOTE: these are allowed to clobber ax, bx, cx, dx, si

libc/include/alloca.h

@@ -6,22 +6,22 @@
 
 // void *alloca(size_t);
 
+int __stackavail(unsigned int size);
+
 #define alloca(s)           (__stackavail(__ALLOCA_ALIGN(s))?   \
                                  __alloca(__ALLOCA_ALIGN(s)): (void *)0)
 
 #define __ALLOCA_ALIGN(s)   (((s)+(sizeof(int)-1))&~(sizeof(int)-1))
 
 #ifdef __GNUC__
-/* The compiler auto-aligns the stack from the parameter somewhat strangely:
+/* The compiler alloca auto-aligns the stack from the parameter somewhat strangely:
  * 0 -> 0, 1 -> 2, 2 -> 4, 3 -> 4, 4 -> 6 etc.
  * Thus, __stackavail should check for two more bytes available than asked for.
  */
 #define __alloca(s)         __builtin_alloca(s)
-#define __stackavail(s)     0           /* temp no stack checking */
 #endif
 
 #ifdef __WATCOMC__
-int __stackavail(unsigned int size);
 #pragma aux __stackavail "*" __modify __nomemory
 
 extern void __based(__segname("_STACK")) *__alloca(unsigned int __size);
@@ -30,8 +30,6 @@ extern void __based(__segname("_STACK")) *__alloca(unsigned int __size);
     __parm __nomemory [__ax]            \
     __value [__sp]                      \
     __modify __exact __nomemory [__sp]
-
 #endif
 
-
 #endif

libc/malloc/Makefile

@@ -29,6 +29,13 @@ OBJS = \
 	fmemalloc.o \
 	fmemfree.o \
 
+IA16OBJS = \
+	stackcheck.o \
+
+ifeq "$(COMPILER)" "ia16"
+OBJS += $(IA16OBJS)
+endif
+
 .PHONY: all
 
 all: $(LIB)

libc/malloc/stackcheck.c

@@ -0,0 +1,51 @@
+/*
+ * IA16 stack checking helper routines
+ *
+ * 5 Dec 2024 Greg Haerr
+ */
+
+#include <sys/cdefs.h>
+#include <alloca.h>
+#include <unistd.h>
+#include <stdio.h>
+
+extern unsigned int __stacklow;
+
+#define __SP()          ((unsigned int)__builtin_frame_address(0))  /* NOTE not exact */
+#define errmsg(str)     write(STDERR_FILENO, str, sizeof(str) - 1)
+
+/* 
+ * Return true if stack can be extended by size bytes,
+ * called by alloca() to check stack available.
+ */
+int __stackavail(unsigned int size)
+{
+    unsigned int remaining = __SP() - __stacklow;
+
+    if ((int)remaining >= 0 && remaining >= size)
+        return 1;
+    errmsg("ALLOCA FAIL, INCREASE STACK\n");
+    return 0;
+}
+
+#if 0
+/* 
+ * Check if size bytes can be allocated from stack,
+ * called from function prologue when -fstack-check set.
+ */
+void __STK(unsigned int size)
+{
+    unsigned int remaining = __SP() - __stacklow;
+    unsigned int curbreak;
+
+    if ((int)remaining >= 0 && remaining >= size)
+        return;
+    curbreak = (unsigned int)sbrk(0);   /* NOTE syscall here will cause SIGSEGV sent */
+#if LATER
+    if (__SP() < curbreak)
+        errmsg("STACK OVERFLOW\n");
+    else
+        errmsg("STACK OVER LIMIT\n");
+#endif
+}
+#endif

libc/system/out.mk

@@ -60,6 +60,7 @@ IA16OBJS = \
 	execlpe.o \
 	program_filename.o \
 	setjmp.o \
+	stacklow.o \
 	syscall01.o \
 	syscall23.o \
 	syscall4.o \

libc/system/stacklow.S

@@ -0,0 +1,15 @@
+// Define and initialize the __stacklow variable, if needed
+// Assume dx = stacksize from entry point, from libc/crt0.S
+
+	.arch i8086, nojumps
+	.code16
+
+	.section .preinit,"ax",@progbits
+
+	mov %sp,%si         // SP will be 4 too high if __argc or __argv used
+	sub %dx,%si
+	mov %si,__stacklow
+
+//------------------------------------------------------------------------------
+
+	.comm __stacklow,2