c-stack: avoid compiler optimizations when provoking overflow

* m4/c-stack.m4 (AC_SYS_XSI_STACK_OVERFLOW_HEURISTIC): Make
recursion harder to optimize, to ensure a stack overflow occurs.
* tests/test-c-stack.c (recurse): Likewise.
Borrowed from libsigsegv.

Signed-off-by: Eric Blake <ebb9@byu.net>

diff --git a/ChangeLog b/ChangeLog
index 1dbbb602e06585fee8420e81a61bb2df5961144f..85861435418e02221357190488598d5ebf670fe8 100644 (file)
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,11 @@
 2008-09-23  Eric Blake  <ebb9@byu.net>
 
+       c-stack: avoid compiler optimizations when provoking overflow
+       * m4/c-stack.m4 (AC_SYS_XSI_STACK_OVERFLOW_HEURISTIC): Make
+       recursion harder to optimize, to ensure a stack overflow occurs.
+       * tests/test-c-stack.c (recurse): Likewise.
+       Borrowed from libsigsegv.
+
        c-stack: work around Irix sigaltstack bug
        * m4/c-stack.m4 (AC_SYS_XSI_STACK_OVERFLOW_HEURISTIC): Check
        whether sigaltstack uses wrong end of stack_t (copied in part from

diff --git a/m4/c-stack.m4 b/m4/c-stack.m4
index f1bda7bb70fa560c7ba2507db871408efe6ac65e..50693331a397e4c9f5791eafaf1260fd4650c6d8 100644 (file)
--- a/m4/c-stack.m4
+++ b/m4/c-stack.m4
@@ -64,15 +64,19 @@ AC_DEFUN([AC_SYS_XSI_STACK_OVERFLOW_HEURISTIC],
           act.sa_handler = segv_handler;
           return sigaction (SIGSEGV, &act, 0);
         }
-
+        static volatile int *
+        recurse_1 (volatile int n, volatile int *p)
+        {
+          if (n >= 0)
+            *recurse_1 (n + 1, p) += n;
+          return p;
+        }
         static int
-        recurse (char *p)
+        recurse (volatile int n)
         {
-          char array[500];
-          array[0] = 1;
-          return *p + recurse (array);
+          int sum = 0;
+          return *recurse_1 (n, &sum);
         }
-
         int
         main ()
         {
@@ -86,7 +90,7 @@ AC_DEFUN([AC_SYS_XSI_STACK_OVERFLOW_HEURISTIC],
           setrlimit (RLIMIT_STACK, &rl);
           #endif
 
-          return c_stack_action () || recurse ("\1");
+          return c_stack_action () || recurse (0);
         }
        ],
        [ac_cv_sys_stack_overflow_works=yes],
@@ -242,15 +246,19 @@ int main ()
           act.sa_sigaction = segv_handler;
           return sigaction (SIGSEGV, &act, 0);
         }
-
+        static volatile int *
+        recurse_1 (volatile int n, volatile int *p)
+        {
+          if (n >= 0)
+            *recurse_1 (n + 1, p) += n;
+          return p;
+        }
         static int
-        recurse (char *p)
+        recurse (volatile int n)
         {
-          char array[500];
-          array[0] = 1;
-          return *p + recurse (array);
+          int sum = 0;
+          return *recurse_1 (n, &sum);
         }
-
         int
         main ()
         {
@@ -264,7 +272,7 @@ int main ()
           setrlimit (RLIMIT_STACK, &rl);
           #endif
 
-          return c_stack_action () || recurse ("\1");
+          return c_stack_action () || recurse (0);
         }
        ],
        [ac_cv_sys_xsi_stack_overflow_heuristic=yes],

diff --git a/tests/test-c-stack.c b/tests/test-c-stack.c
index fe782c95292fac9454d05c6d705eb9b9eeac9a37..96ab152c04639507813515f4743bb31e2ae370d1 100644 (file)
--- a/tests/test-c-stack.c
+++ b/tests/test-c-stack.c
@@ -41,15 +41,22 @@
     }                                                                       \
   while (0)
 
-static long
-recurse (char *p)
+char *program_name;
+
+static volatile int *
+recurse_1 (volatile int n, volatile int *p)
 {
-  char array[500];
-  array[0] = 1;
-  return *p + recurse (array);
+  if (n >= 0)
+    *recurse_1 (n + 1, p) += n;
+  return p;
 }
 
-char *program_name;
+static int
+recurse (volatile int n)
+{
+  int sum = 0;
+  return *recurse_1 (n, &sum);
+}
 
 int
 main (int argc, char **argv)
@@ -72,8 +79,9 @@ main (int argc, char **argv)
          exit_failure = 77;
          ++*argv[argc]; /* Intentionally dereference NULL.  */
        }
-      return recurse ("\1");
+      return recurse (0);
     }
+  fputs ("skipping test: ", stderr);
   perror ("c_stack_action");
   return 77;
 }