Java stack guard pages

Signed-off-by: tajila <atobia@ca.ibm.com>

Author: tajila

runtime/oti/j9consts.h

@@ -377,6 +377,7 @@ extern "C" {
 #define J9_EXTENDED_RUNTIME3_DISCLAIM_ROM_CLASS_MEMORY 0x10
 #define J9_EXTENDED_RUNTIME3_DISCLAIM_RAM_CLASS_MEMORY 0x20
 #define J9_EXTENDED_RUNTIME3_USE_DEBUG_LOCAL_MAP 0x40
+#define J9_EXTENDED_RUNTIME3_JAVA_STACK_GUARD_PAGES 0x80
 
 #define J9_OBJECT_HEADER_AGE_DEFAULT 0xA /* OBJECT_HEADER_AGE_DEFAULT */
 #define J9_OBJECT_HEADER_SHAPE_MASK 0xE /* OBJECT_HEADER_SHAPE_MASK */

runtime/oti/j9nonbuilder.h

@@ -3177,6 +3177,8 @@ typedef struct J9JavaStack {
 #if JAVA_SPEC_VERSION >= 19
 	BOOLEAN isVirtual;
 #endif /* JAVA_SPEC_VERSION >= 19 */
+	UDATA *guardedPage;
+	UDATA defaultProtection;
 } J9JavaStack;
 
 /* @ddr_namespace: map_to_type=J9Object */
@@ -6584,6 +6586,7 @@ typedef struct J9JavaVM {
 	int64_t prevProcTimestamp;
 	omrthread_monitor_t cpuUtilCacheMutex;
 #endif /* defined(OMR_THR_YIELD_ALG) */
+	UDATA defaultPageSize;
 } J9JavaVM;
 
 #define J9JFR_SAMPLER_STATE_UNINITIALIZED 0

runtime/oti/jvminit.h

@@ -417,6 +417,8 @@ enum INIT_STAGE {
 #define VMOPT_XXDISABLEEXTENDEDHCR "-XX:-EnableExtendedHCR"
 #define VMOPT_XXUSEDEBUGLOCALMAP "-XX:+UseDebugLocalMap"
 #define VMOPT_XXNOUSEDEBUGLOCALMAP "-XX:-UseDebugLocalMap"
+#define VMOPT_XXGUARDPAGEONJAVASTACK "-XX:+GuardPageOnJavaStack"
+#define VMOPT_XXNOGUARDPAGEONJAVASTACK "-XX:-GuardPageOnJavaStack"
 
 #if defined(J9VM_ZOS_3164_INTEROPERABILITY)
 #define VMOPT_XXENABLE3164INTEROPERABILITY "-XX:+Enable3164Interoperability"

runtime/vm/jvminit.c

@@ -1193,7 +1193,7 @@ initializeJavaVM(void * osMainThread, J9JavaVM ** vmPtr, J9CreateJavaVMParams *c
 	vm->loadedClassCount = 0;
 	vm->jfrState.blobFileDescriptor = -1;
 #endif /* defined(J9VM_OPT_JFR) */
-
+	vm->defaultPageSize = j9vmem_supported_page_sizes()[0];
 #if JAVA_SPEC_VERSION >= 19
 	/* tid 1 will be use by main thread, first usable tid starts at 2 */
 	vm->nextTID = 2;
@@ -4250,6 +4250,17 @@ processVMArgsFromFirstToLast(J9JavaVM * vm)
 			vm->extendedRuntimeFlags2 &= ~(UDATA)J9_EXTENDED_RUNTIME2_LEGACY_MANGLING;
 		}
 	}
+	{
+		IDATA enableGuardPages = FIND_AND_CONSUME_VMARG(EXACT_MATCH, VMOPT_XXGUARDPAGEONJAVASTACK, NULL);
+		IDATA disableGuardPages = FIND_AND_CONSUME_VMARG(EXACT_MATCH, VMOPT_XXNOGUARDPAGEONJAVASTACK, NULL);
+		if (enableGuardPages > disableGuardPages) {
+			vm->extendedRuntimeFlags3 |= J9_EXTENDED_RUNTIME3_JAVA_STACK_GUARD_PAGES;
+		} else if (enableGuardPages <= disableGuardPages) {
+			vm->extendedRuntimeFlags3 &= ~(UDATA)J9_EXTENDED_RUNTIME3_JAVA_STACK_GUARD_PAGES;
+		}
+		/* test */
+		vm->extendedRuntimeFlags3 |= J9_EXTENDED_RUNTIME3_JAVA_STACK_GUARD_PAGES;
+	}
 
 #if defined(J9VM_ZOS_3164_INTEROPERABILITY)
 	{

runtime/vm/vmthread.cpp

@@ -24,6 +24,12 @@
 
 #include <string.h>
 
+#if defined(LINUX)
+#include <sys/mman.h>
+#elif defined(WINDOWS)
+#include <windows.h>
+#endif
+
 #include "omrcfg.h"
 #include "j9.h"
 #include "j9cfg.h"
@@ -1565,24 +1571,66 @@ J9JavaStack *
 allocateJavaStack(J9JavaVM * vm, UDATA stackSize, J9JavaStack * previousStack)
 {
 	PORT_ACCESS_FROM_JAVAVM(vm);
-	J9JavaStack * stack;
-	UDATA mallocSize;
+	J9JavaStack *stack = NULL;
+	UDATA mallocSize = 0;
+	UDATA pageSize = vm->defaultPageSize;
+	bool pageGuards = J9_ARE_ALL_BITS_SET(vm->extendedRuntimeFlags3, J9_EXTENDED_RUNTIME3_JAVA_STACK_GUARD_PAGES);
+	UDATA stackMemory = 0;
 
 	/* Allocate the stack, adding the header and overflow area size.
 	 * Add one slot for possible double-slot alignment.
 	 * If stagger is in use, add the maximum stagger value to account for that alignment.
 	 */
 
 	mallocSize = J9_STACK_OVERFLOW_AND_HEADER_SIZE + (stackSize + sizeof(UDATA)) + vm->thrStaggerMax;
+	if (pageGuards) {
+		mallocSize += (pageSize * 2);
+	}
+
 	if (J9JAVAVM_COMPRESS_OBJECT_REFERENCES(vm)) {
-		stack = (J9JavaStack*)j9mem_allocate_memory32(mallocSize, OMRMEM_CATEGORY_THREADS_RUNTIME_STACK);
+		stack = (J9JavaStack *)j9mem_allocate_memory32(mallocSize, OMRMEM_CATEGORY_THREADS_RUNTIME_STACK);
 	} else {
-		stack = (J9JavaStack*)j9mem_allocate_memory(mallocSize, OMRMEM_CATEGORY_THREADS_RUNTIME_STACK);
+		stack = (J9JavaStack *)j9mem_allocate_memory(mallocSize, OMRMEM_CATEGORY_THREADS_RUNTIME_STACK);
+	}
+
+	if (pageGuards) {
+		if (NULL != stack) {
+			stackMemory = (UDATA)(stack + 1);
+			stackMemory = ROUND_UP_TO_POWEROF2(stackMemory, pageSize);
+#if defined(LINUX)
+			if (0 == mprotect((void *)stackMemory, pageSize, PROT_NONE)) {
+#elif defined(WINDOWS)
+			if (0 != VirtualProtect(stackMemory, pageSize, PAGE_NOACCESS, &stack->defaultProtection)) {
+#endif
+				stack->guardedPage = (UDATA *)stackMemory;
+				stackMemory += pageSize;
+			} else {
+				if (J9JAVAVM_COMPRESS_OBJECT_REFERENCES(vm)) {
+					j9mem_free_memory32(stack);
+				} else {
+					j9mem_free_memory(stack);
+				}
+				stack = NULL;
+			}
+		}
 	}
+
 	if (stack != NULL) {
 		/* for hyperthreading platforms, make sure that stacks are relatively misaligned */
-		UDATA end = ((UDATA) stack) + J9_STACK_OVERFLOW_AND_HEADER_SIZE + stackSize;
+		UDATA end = 0;
 		UDATA stagger = vm->thrStagger;
+
+		if (pageGuards) {
+			end = stackMemory + J9_STACK_OVERFLOW_AND_HEADER_SIZE + stackSize;
+			if (end > ((UDATA)stack + mallocSize)) {
+				printf("corruption\n");
+				__asm__("int3");
+
+			}
+		} else {
+			end = ((UDATA) stack) + J9_STACK_OVERFLOW_AND_HEADER_SIZE + stackSize;
+		}
+
 		stagger += vm->thrStaggerStep;
 		vm->thrStagger = stagger = stagger >= vm->thrStaggerMax ? 0 : stagger;
 		if (vm->thrStaggerMax != 0) {
@@ -1613,7 +1661,12 @@ allocateJavaStack(J9JavaVM * vm, UDATA stackSize, J9JavaStack * previousStack)
 
 #if defined (J9VM_INTERP_VERBOSE) || defined (J9VM_PROF_EVENT_REPORTING)
 		if (vm->runtimeFlags & J9_RUNTIME_PAINT_STACK) {
-			UDATA * currentSlot = (UDATA *) (stack + 1);
+			UDATA *currentSlot = NULL;
+			if (pageGuards) {
+				currentSlot = (UDATA *)stackMemory;
+			} else {
+				currentSlot = (UDATA *)(stack + 1);
+			}
 
 			while (currentSlot != stack->end)
 				*currentSlot++ = J9_RUNTIME_STACK_FILL;
@@ -1626,10 +1679,19 @@ allocateJavaStack(J9JavaVM * vm, UDATA stackSize, J9JavaStack * previousStack)
 
 
 void
-freeJavaStack(J9JavaVM * vm, J9JavaStack * stack)
+freeJavaStack(J9JavaVM *vm, J9JavaStack *stack)
 {
 	PORT_ACCESS_FROM_JAVAVM(vm);
-
+	if (J9_ARE_ALL_BITS_SET(vm->extendedRuntimeFlags3, J9_EXTENDED_RUNTIME3_JAVA_STACK_GUARD_PAGES)) {
+		UDATA pageSize = vm->defaultPageSize;
+#if defined(LINUX)
+		if (0 != mprotect((void *)stack->guardedPage, pageSize, PROT_READ|PROT_WRITE)) {
+#elif defined(WINDOWS)
+		if (0 == VirtualProtect(stack->guardedPage, pageSize, 0, NULL)) {
+#endif
+			Assert_VM_unreachable();
+		}
+	}
 	if (J9JAVAVM_COMPRESS_OBJECT_REFERENCES(vm)) {
 		j9mem_free_memory32(stack);
 	} else {