|
Description
|
customer is seeing core dumps during garbage collection on their application.
java_g core file is at /net/cores.ebay/cores/62234184/irescores
SunOS kdirep04 5.6 Generic_105181-23 sun4u sparc SUNW,Ultra-60
This server that is crashing is the front end that the clients connect through RMI and then communicates to another box the will do the work.
using mx 128 and mx 128 file descriptors are set at 2001
usually lwp gets to 1030-1040 before dump. there is a gig of memory and 3 gig swap
detected a multithreaded program
xxxxx@xxxxx ( xxxxx@xxxxx ) terminated by signal BUS (invalid address alignment)
dbx: warning: can't find file "/build/javavm/javavm11-1.1.8_12-native/build/solaris/java/java/obj_g/sparc/native_threads/threads_md.o"
dbx: warning: see `help finding-files'
dbx: warning: can't find file "/build/javavm/javavm11-1.1.8_12-native/build/solaris/java/main/obj_g/sparc/native_threads/java.o"
dbx: warning: can't find file "/build/javavm/javavm11-1.1.8_12-native/build/solaris/java/java/obj_g/sparc/native_threads/gc.o"
dbx: warning: can't find file "/build/javavm/javavm11-1.1.8_12-native/build/solaris/java/sunwjit/obj_g/sparc/native_threads/JITSupport.o"
dbx: warning: can't find file "/build/javavm/javavm11-1.1.8_12-native/build/solaris/java/java/obj_g/sparc/native_threads/compiler.o"
dbx: warning: can't find file "/build/javavm/javavm11-1.1.8_12-native/build/solaris/java/java/obj_g/sparc/native_threads/executeJava.o"
dbx: warning: can't find file "/build/javavm/javavm11-1.1.8_12-native/build/solaris/java/java/obj_g/sparc/native_threads/interpreter.o"
dbx: warning: can't find file "/build/javavm/javavm11-1.1.8_12-native/build/solaris/java/java/obj_g/sparc/native_threads/ObjectInputStream.o"
dbx: warning: can't find file "/build/javavm/javavm11-1.1.8_12-native/build/solaris/java/java/obj_g/sparc/native_threads/thread.o"
0xebede6e4: ThreadRT0+0x00ec: call ThreadRT0+0xec
(/tools/SUNWspro/bin/dbx) where
wherewhere: not found
(/tools/SUNWspro/bin/dbx) where
current thread: xxxxx@xxxxx
[1] clear_onproc_flags(0x1, 0x28, 0xe4599990, 0xe2c00000, 0xeabffffc, 0xe459999c), at 0xebf10aa4
[2] sysThreadMulti(0x0, 0x3cf8, 0xe4599990, 0xe2c00000, 0xeabffffc, 0xe459999c), at 0xebf108c4
[3] gc0(0x0, 0x3cf8, 0x2eb34, 0x1, 0x1, 0x0), at 0xebe9d7b0
[4] manageAllocFailure(0x3cf8, 0x1, 0x1, 0xffffc318, 0x398, 0xea115fdc), at 0xebe97288
[5] realObjAlloc(0xdc211c78, 0x1e782, 0x3cf4, 0xdc20ab88, 0x400, 0x63740000), at 0xebe94ad4
[6] ArrayAlloc(0x2, 0xf3c, 0xebdc6c40, 0xdc20abf8, 0x43, 0xea115c38), at 0xebe954a4
[7] JITSupport_anewarray(0xe2c00150, 0xf3c, 0x1, 0xe305db60, 0x0, 0x4), at 0xeb9a90cc
[8] 0x25ef1c(0xe3086850, 0xed9, 0xf3c, 0x64, 0xdc211c78, 0x0), at 0x25ef1b
[9] 0x180564(0xe3086850, 0xe305db58, 0x43, 0x4, 0xe305db50, 0x0), at 0x180563
[10] 0x2f1bcc(0xe3086838, 0xe305db58, 0xebdc6c40, 0xdc20ad30, 0x400, 0xe2c00150), at 0x2f1bcb
[11] 0x2f1f84(0xe3086838, 0x1, 0x0, 0xe305db58, 0x74, 0x0), at 0x2f1f83
[12] JITInvokeCompiledMethod(0xe3086838, 0x1ab8b0, 0x2, 0xdc211c78, 0x73, 0x0), at 0xeb9ed028
[13] invokeCompiledMethod(0xe3086838, 0x1ab8b0, 0x2, 0xdc211c78, 0x400, 0x63740000), at 0xebe8bf84
[14] ExecuteJava(0xdc20b6b0, 0xdc211c78, 0x0, 0x92d8b8, 0x92d8a0, 0xdc20b6b0), at 0xebf077e4
[15] do_execute_java_method_vararg(0xdc211c78, 0xe3086838, 0xebf44a84, 0xebf44a90, 0x0, 0x0), at 0xebea4248
[16] do_execute_java_method(0x0, 0xe3086838, 0xebf44a84, 0xebf44a90, 0x0, 0x0), at 0xebea3614
[17] java_io_ObjectInputStream_inputClassFields(0xe3086838, 0xe3067fb0, 0xe300fa58, 0xe30847f0, 0xad050, 0x0), at 0xebee83ac
[18] JITUnsynchronizedNativeStub(0xe3086838, 0xe3067fb0, 0xe300fa58, 0x0, 0x400, 0x0), at 0xeb9ed0f4
[19] 0x2e93d0(0xe3086838, 0x0, 0xebdc6c40, 0xdc20b8c8, 0x400, 0xe2c00150), at 0x2e93cf
[20] 0x2f228c(0xe3086838, 0x1, 0x0, 0x0, 0x73, 0x0), at 0x2f228b
[21] JITInvokeCompiledMethod(0xe3086838, 0x1ab8b0, 0x2, 0xdc211c78, 0x73, 0x0), at 0xeb9ed028
[22] invokeCompiledMethod(0xe3086838, 0x1ab8b0, 0x2, 0xdc211c78, 0x400, 0x63740000), at 0xebe8bf84
[23] ExecuteJava(0xdc20c248, 0xdc211c78, 0x0, 0x92d848, 0x92d830, 0xdc20c248), at 0xebf077e4
[24] do_execute_java_method_vararg(0xdc211c78, 0xe3086838, 0xebf44a84, 0xebf44a90, 0x0, 0x0), at 0xebea4248
[25] do_execute_java_method(0x0, 0xe3086838, 0xebf44a84, 0xebf44a90, 0x0, 0x0), at 0xebea3614
[26] java_io_ObjectInputStream_inputClassFields(0xe3086838, 0xe3068f50, 0xe30159d0, 0xe3082eb8, 0xad050, 0x0), at 0xebee83ac
[27] JITUnsynchronizedNativeStub(0xe3086838, 0xe3068f50, 0xe30159d0, 0x0, 0x400, 0x0), at 0xeb9ed0f4
[28] 0x2e93d0(0xe3086838, 0x0, 0xebdc6c40, 0xdc20c460, 0x3c696e69, 0xe2c00150), at 0x2e93cf
[29] 0x2f228c(0xe3086838, 0x1, 0x0, 0x0, 0x73, 0x0), at 0x2f228b
[30] JITInvokeCompiledMethod(0xe3086838, 0x1ab8b0, 0x2, 0xdc211c78, 0x73, 0x0), at 0xeb9ed028
[31] invokeCompiledMethod(0xe3086838, 0x1ab8b0, 0x2, 0xdc211c78, 0x400, 0x63740000), at 0xebe8bf84
[32] ExecuteJava(0xdc20cde0, 0xdc211c78, 0x0, 0x92d7d8, 0x92d7c0, 0xdc20cde0), at 0xebf077e4
[33] do_execute_java_method_vararg(0xdc211c78, 0xe3086838, 0xebf44a84, 0xebf44a90, 0x0, 0x0), at 0xebea4248
[34] do_execute_java_method(0x0, 0xe3086838, 0xebf44a84, 0xebf44a90, 0x0, 0x0), at 0xebea3614
[35] java_io_ObjectInputStream_inputClassFields(0xe3086838, 0xe3068108, 0xe302a4e8, 0xe30822f8, 0xad050, 0x0), at 0xebee83ac
[36] JITUnsynchronizedNativeStub(0xe3086838, 0xe3068108, 0xe302a4e8, 0xe2c08280, 0x3c696e69, 0x72656164), at 0xeb9ed0f4
[37] 0x2e93d0(0xe3086838, 0x0, 0xdc211c78, 0xe2c08280, 0x81010100, 0xe2c00150), at 0x2e93cf
[38] 0x2f228c(0xe3086838, 0x1, 0x0, 0x0, 0x73, 0x0), at 0x2f228b
[39] 0x37134c(0xe3086838, 0x1, 0xe3069258, 0xa, 0x400, 0x4), at 0x37134b
[40] 0x2f21e0(0xe3086838, 0x1, 0x0, 0x0, 0x75, 0xe2c00150), at 0x2f21df
[41] JITInvokeCompiledMethod(0xe3086838, 0x1ab8b0, 0x2, 0xdc211c78, 0x73, 0x0), at 0xeb9ed028
[42] invokeCompiledMethod(0xe3086838, 0x1ab8b0, 0x2, 0xdc211c78, 0x400, 0x63740000), at 0xebe8bf84
[43] ExecuteJava(0xdc20db20, 0xdc211c78, 0x0, 0x92d768, 0x92d750, 0xdc20db20), at 0xebf077e4
[44] do_execute_java_method_vararg(0xdc211c78, 0xe3086838, 0xebf44a84, 0xebf44a90, 0x0, 0x0), at 0xebea4248
[45] do_execute_java_method(0x0, 0xe3086838, 0xebf44a84, 0xebf44a90, 0x0, 0x0), at 0xebea3614
[46] java_io_ObjectInputStream_inputClassFields(0xe3086838, 0xe3069240, 0xe2c05f70, 0xe3084a10, 0xad050, 0x0), at 0xebee83ac
[47] JITUnsynchronizedNativeStub(0xe3086838, 0xe3069240, 0xe2c05f70, 0x0, 0x400, 0x0), at 0xeb9ed0f4
[48] 0x2e93d0(0xe3086838, 0x0, 0xebdc6c40, 0xdc20dd38, 0x400, 0xe2c00150), at 0x2e93cf
[49] 0x2f228c(0xe3086838, 0x1, 0x0, 0x0, 0x73, 0x0), at 0x2f228b
[50] JITInvokeCompiledMethod(0xe3086838, 0x1ab8b0, 0x2, 0xdc211c78, 0xa, 0xffffffff), at 0xeb9ed028
[51] invokeCompiledMethod(0xe3086838, 0x1ab8b0, 0x2, 0xdc211c78, 0x400, 0x63740000), at 0xebe8bf84
[52] ExecuteJava(0xdc20e6b8, 0xdc211c78, 0x0, 0x92d6f8, 0x92d6e0, 0xdc20e6b8), at 0xebf077e4
[53] do_execute_java_method_vararg(0xdc211c78, 0xe3086838, 0xebf44a84, 0xebf44a90, 0x0, 0x0), at 0xebea4248
[54] do_execute_java_method(0x0, 0xe3086838, 0xebf44a84, 0xebf44a90, 0x0, 0x0), at 0xebea3614
[55] java_io_ObjectInputStream_inputClassFields(0xe3086838, 0xe307f1e8, 0xe3088f20, 0xe307f1e0, 0xad050, 0x0), at 0xebee83ac
[56] JITUnsynchronizedNativeStub(0xe3086838, 0xe307f1e8, 0xe3088f20, 0x0, 0x400, 0x0), at 0xeb9ed0f4
[57] 0x2e93d0(0xe3086838, 0x0, 0xebdc6c40, 0xdc20e8d0, 0x1, 0xe2c00150), at 0x2e93cf
[58] 0x2f228c(0xe3086838, 0x1, 0x0, 0x0, 0x73, 0x0), at 0x2f228b
[59] JITInvokeCompiledMethod(0xe3086838, 0x1ab8b0, 0x2, 0xdc211c78, 0x6, 0x2), at 0xeb9ed028
[60] invokeCompiledMethod(0xe3086838, 0x1ab8b0, 0x2, 0xdc211c78, 0x400, 0x63740000), at 0xebe8bf84
[61] ExecuteJava(0xdc20f250, 0xdc211c78, 0x0, 0x92d688, 0x92d670, 0xdc20f250), at 0xebf077e4
[62] do_execute_java_method_vararg(0xdc211c78, 0xe3086838, 0xebf44a84, 0xebf44a90, 0x0, 0x0), at 0xebea4248
[63] do_execute_java_method(0x0, 0xe3086838, 0xebf44a84, 0xebf44a90, 0x0, 0x0), at 0xebea3614
[64] java_io_ObjectInputStream_inputClassFields(0xe3086838, 0xe30839f0, 0xe3021820, 0xe3083a68, 0xad050, 0x0), at 0xebee83ac
[65] JITUnsynchronizedNativeStub(0xe3086838, 0xe30839f0, 0xe3021820, 0xe2c08280, 0x1, 0x0), at 0xeb9ed0f4
[66] 0x2e93d0(0xe3086838, 0x0, 0xdc211c78, 0xe2c08280, 0x81010100, 0xe2c00150), at 0x2e93cf
[67] 0x2f228c(0xe3086838, 0x1, 0x0, 0x0, 0x73, 0x0), at 0x2f228b
[68] 0x37134c(0xe3086838, 0x0, 0xebdc6c40, 0xa, 0x400, 0x1), at 0x37134b
[69] 0x2f21e0(0xe3086838, 0x1, 0x0, 0x0, 0x75, 0xe2c00150), at 0x2f21df
[70] JITInvokeCompiledMethod(0xe3086838, 0x1ab8b0, 0x2, 0xdc211c78, 0xb, 0xffffffff), at 0xeb9ed028
[71] invokeCompiledMethod(0xe3086838, 0x1ab8b0, 0x2, 0xdc211c78, 0x400, 0x63740000), at 0xebe8bf84
[72] ExecuteJava(0xdc20ff90, 0xdc211c78, 0x0, 0x92d618, 0x92d600, 0xdc20ff90), at 0xebf077e4
[73] do_execute_java_method_vararg(0xdc211c78, 0xe3086838, 0xebf44a84, 0xebf44a90, 0x0, 0x0), at 0xebea4248
[74] do_execute_java_method(0x0, 0xe3086838, 0xebf44a84, 0xebf44a90, 0x0, 0x0), at 0xebea3614
[75] java_io_ObjectInputStream_inputClassFields(0xe3086838, 0xe3084620, 0xe2c05f70, 0xe3084a10, 0xad050, 0x0), at 0xebee83ac
[76] JITUnsynchronizedNativeStub(0xe3086838, 0xe3084620, 0xe2c05f70, 0x0, 0x400, 0x0), at 0xeb9ed0f4
[77] 0x2e93d0(0xe3086838, 0x0, 0xebdc6c40, 0xdc2101a8, 0x0, 0xe2c00150), at 0x2e93cf
[78] 0x2f228c(0xe3086838, 0x1, 0x0, 0x0, 0x73, 0x0), at 0x2f228b
[79] JITInvokeCompiledMethod(0xe3086838, 0x1ab8b0, 0x2, 0xdc211c78, 0x0, 0x0), at 0xeb9ed028
[80] invokeCompiledMethod(0xe3086838, 0x1ab8b0, 0x2, 0xdc211c78, 0x400, 0x63740000), at 0xebe8bf84
[81] ExecuteJava(0xdc210b28, 0xdc211c78, 0x0, 0x92d5a8, 0x92d590, 0xdc210b28), at 0xebf077e4
[82] do_execute_java_method_vararg(0xdc211c78, 0xe3086838, 0xebf44a84, 0xebf44a90, 0x0, 0x0), at 0xebea4248
[83] do_execute_java_method(0x0, 0xe3086838, 0xebf44a84, 0xebf44a90, 0x0, 0x0), at 0xebea3614
[84] java_io_ObjectInputStream_inputClassFields(0xe3086838, 0xe30848f0, 0xe300fa58, 0xe30847f0, 0xad050, 0x0), at 0xebee83ac
[85] JITUnsynchronizedNativeStub(0xe3086838, 0xe30848f0, 0xe300fa58, 0xdc210e98, 0x8, 0xe8d9e5f1), at 0xeb9ed0f4
[86] 0x2e93d0(0xe3086838, 0x0, 0xebdc6c40, 0xdc210f08, 0x1, 0xe2c00150), at 0x2e93cf
[87] 0x2f228c(0xe3086838, 0x1, 0x1, 0x0, 0x73, 0xe8d9e6f9), at 0x2f228b
[88] 0x2e7558(0xe3086838, 0x191048, 0xdc211c78, 0x50, 0xffff8000, 0xff), at 0x2e7557
[89] 0x3ea694(0xe3abf6c8, 0xe3abf608, 0xe3086830, 0x20, 0x31b3ffc6, 0x8aaed150), at 0x3ea693
[90] 0x30b0f8(0xe3abf680, 0xe3abf608, 0xe3086830, 0x8aaed150, 0x2, 0xe8d9e581), at 0x30b0f7
[91] 0x19a6b8(0xe2c06298, 0xe3086830, 0xe30868a0, 0xe3abf710, 0xe35b5080, 0x359), at 0x19a6b7
[92] 0x1e5008(0xe2c06298, 0xe35bd038, 0x1, 0xe3086830, 0x400, 0xe35bd348), at 0x1e5007
[93] 0x179a2c(0xe369ecd0, 0xe35bcf20, 0xe35bcf38, 0xe35bcf40, 0xe35bcf50, 0xe35bcf58), at 0x179a2b
[94] 0x153e88(0xe35bd348, 0xb5c78, 0xdc211c78, 0x400, 0x1, 0x0), at 0x153e87
[95] JITInvokeCompiledMethod(0xe35bd348, 0xb5c78, 0x1, 0xdc211c78, 0x0, 0x0), at 0xeb9ed028
[96] invokeCompiledMethod(0xe35bd348, 0xb5c78, 0x1, 0xdc211c78, 0x400, 0x72756e00), at 0xebe8bf84
[97] ExecuteJava(0xdc211b50, 0xdc211c78, 0x0, 0x92d53c, 0x92d524, 0xdc211b50), at 0xebf077e4
[98] do_execute_java_method_vararg(0xdc211c78, 0xe35bd348, 0xebf443a8, 0xebf443ac, 0x0, 0x0), at 0xebea4248
[99] execute_java_dynamic_method(0x0, 0xe35bd348, 0xebf443a8, 0xebf443ac, 0x0, 0x87c229), at 0xebea2e24
=>[100] ThreadRT0(0xe35bd348, 0xdc211d28, 0x6, 0xebdc6c40, 0xebdc6c40, 0x2000), at 0xebede6e4
(/tools/SUNWspro/bin/dbx) threads
|
