Application running with Java 5.0u6 64 bit VM under heavy load on Solaris 10.
jvm_args: -Xmx8192m -Xms3072m -XX:PermSize=512m -XX:MaxPermSize=512m -XX:NewSize=2048m -XX:MaxNewSize=2048m -XX:+DisableExplicitGC -XX:+CMSClassUnloadingEnabled -XX:+CMSPermGenSweepingEnabled -XX:+UseCMSCompactAtFullCollection -Dcom.sun.management.jmxremote -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:MaxTenuringThreshold=0 -XX:SurvivorRatio=128

#
# An unexpected error has been detected by HotSpot Virtual Machine:
#
#  Internal Error (4E4D4554484F440E43505001BB 01), pid=17086, tid=99
#
# Java VM: Java HotSpot(TM) 64-Bit Server VM (1.5.0_06-b05 mixed mode)

---------------  T H R E A D  ---------------

Current thread (0x0000000102728da0):  JavaThread "RIWorker [8]" [_thread_in_vm, id=99]



ErrorID 4E4D4554484F440E43505001BB
nmethod.cpp, 443


from the core file:
(dbx) lwp   xxxxx@xxxxx  
  xxxxx@xxxxx   (  xxxxx@xxxxx  ) stopped in __lwp_kill at 0xffffffff7f0ce1b8
0xffffffff7f0ce1b8: __lwp_kill+0x0008:  bcc,a,pt  %icc,__lwp_kill+0x18  ! 0xffffffff7f0ce1c8
(dbx) where
=>[1] __lwp_kill(0x0, 0x6, 0xffffffff7e8393f8, 0x19b760, 0x0, 0x0), at 0xffffffff7f0ce1b8
  [2] raise(0x6, 0x0, 0xffffffffffffffff, 0xffffffff7f1e4000, 0x0, 0x0), at 0xffffffff7f06b1bc
  [3] abort(0x1, 0x1b8, 0xffffffff7e8393f8, 0x19b760, 0x0, 0x0), at 0xffffffff7f0489ac
  [4] os::abort(0x1, 0xf800, 0xffffffff7e55be28, 0xffffffff7ea06000, 0x261a64, 0xb400), at 0xffffffff7e7a45f0
  [5] VMError::report_and_die(0x0, 0xffffffff7eab4a64, 0xffffffff7eab4a30, 0xffffffff7e8f404c, 0xffffffff7eaa8fd0, 0x0), at 0xffffffff7e8393f8
  [6] report_fatal(0xffffffff7e8cdba8, 0x1bb, 0xffffffff7e8cdbe7, 0x0, 0xffffffff31354d40, 0x0), at 0xffffffff7e55be28
  [7] nmethod::scope_desc_at(0xffffffff31354c50, 0xffffffff31354eec, 0x1, 0x0, 0x717cf4, 0xffffffff7e2f0a50), at 0xffffffff7e2ee348
  [8] vframe::new_vframe(0x107f364d8, 0x1, 0x102728da0, 0xfffffffcf0ffc448, 0xffffffff7eaa5f70, 0xffffffff31354c50), at 0xffffffff7e275084
  [9] CompiledRFrame::init(0x107f364d0, 0x0, 0x102728da0, 0xc000, 0x107f36530, 0x0), at 0xffffffff7e2f0a50
  [10] RFrame::caller(0xfffffffcf0ffdfa0, 0x107f364d0, 0x2, 0x107f35fd0, 0x11310, 0x11000), at 0xffffffff7e2a79d0
  [11] StackWalkCompPolicy::findTopInlinableFrame(0xffffffff048fe260, 0x107f35fd0, 0x107f35920, 0xffffffff7ea06000, 0xffffffff7eaa7748, 0x0), at 0xffffffff7e2b82c4
  [12] StackWalkCompPolicy::method_invocation_event(0x107f35920, 0x107f35420, 0x0, 0x2710, 0x0, 0x107f35920), at 0xffffffff7e2b7a3c
  [13] InterpreterRuntime::frequency_counter_overflow(0x0, 0xffffffff7e2b76e0, 0x102b6bfa8, 0x102728da0, 0x102728ec0, 0xffffffff03d6d820), at 0xffffffff7e2939a4
  [14] 0xffffffff3000b830(0xfffffffd3f291490, 0xb6, 0xffffffff189e2938, 0x2000, 0xffffffff03afff20, 0xfffffffcf0ffd5b1), at 0xffffffff3000b82f
  [15] 0xffffffff30005930(0xfffffffd3f291490, 0xb7, 0x0, 0xffffffff30019a80, 0x800, 0xfffffffcf0ffd6b1), at 0xffffffff3000592f
  [16] 0xffffffff30005930(0xfffffffd3f291490, 0xfffffffd3f193cd8, 0x0, 0xffffffff300170a0, 0xffffffff03afff20, 0xfffffffcf0ffd7a1), at 0xffffffff3000592f
  [17] 0xffffffff3008d798(0xfffffffd3f291490, 0xffffffff000d39b0, 0x400, 0x79ff3e, 0x800, 0x0), at 0xffffffff3008d797
  [18] 0xffffffff31354ee4(0xfffffffd3f291490, 0xfffffffd3f193cd8, 0xffffffff000d39b0, 0xffffffff16d7afa8, 0xffffffff03afff20, 0x0), at 0xffffffff31354ee3
  [19] 0xffffffff30e7368c(0x10, 0x4, 0x400, 0x79ff3e, 0x800, 0x0), at 0xffffffff30e7368b
  [20] 0xffffffff307e8c5c(0xfffffffd3f291490, 0xb7, 0xfffffffcf0ffe3f8, 0xffffffff300170d0, 0xfffffffd3f2faaa8, 0x0), at 0xffffffff307e8c5b
  [21] 0xffffffff30005810(0xfffffffd3f291490, 0xb7, 0x0, 0xffffffff300170d0, 0x1, 0xfffffffcf0ffdb41), at 0xffffffff3000580f
  [22] 0xffffffff30005810(0x1, 0x23, 0x0, 0xffffffff300170d0, 0xfffffffd3f2faaa8, 0xfffffffcf0ffdc61), at 0xffffffff3000580f
  [23] 0xffffffff308a63e0(0xfffffffd3f291490, 0xfffffffd3f193cd8, 0xfffffffd3f290af8, 0x79ff3e, 0x1, 0xfffffffcf0ffde61), at 0xffffffff308a63df
  [24] 0xffffffff30a6e484(0x1, 0x23, 0xfffffffd3f291490, 0x102728da0, 0xfffffffd3f2faaa8, 0x3a), at 0xffffffff30a6e483
  [25] 0xffffffff300cad98(0xfffffffd3f291490, 0x54, 0x0, 0xffffffff30017380, 0x1, 0xfffffffcf0ffde61), at 0xffffffff300cad97
  [26] 0xffffffff30005810(0xfffffffd3ed32268, 0xfffffffd8b712320, 0x0, 0xffffffff30019a80, 0x12e08, 0xfffffffcf0ffe0c1), at 0xffffffff3000580f
  [27] 0xffffffff30ba31a8(0xfffffffd8bb696d0, 0xfffffffd3a14f2a0, 0xfffffffd36acc268, 0x0, 0x0, 0x0), at 0xffffffff30ba31a7
  [28] 0xffffffff30a6138c(0xfffffffd3ed32268, 0xfffffffd8b712320, 0x0, 0xfffffffdd45d1ed0, 0x12e08, 0xfffffffcf49), at 0xffffffff30a6138b
  [29] 0xffffffff30db00a0(0xfffffffd3ed32268, 0xffffffff1805b3f8, 0xffffffff000006c8, 0xffffffff01e83d80, 0xfffffffd3f193af8, 0xffffffff001aefc8), at 0xffffffff30db009f
  [30] 0xffffffff30d789d0(0xfffffffd3f193af8, 0xfffffffd8b712320, 0x0, 0xfffffffdd45d1ed0, 0x12e08, 0xfffffffcf49), at 0xffffffff30d789cf
  [31] 0xffffffff314e368c(0x0, 0xa, 0xffffffff000006c8, 0xffffffff01e83d80, 0xfffffffd3f193af8, 0xffffffff001aefc8), at 0xffffffff314e368b
  [32] 0xffffffff30c3bc7c(0xffffffff02496aa8, 0xffffffff0252f6e8, 0xffffffff02490de0, 0xfffffffd3f193af8, 0x12e08, 0xfffffffcf49), at 0xffffffff30c3bc7b
  [33] 0xffffffff30c10e44(0xfffffffd80ac0808, 0xfffffffd8d2b89d0, 0xfffffffd3f193ae8, 0xff7fffff30400000, 0xffffffff0252f6e8, 0x7fffffff), at 0xffffffff30c10e43
  [34] 0xffffffff300aeb54(0xfffffffd80ac0808, 0xb6, 0xfffffffcf0ffefb8, 0xffffffff30016e40, 0x20d0, 0x0), at 0xffffffff300aeb53
  [35] 0xffffffff30005810(0xfffffffd80ac0808, 0xb6, 0xffffffff19ca4800, 0xffffffff30016e40, 0x20d0, 0xfffffffcf0ffe6f1), at 0xffffffff3000580f
  [36] 0xffffffff30005810(0xfffffffd80ac0808, 0xb6, 0xffffffff19ca45b8, 0xffffffff30016e40, 0x20cf, 0xfffffffcf0ffe801), at 0xffffffff3000580f
  [37] 0xffffffff30005810(0xffffffff301ffd60, 0xffffffff310ae3d0, 0xffffffff19ca4170, 0xffffffff30016e40, 0x128d0, 0xfffffffcf0ffe921), at 0xffffffff3000580f
  [38] 0xffffffff30089218(0xfffffffd80ac0808, 0xfffffffd87709b90, 0xfffffffd351d36f8, 0xffffffff15c93da0, 0x2, 0xfffffffcf0ffeb41), at 0xffffffff30089217
  [39] 0xffffffff310ae664(0xffffffff301ffd60, 0xffffffff310ae3d0, 0xffffffff310ae564, 0x102728da0, 0x128d0, 0x0), at 0xffffffff310ae663
  [40] 0xffffffff301ffd58(0xfffffffd45e2c7c8, 0xb7, 0xffffffff19468848, 0xffffffff30016e40, 0xffffffff3000b560, 0xfffffffcf0ffeb41), at 0xffffffff301ffd57
  [41] 0xffffffff30005810(0x1, 0x102728da0, 0x0, 0xffffffff300170a0, 0xfffffffcf0fff648, 0xfffffffcf0ffec41), at 0xffffffff3000580f
  [42] 0xffffffff3000023c(0xfffffffcf0fff628, 0xfffffffcf0fff9b0, 0xa, 0xffffffff15c0a5a0, 0xffffffff3000b560, 0xfffffffcf0fff810), at 0xffffffff3000023b
  [43] JavaCalls::call_helper(0x1, 0x102728da0, 0xa, 0xfffffffd45e2c7c8, 0xfffffffcf0fff648, 0x102b6bf80), at 0xffffffff7e29b4d8
  [44] JavaCalls::call_virtual(0xfffffffd45e2c7c8, 0x102728da0, 0xffffffff15cb1c48, 0xffffffff7eac3538, 0xffffffff7eac36d8, 0xfffffffcf0fff808), at 0xffffffff7e3cf544
  [45] thread_entry(0xffffffff7eab0c78, 0x102728da0, 0xe000, 0xffffffff00042ae8, 0xfffffffd45e2c7c8, 0xffffffff7ea06000), at 0xffffffff7e3ede64
  [46] JavaThread::run(0x102728da0, 0xf000, 0xffffffff7ea9d694, 0x0, 0x0, 0xffffffff7ea06000), at 0xffffffff7e3e8384
  [47] _start(0x102728da0, 0xd800, 0xb000, 0xb138, 0xffffffff7eaa889c, 0xffffffff7ea06000), at 0xffffffff7e7a4080

(dbx) x 0xffffffff31354c50/4
0xffffffff31354c50:      0xffffffff 0x7eaa5f70 0xffffffff 0x7e8cdba0
(dbx) x  0xffffffff31354eec/4
0xffffffff31354eec:      0xa6100018 0xc24e2114 0xa8100008 0x2b3f4cc8
(dbx) x  0xffffffff31354eec
0xffffffff31354eec:      0xa6100018
(dbx) frame 7
0xffffffff7e2ee348: scope_desc_at+0x0048:       call     report_fatal   ! 0xffffffff7e55be00
(dbx) dis
0xffffffff7e2ee34c: scope_desc_at+0x004c:       add      %o2, -63, %o0
0xffffffff7e2ee350: scope_desc_at+0x0050:       call      customer      ! 0xffffffff7e7a3e50
0xffffffff7e2ee354: scope_desc_at+0x0054:       nop
0xffffffff7e2ee358: scope_desc_at+0x0058:       call     resource_allocate_bytes        ! 0xffffffff7e1e4770
0xffffffff7e2ee35c: scope_desc_at+0x005c:       mov      40, %o0
0xffffffff7e2ee360: scope_desc_at+0x0060:       orcc     %g0, %o0, %i5
0xffffffff7e2ee364: scope_desc_at+0x0064:       be,pn    %xcc,scope_desc_at+0x98        ! 0xffffffff7e2ee398
0xffffffff7e2ee368: scope_desc_at+0x0068:       mov      %o0, %i5
0xffffffff7e2ee36c: scope_desc_at+0x006c:       ld       [%l2 + 4], %l6
0xffffffff7e2ee370: scope_desc_at+0x0070:       cmp      %l6, 0
(dbx) regs
current thread:   xxxxx@xxxxx  
current frame:  [7]
g0-g1    0x0000000000000000 0x00000000000000a3
g2-g3    0x000000000000e400 0xffffffff7eaa8f40
g4-g5    0x0000000000000000 0x0000000000000004
g6-g7    0x0000000000000000 0xfffffffcf4907400
o0-o1    0xffffffff7e8cdba8 0x00000000000001bb
o2-o3    0xffffffff7e8cdbe7 0x0000000000000000
o4-o5    0xffffffff31354d40 0x0000000000000000
o6-o7    0xfffffffcf0ffb561 0xffffffff7e2ee348
l0-l1    0xffffffff7ea06000 0x0000000000000004
l2-l3    0x0000000000000000 0xffffffff7ea06000
l4-l5    0x000000000000adb0 0x000000000000ac00
l6-l7    0x0000000000000000 0xffffffff7e247358
i0-i1    0xffffffff31354c50 0xffffffff31354eec
i2-i3    0x0000000000000001 0x0000000000000000
i4-i5    0x0000000000717cf4 0xffffffff7e2f0a50
i6-i7    0xfffffffcf0ffb611 0xffffffff7e275084
y        0x0000000000000000
ccr      0x0000000000000098
Posted Date : 2006-06-19 17:53:15.0

Use -XX:+UseInlineCaches to reenable compiled ics for Niagara.

It looks to me like the code for MachCallDynamicJavaNode::ret_addr_offset is wrong for -UseInlineCaches if vtable_index fits into a hi22.

int MachCallDynamicJavaNode::ret_addr_offset() {
  int vtable_index = this->_vtable_index;
  if (vtable_index == -1) {
    return (NativeMovConstReg::instruction_size +
           NativeCall::instruction_size);  // sethi; setlo; call; delay slot
  } else {
    assert(!UseInlineCaches, "expect vtable calls only if not using ICs");
    int entry_offset = instanceKlass::vtable_start_offset() + vtable_index*vtableEntry::size();
    int v_off = entry_offset*wordSize + vtableEntry::method_offset_in_bytes();
    if( Assembler::is_simm13(v_off) ) {
      return (3*BytesPerInstWord +           // ld_ptr, ld_ptr, ld_ptr
             NativeCall::instruction_size);  // call; delay slot
    } else {
      return (5*BytesPerInstWord +           // ld_ptr, set_hi, set, ld_ptr, ld_ptr
             NativeCall::instruction_size);  // call; delay slot
    }
  }
}


It assumes that for a non simm13 vtable_index 2 instructions get emitted but that's not happening so the scope desc ends up on the wrong pc.

0xffffffff31354ed4:    ldx      [%o0 + 0x8], %g3
0xffffffff31354ed8:    sethi    %hi(0x1800), %g5
0xffffffff31354edc:    ldx      [%g3 + %g5], %g5
0xffffffff31354ee0:    ldx      [%g5 + 0x60], %g3
0xffffffff31354ee4:    call     %g3 + %g0
0xffffffff31354ee8:    nop
0xffffffff31354eec:    mov      %i0, %l3
public boolean setAccountingPolicyObjectPointer(com.timetra.nms.common.ifs.DeploymentRuleInterface, java.lang\
.String) @0xffffffff04935190 of public abstract class com.timetra.nms.server.generated.service.pso.PsoAccessI\
nterface @0xffffffff04979748 @ bci = 6, line = 7782

There's only a single sethi so the debug info ends up in the wrong place.  The emit code appears to assume that 2 will always be emitted.

      } else {
        // This will generate 2 instructions
        __ set(v_off, G5_method);


The fix is either to correct the offset logic to match the code emission or to always force emission of 2 instructions.
Posted Date : 2006-06-19 20:47:36.0

I constructed a test which indicates you can die in another way.  Here's an alternate crashing stack:

  ---- called from signal handler with signal 11 (SIGSEGV) ------
  [8] vframeStreamCommon::fill_from_frame(0xffffffff7fffd1f8, 0xffffffff75cc5d10, 0xffffffff7e24af2c, 0x11ecb8, 0x823adc, 0xffffffff75c6e6b0), at 0xffffffff7e24b020
  [9] vframeStream::vframeStream(0xffffffff7fffd1f8, 0xffffffff7eac70c0, 0x1, 0xffffffff7e2faae4, 0x10011e5f0, 0x0), at 0xffffffff7e2b13a0
  [10] SharedRuntime::find_callee_method(0x10011e4d0, 0x10011e4d0, 0xf378, 0x10011ebb8, 0x10011e7e0, 0x0), at 0xffffffff7e3b9958
  [11] OptoRuntime::lazy_c2i_adapter_generation_C(0x10011e4d0, 0x10011e4d0, 0x5bad60, 0xffffffff7e45d4ec, 0x6, 0x0), at 0xffffffff7e45d2bc
Posted Date : 2006-06-20 18:07:08.0