12:25 p.m.
Using an AS4 i386 based system with a 2.6.9-22ELsmp kernel. Currently BUG()
gets defined as the following:
#define BUG() \
__asm__ __volatile__( "ud2\n" \
"\t.word %c0\n" \
"\t.long %c1\n" \
: : "i" (__LINE__), "i" (__FILE__))
So after the ud2 opcode it places __LINE__ and __FILE__ in the next 6 bytes.
The trap handler for ud2 uses these to print a message saying where BUG() was
used.
Crash has no knowledge of this convention so it thinks the byte after the ud2
opcode is the start of the next instruction. This results in bad disassemblies
being generated. The example where I ran into it was flush_tlb_others() .
Below I have included the source for flush_tlb_others , the disassembly from
crash, the raw code for flush_tlb_others, and what I think the disassembly
should be if one takes into account the convention used in BUG(). What
initially made me suspicous was that I didn't see "call <_spin_lock>"
anywhere, and the offsets for jumps didn't line up with instructions. From
what I can tell this would probably have to be dealt with in print_insn() in
gdb/opcodes/i386-dis.c . Not sure how to go about it, or what should be done
since newer kernels allow you to configure whether those bytes get encoded
after the ud2 opcode with CONFIG_DEBUG_VERBOSE.
Any ideas on solving this?
source for flush_tlb_others:
static void flush_tlb_others(cpumask_t cpumask, struct mm_struct *mm,
unsigned long va)
{
cpumask_t tmp;
/*
* A couple of (to be removed) sanity checks:
*
* - we do not send IPIs to not-yet booted CPUs.
* - current CPU must not be in mask
* - mask must exist :)
*/
BUG_ON(cpus_empty(cpumask));
cpus_and(tmp, cpumask, cpu_online_map);
BUG_ON(!cpus_equal(cpumask, tmp));
BUG_ON(cpu_isset(smp_processor_id(), cpumask));
BUG_ON(!mm);
/*
* i'm not happy about this global shared spinlock in the
* MM hot path, but we'll see how contended it is.
* Temporarily this turns IRQs off, so that lockups are
* detected by the NMI watchdog.
*/
spin_lock(&tlbstate_lock);
flush_mm = mm;
flush_va = va;
#if NR_CPUS <= BITS_PER_LONG
atomic_set_mask(cpumask, &flush_cpumask);
#else
{
int k;
unsigned long *flush_mask = (unsigned long *)&flush_cpumask;
unsigned long *cpu_mask = (unsigned long *)&cpumask;
for (k = 0; k < BITS_TO_LONGS(NR_CPUS); ++k)
atomic_set_mask(cpu_mask[k], &flush_mask[k]);
}
#endif
/*
* We have to send the IPI only to
* CPUs affected.
*/
send_IPI_mask(cpumask, INVALIDATE_TLB_VECTOR);
while (!cpus_empty(flush_cpumask))
/* nothing. lockup detection does not belong here */
mb();
flush_mm = NULL;
flush_va = 0;
spin_unlock(&tlbstate_lock);
}
========
crash disassembly of flush_tlb_others:
crash> dis flush_tlb_others
0xc011681a <flush_tlb_others>: push %esi
0xc011681b <flush_tlb_others+0x1>: test %eax,%eax
0xc011681d <flush_tlb_others+0x3>: push %ebx
0xc011681e <flush_tlb_others+0x4>: mov %edx,%ebx
0xc0116820 <flush_tlb_others+0x6>: mov %eax,%edx
0xc0116822 <flush_tlb_others+0x8>: push %esi
0xc0116823 <flush_tlb_others+0x9>: mov %ecx,%esi
0xc0116825 <flush_tlb_others+0xb>: mov %eax,(%esp)
0xc0116828 <flush_tlb_others+0xe>: jne 0xc0116832
<flush_tlb_others+0x18>
0xc011682a <flush_tlb_others+0x10>: ud2a
0xc011682c <flush_tlb_others+0x12>: add %sp,(%esi)
0xc011682f <flush_tlb_others+0x15>: add (%esi),%ebp
0xc0116831 <flush_tlb_others+0x17>: rorb $0x91,0x880523d0(%ecx)
0xc0116838 <flush_tlb_others+0x1e>: inc %eax
0xc0116839 <flush_tlb_others+0x1f>: sarb $0xc2,(%ecx)
0xc011683c <flush_tlb_others+0x22>: je 0xc0116846
<flush_tlb_others+0x2c>
0xc011683e <flush_tlb_others+0x24>: ud2a
0xc0116840 <flush_tlb_others+0x26>: imul $0xc02e0326,(%ecx),%eax
0xc0116846 <flush_tlb_others+0x2c>: mov $0xfffff000,%eax
0xc011684b <flush_tlb_others+0x31>: and %esp,%eax
0xc011684d <flush_tlb_others+0x33>: mov 0x10(%eax),%eax
0xc0116850 <flush_tlb_others+0x36>: bt %eax,(%esp)
0xc0116854 <flush_tlb_others+0x3a>: sbb %eax,%eax
0xc0116856 <flush_tlb_others+0x3c>: test %eax,%eax
0xc0116858 <flush_tlb_others+0x3e>: je 0xc0116862
<flush_tlb_others+0x48>
0xc011685a <flush_tlb_others+0x40>: ud2a
0xc011685c <flush_tlb_others+0x42>: push $0x1
0xc011685e <flush_tlb_others+0x44>: add %es:(%esi),%ebp
0xc0116861 <flush_tlb_others+0x47>: rolb $0xb,0xf0875db(%ebp)
0xc0116868 <flush_tlb_others+0x4e>: imul $0x26,(%ecx),%eax
0xc011686b <flush_tlb_others+0x51>: add (%esi),%ebp
0xc011686d <flush_tlb_others+0x53>: sarb $0xe8,0xc0321c3c(%eax)
0xc0116874 <flush_tlb_others+0x5a>: int $0x93
0xc0116876 <flush_tlb_others+0x5c>: sbb (%eax),%eax
0xc0116878 <flush_tlb_others+0x5e>: mov %ebx,0xc0409160
0xc011687e <flush_tlb_others+0x64>: mov (%esp),%eax
0xc0116881 <flush_tlb_others+0x67>: mov %esi,0xc0409164
0xc0116887 <flush_tlb_others+0x6d>: lock or %eax,0xc040915c
0xc011688e <flush_tlb_others+0x74>: mov 0xc031f300,%ecx
0xc0116894 <flush_tlb_others+0x7a>: mov $0xfd,%edx
0xc0116899 <flush_tlb_others+0x7f>: mov (%esp),%eax
0xc011689c <flush_tlb_others+0x82>: call *0x80(%ecx)
0xc01168a2 <flush_tlb_others+0x88>:
jmp 0xc01168aa <flush_tlb_others+0x90>
0xc01168a4 <flush_tlb_others+0x8a>: mfence
0xc01168a7 <flush_tlb_others+0x8d>: lea 0x0(%esi),%esi
0xc01168aa <flush_tlb_others+0x90>: cmpl $0x0,0xc040915c
0xc01168b1 <flush_tlb_others+0x97>:
jne 0xc01168a4 <flush_tlb_others+0x8a>
0xc01168b3 <flush_tlb_others+0x99>: movl $0x0,0xc0409160
0xc01168bd <flush_tlb_others+0xa3>: mov $0xc0321c3c,%eax
0xc01168c2 <flush_tlb_others+0xa8>: movl $0x0,0xc0409164
0xc01168cc <flush_tlb_others+0xb2>: call 0xc02cfcb6 <_spin_unlock>
0xc01168d1 <flush_tlb_others+0xb7>: pop %ebx
0xc01168d2 <flush_tlb_others+0xb8>: pop %ebx
0xc01168d3 <flush_tlb_others+0xb9>: pop %esi
0xc01168d4 <flush_tlb_others+0xba>: ret
========
raw code for flush_tlb_others:
crash> rd flush_tlb_others 0x2f
c011681a: 53c08556 c289d389 89ce8956 08752404 V..S....V....$u.
c011682a: 01660b0f c02e0326 0523d089 c0409188 ..f.&.....#...@.
c011683a: 0874c239 01690b0f c02e0326 fff000b8 9.t...i.&.......
c011684a: 8be021ff a30f1040 c0192404 0874c085 .!..@....$....t.
c011685a: 016a0b0f c02e0326 0875db85 016b0b0f ..j.&.....u...k.
c011686a: c02e0326 321c3cb8 93cde8c0 1d89001b &....<.2........
c011687a: c0409160 8924048b 40916435 0509f0c0 `.@...$.5d.@....
c011688a: c040915c f3000d8b fdbac031 8b000000 \.@.....1.......
c011689a: 91ff2404 00000080 ae0f06eb 00768df0 .$............v.
c01168aa: 915c3d83 7500c040 6005c7f1 00c04091 .=\.@..u...`.@..
c01168ba: b8000000 c0321c3c 916405c7 0000c040 ....<.2...d.@...
c01168ca: e5e80000 5b001b93 52c35e5b .......[[^.R
========
hand disassembly of flush_tlb_others taking into account
kernel convention of encoding __LINE__ and __FILE__ after
the ud2a opcode:
flush_tlb_others+0x000: 56 push %esi;
flush_tlb_others+0x001: 85c0 test %eax, %eax;
flush_tlb_others+0x003: 53 push %ebx;
flush_tlb_others+0x004: 89d3 mov %edx, %ebx;
flush_tlb_others+0x006: 89c2 mov %eax, %edx;
flush_tlb_others+0x008: 56 push %esi;
flush_tlb_others+0x009: 89ce mov %ecx, %esi;
flush_tlb_others+0x00b: 890424 mov %eax, (%esp);
flush_tlb_others+0x00e: 7508 jne flush_tlb_others+0x18
flush_tlb_others+0x010: 0f0b ud2;
flush_tlb_others+0x012: 660126032ec0 ; put __LINE__ __FILE__ here for trap
handler
358 arch/i386/kernel/smp.c
flush_tlb_others+0x018: 89d0 mov %edx, %eax;
flush_tlb_others+0x01a: 2305889140c0 and 0xc0409188, %eax;
flush_tlb_others+0x020: 39c2 cmp %eax, %edx;
flush_tlb_others+0x022: 7408 je flush_tlb_others+0x2c
flush_tlb_others+0x024: 0f0b ud2;
flush_tlb_others+0x026: 690126032ec0 ; put __LINE__ __FILE__ here for trap
handler
361 arch/i386/kernel/smp.c
flush_tlb_others+0x02c: b8000fffff mov $0xfffff000, %eax;
flush_tlb_others+0x031: 21e0 and %esp, %eax;
flush_tlb_others+0x033: 8b4010 mov 0x10(%eax), %eax;
flush_tlb_others+0x036: 0fa30424 bt %eax, (%esp);
flush_tlb_others+0x03a: 19c0 sbb %eax, %eax;
flush_tlb_others+0x03c: 85c0 test %eax, %eax;
flush_tlb_others+0x03e: 7408 je flush_tlb_others+0x48
flush_tlb_others+0x040: 0f0b ud2;
flush_tlb_others+0x042: 6a0126032ec0 ; put __LINE__ __FILE__ here for trap
handler
362 arch/i386/kernel/smp.c
flush_tlb_others+0x048: 85db test %ebx, %ebx;
flush_tlb_others+0x04a: 7508 jne flush_tlb_others+0x54
flush_tlb_others+0x04c: 0f0b ud2;
flush_tlb_others+0x04e: 6b0126032ec0 ; put __LINE__ __FILE__ here for trap
handler
363 arch/i386/kernel/smp.c
flush_tlb_others+0x054: b83c1c32c0 mov $0xc0321c3c, %eax;
flush_tlb_others+0x059: e8cd931b00 call <_spin_lock>
flush_tlb_others+0x05e: 891d609140c0 mov 0xc0409160, %ebx;
flush_tlb_others+0x064: 8b0424 mov (%esp),%eax;
flush_tlb_others+0x067: 8935649140c0 mov %esi, 0xc0409164;
flush_tlb_others+0x06d: f009055c9140c0 lock or %eax, 0xc040915c;
flush_tlb_others+0x074: 8b0d00f331c0 mov 0xc031f300, %ecx;
flush_tlb_others+0x07a: bafd000000 mov $0xfd, %edx;
flush_tlb_others+0x07f: 8b0424 mov (%esp), %eax;
flush_tlb_others+0x082: ff9180000000 call *0x80(%ecx);
flush_tlb_others+0x088: eb06 jmp flush_tlb_others+0x90
flush_tlb_others+0x08a: 0faef0 mfence
flush_tlb_others+0x08d: 8d7600 lea 0x0(%esi), %esi;
flush_tlb_others+0x090: 833d5c9140c000 cmp 0x0, 0xc040915c;
flush_tlb_others+0x097: 75f1 jne flush_tlb_others+0x8a
flush_tlb_others+0x099: c705609140c000000000 mov $0x0, 0xc0409160;
flush_tlb_others+0x0a3: b83c1c32c0 mov 0xc0321c3c, %eax
flush_tlb_others+0x0a8: c705649140c000000000 mov $0x0, 0xc0409164;
flush_tlb_others+0x0b2: e8e5931b00 call <_spin_unlock>
flush_tlb_others+0x0b7: 5b pop %ebx;
flush_tlb_others+0x0b8: 5b pop %ebx;
flush_tlb_others+0x0b9: 5e pop %esi;
flush_tlb_others+0x0ba: c3 ret;