1、安装kernel-debuginfo-common 和 kernel-debuginfo

[root@wds ~]# uname -r

3.10.0-693.el7.x86_64

rpm -ivh kernel-debuginfo-3.10.0-514.16.1.el7.4.x86_64.rpm

rpm -ivh rpm -ivhkernel-debuginfo-common-x86_64-3.10.0-514.16.1.el7.4.x86_64.rpm

2、安装crash

yum install crash

3、分析dump(文件名vmcore )，可参考：

命令操作如下：

4、打开dump 文件

[root@wds ]# crash /usr/lib/debug/lib/modules/3.10.0-514.16.1.el7.4.x86_64/vmlinux vmcore

crash 7.1.9-2.el7

Copyright (C) 2002-2016 Red Hat, Inc.

Copyright (C) 2004, 2005, 2006, 2010 IBM Corporation

Copyright (C) 1999-2006 Hewlett-Packard Co

Copyright (C) 2005, 2006, 2011, 2012 Fujitsu Limited

Copyright (C) 2006, 2007 VA Linux Systems Japan K.K.

Copyright (C) 2005, 2011 NEC Corporation

Copyright (C) 1999, 2002, 2007 Silicon Graphics, Inc.

Copyright (C) 1999, 2000, 2001, 2002 Mission Critical Linux, Inc.

This program is free software, covered by the GNU General Public License,

and you are welcome to change it and/or distribute copies of it under

certain conditions. Enter "help copying" to see the conditions.

This program has absolutely no warranty. Enter "help warranty" for details.

GNU gdb (GDB) 7.6

Copyright (C) 2013 Free Software Foundation, Inc.

License GPLv3+: GNU GPL version 3 or later

This is free software: you are free to change and redistribute it.

There is NO WARRANTY, to the extent permitted by law. Type "show copying"

and "show warranty" for details.

This GDB was configured as "x86_64-unknown-linux-gnu"...

KERNEL: /usr/lib/debug/lib/modules/3.10.0-514.16.1.el7.4.x86_64/vmlinux

DUMPFILE: vmcore [PARTIAL DUMP]

CPUS: 48

DATE: Sun Oct 25 11:27:01 2020

UPTIME: 413 days, 09:42:32

LOAD AVERAGE: 237.18, 232.96, 232.37

TASKS: 2266

NODENAME: node-18.domain.tld

RELEASE: 3.10.0-514.16.1.el7.4.x86_64

VERSION: #1 SMP Fri Jun 23 22:55:02 CST 2017

MACHINE: x86_64 (2194 Mhz)

MEMORY: 767.9 GB

PANIC: "BUG: unable to handle kernel paging request at 00000000fc173000"

PID: 2826493

COMMAND: "qemu-kvm"

TASK: ffff885ef90dedd0 [THREAD_INFO: ffff883a14350000]

CPU: 16

STATE: TASK_RUNNING (PANIC)

crash> bt

PID: 2826493 TASK: ffff885ef90dedd0 CPU: 16 COMMAND: "qemu-kvm"

#0 [ffff883a143538a8] machine_kexec at ffffffff81059bdb

#1 [ffff883a14353908] __crash_kexec at ffffffff811057c2

#2 [ffff883a143539d8] crash_kexec at ffffffff811058b0

#3 [ffff883a143539f0] oops_end at ffffffff8168f888

#4 [ffff883a14353a18] no_context at ffffffff8167f526

#5 [ffff883a14353a68] __bad_area_nosemaphore at ffffffff8167f5bc

#6 [ffff883a14353ab0] bad_area_nosemaphore at ffffffff8167f726

#7 [ffff883a14353ac0] __do_page_fault at ffffffff816926ce

#8 [ffff883a14353b20] do_page_fault at ffffffff81692875

#9 [ffff883a14353b50] page_fault at ffffffff8168ea88

[exception RIP: kvm_zap_rmapp+52]

RIP: ffffffffa07c4b14 RSP: ffff883a14353c00 RFLAGS: 00010206

RAX: 0000000000000000 RBX: ffffc900a77b0520 RCX: 000000000006f1a4

RDX: 00000000fc173000 RSI: 00000000fc173000 RDI: ffff887e9e3a8000

RBP: ffff883a14353c10 R8: 0000000000000001 R9: 0000000000000000

R10: 0000000000000000 R11: 000000000000be8c R12: ffff887e9e3a8000

R13: ffffffffa07c4b40 R14: 0000000000000000 R15: ffffc90066d171e8

ORIG_RAX: ffffffffffffffff CS: 0010 SS: 0000

#10 [ffff883a14353c18] kvm_unmap_rmapp at ffffffffa07c4b4e [kvm]

#11 [ffff883a14353c28] kvm_handle_hva_range at ffffffffa07c0c74 [kvm]

#12 [ffff883a14353cc8] kvm_unmap_hva_range at ffffffffa07cd3d7 [kvm]

#13 [ffff883a14353cd8] kvm_mmu_notifier_invalidate_range_start at ffffffffa07a30c3 [kvm]

#14 [ffff883a14353d18] __mmu_notifier_invalidate_range_start at ffffffff811d4374

#15 [ffff883a14353d58] change_protection_range at ffffffff811b8fa1

#16 [ffff883a14353e60] change_protection at ffffffff811b9015

#17 [ffff883a14353e98] change_prot_numa at ffffffff811d0eeb

#18 [ffff883a14353ea8] task_numa_work at ffffffff810c9f16

#19 [ffff883a14353f00] task_work_run at ffffffff810ad1e7

#20 [ffff883a14353f30] do_notify_resume at ffffffff8102ab22

#21 [ffff883a14353f50] retint_signal at ffffffff8168e87c

RIP: 00007fee2a82c6d3 RSP: 00007ffcf02a2dc0 RFLAGS: 00000246

RAX: 0000000000000000 RBX: 00007fee43fc8b00 RCX: 0000000000000000

RDX: 00007fee44033b00 RSI: 00007fee44033b00 RDI: 00007fee43fc8b00

RBP: 00007ffcf02a2df0 R8: 0000000000000000 R9: 00000000002b20fd

R10: 0000000000001010 R11: 0000000000001010 R12: 00007ffcf02a2de8

R13: 0000000000000001 R14: 0000000000000000 R15: 00007fee41fa96a0

ORIG_RAX: ffffffffffffffff CS: 0033 SS: 002b

crash>

a./usr/lib/debug/lib/modules/3.10.0-514.16.1.el7.4.x86_64/vmlinux 为指定的vmlinux文件，不同版本下不同的vmlinux，vmcore是kdump出来的vmcore文件的路径。

b.不同的vmcore的需要安装相应的kernel版本。

c.主要命令有bt、log、sym ，使用help命令可以查看支持crash工具下支持哪些命令

来源：51CTO

作者：Darren_Wen

链接：https://blog.51cto.com/wendashuai/2544437