鍓嶈█浼楁墍鍛ㄧ煡锛孡inux绯荤粺鍙兘浼氬洜涓烘寚浠ゅ紓甯搞€佽闂棤鏁堝唴瀛樺湴鍧€銆佹閿侊紙杞攣姝伙級绛夊師鍥犺€屽穿婧冦€傜郴缁熷穿婧冨悗锛屾垜浠彲浠ラ€氳繃kdump杞偍褰撴椂鐨勫唴瀛樹俊鎭紝鐢熸垚涓€涓浆鍌ㄦ枃浠秜mcore.dump銆傚悓鏃禼rash宸ュ叿鍙互甯姪鎴戜滑鍒嗘瀽vmcore鏂囦欢锛屾壘鍒伴棶棰樻墍鍦ㄣ€傛湰鏂囧皢绠€瑕佷粙缁峜rash宸ュ叿鐨勫畨瑁呭拰甯哥敤鍛戒护锛屽苟閫氳繃瀹炰緥璇存槑濡備綍浣跨敤crash宸ュ叿瀹氫綅鍐呮牳闂銆備粈涔堟槸宕╂簝宸ュ叿锛熷穿婧冨伐鍏锋槸Redhat鎻愪緵鐨勫紑婧愬伐鍏枫€備富瑕佺敤浜庣绾垮垎鏋怢inux鍐呮牳杞偍鏂囦欢銆傚畠闆嗘垚浜唃db宸ュ叿锛屽彲浠ユ煡鐪嬪爢鏍堛€乨mesg鏃ュ織銆佸唴鏍告暟鎹粨鏋勩€佸弽姹囩紪绛夛紝鍔熻兘闈炲父寮哄ぇ銆傚畨瑁呭穿婧冨伐鍏?瀹夎宕╂簝宸ュ叿yuminstallcrash#瀹夎璋冭瘯淇℃伅鍖卍ebuginfo-installkernel#杩愯宕╂簝宸ュ叿crash/usr/lib/debug/lib/modules//vmlinux/var/crash//vmcorecrashtool甯哥敤鍛戒护CPU瀵勫瓨鍣ㄥ拰鍑芥暟鏍堝抚缁撴瀯鍦ㄤ娇鐢╟rash宸ュ叿鍒嗘瀽vmcore鏂囦欢涔嬪墠锛屾垜浠繕闇€瑕佺畝鍗曚簡瑙ｄ竴涓婥PU涓瘎瀛樺櫒鍜屽嚱鏁版爤甯х殑缁撴瀯锛屽洜涓烘垜浠粡甯搁渶瑕佹煡鐪嬬殑鍊煎畾浣嶉棶棰樻椂鐨勪竴涓彉閲忔垨鍙傛暟锛岃€岃繖浜涚嚎绱㈠線寰€瀛樺湪浜庡瘎瀛樺櫒鎴栧嚱鏁版爤涓€傚彧鏈変簡瑙ｄ簡瀹冧滑鐨勫姛鑳藉拰缁撴瀯锛屾墠鑳藉噯纭壘鍒板搴旂殑瀛樺偍浣嶇疆銆傚彟澶栵紝浣犻渶瑕佺煡閬撲竴浜涚畝鍗曠殑缁勮璇存槑锛屼綘鍙互鑷繁鍘绘煡銆倄86-64瀵勫瓨鍣ㄤ綔鐢?rax锛氬瓨鍌ㄥ嚱鏁拌繑鍥炲€?rsp锛氭爤鎸囬拡瀵勫瓨鍣紝鎸囧悜鏍堥《%rdi銆?rsi銆?rdx銆?rcx銆?r8銆?r9锛氬嚱鏁板弬鏁帮紝瀵瑰簲param1,param2....%rbx,%rbp,%r12~%r15:鏁版嵁瀛樺偍锛岄伒寰璋冪敤鑰呭師鍒?r10,%r11:鏁版嵁瀛樺偍锛岄伒寰皟鐢ㄨ€呭師鍒欎娇鐢ㄨ璋冪敤鑰呭師鍒欙細褰撹皟鐢ㄥ嚱鏁版椂锛岃皟鐢ㄨ€呬笉闇€瑕佷繚瀛樿繖浜涘瘎瀛樺櫒鐨勫€硷紝鐩存帴璋冪敤瀛愬嚱鏁般€傝繘鍏ュ瓙鍑芥暟鍚庯紝瀛愬嚱鏁板湪瑕嗙洊杩欎簺瀵勫瓨鍣ㄤ箣鍓嶉渶瑕佷繚瀛樿繖浜涘瘎瀛樺櫒鐨勫€硷紝鍗宠繖浜涘瘎瀛樺櫒鐨勫€肩敱琚皟鐢ㄨ€呬繚瀛樺拰鎭㈠銆傝皟鐢ㄨ€呬娇鐢ㄥ師鍒欙細鍦ㄨ皟鐢ㄤ竴涓瓙鍑芥暟涔嬪墠锛岃皟鐢ㄨ€呴渶瑕佹彁鍓嶄繚鎶ゅソ杩欎簺瀵勫瓨鍣ㄧ殑鍊笺€備繚瀛樻柟寮忛€氬父鏄皢瀵勫瓨鍣ㄧ殑鍊煎帇鍏ユ爤涓€傝皟鐢ㄨ€呬繚瀛樺悗锛屽彲浠ュ湪瀛愬嚱鏁颁腑闅忔剰瑕嗙洊杩欎簺瀵勫瓨鍣ㄧ殑鍊笺€傚嚱鏁版爤甯х粨鏋勫嚱鏁拌皟鐢ㄦ爤浠庨珮鍦板潃鍚戜綆鍦板潃澧為暱锛屽叾涓璻sp鍜宺bp鍒嗗埆鎸囧悜鏍堥《鍜屾爤搴曘€傝繘琛屽嚱鏁拌皟鐢ㄦ椂锛屼細灏嗗弬鏁拌祴鍊肩粰鐩稿簲鐨勫瘎瀛樺櫒锛屽苟灏嗙埗鍑芥暟鐨剅bp鍦板潃鍘嬫爤锛岀劧鍚庡皢rsp璧嬪€肩粰rbp锛屽啀灏嗗叾浠栦俊鎭帇鏍堝苟渚濇淇濆瓨锛岃繖鏍锋垜浠氨鍙互閫氳繃bt-F鍛戒护缁撳悎姹囩紪鎸囦护锛屽氨鍙互鐭ラ亾鍘嬪叆鏍堢殑淇℃伅浜嗐€備妇涓牀瀛愷煂板皾璇曞垎鏋愪笅闈㈢殑vmcore鏂囦欢锛屾垜浠厛鐢╟rash宸ュ叿鎵撳紑vmcore鏂囦欢锛岀劧鍚庣敤log鎴栬€卋t鍛戒护鏌ョ湅鏈€缁堢殑鍫嗘爤鍜屽瘎瀛樺櫒淇℃伅銆備粠绗竴琛屸€渦nabletohandlekernelpagingrequest鈥濆彲浠ュ垽鏂繖鏄闂潪娉曞湴鍧€瀵艰嚧鐨刢rash闂銆傛帴涓嬫潵锛屾垜浠渶瑕佸畾浣嶉潪娉曞湴鍧€鐨勬潵婧愩€俢rash>log...BUG:unabletohandlekernelpagingrequestat0000000000019600[2221616.682227]IP:[]native_queued_spin_lock_slowpath+0x110/0x200[2221616.682565]PGD0[2221616.682816]Oops:0002[#1]SMP[2221616.683060]Moduleslinkedin:binfmt_miscnfsv3rpcsec_gss_krb5nfsv4dns_resolvernfsfscacheipt_MASQUERADEnf_nat_masquerade_ipv4xt_addrtypeiptable_filterxt_natiptable_natnf_conntrack_ipv4nf_defrag_ipv4nf_nat_ipv4nf_natnf_conntrackbondingemcpdm(POE)emcpgpx(POE)emcpmpx(POE)emcp(POE)vfatfatiTCO_wdtiTCO_vendor_supportmxm_wmidcdbassb_edacedac_coreintel_powerclampcoretempintel_rapliosf_mbikvm_intelkvmirqbypasscrc32_pclmulghash_clmulni_intelaesni_intellrwgf128mulglue_helperablk_helpercryptdpcspkrcdc_etherusbnetmiimei_memeilpc_ichacpi_power_metersgshpchpwmiipmi_siipmi_devintfipmi_msghandlernfsdauth_rpcgssnfs_acllockdgracesunrpcip_tablesxfslibcrc32csd_modmgag200i2c_algo_bitdrm_kms_helperlpfcsyscopyareasysfillrectsysimgbltfb_sys_fops[2221616.685213]ttmdrmcrc32c_intelahcii40elibahcicrc_t10difcrct10dif_genericlibatacrct10dif_pclmulscsi_transport_fcmegaraid_sasi2c_corescsi_tgtptppps_corecrct10dif_commondm_mirrordm_region_hashdm_logdm_modlastunloaded:emcpiocCPU:39PID:101271Comm:systemd-cgtopTainted:POE----------3.10.0-693.21.1.el7.x86_64#1[2221616.687011]纭欢鍚嶇О锛欴ellInc.PowerEdgeR630/02C2CP锛孊IOS2.7.1001/22/2018[2221616.687509]task:ffff88158c812f70ti:ffff88a4b502c000task.ti:ffff88a4b502c000[2221616.687982]RIP:0010:[][]native_queued_spin_lock_slowpath+0x110/0x200[2221616.688519]RSP:0018:ffff88a4b502fdc8EFLAGS:00010006[2221616.689011]RAX锛?000000000001fd4RBX锛歠fffffff81abb3a0RCX锛?000000001390000[2221616.689520]RDX锛?000000000019600RSI锛?0000000fea18bd0RDI锛歠fff88b68e104f98[2221616.690044]RBP:ffff88a4b502fdc8R08:ffff88befe2d9600R09:0000000000000000[2221616.690618]R10:ffff8801bfc07300R11:0000000000000246R12:0000000000000000[2221616.691175]R13:ffff88748b81be00R14:ffffffff81318e90R15:ffff88bed0d860c0[2221616.691743]FS:00007efd99b72740(0000)GS:ffff88befe2c0000(0000)knlGS:0000000000000000[2221616.692299]CS:0010DS:0000ES:0000CR0:0000000080050033[2221616.692870]CR2:0000000000019600CR3:0000009284026000CR4:00000000003607e0[2221616.693498]DR0:0000000000000000DR1:0000000000000000DR2:0000000000000000[2221616.694113]DR3:0000000000000000DR6:00000000fffe0ff0DR7:0000000000000400[2221616.694719]CallTrace:[2221616.695352][]queued_spin_lock_slowpath+0xb/0xf[2221616.695956][]_raw_spin_lock_irq+0x28/0x30[2221616.696615][]blkcg_print_blkgs+0x74/0xe0[2221616.697260][]cfqg_print_rwstat+0x36/0x40[2221616.697899][]cgroup_seqfile_show+0x76/0x80[2221616.698527][]?seq_buf_alloc+0x17/0x40[2221616.699185][]seq_read+0x110/0x3e0[2221616.699849][]vfs_read+0x9f/0x170[2221616.700539][]SyS_read+0x7f/0xe0[2221616.701184][]system_call_fastpath+0x1c/0x21[2221616.701834]Code:874702c1e0104531c985c074444889c2c1e81348c1ea0d489883e2304881c200960100480314c5a010b281<4c>8902418b400885c0750f04001f04f390418b40[2221616.703308]RIP[]native_queued_spin_lock_slowpath+0x110/0x200[2221616.704014]RSP[2221616.704702]CR2:0000000000019600閫氬父鎴戜滑浼氬厛鍙嶆眹缂朢IP瀵勫瓨鍣ㄤ腑鐨勬寚浠わ紝鍥犱负杩欐槸鍦ㄧ郴缁熷穿婧僒heinstructionexecutedatthetimeistheproblemwhenexecutingthemovinstruction宕╂簝>dis-lrffffffff810fdfa0|tail-n10/usr/src/debug/kernel-3.10.0-693.21.1.el7/linux-3.10.0-693.21.1.el7.x86_64/kernel/qspinlock.c:1070xffffffff810fdf85:shr$0x13,%eax/usr/src/debug/kernel-3.10.0-693.21.1.el7/linux-3.10.0-693.21.1.el7.x86_64/kernel/qspinlock.c:1100xffffffff810fdf88:shr$0xd,%rdx0xffffffff810fdf8c:cltq0xffffffff810fdf8e:and$0x30,%edx0xffffffff810fdf91:add$0x19600,%rdx0xffffffff810fdf98:add-0x7e4def60(,%rax,8),%rdx/usr/src/debug/kernel-3.10.0-693.21.1.el7/linux-3.10.0-693.21.1.el7.x86_64/include/linux/compiler.h:2080xffffffff810fdfa0:mov%r8,(%rdx)<---鍦ㄨ繖閲屽穿婧冨搴斿湪鐗堟湰鍐呮牳浠ｇ爜涓紝鎴戜滑鎵惧埌鍑芥暟blkcg_print_blkgs锛屽湪璋冪敤spin_lock_irq鐨勫湴鏂圭湅鍒拌嚜鏃嬮攣鏉ヨ嚜鈥渂lkcg_gq鈥濈粨鏋勪綋銆傛帴涓嬫潵鎴戜滑甯屾湜鎵惧埌blkg->q->queue_lock鐨勫湴鍧€锛屽垽鏂繖涓湴鍧€鏄惁鏈夐棶棰榠nt),conststructblkcg_policy*pol,intdata,boolshow_total){structblkcgg_gq*6鎬昏=0;rcu_read_lock();hlist_for_each_entry_rcu(blkg,&blkcg->blkg_list,blkcg_node){spin_lock_irq(blkg->q->queue_lock);濡傛灉(blkcg_policy_enabled(blkg->q,pol))total+=prfill(sf,blkg->pd[pol->plid],data);spin_unlock_irq(blkg->q->queue_lock);}rcu_read_unlock();if(show_total)seq_printf(sf,"Total%llu\n",(unsignedlonglong)total);}缁х画鐢╠is鍛戒护鍙嶆眹缂朾lkcg_print_blkgs鎸囦护銆傞€氳繃鈥渓ea-0x18(%rcx),%r15鈥濇寚浠わ紝鎴戜滑鍙互鐭ラ亾blk鐨勫湴鍧€瀛樻斁鍦╮15瀵勫瓨鍣ㄤ腑銆傜煡閬撲簡blkg鐨勫湴鍧€锛屾垜浠氨鍙互鎵惧埌鏈€鍚庝竴涓猶ueue_lock鐨勫湴鍧€銆傚穿婧?dis-lrffffffff81319654|tail-n100xffffffff8131963c:test%rcx,%rcx0xffffffff8131963f:je0xffffffff813196880xffffffff81319641:lea-0x18(%rcx),%r15/usr/src/debug/kernel-3.10.0-693.21.1.el7/linux-3.10.0-693.21.1.el7.x86_64/block/blk-cgroup.c:5040xffffffff81319645:mov(%r15),%rax/usr/src/debug/kernel-3.10.0-693.21.1.el7/linux-3.10.0-693.21.1.el7.x86_64/include/linux/spinlock.h:3310xffffffff81319648:mov0x458(%rax),%rdi0xffffffff8131964f:callq0xffffffff816b6730<_raw_spin_lock_irq>/usr/src/debug/kernel-3.10.0-693.21.1.el7/linux-3.10銆?-693.21.1.el7.x86_64/block/blk-cgroup.c:5050xffffffff81319654:mov(%r15),%rdi閫氳繃blkcg_gq缁撴瀯浣撶殑鍦板潃锛屾垜浠彲浠ュ厛鎵惧埌q鐨勫湴鍧€銆俿tructblkcg_gq{/*鎸囧悜鐩稿叧璇锋眰闃熷垪鐨勬寚閽?/structrequest_queue*q;缁撴瀯list_headq_node锛涚粨鏋刪list_nodeblkcg_node锛涚粨鏋刡lkcg*blkcg;...};鐨勫湴鍧€銆俿tructrequest_queue{/**涓巕ueue_head涓€璧风敤浜巆acheline鍏变韩*structlist_headqueue_head;缁撴瀯璇锋眰*last_merge;缁撴瀯鐢垫闃熷垪*鐢垫锛沬ntnr_rqs[2];/*#鍒嗛厤[a]syncrqs*/intnr_rqs_elvpriv;/*鍒嗛厤rqsw/elvpriv*/...spinlock_t__queue_lock;spinlock_t*闃熷垪閿侊紱...};crash>structrequest_queue.queue_lock0xffff88bef3f1b4e0queue_lock=0xffff88b68e104f98浣跨敤kmem鍛戒护妫€鏌ュ綋鍓嶅唴瀛樺垎閰嶃€傚垎閰嶇殑鍦板潃锛岀‘瀹炴槸闈炴硶鍦板潃銆俢rash>kmem-s0xffff88b68e104f98CACHENAMEOBJSIZEALLOCATEDTOTALSLABSSSIZEffff8801bfc07a00kmalloc-969623531347768284kSLABMEMORYNODETOTALALLOCATEDFREEffffea02da384100ffff88b68e1040001421032FREE/[ALLOCATED]ffff88b68e104f60鏌ョ湅requset_queue缁撴瀯浣擄紝杩欎釜闃熷垪宸叉病鏈夎澶囧湪浣跨敤,butfromrequest_fnitcanbeconfirmedthatitisusedbytheemcpmodule.crash>dev-d|grepffff88bef3f1b4e0crash>request_queue.dev,request_fn,nr_queues0xffff88bef3f1b4e0dev=0x0request_fn=0xffffffffc09348b0nr_queues=0x0crash>sym0xffffffffc09348b0ffffffffc09348b0(t)emcp_native_strategy_request_fn[emcp]鍥犳锛屾垜浠彲浠ュ緱鍑虹粨璁猴細鐢变簬emcp妯″潡娌℃湁姝ｇ‘鍒犻櫎request_queue,causingaspinlockoperationtoaccessaninvalidaddress.SummaryFromtheabove,itcanbeseenthatthecrashtoolisverypowerful,andlearningtousethecrashtoolcannotonlyhelpuslocateproblemsmorequickly,butalsohelpusdeeplyunderstandtheLinuxkernel.
