该问题，疑似rootkit或者哪个软件捆绑安装的模块（apache？），不过谷歌并搜不到相关信息。
【如何判断】
- 查看dmesg，堆栈有tampercore、tampercfg内核模块打印
2. 如果没有直接打印出tampercore、tampercfg，但是每次重启都是不同的位置踩内存（比如BUG、gpf等）
打开vmcore 后执行mod -t命令可以看到tampercfg、tampercore模块:
代码语言：javascript复制crash> mod -t
NAME TAINTS
tampercfg OE
tampercore OE【解决方案】
这个内核模块用rmmod是无法完全清理掉的，过一段时间又会自己装回来（目前暂无环境继续分析是谁装回来的）。
所以建议直接备份重装、并使用强密码。
【其他一些细节】
这个模块会hook系统调用：
代码语言：javascript复制crash> p sys_call_table
sys_call_table = $2 =
/*
* 提示：该行代码过长，系统自动注释不进行高亮。一键复制会移除系统注释
* {0xffffffff95c4de60, 0xffffffffc0a007d0, 0xffffffffc0a00d30, 0xffffffff95c4c1f0, 0xffffffff95c534d0, 0xffffffff95c53500, 0xffffffff95c534e0, 0xffffffff95c64e40, 0xffffffff95c4cc80, 0xffffffff95a31ef0, 0xffffffff95bfea30, 0xffffffff95bfcce0, 0xffffffff95bfbf40, 0xffffffff95ab5430, 0xffffffff95ab42f0, 0xffffffff961934b0, 0xffffffff95c62ad0, 0xffffffff95c4e040, 0xffffffff95c4e100, 0xffffffff95c4e620, 0xffffffff95c4e730, 0xffffffff95c4b6f0, 0xffffffff95c57c30, 0xffffffff95c644a0, 0xffffffff95ada780, 0xffffffff95bffae0, 0xffffffff95c00200, 0xffffffff95bf74d0, 0xffffffff95becf50, 0xffffffff95cf50c0, 0xffffffff95cf58e0, 0xffffffff95cf5150, 0xffffffff95c6ec30, 0xffffffff95c6ebd0, 0xffffffff95ab57d0, 0xffffffff95acb600, 0xffffffff95aa3430, 0xffffffff95aafa30, 0xffffffff95aa3810, 0xffffffff95ab8bd0, 0xffffffff95c4ee80, 0xffffffff96035190, 0xffffffff960355b0, 0xffffffff96035590, 0xffffffff960355e0, 0xffffffff96035610, 0xffffffff96035960, 0xffffffff96035bd0, 0xffffffff96035830, 0xffffffff960354c0, 0xffffffff960354d0, 0xffffffff960355c0, 0xffffffff960355d0, 0xffffffff96035280, 0xffffffff96035640, 0xffffffff96035740, 0xffffffff96193270, 0xffffffff961932e0, 0xffffffff96193350, 0xffffffff96193430, 0xffffffff95aa2b70, 0xffffffff95aa2e10, 0xffffffff95ab4bb0, 0xffffffff95ab91e0, 0xffffffff95cf3720, 0xffffffff95cf3a60, 0xffffffff95cf37c0, 0xffffffff95cf5930, 0xffffffff95cf0320, 0xffffffff95cf07e0, 0xffffffff95cf0d70, 0xffffffff95cf03a0, 0xffffffff95c61200, 0xffffffff95cad390, 0xffffffff95c83230, 0xffffffff95c83250, 0xffffffff95c4b3c0, 0xffffffff95c4b3f0, 0xffffffff95c630a0, 0xffffffff95c699a0, 0xffffffff95c4b710, 0xffffffff95c4b7e0, 0xffffffffc0a00ad0, 0xffffffffc0a01140, 0xffffffffc0a00e80, 0xffffffffc0a00790, 0xffffffff95c60e70, 0xffffffffc0a009a0, 0xffffffff95c60b40, 0xffffffff95c53660, 0xffffffff95c4bb30, 0xffffffff95c4b9f0, 0xffffffff95c4bc70, 0xffffffff95c4bcc0, 0xffffffff95c4bc90, 0xffffffff95ab9e30, 0xffffffff95aa4090, 0xffffffff95ab9a30, 0xffffffff95ab9de0, 0xffffffff95aba3c0, 0xffffffff95ab8dd0, 0xffffffff95aab8d0, 0xffffffff95ab8c50, 0xffffffff95a9e0e0, 0xffffffff95ab8cb0, 0xffffffff95ab8420, 0xffffffff95ab8190, 0xffffffff95ab8c80, 0xffffffff95ab8ce0, 0xffffffff95ab8e60, 0xffffffff95ab8c20, 0xffffffff95ab9090, 0xffffffff95ab9120, 0xffffffff95ab8270, 0xffffffff95ab8010, 0xffffffff95acea40, 0xffffffff95aceb80, 0xffffffff95ab8540, 0xffffffff95ab8750, 0xffffffff95ab87f0, 0xffffffff95ab8980, 0xffffffff95ab9020, 0xffffffff95ab8a20, 0xffffffff95ab8b00, 0xffffffff95ab90b0, 0xffffffff95aaa070, 0xffffffff95aaa210, 0xffffffff95ab4410, 0xffffffff95ab4ba0, 0xffffffff95ab4c20, 0xffffffff95ab5890, 0xffffffff95ab4fd0, 0xffffffff95c83760, 0xffffffff95c60870, 0xffffffff95ac78b0, 0xffffffff95a9b7e0, 0xffffffff95c84840, 0xffffffff95c846c0, 0xffffffff95c846e0, 0xffffffff95c6f2b0, 0xffffffff95ab79d0, 0xffffffff95ab7750, 0xffffffff95ada070, 0xffffffff95ada2e0, 0xffffffff95ada040, 0xffffffff95ada260, 0xffffffff95ada850, 0xffffffff95ada890, 0xffffffff95ada8d0, 0xffffffff95bf8880, 0xffffffff95bf88e0, 0xffffffff95bf8960, 0xffffffff95bf8b00, 0xffffffff95c4c240, 0xffffffff95a30440, 0xffffffff95c73e70, 0xffffffff95aa9bf0, 0xffffffff95ab9e80, 0xffffffff95a2c1f0, 0xffffffff95aa4350, 0xffffffff95ab9cd0, 0xffffffff95c4b8b0, 0xffffffff95c83080, 0xffffffff95b21530, 0xffffffff95aa4260, 0xffffffff95c73ce0, 0xffffffff95c71cb0, 0xffffffff95c0e380, 0xffffffff95c0dca0, 0xffffffff95ab7fb0, 0xffffffff95ab9410, 0xffffffff95ab95e0, 0xffffffff961933c0, 0xffffffff95a2fc70, 0xffffffff95ac78b0, 0xffffffff95b1f2f0, 0xffffffff95b1c320, 0xffffffff95ac78b0, 0xffffffff95ac78b0, 0xffffffff95cc20c0, 0xffffffff95ac78b0, 0xffffffff95ac78b0, 0xffffffff95ac78b0, 0xffffffff95ac78b0, 0xffffffff95ac78b0, 0xffffffff95ac78b0, 0xffffffff95ab8c00, 0xffffffff95bcbd10, 0xffffffff95c76ee0, 0xffffffff95c76fe0, 0xffffffff95c770d0, 0xffffffff95c771e0, 0xffffffff95c772a0, 0xffffffff95c77350, 0xffffffff95c77420, 0xffffffff95c774d0, 0xffffffff95c77570, 0xffffffff95c77640, 0xffffffff95c77710, 0xffffffff95c777e0, 0xffffffff95ab4bf0, 0xffffffff95aa3fc0, 0xffffffff95b15bb0, 0xffffffff95adbf00, 0xffffffff95ada690, 0xffffffff95ac78b0, 0xffffffff95ca45c0, 0xffffffff95ca4950, 0xffffffff95ca5470, 0xffffffff95ca52f0, 0xffffffff95ca5310, 0xffffffff95ac78b0, 0xffffffff95ce0c50, 0xffffffff95c9ca00, 0xffffffff95ac78b0, 0xffffffff95ac78b0, 0xffffffff95bea0e0, 0xffffffff95c631d0, 0xffffffff95a9aba0, 0xffffffff95ab4060, 0xffffffff95cf3a50, 0xffffffff95bc30e0, 0xffffffff95ac5460, 0xffffffff95ac5ae0, 0xffffffff95ac5960, 0xffffffff95ac5a70, 0xffffffff95ac5cb0, 0xffffffff95ac5ec0, 0xffffffff95ac5f70, 0xffffffff95ac6030, 0xffffffff95ac60e0, 0xffffffff95aa2c30, 0xffffffff95c9d6b0, 0xffffffff95c9ca30, 0xffffffff95ab4bc0, 0xffffffff95c839d0, 0xffffffff95ac78b0, 0xffffffff95c1ce30, 0xffffffff95c1b950, 0xffffffff95c1b970, 0xffffffff95cf73f0, 0xffffffff95cf7790, 0xffffffff95cf7910, 0xffffffff95cf7c90, 0xffffffff95cf81d0, 0xffffffff95cf8620, 0xffffffff95b23560, 0xffffffff95aa2c80, 0xffffffff95cfc1c0, 0xffffffff95cfc3e0, 0xffffffff95cfdc40, 0xffffffff95c93dc0, 0xffffffff95c940f0, 0xffffffff95c993e0, 0xffffffff95c99400, 0xffffffff95c99750...}
*/
crash> sym 0xffffffff95c4de60
ffffffff95c4de60 (T) sys_read /usr/src/debug/kernel-3.10.0-1127.19.1.el7/linux-3.10.0-1127.19.1.el7.x86_64/fs/read_write.c: 569
crash> sym 0xffffffffc0a00d30
ffffffffc0a00d30 (t) tamperhack_open [tampercore] 启动加载日志可以看到：
代码语言：javascript复制crash> log | grep tainting
[ 15.908686] tampercfg: module verification failed: signature and/or required key missing - tainting kernel一次踩内存的分析举例：
代码语言：javascript复制crash> bt
PID: 3141 TASK: ffff9413fba320e0 CPU: 3 COMMAND: "YDService"
#0 [ffff9413fba7b960] machine_kexec at ffffffff95a66254
#1 [ffff9413fba7b9c0] __crash_kexec at ffffffff95b22ef2
#2 [ffff9413fba7ba90] crash_kexec at ffffffff95b22fe0
#3 [ffff9413fba7baa8] oops_end at ffffffff9618a798
#4 [ffff9413fba7bad0] no_context at ffffffff95a75d74
#5 [ffff9413fba7bb20] __bad_area_nosemaphore at ffffffff95a76042
#6 [ffff9413fba7bb70] bad_area_nosemaphore at ffffffff95a76164
#7 [ffff9413fba7bb80] __do_page_fault at ffffffff9618d750
#8 [ffff9413fba7bbf0] trace_do_page_fault at ffffffff9618da26
#9 [ffff9413fba7bc30] do_async_page_fault at ffffffff9618cfa2
#10 [ffff9413fba7bc50] async_page_fault at ffffffff961897a8
[exception RIP: exit_creds 27]
RIP: ffffffff95acd8ab RSP: ffff9413fba7bd08 RFLAGS: 00010246
RAX: 0000000000000000 RBX: ffff9414409f20e0 RCX: 0000000000000000
RDX: ffffffff966c0ce0 RSI: ffff9414409f20e0 RDI: 0000000000000000
RBP: ffff9413fba7bd10 R8: 0000000000000000 R9: ffff94146b229c00
R10: 00000000000002b1 R11: 0000000000000000 R12: ffff9414409f20e0
R13: ffff9414409f20e0 R14: 0000000000000001 R15: ffffa23101b3a170
ORIG_RAX: ffffffffffffffff CS: 0010 SS: 0018
#11 [ffff9413fba7bd18] __put_task_struct at ffffffff95a9839a
#12 [ffff9413fba7bd38] wake_up_q at ffffffff95adb960
#13 [ffff9413fba7bd60] futex_wake_op at ffffffff95b13193
#14 [ffff9413fba7be48] do_futex at ffffffff95b1578a
#15 [ffff9413fba7bed8] sys_futex at ffffffff95b15c30
#16 [ffff9413fba7bf50] system_call_fastpath at ffffffff96192ed2
RIP: 00007f49c18200f6 RSP: 00007f4990060de8 RFLAGS: 00000202
RAX: 00000000000000ca RBX: 00000000025cdbe0 RCX: 0000000000000000
RDX: 0000000000000001 RSI: 0000000000000085 RDI: 00000000025cdc5c
RBP: 0000000000511800 R8: 00000000025cdc58 R9: 0000000004000001
R10: 0000000000000001 R11: 0000000000000283 R12: 0000000000000400
R13: 00007f4990060e50 R14: 00000000025b8620 R15: 0000000000000000
ORIG_RAX: 00000000000000ca CS: 0033 SS: 002b
crash> dis -r ffffffff95acd8ab
0xffffffff95acd890 <exit_creds>: nopl 0x0(%rax,%rax,1) [FTRACE NOP]
0xffffffff95acd895 <exit_creds 5>: push %rbp
0xffffffff95acd896 <exit_creds 6>: mov %rsp,%rbp
0xffffffff95acd899 <exit_creds 9>: push %rbx
0xffffffff95acd89a <exit_creds 10>: mov 0x670(%rdi),%rax
0xffffffff95acd8a1 <exit_creds 17>: mov %rdi,%rbx
0xffffffff95acd8a4 <exit_creds 20>: mov 0x668(%rdi),%rdi
0xffffffff95acd8ab <exit_creds 27>: mov (%rax),�x
rax、rdi是0，对应的task结构体被置空
crash> wake_q_head ffff9413fba7bdd0
struct wake_q_head {
first = 0xffff9414409f30f8,
lastp = 0xffff9414409f30f8
}
crash> wake_q_node 0xffff9414409f30f8
struct wake_q_node {
next = 0x0
}
这里为0不符合预期
对应代码：
void wake_up_q(struct wake_q_head *head)
{
struct wake_q_node *node = head->first;
while (node != WAKE_Q_TAIL) {
struct task_struct *task;
task = container_of(node, struct task_struct, wake_q);
BUG_ON(!task); //注意，这里通过了BUG检查，说明这时候还不是0
/* task can safely be re-inserted now */
node = node->next;
task->wake_q.next = NULL;
/*
* wake_up_process() implies a wmb() to pair with the queueing
* in wake_q_add() so as not to miss wakeups.
*/
wake_up_process(task);
put_task_struct(task); //这里进入这个函数后，就再没有task为0的检查，后面被并发改成0了
}
}所以就是这个内核模块本身写的也有点问题，会去并发的改task结构体，导致其他的正常执行逻辑里，task为0跪了。
