Kernel panic and multiple performance regression on the new kernel #2472
Description
Describe the issue
Kernel BUG after migration on the new kernel
Also there are a lot of performance regressions for example
To Reproduce
Start nginx with a large response on SUT
Start Temoesta FW
Run curl -k https://tempesta-tech.com:4443
Run h2load -c1000 -m95 -t100 -D50 https://tempesta-tech.com:4443
Configuration file
listen 192.168.2.1:800 proto=http;
listen 192.168.2.1:4443 proto=h2,https;
cache 2;
cache_fulfill * *;
tls_match_any_server_name;
tls_certificate /home/evgenii_mekhanik/certs/tempesta-tech.test.crt;
tls_certificate_key /home/evgenii_mekhanik/certs/tempesta-tech.test.key;
req_hdr_set X-Forwarded-Proto https;
frang_limits {
concurrent_tcp_connections 100000;
}
srv_group default {
server 127.0.0.1:8000 conns_n=32;
}
vhost tempesta-tech.com {
proxy_pass default;
}
http_chain {
-> tempesta-tech.com;
}
Version or commit hash
e8af77e
Stacktrace or debug log
[ 90.246076] BUG: Bad page state in process swapper/18 pfn:208818e
[ 90.252573] page: refcount:0 mapcount:0 mapping:0000000000000000 index:0x208818e000 pfn:0x208818e
[ 90.254763] BUG: Bad page state in process swapper/54 pfn:208a430
[ 90.261446] flags: 0x57ffffc0000000(node=1|zone=2|lastcpupid=0x1fffff)
[ 90.261451] raw: 0057ffffc0000000 dead000000000040 ffff9c30e2f97800 0000000000000000
[ 90.267633] page: refcount:0 mapcount:0 mapping:0000000000000000 index:0x208a430000 pfn:0x208a430
[ 90.274162] raw: 000000208818e000 0000000000000040 00000000ffffffff 0000000000000000
[ 90.274167] page dumped because: page_pool leak
[ 90.274169] Modules linked in: tempesta_fw(OE)
[ 90.274338] BUG: Bad page state in process grep pfn:20898de
[ 90.274342] page: refcount:0 mapcount:0 mapping:0000000000000000 index:0x20898de000 pfn:0x20898de
[ 90.274344] flags: 0x57ffffc0000000(node=1|zone=2|lastcpupid=0x1fffff)
[ 90.274347] raw: 0057ffffc0000000 dead000000000040 ffff9c30e43e6800 0000000000000000
[ 90.274349] raw: 00000020898de000 0000000000000040 00000000ffffffff 0000000000000000
[ 90.274350] page dumped because: page_pool leak
[ 113.953323] RIP: 0010:cpuidle_enter_state+0xcc/0x420
[ 113.958451] Code: ba 84 14 ff e8 95 f3 ff ff 8b 53 04 49 89 c5 0f 1f 44 00 00 31 ff e8 e3 52 13 ff 45 84 ff 0f 85 92 01 00 00 fb 0f 1f 44 00 00 <45> 85 f6 0f 88 69 01 00 00 49 63 d6 4c 2b 2c 24 48 8d 04 52 48 8d
[ 113.977544] RSP: 0000:ffffc0598841fe90 EFLAGS: 00000246
[ 113.982946] RAX: ffff9c4f3c500000 RBX: ffffe0597e3041b0 RCX: 000000000000001f
[ 113.990265] RDX: 0000000000000005 RSI: 000000003158b49a RDI: 0000000000000000
[ 113.997582] RBP: 0000000000000003 R08: 00000015186f7565 R09: 00000049e3e44300
[ 114.004897] R10: 0000000000000628 R11: ffff9c4f3c5349c4 R12: ffffffffac041b80
[ 114.012207] R13: 00000015186f7565 R14: 0000000000000003 R15: 0000000000000000
[ 114.019521] cpuidle_enter+0x2d/0x40
[ 114.023278] do_idle+0x1e7/0x240
[ 114.026681] cpu_startup_entry+0x29/0x40
[ 114.030781] start_secondary+0x11e/0x140
[ 114.034882] common_startup_64+0x13e/0x141
[ 114.039155]
[ 114.041519] CPU: 28 UID: 0 PID: 0 Comm: swapper/28 Kdump: loaded Tainted: G B OE 6.12.12.tfw-1493f1e #1
[ 114.052263] Tainted: [B]=BAD_PAGE, [O]=OOT_MODULE, [E]=UNSIGNED_MODULE
[ 114.059014] Hardware name: Dell Inc. PowerEdge R650/0TCW38, BIOS 1.12.1 09/13/2023
[ 114.066780] Call Trace:
[ 114.069427]
[ 114.071633] dump_stack_lvl+0x66/0x80
[ 114.075489] bad_page+0x6c/0x100
[ 114.078910] free_unref_page+0x3e9/0x5c0
[ 114.083017] ? ttls_decrypt+0x277/0x5c0 [tempesta_tls]
[ 114.088349] __split_pgfrag_del_w_frag+0x622/0x6a0 [tempesta_fw]
[ 114.094565] ss_skb_chop_head_tail+0x170/0x1e0 [tempesta_fw]
[ 114.100434] ? ss_skb_process+0xd4/0x160 [tempesta_fw]
[ 114.105783] tfw_tls_connection_recv+0x191/0x420 [tempesta_fw]
[ 114.111827] ss_tcp_process_data+0x217/0x520 [tempesta_fw]
[ 114.117522] ss_tcp_data_ready+0x56/0x100 [tempesta_fw]
[ 114.122958] tcp_data_queue+0x957/0xe20
[ 114.126996] tcp_rcv_established+0x21e/0x700
[ 114.131466] tcp_v4_do_rcv+0x16b/0x2c0
[ 114.135410] tcp_v4_rcv+0xf9c/0x1260
[ 114.139189] ? raw_local_deliver+0xc7/0x260
[ 114.143570] ip_protocol_deliver_rcu+0x32/0x180
[ 114.148295] ip_local_deliver_finish+0x77/0xa0
[ 114.152930] ip_sublist_rcv_finish+0x64/0x80
[ 114.157392] ip_sublist_rcv+0x179/0x220
[ 114.161423] ? __pfx_ip_rcv_finish+0x20/0x20
[ 114.165886] ip_list_rcv+0xfd/0x140
[ 114.169568] __netif_receive_skb_list_core+0x224/0x260
[ 114.174900] netif_receive_skb_list_internal+0x198/0x2c0
[ 114.180404] napi_complete_done+0x73/0x1c0
[ 114.184695] mlx5e_napi_poll+0x176/0x720 [mlx5_core]
[ 114.189933] __napi_poll+0x29/0x1c0
[ 114.193622] net_rx_action+0x325/0x3a0
[ 114.197570] handle_softirqs+0xcd/0x2c0
[ 114.201606] __irq_exit_rcu+0x63/0x80
[ 114.205455] common_interrupt+0x83/0xa0
[ 114.209479]
[ 114.211760]
[ 114.214035] asm_common_interrupt+0x26/0x40
[ 114.218395] RIP: 0010:cpuidle_enter_state+0xcc/0x420
[ 114.223536] Code: ba 84 14 ff e8 95 f3 ff ff 8b 53 04 49 89 c5 0f 1f 44 00 00 31 ff e8 e3 52 13 ff 45 84 ff 0f 85 92 01 00 00 fb 0f 1f 44 00 00 <45> 85 f6 0f 88 69 01 00 00 49 63 d6 4c 2b 2c 24 48 8d 04 52 48 8d
[ 114.242651] RSP: 0018:ffffc059884d7e90 EFLAGS: 00000246
[ 114.248057] RAX: ffff9c303ef00000 RBX: ffffe03a80d041b0 RCX: 000000000000001f
[ 114.255376] RDX: 000000000000001c RSI: 000000003158b49a RDI: 0000000000000000
[ 114.262702] RBP: 0000000000000003 R08: 000000151dac7ea2 R09: 00000049e3e44300
[ 114.270027] R10: 0000000000000019 R11: ffff9c303ef349c4 R12: ffffffffac041b80
[ 114.277353] R13: 000000151dac7ea2 R14: 0000000000000003 R15: 0000000000000000
[ 114.284677] cpuidle_enter+0x2d/0x40
[ 114.288451] do_idle+0x1e7/0x240
[ 114.291865] cpu_startup_entry+0x29/0x40
[ 114.295972] start_secondary+0x11e/0x140
[ 114.300076] common_startup_64+0x13e/0x141
[ 114.304351]
[ 114.306719] CPU: 18 UID: 0 PID: 0 Comm: swapper/18 Kdump: loaded Tainted: G B OE 6.12.12.tfw-1493f1e #1
[ 114.317415] Tainted: [B]=BAD_PAGE, [O]=OOT_MODULE, [E]=UNSIGNED_MODULE
[ 114.324138] Hardware name: Dell Inc. PowerEdge R650/0TCW38, BIOS 1.12.1 09/13/2023
[ 114.331896] Call Trace:
[ 114.334546]
[ 114.336750] dump_stack_lvl+0x66/0x80
[ 114.340605] bad_page+0x6c/0x100
[ 114.344026] free_unref_page+0x3e9/0x5c0
[ 114.348133] ? ttls_decrypt+0x277/0x5c0 [tempesta_tls]
[ 114.353465] __split_pgfrag_del_w_frag+0x622/0x6a0 [tempesta_fw]
[ 114.359681] ss_skb_chop_head_tail+0x170/0x1e0 [tempesta_fw]
[ 114.365540] ? ss_skb_process+0xd4/0x160 [tempesta_fw]
[ 114.370880] tfw_tls_connection_recv+0x191/0x420 [tempesta_fw]
[ 114.376914] ss_tcp_process_data+0x217/0x520 [tempesta_fw]
[ 114.382612] ss_tcp_data_ready+0x56/0x100 [tempesta_fw]
[ 114.388046] tcp_data_queue+0x957/0xe20
[ 114.392082] tcp_rcv_established+0x21e/0x700
[ 114.396546] tcp_v4_do_rcv+0x16b/0x2c0
[ 114.400492] tcp_v4_rcv+0xf9c/0x1260
[ 114.404268] ? raw_local_deliver+0xc7/0x260
[ 114.408647] ip_protocol_deliver_rcu+0x32/0x180
[ 114.413379] ip_local_deliver_finish+0x77/0xa0
[ 114.418012] ip_sublist_rcv_finish+0x64/0x80
[ 114.422477] ip_sublist_rcv+0x179/0x220
[ 114.426506] ? get_nohz_timer_target+0x110/0x140
[ 114.431314] ? __pfx_ip_rcv_finish+0x20/0x20
[ 114.435777] ip_list_rcv+0xfd/0x140
[ 114.439462] __netif_receive_skb_list_core+0x224/0x260
[ 114.444792] netif_receive_skb_list_internal+0x198/0x2c0
[ 114.450299] napi_complete_done+0x73/0x1c0
[ 114.454587] mlx5e_napi_poll+0x176/0x720 [mlx5_core]
[ 114.459837] ? update_curr+0x120/0x260
[ 114.463787] __napi_poll+0x29/0x1c0
[ 114.467477] net_rx_action+0x325/0x3a0
[ 114.471414] ? notifier_call_chain+0x5b/0xe0
[ 114.475871] ? atomic_notifier_call_chain+0x3b/0x60
[ 114.480930] handle_softirqs+0xcd/0x2c0
[ 114.484947] __irq_exit_rcu+0x63/0x80
[ 114.488785] common_interrupt+0x83/0xa0
[ 114.492798]
[ 114.495072]
[ 114.497339] asm_common_interrupt+0x26/0x40
[ 114.501690] RIP: 0010:cpuidle_enter_state+0xcc/0x420
[ 114.506822] Code: ba 84 14 ff e8 95 f3 ff ff 8b 53 04 49 89 c5 0f 1f 44 00 00 31 ff e8 e3 52 13 ff 45 84 ff 0f 85 92 01 00 00 fb 0f 1f 44 00 00 <45> 85 f6 0f 88 69 01 00 00 49 63 d6 4c 2b 2c 24 48 8d 04 52 48 8d
[ 114.525931] RSP: 0018:ffffc05988487e90 EFLAGS: 00000246
[ 114.531342] RAX: ffff9c303ec80000 RBX: ffffe03a80a841b0 RCX: 000000000000001f
[ 114.538662] RDX: 0000000000000012 RSI: 000000003158b49a RDI: 0000000000000000
[ 114.545983] RBP: 0000000000000003 R08: 00000015031536ee R09: 0000000000000001
[ 114.553301] R10: ffff9c303ed26640 R11: 0000000000000080 R12: ffffffffac041b80
[ 114.560616] R13: 00000015031536ee R14: 0000000000000003 R15: 0000000000000000
[ 114.567930] cpuidle_enter+0x2d/0x40
[ 114.571689] do_idle+0x1e7/0x240
[ 114.575102] cpu_startup_entry+0x29/0x40
[ 114.579213] start_secondary+0x11e/0x140
[ 114.583320] common_startup_64+0x13e/0x141
[ 114.587600]
[ 114.589963] CPU: 40 UID: 0 PID: 0 Comm: swapper/40 Kdump: loaded Tainted: G B OE 6.12.12.tfw-1493f1e #1
[ 114.594346] sched: DL replenish lagged too much
[ 114.605353] Tainted: [B]=BAD_PAGE, [O]=OOT_MODULE, [E]=UNSIGNED_MODULE
[ 114.612068] Hardware name: Dell Inc. PowerEdge R650/0TCW38, BIOS 1.12.1 09/13/2023
[ 114.619827] Call Trace:
[ 114.622469]
[ 114.624672] dump_stack_lvl+0x66/0x80
[ 114.628524] bad_page+0x6c/0x100
[ 114.631946] free_unref_page+0x3e9/0x5c0
[ 114.636055] ? ttls_decrypt+0x277/0x5c0 [tempesta_tls]
[ 114.641396] __split_pgfrag_del_w_frag+0x622/0x6a0 [tempesta_fw]
[ 114.647614] ss_skb_chop_head_tail+0x170/0x1e0 [tempesta_fw]
[ 114.653482] ? ss_skb_process+0xd4/0x160 [tempesta_fw]
[ 114.658828] tfw_tls_connection_recv+0x191/0x420 [tempesta_fw]
[ 114.664910] ss_tcp_process_data+0x217/0x520 [tempesta_fw]
[ 114.670719] ss_tcp_data_ready+0x56/0x100 [tempesta_fw]
[ 114.676266] tcp_data_queue+0x957/0xe20
[ 114.680403] tcp_rcv_established+0x21e/0x700
[ 114.684966] tcp_v4_do_rcv+0x16b/0x2c0
[ 114.689019] tcp_v4_rcv+0xf9c/0x1260
[ 114.692897] ? raw_local_deliver+0xc7/0x260
[ 114.697376] ip_protocol_deliver_rcu+0x32/0x180
[ 114.702192] ip_local_deliver_finish+0x77/0xa0
[ 114.706931] ip_sublist_rcv_finish+0x64/0x80
[ 114.711507] ip_sublist_rcv+0x179/0x220
[ 114.715623] ? __pfx_ip_rcv_finish+0x20/0x20
[ 114.720199] ip_list_rcv+0xfd/0x140
[ 114.723980] __netif_receive_skb_list_core+0x224/0x260
[ 114.729411] netif_receive_skb_list_internal+0x198/0x2c0
[ 114.735027] napi_complete_done+0x73/0x1c0
[ 114.739427] mlx5e_napi_poll+0x176/0x720 [mlx5_core]
[ 114.744773] ? update_curr+0x120/0x260
[ 114.748823] __napi_poll+0x29/0x1c0
[ 114.752601] net_rx_action+0x325/0x3a0
[ 114.756647] ? notifier_call_chain+0x5b/0xe0
[ 114.761199] ? atomic_notifier_call_chain+0x3b/0x60
[ 114.766347] handle_softirqs+0xcd/0x2c0
[ 114.770457] __irq_exit_rcu+0x63/0x80
[ 114.774390] common_interrupt+0x83/0xa0
[ 114.778499]
[ 114.780876]
[ 114.783236] asm_common_interrupt+0x26/0x40
[ 114.787690] RIP: 0010:cpuidle_enter_state+0xcc/0x420
[ 114.792919] Code: ba 84 14 ff e8 95 f3 ff ff 8b 53 04 49 89 c5 0f 1f 44 00 00 31 ff e8 e3 52 13 ff 45 84 ff 0f 85 92 01 00 00 fb 0f 1f 44 00 00 <45> 85 f6 0f 88 69 01 00 00 49 63 d6 4c 2b 2c 24 48 8d 04 52 48 8d
[ 114.812125] RSP: 0018:ffffc05988537e90 EFLAGS: 00000246
[ 114.817645] RAX: ffff9c303f200000 RBX: ffffe03a810041b0 RCX: 000000000000001f
[ 114.825061] RDX: 0000000000000028 RSI: 000000003158b49a RDI: 0000000000000000
[ 114.832469] RBP: 0000000000000003 R08: 0000001508274dc8 R09: 00000014a0419b00
[ 114.839882] R10: 000000000000000e R11: ffff9c303f2349c4 R12: ffffffffac041b80
[ 114.847295] R13: 0000001508274dc8 R14: 0000000000000003 R15: 0000000000000000
[ 114.854706] cpuidle_enter+0x2d/0x40
[ 114.858568] do_idle+0x1e7/0x240
[ 114.862082] cpu_startup_entry+0x29/0x40
[ 114.866291] start_secondary+0x11e/0x140
[ 114.870501] common_startup_64+0x13e/0x141
[ 114.874887]
[ 114.877362] CPU: 48 UID: 0 PID: 0 Comm: swapper/48 Kdump: loaded Tainted: G B OE 6.12.12.tfw-1493f1e #1
[ 114.888175] Tainted: [B]=BAD_PAGE, [O]=OOT_MODULE, [E]=UNSIGNED_MODULE
[ 114.894999] Hardware name: Dell Inc. PowerEdge R650/0TCW38, BIOS 1.12.1 09/13/2023
[ 114.902862] Call Trace:
[ 114.905589]
[ 114.907876] dump_stack_lvl+0x66/0x80
[ 114.911819] bad_page+0x6c/0x100
[ 114.915338] free_unref_page+0x3e9/0x5c0
[ 114.919540] ? ttls_decrypt+0x277/0x5c0 [tempesta_tls]
[ 114.924975] __split_pgfrag_del_w_frag+0x622/0x6a0 [tempesta_fw]
[ 114.931289] ss_skb_chop_head_tail+0x170/0x1e0 [tempesta_fw]
[ 114.937253] ? ss_skb_process+0xd4/0x160 [tempesta_fw]
[ 114.942710] tfw_tls_connection_recv+0x191/0x420 [tempesta_fw]
[ 114.948852] ss_tcp_process_data+0x217/0x520 [tempesta_fw]
[ 114.954649] ss_tcp_data_ready+0x56/0x100 [tempesta_fw]
[ 114.960184] tcp_data_queue+0x957/0xe20
[ 114.964323] tcp_rcv_established+0x21e/0x700
[ 114.968895] tcp_v4_do_rcv+0x16b/0x2c0
[ 114.972948] tcp_v4_rcv+0xf9c/0x1260
[ 114.976826] ? raw_local_deliver+0xc7/0x260
[ 114.981297] ip_protocol_deliver_rcu+0x32/0x180
[ 114.986113] ip_local_deliver_finish+0x77/0xa0
[ 114.990832] ip_sublist_rcv_finish+0x64/0x80
[ 114.995396] ip_sublist_rcv+0x179/0x220
[ 114.999526] ? __pfx_ip_rcv_finish+0x20/0x20
[ 115.004089] ip_list_rcv+0xfd/0x140
[ 115.007874] __netif_receive_skb_list_core+0x224/0x260
[ 115.013316] netif_receive_skb_list_internal+0x198/0x2c0
[ 115.018911] napi_complete_done+0x73/0x1c0
[ 115.023295] mlx5e_napi_poll+0x176/0x720 [mlx5_core]
[ 115.028632] __napi_poll+0x29/0x1c0
[ 115.032429] net_rx_action+0x325/0x3a0
[ 115.036485] handle_softirqs+0xcd/0x2c0
[ 115.040614] __irq_exit_rcu+0x63/0x80
[ 115.044561] common_interrupt+0x83/0xa0
[ 115.048695]
[ 115.051074]
[ 115.053447] asm_common_interrupt+0x26/0x40
[ 115.057896] RIP: 0010:cpuidle_enter_state+0xcc/0x420
[ 115.063123] Code: ba 84 14 ff e8 95 f3 ff ff 8b 53 04 49 89 c5 0f 1f 44 00 00 31 ff e8 e3 52 13 ff 45 84 ff 0f 85 92 01 00 00 fb 0f 1f 44 00 00 <45> 85 f6 0f 88 69 01 00 00 49 63 d6 4c 2b 2c 24 48 8d 04 52 48 8d
[ 115.082340] RSP: 0018:ffffc05988577e90 EFLAGS: 00000246
[ 115.087853] RAX: ffff9c303f400000 RBX: ffffe03a812041b0 RCX: 000000000000001f
[ 115.095276] RDX: 0000000000000030 RSI: 000000003158b49a RDI: 0000000000000000
[ 115.102701] RBP: 0000000000000003 R08: 0000001506cc9646 R09: 0000000000000018
[ 115.110116] R10: 00000000000001d5 R11: ffff9c303f4349c4 R12: ffffffffac041b80
[ 115.117540] R13: 0000001506cc9646 R14: 0000000000000003 R15: 0000000000000000
[ 115.124974] cpuidle_enter+0x2d/0x40
[ 115.128836] do_idle+0x1e7/0x240
[ 115.132344] cpu_startup_entry+0x29/0x40
[ 115.136539] start_secondary+0x11e/0x140
[ 115.140742] common_startup_64+0x13e/0x141
[ 115.145119]
[ 115.147585] CPU: 35 UID: 0 PID: 0 Comm: swapper/35 Kdump: loaded Tainted: G B OE 6.12.12.tfw-1493f1e #1
[ 115.158427] Tainted: [B]=BAD_PAGE, [O]=OOT_MODULE, [E]=UNSIGNED_MODULE
[ 115.165307] Hardware name: Dell Inc. PowerEdge R650/0TCW38, BIOS 1.12.1 09/13/2023
[ 115.173210] Call Trace:
[ 115.175969]
[ 115.178278] dump_stack_lvl+0x66/0x80
[ 115.182239] bad_page+0x6c/0x100
[ 115.185765] free_unref_page+0x3e9/0x5c0
[ 115.189993] ? ttls_decrypt+0x277/0x5c0 [tempesta_tls]
[ 115.195432] __split_pgfrag_del_w_frag+0x622/0x6a0 [tempesta_fw]
[ 115.201752] ss_skb_chop_head_tail+0x170/0x1e0 [tempesta_fw]
[ 115.207761] ? ss_skb_process+0xd4/0x160 [tempesta_fw]
[ 115.213251] tfw_tls_connection_recv+0x191/0x420 [tempesta_fw]
[ 115.219449] ss_tcp_process_data+0x217/0x520 [tempesta_fw]
[ 115.225261] ss_tcp_data_ready+0x56/0x100 [tempesta_fw]
[ 115.230818] tcp_data_queue+0x957/0xe20
[ 115.234975] tcp_rcv_established+0x21e/0x700
[ 115.239580] tcp_v4_do_rcv+0x16b/0x2c0
[ 115.243661] tcp_v4_rcv+0xf9c/0x1260
[ 115.247570] ? raw_local_deliver+0xc7/0x260
[ 115.252092] ip_protocol_deliver_rcu+0x32/0x180
[ 115.256947] ip_local_deliver_finish+0x77/0xa0
[ 115.261685] ip_sublist_rcv_finish+0x64/0x80
[ 115.266283] ip_sublist_rcv+0x179/0x220
[ 115.270439] ? __pfx_ip_rcv_finish+0x20/0x20
[ 115.275030] ip_list_rcv+0xfd/0x140
[ 115.278844] __netif_receive_skb_list_core+0x224/0x260
[ 115.284307] netif_receive_skb_list_internal+0x198/0x2c0
[ 115.289936] napi_gro_receive+0xce/0x220
[ 115.294188] mlx5e_handle_rx_cqe_mpwrq+0x125/0x220 [mlx5_core]
[ 115.300452] mlx5e_poll_rx_cq+0xb2/0x7e0 [mlx5_core]
[ 115.305847] mlx5e_napi_poll+0xe6/0x720 [mlx5_core]
[ 115.311117] __napi_poll+0x29/0x1c0
[ 115.314911] net_rx_action+0x325/0x3a0
[ 115.318958] ? notifier_call_chain+0x5b/0xe0
[ 115.323527] ? atomic_notifier_call_chain+0x3b/0x60
[ 115.328787] handle_softirqs+0xcd/0x2c0
[ 115.332934] __irq_exit_rcu+0x63/0x80
[ 115.336884] common_interrupt+0x83/0xa0
[ 115.341006]
[ 115.343400]
[ 115.345815] asm_common_interrupt+0x26/0x40
[ 115.350290] RIP: 0010:cpuidle_enter_state+0xcc/0x420
[ 115.355586] Code: ba 84 14 ff e8 95 f3 ff ff 8b 53 04 49 89 c5 0f 1f 44 00 00 31 ff e8 e3 52 13 ff 45 84 ff 0f 85 92 01 00 00 fb 0f 1f 44 00 00 <45> 85 f6 0f 88 69 01 00 00 49 63 d6 4c 2b 2c 24 48 8d 04 52 48 8d
[ 115.374868] RSP: 0000:ffffc0598850fe90 EFLAGS: 00000246
[ 115.380393] RAX: ffff9c4f3cc80000 RBX: ffffe0597ea841b0 RCX: 000000000000001f
[ 115.387847] RDX: 0000000000000023 RSI: 000000003158b49a RDI: 0000000000000000
[ 115.395302] RBP: 0000000000000003 R08: 00000015057269da R09: 0000000000000000
[ 115.402804] R10: 0000000000001686 R11: ffff9c4f3ccb49c4 R12: ffffffffac041b80
[ 115.410250] R13: 00000015057269da R14: 0000000000000003 R15: 0000000000000000
[ 115.417688] cpuidle_enter+0x2d/0x40
[ 115.421568] do_idle+0x1e7/0x240
[ 115.425091] cpu_startup_entry+0x29/0x40
[ 115.429323] start_secondary+0x11e/0x140
[ 115.433564] common_startup_64+0x13e/0x141
[ 115.437970]
[ 115.440463] CPU: 26 UID: 0 PID: 177 Comm: ksoftirqd/26 Kdump: loaded Tainted: G B OE 6.12.12.tfw-1493f1e #1
[ 115.451631] Tainted: [B]=BAD_PAGE, [O]=OOT_MODULE, [E]=UNSIGNED_MODULE
[ 115.458452] Hardware name: Dell Inc. PowerEdge R650/0TCW38, BIOS 1.12.1 09/13/2023
[ 115.466329] Call Trace:
[ 115.469072]
[ 115.471531] dump_stack_lvl+0x66/0x80
[ 115.475479] bad_page+0x6c/0x100
[ 115.478988] free_unref_page+0x3e9/0x5c0
[ 115.483226] ? ttls_decrypt+0x277/0x5c0 [tempesta_tls]
[ 115.488679] __split_pgfrag_del_w_frag+0x622/0x6a0 [tempesta_fw]
[ 115.495011] ss_skb_chop_head_tail+0x170/0x1e0 [tempesta_fw]
[ 115.500999] ? ss_skb_process+0xd4/0x160 [tempesta_fw]
[ 115.506453] tfw_tls_connection_recv+0x191/0x420 [tempesta_fw]
[ 115.512596] ss_tcp_process_data+0x217/0x520 [tempesta_fw]
[ 115.518406] ss_tcp_data_ready+0x56/0x100 [tempesta_fw]
[ 115.523979] tcp_data_queue+0x957/0xe20
[ 115.528110] tcp_rcv_established+0x21e/0x700
[ 115.532690] tcp_v4_do_rcv+0x16b/0x2c0
[ 115.536737] tcp_v4_rcv+0xf9c/0x1260
[ 115.540646] ? raw_local_deliver+0xc7/0x260
[ 115.545121] ip_protocol_deliver_rcu+0x32/0x180
[ 115.549963] ip_local_deliver_finish+0x77/0xa0
[ 115.554706] ip_sublist_rcv_finish+0x64/0x80
[ 115.559283] ip_sublist_rcv+0x179/0x220
[ 115.563419] ? __pfx_ip_rcv_finish+0x20/0x20
[ 115.567972] ip_list_rcv+0xfd/0x140
[ 115.571741] __netif_receive_skb_list_core+0x224/0x260
[ 115.577165] netif_receive_skb_list_internal+0x198/0x2c0
[ 115.582775] ? kmem_cache_free+0x349/0x3e0
[ 115.587176] napi_gro_receive+0xce/0x220
[ 115.591406] mlx5e_handle_rx_cqe_mpwrq+0x125/0x220 [mlx5_core]
[ 115.597610] mlx5e_poll_rx_cq+0xb2/0x7e0 [mlx5_core]
[ 115.602936] mlx5e_napi_poll+0xe6/0x720 [mlx5_core]
[ 115.608187] __napi_poll+0x29/0x1c0
[ 115.611971] net_rx_action+0x325/0x3a0
[ 115.616013] ? __run_timers+0x100/0x280
[ 115.620170] handle_softirqs+0xcd/0x2c0
[ 115.624305] ? __pfx_smpboot_thread_fn+0x20/0x20
[ 115.629204] run_ksoftirqd+0x2f/0x40
[ 115.633078] smpboot_thread_fn+0xda/0x1e0
[ 115.637364] kthread+0xd0/0x100
[ 115.640783] ? __pfx_kthread+0x20/0x20
[ 115.644802] ret_from_fork+0x31/0x60
[ 115.648649] ? __pfx_kthread+0x20/0x20
[ 115.652673] ret_from_fork_asm+0x22/0x60
[ 115.656869]
[ 115.659344] CPU: 16 UID: 0 PID: 28568 Comm: grep Kdump: loaded Tainted: G B OE 6.12.12.tfw-1493f1e #1
[ 115.669974] Tainted: [B]=BAD_PAGE, [O]=OOT_MODULE, [E]=UNSIGNED_MODULE
[ 115.676790] Hardware name: Dell Inc. PowerEdge R650/0TCW38, BIOS 1.12.1 09/13/2023
[ 115.684620] Call Trace:
[ 115.687338]
[ 115.689630] dump_stack_lvl+0x66/0x80
[ 115.693558] bad_page+0x6c/0x100
[ 115.697060] free_unref_page+0x3e9/0x5c0
[ 115.701254] ? ttls_decrypt+0x277/0x5c0 [tempesta_tls]
[ 115.706686] __split_pgfrag_del_w_frag+0x622/0x6a0 [tempesta_fw]
[ 115.712988] ss_skb_chop_head_tail+0x170/0x1e0 [tempesta_fw]
[ 115.718923] ? ss_skb_process+0xd4/0x160 [tempesta_fw]
[ 115.724349] tfw_tls_connection_recv+0x191/0x420 [tempesta_fw]
[ 115.730506] ss_tcp_process_data+0x217/0x520 [tempesta_fw]
[ 115.736281] ss_tcp_data_ready+0x56/0x100 [tempesta_fw]
[ 115.741797] tcp_data_queue+0x957/0xe20
[ 115.745898] tcp_rcv_established+0x21e/0x700
[ 115.750439] tcp_v4_do_rcv+0x16b/0x2c0
[ 115.754482] tcp_v4_rcv+0xf9c/0x1260
[ 115.758330] ? raw_local_deliver+0xc7/0x260
[ 115.762788] ip_protocol_deliver_rcu+0x32/0x180
[ 115.767589] ip_local_deliver_finish+0x77/0xa0
[ 115.772309] ip_sublist_rcv_finish+0x64/0x80
[ 115.776850] ip_sublist_rcv+0x179/0x220
[ 115.780951] ? __pfx_ip_rcv_finish+0x20/0x20
[ 115.785492] ip_list_rcv+0xfd/0x140
[ 115.789268] __netif_receive_skb_list_core+0x224/0x260
[ 115.794673] netif_receive_skb_list_internal+0x198/0x2c0
[ 115.800291] napi_complete_done+0x73/0x1c0
[ 115.804674] mlx5e_napi_poll+0x176/0x720 [mlx5_core]
[ 115.810007] ? update_curr+0x120/0x260
[ 115.814033] __napi_poll+0x29/0x1c0
[ 115.817789] net_rx_action+0x325/0x3a0
[ 115.821808] ? notifier_call_chain+0x5b/0xe0
[ 115.826345] ? atomic_notifier_call_chain+0x3b/0x60
[ 115.831504] handle_softirqs+0xcd/0x2c0
[ 115.835612] __irq_exit_rcu+0x63/0x80
[ 115.839550] common_interrupt+0x83/0xa0
[ 115.843688]
[ 115.846059]
[ 115.848430] asm_common_interrupt+0x26/0x40
[ 115.852879] RIP: 0010:up_write+0x24/0x60
[ 115.857072] Code: 90 90 90 90 90 90 f3 0f 1e fa 0f 1f 44 00 00 65 ff 05 b8 91 cb 55 48 c7 c0 ff ff ff ff 48 c7 47 08 00 00 00 00 f0 48 0f c1 07 02 75 18 65 ff 0d 99 91 cb 55 74 05 c3 cc cc cc cc 0f 1f 44 00
[ 115.876281] RSP: 0018:ffffc059b0f2fbf8 EFLAGS: 00000257
[ 115.881788] RAX: 0000000000000001 RBX: ffff9c11f437ec60 RCX: 0000000000000001
[ 115.889213] RDX: 0000000000000001 RSI: 0000000000000000 RDI: ffff9c1173f01780
[ 115.896651] RBP: 0000000000118173 R08: 0000000000000001 R09: 0000000000000010
[ 115.904078] R10: ffffc059b0f2fc88 R11: 0000000000000000 R12: 0000000000000001
[ 115.911508] R13: 0000000008118173 R14: 00007fffffffe000 R15: 00007ffffffff000
[ 115.918927] mprotect_fixup+0x163/0x380
[ 115.923056] setup_arg_pages+0x1a3/0x380
[ 115.927283] ? security_bprm_committed_creds+0x91/0xa0
[ 115.932704] ? arch_pick_mmap_layout+0x149/0x180
[ 115.937602] load_elf_binary+0x38b/0x1680
[ 115.941889] ? tomoyo_find_next_domain+0x293/0x880
[ 115.946991] ? load_misc_binary+0x25c/0x340 [binfmt_misc]
[ 115.952698] bprm_execve+0x238/0x600
[ 115.956578] do_execveat_common.isra.0+0x17f/0x1e0
[ 115.961686] __x64_sys_execve+0x37/0x60
[ 115.965817] do_syscall_64+0x51/0x120
[ 115.969770] entry_SYSCALL_64_after_hwframe+0x76/0x7e
[ 115.975118] RIP: 0033:0x7ffff7ceef3b
[ 115.978986] Code: Unable to access opcode bytes at 0x7ffff7ceef11.
[ 115.985451] RSP: 002b:00007fffffffe3a8 EFLAGS: 00000202 ORIG_RAX: 000000000000003b
[ 115.993294] RAX: ffffffffffffffda RBX: 00007fffffffefb3 RCX: 00007ffff7ceef3b
[ 116.000744] RDX: 00007fffffffec00 RSI: 0000555555589f10 RDI: 00007fffffffe3b0
[ 116.008179] RBP: 00007fffffffe490 R08: 00000000ffffffff R09: 0000000000000000
[ 116.015599] R10: 00007ffff7ffe0b8 R11: 0000000000000202 R12: 00007ffff7ec4010
[ 116.023014] R13: 00007fffffffefab R14: 0000000000000005 R15: 0000555555589f10
[ 116.030426]
[ 116.032904] CPU: 46 UID: 0 PID: 0 Comm: swapper/46 Kdump: loaded Tainted: G B OE 6.12.12.tfw-1493f1e #1
[ 116.043705] Tainted: [B]=BAD_PAGE, [O]=OOT_MODULE, [E]=UNSIGNED_MODULE
[ 116.050575] Hardware name: Dell Inc. PowerEdge R650/0TCW38, BIOS 1.12.1 09/13/2023
[ 116.058476] Call Trace:
[ 116.061222]
[ 116.063470] dump_stack_lvl+0x66/0x80
[ 116.067448] bad_page+0x6c/0x100
[ 116.070998] free_unref_page+0x3e9/0x5c0
[ 116.075235] ? ttls_decrypt+0x277/0x5c0 [tempesta_tls]
[ 116.080691] __split_pgfrag_del_w_frag+0x622/0x6a0 [tempesta_fw]
[ 116.087026] ss_skb_chop_head_tail+0x170/0x1e0 [tempesta_fw]
[ 116.093000] ? ss_skb_process+0xd4/0x160 [tempesta_fw]
[ 116.098467] tfw_tls_connection_recv+0x191/0x420 [tempesta_fw]
[ 116.104651] ss_tcp_process_data+0x217/0x520 [tempesta_fw]
[ 116.110492] ss_tcp_data_ready+0x56/0x100 [tempesta_fw]
[ 116.116042] tcp_data_queue+0x957/0xe20
[ 116.120193] tcp_rcv_established+0x21e/0x700
[ 116.124819] tcp_v4_do_rcv+0x16b/0x2c0
[ 116.128913] tcp_v4_rcv+0xf9c/0x1260
[ 116.132814] ? raw_local_deliver+0xc7/0x260
[ 116.137296] ip_protocol_deliver_rcu+0x32/0x180
[ 116.142133] ip_local_deliver_finish+0x77/0xa0
[ 116.146878] ip_sublist_rcv_finish+0x64/0x80
[ 116.151446] ip_sublist_rcv+0x179/0x220
[ 116.155569] ? __pfx_ip_rcv_finish+0x20/0x20
[ 116.160131] ip_list_rcv+0xfd/0x140
[ 116.163906] __netif_receive_skb_list_core+0x224/0x260
[ 116.169323] netif_receive_skb_list_internal+0x198/0x2c0
[ 116.174945] napi_complete_done+0x73/0x1c0
[ 116.179329] mlx5e_napi_poll+0x176/0x720 [mlx5_core]
[ 116.184663] ? update_curr+0x120/0x260
[ 116.188711] __napi_poll+0x29/0x1c0
[ 116.192485] net_rx_action+0x325/0x3a0
[ 116.196532] ? notifier_call_chain+0x5b/0xe0
[ 116.201098] ? atomic_notifier_call_chain+0x3b/0x60
[ 116.206256] handle_softirqs+0xcd/0x2c0
[ 116.210384] __irq_exit_rcu+0x63/0x80
[ 116.214337] common_interrupt+0x83/0xa0
[ 116.218474]
[ 116.220854]
[ 116.223216] asm_common_interrupt+0x26/0x40
[ 116.227678] RIP: 0010:cpuidle_enter_state+0xcc/0x420
[ 116.232939] Code: ba 84 14 ff e8 95 f3 ff ff 8b 53 04 49 89 c5 0f 1f 44 00 00 31 ff e8 e3 52 13 ff 45 84 ff 0f 85 92 01 00 00 fb 0f 1f 44 00 00 <45> 85 f6 0f 88 69 01 00 00 49 63 d6 4c 2b 2c 24 48 8d 04 52 48 8d
[ 116.252167] RSP: 0018:ffffc05988567e90 EFLAGS: 00000246
[ 116.257684] RAX: ffff9c303f380000 RBX: ffffe03a811841b0 RCX: 000000000000001f
[ 116.265099] RDX: 000000000000002e RSI: 000000003158b49a RDI: 0000000000000000
[ 116.272525] RBP: 0000000000000003 R08: 0000001508ee6322 R09: 00000014a229e300
[ 116.279931] R10: 0000000000000135 R11: ffff9c303f3b49c4 R12: ffffffffac041b80
[ 116.287356] R13: 0000001508ee6322 R14: 0000000000000003 R15: 0000000000000000
[ 116.294777] cpuidle_enter+0x2d/0x40
[ 116.298634] do_idle+0x1e7/0x240
[ 116.302147] cpu_startup_entry+0x29/0x40
[ 116.306363] start_secondary+0x11e/0x140
[ 116.310564] common_startup_64+0x13e/0x141
[ 116.314941]
[ 116.317414] watchdog: BUG: soft lockup - CPU#43 stuck for 26s! [swapper/43:0]