From 81490d59e9a66e188d6b204fe1dbcabbc087dadc Mon Sep 17 00:00:00 2001 From: swananan Date: Sat, 13 Jun 2026 20:56:34 +0800 Subject: [PATCH] fix: resolve backtrace frame modules at runtime Record proc module base ranges in the offsets map so eBPF can match unwound frame PCs to their real mapped module before writing wire fields. This keeps module cookies and normalized PCs aligned with the actual frame module instead of the probe module. --- .../src/ebpf/codegen/backtrace.rs | 387 ++++++++++++++++-- ghostscope-compiler/src/ebpf/context.rs | 3 +- .../src/ebpf/helper_functions.rs | 222 +++++++--- ghostscope-compiler/src/ebpf/maps.rs | 4 +- ghostscope-process/src/offsets.rs | 21 +- ghostscope-process/src/pinned_bpf_maps.rs | 52 ++- ghostscope-process/src/sysmon.rs | 20 +- ghostscope-protocol/src/bpf_abi.rs | 14 +- ghostscope-protocol/src/lib.rs | 7 +- ghostscope/src/script/compiler.rs | 23 +- 10 files changed, 609 insertions(+), 144 deletions(-) diff --git a/ghostscope-compiler/src/ebpf/codegen/backtrace.rs b/ghostscope-compiler/src/ebpf/codegen/backtrace.rs index 1baa3f4..9a95803 100644 --- a/ghostscope-compiler/src/ebpf/codegen/backtrace.rs +++ b/ghostscope-compiler/src/ebpf/codegen/backtrace.rs @@ -16,6 +16,7 @@ const BPF_INLINE_BACKTRACE_FRAME_LIMIT: u8 = 5; const BPF_BACKTRACE_FRAMES_PER_TAIL_CALL: u8 = 4; const BPF_BACKTRACE_MAX_STEP_INVOCATIONS: u8 = 32; const BPF_BACKTRACE_STEP_PROG_INDEX: u32 = 0; +const BPF_BACKTRACE_MODULE_CANDIDATE_LIMIT: usize = 16; struct RuntimeBtUnwindRow<'ctx> { found: IntValue<'ctx>, @@ -29,6 +30,12 @@ struct RuntimeBtUnwindRow<'ctx> { rbp_offset: IntValue<'ctx>, } +struct BtFrameModule<'ctx> { + cookie: IntValue<'ctx>, + bias: IntValue<'ctx>, + found: IntValue<'ctx>, +} + struct RuntimeBtRowScratch<'ctx> { found_ptr: PointerValue<'ctx>, cfa_register_ptr: PointerValue<'ctx>, @@ -77,6 +84,7 @@ enum BacktraceUnwindRowForPc { impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { pub(crate) fn prepare_backtrace_unwind_rows(&mut self, statements: &[Statement]) { self.backtrace_unwind_rows.clear(); + self.backtrace_module_cookies.clear(); self.backtrace_unwind_rows_use_runtime_pcs = false; self.backtrace_tail_call_slots = 1; self.next_backtrace_tail_call_slot = 0; @@ -132,7 +140,7 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { } fn runtime_backtrace_unwind_rows( - &self, + &mut self, analyzer: &ghostscope_dwarf::DwarfAnalyzer, ) -> Vec { let started_at = Instant::now(); @@ -156,16 +164,31 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { wire.pc_end = wire.pc_end.checked_add(module_bias)?; Some(wire) })); + let bpf_rows = rows.len().saturating_sub(row_start); + if bpf_rows > 0 { + let module_path = module.module_path.to_string_lossy(); + let cookie = self.cookie_for_module_or_fallback(&module_path); + if !self.backtrace_module_cookies.contains(&cookie) { + self.backtrace_module_cookies.push(cookie); + } + } modules += 1; tracing::info!( module = %module.module_path.display(), compact_rows = table.rows.len(), - bpf_rows = rows.len().saturating_sub(row_start), + bpf_rows, elapsed_ms = module_started_at.elapsed().as_millis(), "Prepared bt unwind rows for module" ); } rows.sort_by_key(|row| (row.pc_start, row.pc_end)); + if self.backtrace_module_cookies.len() > BPF_BACKTRACE_MODULE_CANDIDATE_LIMIT { + tracing::warn!( + candidates = self.backtrace_module_cookies.len(), + limit = BPF_BACKTRACE_MODULE_CANDIDATE_LIMIT, + "Limiting bt frame module range lookup candidates" + ); + } tracing::info!( modules, rows = rows.len(), @@ -269,6 +292,7 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { }; let module_cookie = self.cookie_for_module_or_fallback(&compile_ctx.module_path); + let module_cookie_value = self.context.i64_type().const_int(module_cookie, false); let pt_regs = self.get_pt_regs_parameter()?; let raw_ip = self.load_dwarf_register_i64(X86_64_DWARF_RIP, pt_regs)?; let (module_bias, offsets_found) = self.generate_runtime_address_from_offsets( @@ -277,8 +301,20 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { module_cookie, )?; let normalized_pc = self.normalized_pc_from_raw(raw_ip, module_bias, offsets_found)?; + let caller_fallback_found = if self.backtrace_unwind_rows_use_runtime_pcs { + self.context.bool_type().const_zero() + } else { + offsets_found + }; - self.store_backtrace_frame(inst_buffer, 0, module_cookie, normalized_pc, raw_ip, 0)?; + self.store_backtrace_frame( + inst_buffer, + 0, + module_cookie_value, + normalized_pc, + raw_ip, + 0, + )?; if depth == 1 { let status = @@ -368,8 +404,16 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; self.builder.position_at_end(initial_store_block); - let next_pc = self.normalized_pc_from_raw(next.ip, module_bias, offsets_found)?; - self.store_backtrace_frame(inst_buffer, 1, module_cookie, next_pc, next.ip, 0)?; + let frame_module = self.resolve_backtrace_frame_module( + next.ip, + module_cookie_value, + module_bias, + caller_fallback_found, + "bt_initial_frame_module", + )?; + let next_pc = + self.normalized_pc_from_raw(next.ip, frame_module.bias, frame_module.found)?; + self.store_backtrace_frame(inst_buffer, 1, frame_module.cookie, next_pc, next.ip, 0)?; self.store_u8_const( inst_buffer, data_base + BACKTRACE_DATA_FRAME_COUNT_OFFSET, @@ -485,11 +529,19 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; self.builder.position_at_end(store_block); - let next_pc = self.normalized_pc_from_raw(next.ip, module_bias, offsets_found)?; + let frame_module = self.resolve_backtrace_frame_module( + next.ip, + module_cookie_value, + module_bias, + caller_fallback_found, + &format!("bt_frame_{frame_index}_module"), + )?; + let next_pc = + self.normalized_pc_from_raw(next.ip, frame_module.bias, frame_module.found)?; self.store_backtrace_frame( inst_buffer, frame_index as usize, - module_cookie, + frame_module.cookie, next_pc, next.ip, 0, @@ -619,6 +671,7 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { }; let module_cookie = self.cookie_for_module_or_fallback(&compile_ctx.module_path); + let module_cookie_value = self.context.i64_type().const_int(module_cookie, false); let pt_regs = self.get_pt_regs_parameter()?; let raw_ip = self.load_dwarf_register_i64(X86_64_DWARF_RIP, pt_regs)?; let initial_rsp = self.load_dwarf_register_i64(X86_64_DWARF_RSP, pt_regs)?; @@ -629,8 +682,20 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { module_cookie, )?; let normalized_pc = self.normalized_pc_from_raw(raw_ip, module_bias, offsets_found)?; + let caller_fallback_found = if self.backtrace_unwind_rows_use_runtime_pcs { + self.context.bool_type().const_zero() + } else { + offsets_found + }; - self.store_backtrace_frame(inst_buffer, 0, module_cookie, normalized_pc, raw_ip, 0)?; + self.store_backtrace_frame( + inst_buffer, + 0, + module_cookie_value, + normalized_pc, + raw_ip, + 0, + )?; if depth == 1 { let status = @@ -691,6 +756,15 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { let ip_ptr = self.build_entry_alloca(i64_type, "bt_tail_prefix_ip")?; let rsp_ptr = self.build_entry_alloca(i64_type, "bt_tail_prefix_rsp")?; let rbp_ptr = self.build_entry_alloca(i64_type, "bt_tail_prefix_rbp")?; + let module_bias_ptr = self.build_entry_alloca(i64_type, "bt_tail_prefix_module_bias")?; + let module_cookie_ptr = + self.build_entry_alloca(i64_type, "bt_tail_prefix_module_cookie")?; + self.builder + .build_store(module_bias_ptr, module_bias) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(module_cookie_ptr, module_cookie_value) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; let runtime_row = self.runtime_row_from_static(row); let state = BtRegisterState { @@ -733,8 +807,16 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; self.builder.position_at_end(initial_store_block); - let next_pc = self.normalized_pc_from_raw(next.ip, module_bias, offsets_found)?; - self.store_backtrace_frame(inst_buffer, 1, module_cookie, next_pc, next.ip, 0)?; + let frame_module = self.resolve_backtrace_frame_module( + next.ip, + module_cookie_value, + module_bias, + caller_fallback_found, + "bt_tail_initial_frame_module", + )?; + let next_pc = + self.normalized_pc_from_raw(next.ip, frame_module.bias, frame_module.found)?; + self.store_backtrace_frame(inst_buffer, 1, frame_module.cookie, next_pc, next.ip, 0)?; self.store_u8_const( inst_buffer, data_base + BACKTRACE_DATA_FRAME_COUNT_OFFSET, @@ -762,6 +844,12 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { self.builder .build_store(rbp_ptr, next.rbp) .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(module_bias_ptr, frame_module.bias) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(module_cookie_ptr, frame_module.cookie) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; if depth == 2 { self.builder @@ -774,9 +862,11 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { let prefix_depth = depth.min(BPF_INLINE_BACKTRACE_FRAME_LIMIT); for frame_index in 2..prefix_depth { let current_ip = self.load_i64(ip_ptr, "bt_tail_prefix_lookup_ip")?; + let current_module_bias = + self.load_i64(module_bias_ptr, "bt_tail_prefix_lookup_module_bias")?; let lookup_raw = self.add_signed_offset(current_ip, -1, "bt_tail_prefix_lookup_raw")?; let lookup_pc = - self.backtrace_lookup_pc_from_raw(lookup_raw, module_bias, offsets_found)?; + self.backtrace_lookup_pc_from_raw(lookup_raw, current_module_bias, offsets_found)?; let runtime_row = self.lookup_backtrace_unwind_row(lookup_pc, &scratch.row)?; let found_block = self .context @@ -842,11 +932,22 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; self.builder.position_at_end(store_block); - let next_pc = self.normalized_pc_from_raw(next.ip, module_bias, offsets_found)?; + let fallback_cookie = + self.load_i64(module_cookie_ptr, "bt_tail_prefix_module_cookie")?; + let fallback_bias = self.load_i64(module_bias_ptr, "bt_tail_prefix_module_bias")?; + let frame_module = self.resolve_backtrace_frame_module( + next.ip, + fallback_cookie, + fallback_bias, + caller_fallback_found, + &format!("bt_tail_prefix_frame_{frame_index}_module"), + )?; + let next_pc = + self.normalized_pc_from_raw(next.ip, frame_module.bias, frame_module.found)?; self.store_backtrace_frame( inst_buffer, frame_index as usize, - module_cookie, + frame_module.cookie, next_pc, next.ip, 0, @@ -878,6 +979,12 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { self.builder .build_store(rbp_ptr, next.rbp) .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(module_bias_ptr, frame_module.bias) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(module_cookie_ptr, frame_module.cookie) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; if frame_index + 1 == depth { self.builder @@ -903,7 +1010,6 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { self.pending_backtrace_tail_call = Some(crate::ebpf::context::PendingBacktraceTailCall { step_program_name, - module_cookie, depth, instruction_size, }); @@ -944,6 +1050,8 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { let tail_ip = self.load_i64(ip_ptr, "bt_tail_state_prefix_ip")?; let tail_rsp = self.load_i64(rsp_ptr, "bt_tail_state_prefix_rsp")?; let tail_rbp = self.load_i64(rbp_ptr, "bt_tail_state_prefix_rbp")?; + let tail_module_bias = self.load_i64(module_bias_ptr, "bt_tail_state_module_bias")?; + let tail_module_cookie = self.load_i64(module_cookie_ptr, "bt_tail_state_module_cookie")?; self.store_state_i64( state_ptr, crate::BACKTRACE_TAIL_STATE_CURRENT_IP_OFFSET, @@ -965,13 +1073,13 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { self.store_state_i64( state_ptr, crate::BACKTRACE_TAIL_STATE_MODULE_BIAS_OFFSET, - module_bias, + tail_module_bias, "bt_state_module_bias", )?; - self.store_u64_const( + self.store_u64_value( state_ptr, crate::BACKTRACE_TAIL_STATE_MODULE_COOKIE_OFFSET, - module_cookie, + tail_module_cookie, "bt_state_module_cookie", )?; self.store_state_i32( @@ -1274,7 +1382,6 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { for _ in 0..BPF_BACKTRACE_FRAMES_PER_TAIL_CALL { self.generate_backtrace_tail_call_step_iteration( plan.depth, - plan.module_cookie, state_ptr, inst_buffer, &scratch, @@ -1450,7 +1557,6 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { fn generate_backtrace_tail_call_step_iteration( &mut self, depth: u8, - module_cookie: u64, state_ptr: PointerValue<'ctx>, inst_buffer: PointerValue<'ctx>, scratch: &BtScratch<'ctx>, @@ -1513,6 +1619,11 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { crate::BACKTRACE_TAIL_STATE_MODULE_BIAS_OFFSET, "bt_step_module_bias", )?; + let module_cookie = self.load_row_i64( + state_ptr, + crate::BACKTRACE_TAIL_STATE_MODULE_COOKIE_OFFSET, + "bt_step_module_cookie", + )?; let offsets_found_u8 = self.load_row_i8( state_ptr, crate::BACKTRACE_TAIL_STATE_OFFSETS_FOUND_OFFSET, @@ -1527,6 +1638,11 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { "bt_step_offsets_found", ) .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let caller_fallback_found = if self.backtrace_unwind_rows_use_runtime_pcs { + self.context.bool_type().const_zero() + } else { + offsets_found + }; let is_first_unwind = self .builder .build_int_compare( @@ -1610,12 +1726,20 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; self.builder.position_at_end(store_block); - let next_pc = self.normalized_pc_from_raw(next.ip, module_bias, offsets_found)?; + let frame_module = self.resolve_backtrace_frame_module( + next.ip, + module_cookie, + module_bias, + caller_fallback_found, + "bt_step_frame_module", + )?; + let next_pc = + self.normalized_pc_from_raw(next.ip, frame_module.bias, frame_module.found)?; self.store_backtrace_frame_dynamic( inst_buffer, frame_count, depth.saturating_sub(1), - module_cookie, + frame_module.cookie, next_pc, next.ip, )?; @@ -1657,6 +1781,18 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { next.rbp, "bt_step_state_next_rbp", )?; + self.store_state_i64( + state_ptr, + crate::BACKTRACE_TAIL_STATE_MODULE_BIAS_OFFSET, + frame_module.bias, + "bt_step_state_next_module_bias", + )?; + self.store_state_i64( + state_ptr, + crate::BACKTRACE_TAIL_STATE_MODULE_COOKIE_OFFSET, + frame_module.cookie, + "bt_step_state_next_module_cookie", + )?; let reached_depth = self .builder @@ -3243,6 +3379,129 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { .map_err(|e| CodeGenError::LLVMError(e.to_string())) } + fn resolve_backtrace_frame_module( + &mut self, + raw_ip: IntValue<'ctx>, + fallback_cookie: IntValue<'ctx>, + fallback_bias: IntValue<'ctx>, + fallback_found: IntValue<'ctx>, + name_prefix: &str, + ) -> Result> { + if !self.backtrace_unwind_rows_use_runtime_pcs || self.backtrace_module_cookies.is_empty() { + return Ok(BtFrameModule { + cookie: fallback_cookie, + bias: fallback_bias, + found: fallback_found, + }); + } + + let i64_type = self.context.i64_type(); + let mut cookie = fallback_cookie; + let mut bias = fallback_bias; + let mut found = fallback_found; + let candidates = self.backtrace_module_cookies.clone(); + for (idx, candidate_cookie) in candidates + .into_iter() + .take(BPF_BACKTRACE_MODULE_CANDIDATE_LIMIT) + .enumerate() + { + let lookup = self.lookup_proc_module_offsets_value( + candidate_cookie, + &format!("{name_prefix}_{idx}"), + )?; + let end = self + .builder + .build_int_add( + lookup.base, + lookup.size, + &format!("{name_prefix}_{idx}_end"), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let non_empty = self + .builder + .build_int_compare( + inkwell::IntPredicate::NE, + lookup.size, + i64_type.const_zero(), + &format!("{name_prefix}_{idx}_non_empty"), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let at_or_after_base = self + .builder + .build_int_compare( + inkwell::IntPredicate::UGE, + raw_ip, + lookup.base, + &format!("{name_prefix}_{idx}_after_base"), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let before_end = self + .builder + .build_int_compare( + inkwell::IntPredicate::ULT, + raw_ip, + end, + &format!("{name_prefix}_{idx}_before_end"), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let in_bounds = self + .builder + .build_and( + at_or_after_base, + before_end, + &format!("{name_prefix}_{idx}_in_bounds"), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let valid_range = self + .builder + .build_and( + lookup.found, + non_empty, + &format!("{name_prefix}_{idx}_range_has_offsets"), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let in_range = self + .builder + .build_and( + valid_range, + in_bounds, + &format!("{name_prefix}_{idx}_in_range"), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + cookie = self + .builder + .build_select::, _>( + in_range, + i64_type.const_int(candidate_cookie, false).into(), + cookie.into(), + &format!("{name_prefix}_{idx}_cookie"), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))? + .into_int_value(); + bias = self + .builder + .build_select::, _>( + in_range, + lookup.text.into(), + bias.into(), + &format!("{name_prefix}_{idx}_bias"), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))? + .into_int_value(); + found = self + .builder + .build_or(found, in_range, &format!("{name_prefix}_{idx}_found")) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + } + + Ok(BtFrameModule { + cookie, + bias, + found, + }) + } + fn backtrace_lookup_pc_from_raw( &self, raw_ip: IntValue<'ctx>, @@ -3349,14 +3608,14 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { &self, inst_buffer: PointerValue<'ctx>, frame_index: usize, - module_cookie: u64, + module_cookie: IntValue<'ctx>, pc: IntValue<'ctx>, raw_ip: IntValue<'ctx>, flags: u16, ) -> Result<()> { let frame_base = INSTRUCTION_HEADER_SIZE + BACKTRACE_DATA_SIZE + frame_index * BACKTRACE_FRAME_DATA_SIZE; - self.store_u64_const( + self.store_u64_value( inst_buffer, frame_base + BACKTRACE_FRAME_MODULE_COOKIE_OFFSET, module_cookie, @@ -3387,7 +3646,7 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { inst_buffer: PointerValue<'ctx>, frame_index: IntValue<'ctx>, max_frame_index: u8, - module_cookie: u64, + module_cookie: IntValue<'ctx>, pc: IntValue<'ctx>, raw_ip: IntValue<'ctx>, ) -> Result<()> { @@ -3436,7 +3695,7 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { let frame_base = self.dynamic_byte_gep(inst_buffer, frame_base_offset, "bt_dynamic_frame")?; - self.store_u64_const( + self.store_u64_value( frame_base, BACKTRACE_FRAME_MODULE_COOKIE_OFFSET, module_cookie, @@ -3523,21 +3782,6 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { Ok(()) } - fn store_u64_const( - &self, - base: PointerValue<'ctx>, - offset: usize, - value: u64, - name: &str, - ) -> Result<()> { - self.store_u64_value( - base, - offset, - self.context.i64_type().const_int(value, false), - name, - ) - } - fn store_u64_value( &self, base: PointerValue<'ctx>, @@ -3755,7 +3999,9 @@ fn backtrace_flags(stmt: &BacktraceStatement) -> u8 { #[cfg(test)] mod tests { - use super::backtrace_row_binary_search_steps; + use super::*; + use crate::CompileOptions; + use inkwell::AddressSpace; #[test] fn binary_search_steps_cover_power_of_two_row_counts() { @@ -3772,4 +4018,63 @@ mod tests { assert_eq!(backtrace_row_binary_search_steps(5), 4); assert_eq!(backtrace_row_binary_search_steps(9), 5); } + + #[test] + fn runtime_backtrace_frame_module_resolution_generates_range_lookups() { + let context = inkwell::context::Context::create(); + let opts = CompileOptions::default(); + let mut ctx = + EbpfContext::new(&context, "bt_frame_module_test", Some(0), &opts).expect("ctx"); + let i64_type = context.i64_type(); + let map_global = ctx.module.add_global( + i64_type, + Some(AddressSpace::default()), + "proc_module_offsets", + ); + map_global.set_initializer(&i64_type.const_zero()); + + let fn_type = context.i32_type().fn_type(&[], false); + let function = ctx.module.add_function("bt_frame_module", fn_type, None); + let entry = context.append_basic_block(function, "entry"); + ctx.builder.position_at_end(entry); + let key_type = context.i32_type().array_type(4); + ctx.pm_key_alloca = Some( + ctx.builder + .build_alloca(key_type, "pm_key") + .expect("pm_key alloca"), + ); + ctx.backtrace_unwind_rows_use_runtime_pcs = true; + ctx.backtrace_module_cookies = vec![0x1111, 0x2222]; + + let frame_module = ctx + .resolve_backtrace_frame_module( + i64_type.const_int(0x7f00_1234, false), + i64_type.const_int(0x1111, false), + i64_type.const_zero(), + context.bool_type().const_zero(), + "test_bt_frame_module", + ) + .expect("resolve frame module"); + ctx.store_u64_value( + ctx.pm_key_alloca.expect("pm key"), + 0, + frame_module.cookie, + "selected_cookie", + ) + .expect("store selected cookie"); + + let ir = ctx.module.print_to_string().to_string(); + assert!( + ir.contains("test_bt_frame_module_0_base") + && ir.contains("test_bt_frame_module_0_size") + && ir.contains("test_bt_frame_module_1_base") + && ir.contains("test_bt_frame_module_1_size"), + "resolver should load base/size for every module candidate\nIR:\n{ir}" + ); + assert!( + ir.contains("test_bt_frame_module_0_cookie") + && ir.contains("test_bt_frame_module_1_cookie"), + "resolver should select a per-frame module cookie\nIR:\n{ir}" + ); + } } diff --git a/ghostscope-compiler/src/ebpf/context.rs b/ghostscope-compiler/src/ebpf/context.rs index b615a5a..5fbd526 100644 --- a/ghostscope-compiler/src/ebpf/context.rs +++ b/ghostscope-compiler/src/ebpf/context.rs @@ -34,7 +34,6 @@ pub struct BacktraceTailCallProgram { #[derive(Debug, Clone)] pub(crate) struct PendingBacktraceTailCall { pub step_program_name: String, - pub module_cookie: u64, pub depth: u8, pub instruction_size: usize, } @@ -162,6 +161,7 @@ pub struct EbpfContext<'ctx, 'dw> { // === DWARF compact unwind rows for bt === pub backtrace_unwind_rows: Vec, + pub(crate) backtrace_module_cookies: Vec, pub(crate) backtrace_unwind_rows_use_runtime_pcs: bool, pub(crate) backtrace_tail_call_slots: u8, pub(crate) next_backtrace_tail_call_slot: u8, @@ -273,6 +273,7 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { string_vars: HashMap::new(), // Backtrace compact unwind rows backtrace_unwind_rows: Vec::new(), + backtrace_module_cookies: Vec::new(), backtrace_unwind_rows_use_runtime_pcs: false, backtrace_tail_call_slots: 1, next_backtrace_tail_call_slot: 0, diff --git a/ghostscope-compiler/src/ebpf/helper_functions.rs b/ghostscope-compiler/src/ebpf/helper_functions.rs index b18ad69..2647334 100644 --- a/ghostscope-compiler/src/ebpf/helper_functions.rs +++ b/ghostscope-compiler/src/ebpf/helper_functions.rs @@ -22,6 +22,16 @@ struct ProbeReadResult<'ctx> { not_found: IntValue<'ctx>, } +pub(crate) struct ProcModuleOffsetsLookup<'ctx> { + pub(crate) found: IntValue<'ctx>, + pub(crate) text: IntValue<'ctx>, + pub(crate) rodata: IntValue<'ctx>, + pub(crate) data: IntValue<'ctx>, + pub(crate) bss: IntValue<'ctx>, + pub(crate) base: IntValue<'ctx>, + pub(crate) size: IntValue<'ctx>, +} + impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { fn get_or_create_tls_scratch_buffer(&mut self) -> Result> { if let Some(alloca) = self.tls_scratch_alloca { @@ -420,14 +430,11 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { .into_int_value(); Ok((buf_global, status, arr_ty)) } - /// Compute runtime address from link-time address using proc_module_offsets map - /// section_type: 0=text, 1=rodata, 2=data, 3=bss; other values fallback to data - pub fn generate_runtime_address_from_offsets( + pub(crate) fn lookup_proc_module_offsets_value( &mut self, - link_addr: IntValue<'ctx>, - section_type: u8, module_cookie: u64, - ) -> Result<(IntValue<'ctx>, IntValue<'ctx>)> { + name_prefix: &str, + ) -> Result> { const BPF_FUNC_GET_NS_CURRENT_PID_TGID: u64 = 120; const BPF_PIDNS_INFO_SIZE: u64 = 8; // struct { u32 pid; u32 tgid; } @@ -444,7 +451,7 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { let map_ptr = map_global.as_pointer_value(); let map_ptr_cast = self .builder - .build_bit_cast(map_ptr, ptr_type, "map_ptr") + .build_bit_cast(map_ptr, ptr_type, &format!("{name_prefix}_map_ptr")) .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; // Use per-invocation key buffer [4 x u32] pre-allocated in entry block @@ -461,11 +468,16 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { let helper_fn_type = i64_type.fn_type(&[], false); let helper_fn_ptr = self .builder - .build_int_to_ptr(helper_id, ptr_type, "get_pid_fn") + .build_int_to_ptr(helper_id, ptr_type, &format!("{name_prefix}_get_pid_fn")) .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; let pid_tgid = self .builder - .build_indirect_call(helper_fn_type, helper_fn_ptr, &[], "pid_tgid") + .build_indirect_call( + helper_fn_type, + helper_fn_ptr, + &[], + &format!("{name_prefix}_pid_tgid"), + ) .map_err(|e| CodeGenError::LLVMError(e.to_string()))? .try_as_basic_value() .left() @@ -476,10 +488,15 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { // pid = upper 32 bits let shifted = self .builder - .build_right_shift(v, i64_type.const_int(32, false), false, "pid_shift") + .build_right_shift( + v, + i64_type.const_int(32, false), + false, + &format!("{name_prefix}_pid_shift"), + ) .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; self.builder - .build_int_truncate(shifted, i32_type, "pid32") + .build_int_truncate(shifted, i32_type, &format!("{name_prefix}_pid32")) .map_err(|e| CodeGenError::LLVMError(e.to_string()))? } else { return Err(CodeGenError::LLVMError( @@ -499,7 +516,11 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; let pidns_info_ptr = self .builder - .build_bit_cast(key_alloca, ptr_type, "offset_pidns_info_ptr") + .build_bit_cast( + key_alloca, + ptr_type, + &format!("{name_prefix}_pidns_info_ptr"), + ) .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; let helper_args = [ i64_type.const_int(pid_ns_dev, false).into(), @@ -511,7 +532,7 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { BPF_FUNC_GET_NS_CURRENT_PID_TGID, &helper_args, i64_type.into(), - "offset_ns_pid_tgid_ret", + &format!("{name_prefix}_ns_pid_tgid_ret"), )?; let helper_ret = match helper_ret { BasicValueEnum::IntValue(v) => v, @@ -527,7 +548,7 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { inkwell::IntPredicate::EQ, helper_ret, i64_type.const_zero(), - "offset_ns_helper_ok", + &format!("{name_prefix}_ns_helper_ok"), ) .map_err(|e| CodeGenError::Builder(e.to_string()))?; // SAFETY: key_alloca temporarily holds the two-field pid namespace @@ -537,13 +558,13 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { key_arr_ty, key_alloca, &[i32_type.const_zero(), i32_type.const_int(1, false)], - "offset_ns_tgid_ptr", + &format!("{name_prefix}_ns_tgid_ptr"), ) } .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; let ns_tgid = self .builder - .build_load(i32_type, ns_tgid_ptr, "offset_ns_tgid") + .build_load(i32_type, ns_tgid_ptr, &format!("{name_prefix}_ns_tgid")) .map_err(|e| CodeGenError::LLVMError(e.to_string()))? .into_int_value(); // The proc_module_offsets map is populated from `/proc//maps`, @@ -551,13 +572,18 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { // those `/proc` reads. This is *not* necessarily the same namespace // used for `$pid`/`$tid` or NamespaceTgid filtering. self.builder - .build_select(helper_ok, ns_tgid, host_tgid, "offset_pid_key") + .build_select( + helper_ok, + ns_tgid, + host_tgid, + &format!("{name_prefix}_pid_key"), + ) .map_err(|e| CodeGenError::Builder(e.to_string()))? .into_int_value() } else { host_tgid }; - let pid = self.lookup_proc_pid_alias(runtime_pid, "offset_pid")?; + let pid = self.lookup_proc_pid_alias(runtime_pid, name_prefix)?; let store_key_u32 = |offset: usize, value: IntValue<'ctx>, @@ -581,13 +607,13 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { store_key_u32( ghostscope_protocol::PROC_MODULE_KEY_PID_OFFSET, pid, - "pm_key_pid_ptr", + &format!("{name_prefix}_pm_key_pid_ptr"), self, )?; store_key_u32( ghostscope_protocol::PROC_MODULE_KEY_PAD_OFFSET, self.context.i32_type().const_zero(), - "pm_key_pad_ptr", + &format!("{name_prefix}_pm_key_pad_ptr"), self, )?; @@ -596,13 +622,13 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { store_key_u32( ghostscope_protocol::PROC_MODULE_KEY_COOKIE_LO_OFFSET, cookie_lo, - "pm_key_cookie_lo_ptr", + &format!("{name_prefix}_pm_key_cookie_lo_ptr"), self, )?; store_key_u32( ghostscope_protocol::PROC_MODULE_KEY_COOKIE_HI_OFFSET, cookie_hi, - "pm_key_cookie_hi_ptr", + &format!("{name_prefix}_pm_key_cookie_hi_ptr"), self, )?; @@ -611,17 +637,22 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { let lookup_fn_type = ptr_type.fn_type(&[ptr_type.into(), ptr_type.into()], false); let lookup_fn_ptr = self .builder - .build_int_to_ptr(lookup_id, ptr_type, "lookup_fn") + .build_int_to_ptr(lookup_id, ptr_type, &format!("{name_prefix}_lookup_fn")) .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; // Pass pointer to the beginning of the key buffer (void*) let key_arg = self .builder - .build_bit_cast(key_alloca, ptr_type, "key_arg") + .build_bit_cast(key_alloca, ptr_type, &format!("{name_prefix}_key_arg")) .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; let args: Vec = vec![map_ptr_cast.into(), key_arg.into()]; let val_ptr_any = self .builder - .build_indirect_call(lookup_fn_type, lookup_fn_ptr, &args, "val_ptr_any") + .build_indirect_call( + lookup_fn_type, + lookup_fn_ptr, + &args, + &format!("{name_prefix}_val_ptr_any"), + ) .map_err(|e| CodeGenError::LLVMError(e.to_string()))? .try_as_basic_value() .left() @@ -629,9 +660,15 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { let null_ptr = ptr_type.const_null(); let current_fn = self.current_function("generate module offset lookup")?; - let found_block = self.context.append_basic_block(current_fn, "found_offsets"); - let miss_block = self.context.append_basic_block(current_fn, "miss_offsets"); - let cont_block = self.context.append_basic_block(current_fn, "cont_offsets"); + let found_block = self + .context + .append_basic_block(current_fn, &format!("{name_prefix}_found_offsets")); + let miss_block = self + .context + .append_basic_block(current_fn, &format!("{name_prefix}_miss_offsets")); + let cont_block = self + .context + .append_basic_block(current_fn, &format!("{name_prefix}_cont_offsets")); // Compare against NULL let val_ptr = if let BasicValueEnum::PointerValue(p) = val_ptr_any { @@ -644,21 +681,22 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { .build_int_compare( inkwell::IntPredicate::EQ, self.builder - .build_ptr_to_int(val_ptr, i64_type, "val_ptr_i64") + .build_ptr_to_int(val_ptr, i64_type, &format!("{name_prefix}_val_ptr_i64")) .map_err(|e| CodeGenError::LLVMError(e.to_string()))?, i64_type.const_zero(), - "is_null_offsets", + &format!("{name_prefix}_is_null_offsets"), ) .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; self.builder .build_conditional_branch(is_null, miss_block, found_block) .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - // Found: load the requested field from ProcModuleOffsetsValue. The + // Found: load fields from ProcModuleOffsetsValue. The // byte offsets are shared through ghostscope_protocol::bpf_abi because // this map is an ABI between generated eBPF and userspace. self.builder.position_at_end(found_block); let load_field = |offset: usize, + field_name: &str, ctx: &mut EbpfContext<'ctx, 'dw>, base: PointerValue<'ctx>| -> Result> { @@ -672,7 +710,7 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { }; let loaded = ctx .builder - .build_load(ctx.context.i64_type(), field_ptr, "loaded_offset") + .build_load(ctx.context.i64_type(), field_ptr, field_name) .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; if let BasicValueEnum::IntValue(iv) = loaded { Ok(iv) @@ -680,32 +718,103 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { Err(CodeGenError::LLVMError("offset load failed".to_string())) } }; - let st = section_type; let off_text = load_field( ghostscope_protocol::PROC_MODULE_OFFSETS_VALUE_TEXT_OFFSET, + &format!("{name_prefix}_text"), self, val_ptr, )?; let off_rodata = load_field( ghostscope_protocol::PROC_MODULE_OFFSETS_VALUE_RODATA_OFFSET, + &format!("{name_prefix}_rodata"), self, val_ptr, )?; let off_data = load_field( ghostscope_protocol::PROC_MODULE_OFFSETS_VALUE_DATA_OFFSET, + &format!("{name_prefix}_data"), self, val_ptr, )?; let off_bss = load_field( ghostscope_protocol::PROC_MODULE_OFFSETS_VALUE_BSS_OFFSET, + &format!("{name_prefix}_bss"), + self, + val_ptr, + )?; + let base = load_field( + ghostscope_protocol::PROC_MODULE_OFFSETS_VALUE_BASE_OFFSET, + &format!("{name_prefix}_base"), + self, + val_ptr, + )?; + let size = load_field( + ghostscope_protocol::PROC_MODULE_OFFSETS_VALUE_SIZE_OFFSET, + &format!("{name_prefix}_size"), self, val_ptr, )?; + self.builder + .build_unconditional_branch(cont_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let found_end = self.current_insert_block("finish proc offsets found block")?; + + self.builder.position_at_end(miss_block); + self.builder + .build_unconditional_branch(cont_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let miss_end = self.current_insert_block("finish proc offsets miss block")?; + + self.builder.position_at_end(cont_block); + let zero_i64 = i64_type.const_zero(); + let phi_i64 = |ctx: &mut EbpfContext<'ctx, 'dw>, + name: &str, + hit_value: IntValue<'ctx>| + -> Result> { + let phi = ctx + .builder + .build_phi(i64_type, name) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + phi.add_incoming(&[(&hit_value, found_end), (&zero_i64, miss_end)]); + Ok(phi.as_basic_value().into_int_value()) + }; + let found_type = self.context.bool_type(); + let found_phi = self + .builder + .build_phi(found_type, &format!("{name_prefix}_found_phi")) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + found_phi.add_incoming(&[ + (&found_type.const_int(1, false), found_end), + (&found_type.const_zero(), miss_end), + ]); + + Ok(ProcModuleOffsetsLookup { + found: found_phi.as_basic_value().into_int_value(), + text: phi_i64(self, &format!("{name_prefix}_text_phi"), off_text)?, + rodata: phi_i64(self, &format!("{name_prefix}_rodata_phi"), off_rodata)?, + data: phi_i64(self, &format!("{name_prefix}_data_phi"), off_data)?, + bss: phi_i64(self, &format!("{name_prefix}_bss_phi"), off_bss)?, + base: phi_i64(self, &format!("{name_prefix}_base_phi"), base)?, + size: phi_i64(self, &format!("{name_prefix}_size_phi"), size)?, + }) + } + + /// Compute runtime address from link-time address using proc_module_offsets map + /// section_type: 0=text, 1=rodata, 2=data, 3=bss; other values fallback to data + pub fn generate_runtime_address_from_offsets( + &mut self, + link_addr: IntValue<'ctx>, + section_type: u8, + module_cookie: u64, + ) -> Result<(IntValue<'ctx>, IntValue<'ctx>)> { + let i32_type = self.context.i32_type(); + let offsets = self.lookup_proc_module_offsets_value(module_cookie, "offset")?; + // Build a bottom-up cascade to preserve earlier choices: // tmp = (section==data) ? off_data : off_bss // tmp2 = (section==rodata) ? off_rodata : tmp // off = (section==text) ? off_text : tmp2 - let st_val = i32_type.const_int(st as u64, false); + let st_val = i32_type.const_int(section_type as u64, false); let eq_text = self .builder .build_int_compare( @@ -736,56 +845,37 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { let tmp_any = self .builder - .build_select(eq_da, off_data, off_bss, "sel_data_bss") + .build_select(eq_da, offsets.data, offsets.bss, "sel_data_bss") .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; let tmp = tmp_any.into_int_value(); let tmp2_any = self .builder - .build_select(eq_ro, off_rodata, tmp, "sel_rodata_else") + .build_select(eq_ro, offsets.rodata, tmp, "sel_rodata_else") .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; let tmp2 = tmp2_any.into_int_value(); let off_final_any = self .builder - .build_select(eq_text, off_text, tmp2, "sel_text_else") + .build_select(eq_text, offsets.text, tmp2, "sel_text_else") .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; let off_final = off_final_any.into_int_value(); let rt_addr = self .builder .build_int_add(link_addr, off_final, "runtime_addr") .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - self.builder - .build_unconditional_branch(cont_block) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - - // Miss: return link_addr as-is (will likely fault and set ReadError) - self.builder.position_at_end(miss_block); - self.builder - .build_unconditional_branch(cont_block) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - - // Phi to merge address - self.builder.position_at_end(cont_block); - let phi = self - .builder - .build_phi(i64_type, "addr_phi") - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - phi.add_incoming(&[(&rt_addr, found_block), (&link_addr, miss_block)]); - let final_addr = phi.as_basic_value().into_int_value(); - - // Phi to merge found-flag (i1): 1 on found, 0 on miss - let i1_type = self.context.bool_type(); - let one = i1_type.const_int(1, false); - let zero = i1_type.const_int(0, false); - let flag_phi = self + let final_addr = self .builder - .build_phi(i1_type, "off_found_phi") - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - flag_phi.add_incoming(&[(&one, found_block), (&zero, miss_block)]); - let found_flag = flag_phi.as_basic_value().into_int_value(); + .build_select::, _>( + offsets.found, + rt_addr.into(), + link_addr.into(), + "addr_or_link", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))? + .into_int_value(); - Ok((final_addr, found_flag)) + Ok((final_addr, offsets.found)) } /// Load a register value from pt_regs pub fn load_register_value( diff --git a/ghostscope-compiler/src/ebpf/maps.rs b/ghostscope-compiler/src/ebpf/maps.rs index dc6414e..4ef25e1 100644 --- a/ghostscope-compiler/src/ebpf/maps.rs +++ b/ghostscope-compiler/src/ebpf/maps.rs @@ -259,7 +259,7 @@ impl<'ctx> MapManager<'ctx> { max_entries: u64, ) -> Result<()> { // Key: {pid:u32, pad:u32, cookie:u64} => 16 bytes => 128 bits - // Value: {text, rodata, data, bss: u64} => 32 bytes => 256 bits + // Value: {text, rodata, data, bss, base, size: u64} => 48 bytes => 384 bits self.create_map_definition( module, di_builder, @@ -268,7 +268,7 @@ impl<'ctx> MapManager<'ctx> { BpfMapType::Hash, max_entries, SizedType::integer(128), - SizedType::integer(256), + SizedType::integer(384), ) } diff --git a/ghostscope-process/src/offsets.rs b/ghostscope-process/src/offsets.rs index f09f9b8..0dfef72 100644 --- a/ghostscope-process/src/offsets.rs +++ b/ghostscope-process/src/offsets.rs @@ -50,6 +50,8 @@ struct CachedEntry { pid: u32, cookie: u64, offsets: SectionOffsets, + base: u64, + size: u64, } #[derive(Debug, Clone, Default)] @@ -255,11 +257,13 @@ impl ProcessManager { 3, std::time::Duration::from_millis(75), ) { - Ok((cookie, offsets, _base, _size)) => { + Ok((cookie, offsets, base, size)) => { cached.push(CachedEntry { pid, cookie, offsets, + base, + size, }); new_count += 1; } @@ -276,10 +280,17 @@ impl ProcessManager { Ok(new_count) } - pub fn cached_offsets_for_module(&self, module_path: &str) -> Vec<(u32, u64, SectionOffsets)> { + pub fn cached_offsets_for_module( + &self, + module_path: &str, + ) -> Vec<(u32, u64, SectionOffsets, u64, u64)> { self.module_cache .get(module_path) - .map(|v| v.iter().map(|e| (e.pid, e.cookie, e.offsets)).collect()) + .map(|v| { + v.iter() + .map(|e| (e.pid, e.cookie, e.offsets, e.base, e.size)) + .collect() + }) .unwrap_or_default() } @@ -649,11 +660,15 @@ mod tests { pid: 42, cookie: 1, offsets: SectionOffsets::default(), + base: 0, + size: 0, }, CachedEntry { pid: 7, cookie: 2, offsets: SectionOffsets::default(), + base: 0, + size: 0, }, ], ); diff --git a/ghostscope-process/src/pinned_bpf_maps.rs b/ghostscope-process/src/pinned_bpf_maps.rs index e2fb0bc..5a05a1e 100644 --- a/ghostscope-process/src/pinned_bpf_maps.rs +++ b/ghostscope-process/src/pinned_bpf_maps.rs @@ -206,6 +206,19 @@ pub fn bpffs_mount_hint_for_pin_path(pin_path: &Path) -> Option { pub use ghostscope_protocol::{PidAliasValue, ProcModuleKey, ProcModuleOffsetsValue}; +fn proc_offsets_pin_layout_matches(map: &MapData) -> bool { + match map.info() { + Ok(info) => { + info.key_size() == ghostscope_protocol::PROC_MODULE_KEY_SIZE as u32 + && info.value_size() == ghostscope_protocol::PROC_MODULE_OFFSETS_VALUE_SIZE as u32 + } + Err(e) => { + warn!("Unable to inspect pinned proc_module_offsets map layout: {e}"); + false + } + } +} + fn ensure_pin_dir(path: &Path) -> std::io::Result<()> { if let Some(dir) = path.parent() { std::fs::create_dir_all(dir) @@ -232,17 +245,28 @@ pub fn ensure_pinned_proc_offsets_exists(max_entries: u32) -> anyhow::Result<()> ) })?; - // If pinned file already exists, try to reuse it directly (idempotent) + // If pinned file already exists, reuse it only when the ABI layout matches. if pin_path.exists() { - if MapData::from_pin(&pin_path).is_ok() { - info!( - "Reusing existing pinned map at {} (no recreate)", - pin_path.display() - ); - return Ok(()); - } else { - // Stale/corrupted pin path, remove and recreate - let _ = std::fs::remove_file(&pin_path); + match MapData::from_pin(&pin_path) { + Ok(map) if proc_offsets_pin_layout_matches(&map) => { + info!( + "Reusing existing pinned map at {} (layout ok)", + pin_path.display() + ); + return Ok(()); + } + Ok(_) => { + warn!( + "Pinned {} at {} has stale ABI layout; recreating", + PROC_OFFSETS_MAP_NAME, + pin_path.display() + ); + let _ = std::fs::remove_file(&pin_path); + } + Err(_) => { + // Stale/corrupted pin path, remove and recreate + let _ = std::fs::remove_file(&pin_path); + } } } @@ -280,7 +304,7 @@ pub fn ensure_pinned_proc_offsets_exists(max_entries: u32) -> anyhow::Result<()> Err(e) => { // If another thread/process pinned concurrently, reuse the existing pin match MapData::from_pin(&pin_path) { - Ok(_) => { + Ok(map) if proc_offsets_pin_layout_matches(&map) => { info!( "Pin path {} already exists; reusing existing map ({}).", pin_path.display(), @@ -288,7 +312,7 @@ pub fn ensure_pinned_proc_offsets_exists(max_entries: u32) -> anyhow::Result<()> ); Ok(()) } - Err(_) => { + Ok(_) | Err(_) => { // Best-effort cleanup and propagate error let _ = std::fs::remove_file(&pin_path); let hint = bpffs_mount_hint_for_pin_path(&pin_path) @@ -580,8 +604,8 @@ pub fn insert_offsets_for_pid( match map.insert(key, *off, 0) { Ok(()) => { tracing::debug!( - "proc_module_offsets insert ok: pid={} cookie=0x{:08x}{:08x} text=0x{:x} rodata=0x{:x} data=0x{:x} bss=0x{:x}", - pid, key.cookie_hi, key.cookie_lo, off.text, off.rodata, off.data, off.bss + "proc_module_offsets insert ok: pid={} cookie=0x{:08x}{:08x} text=0x{:x} rodata=0x{:x} data=0x{:x} bss=0x{:x} base=0x{:x} size=0x{:x}", + pid, key.cookie_hi, key.cookie_lo, off.text, off.rodata, off.data, off.bss, off.base, off.size ); inserted += 1 } diff --git a/ghostscope-process/src/sysmon.rs b/ghostscope-process/src/sysmon.rs index 6ca3b1e..268d868 100644 --- a/ghostscope-process/src/sysmon.rs +++ b/ghostscope-process/src/sysmon.rs @@ -634,10 +634,12 @@ fn run_sysmon_loop( use std::collections::HashMap; let mut by_pid: HashMap> = HashMap::new(); - for (pid, cookie, off) in entries { + for (pid, cookie, off, base, size) in entries { by_pid.entry(pid).or_default().push(( cookie, - ProcModuleOffsetsValue::new(off.text, off.rodata, off.data, off.bss), + ProcModuleOffsetsValue::new( + off.text, off.rodata, off.data, off.bss, base, size, + ), )); } let mut total = 0usize; @@ -954,10 +956,14 @@ fn prefill_offsets_for_pid( } let mut by_pid: HashMap> = HashMap::new(); - for (pid, cookie, off) in guard.cached_offsets_for_module(&module_path) { + for (pid, cookie, off, base, size) in + guard.cached_offsets_for_module(&module_path) + { by_pid.entry(pid).or_default().push(( cookie, - ProcModuleOffsetsValue::new(off.text, off.rodata, off.data, off.bss), + ProcModuleOffsetsValue::new( + off.text, off.rodata, off.data, off.bss, base, size, + ), )); } for (pid, items) in by_pid { @@ -1023,6 +1029,8 @@ fn prefill_offsets_for_pid( e.offsets.rodata, e.offsets.data, e.offsets.bss, + e.base, + e.size, ), ) }) @@ -1094,10 +1102,10 @@ fn refresh_target_module_offsets( ); return; } - for (pid, cookie, off) in guard.cached_offsets_for_module(&module_path) { + for (pid, cookie, off, base, size) in guard.cached_offsets_for_module(&module_path) { by_pid.entry(pid).or_default().push(( cookie, - ProcModuleOffsetsValue::new(off.text, off.rodata, off.data, off.bss), + ProcModuleOffsetsValue::new(off.text, off.rodata, off.data, off.bss, base, size), )); } } diff --git a/ghostscope-protocol/src/bpf_abi.rs b/ghostscope-protocol/src/bpf_abi.rs index 2ee0182..df3edfe 100644 --- a/ghostscope-protocol/src/bpf_abi.rs +++ b/ghostscope-protocol/src/bpf_abi.rs @@ -38,15 +38,19 @@ pub struct ProcModuleOffsetsValue { pub rodata: u64, pub data: u64, pub bss: u64, + pub base: u64, + pub size: u64, } impl ProcModuleOffsetsValue { - pub fn new(text: u64, rodata: u64, data: u64, bss: u64) -> Self { + pub fn new(text: u64, rodata: u64, data: u64, bss: u64, base: u64, size: u64) -> Self { Self { text, rodata, data, bss, + base, + size, } } } @@ -59,6 +63,10 @@ pub const PROC_MODULE_OFFSETS_VALUE_DATA_OFFSET: usize = std::mem::offset_of!(ProcModuleOffsetsValue, data); pub const PROC_MODULE_OFFSETS_VALUE_BSS_OFFSET: usize = std::mem::offset_of!(ProcModuleOffsetsValue, bss); +pub const PROC_MODULE_OFFSETS_VALUE_BASE_OFFSET: usize = + std::mem::offset_of!(ProcModuleOffsetsValue, base); +pub const PROC_MODULE_OFFSETS_VALUE_SIZE_OFFSET: usize = + std::mem::offset_of!(ProcModuleOffsetsValue, size); pub const PROC_MODULE_OFFSETS_VALUE_SIZE: usize = std::mem::size_of::(); /// Value for the pinned `pid_aliases` map. @@ -263,11 +271,13 @@ mod tests { assert_eq!(PROC_MODULE_KEY_COOKIE_LO_OFFSET, 8); assert_eq!(PROC_MODULE_KEY_COOKIE_HI_OFFSET, 12); - assert_eq!(PROC_MODULE_OFFSETS_VALUE_SIZE, 32); + assert_eq!(PROC_MODULE_OFFSETS_VALUE_SIZE, 48); assert_eq!(PROC_MODULE_OFFSETS_VALUE_TEXT_OFFSET, 0); assert_eq!(PROC_MODULE_OFFSETS_VALUE_RODATA_OFFSET, 8); assert_eq!(PROC_MODULE_OFFSETS_VALUE_DATA_OFFSET, 16); assert_eq!(PROC_MODULE_OFFSETS_VALUE_BSS_OFFSET, 24); + assert_eq!(PROC_MODULE_OFFSETS_VALUE_BASE_OFFSET, 32); + assert_eq!(PROC_MODULE_OFFSETS_VALUE_SIZE_OFFSET, 40); assert_eq!(PID_ALIAS_VALUE_SIZE, 4); assert_eq!(PID_ALIAS_VALUE_PROC_PID_OFFSET, 0); diff --git a/ghostscope-protocol/src/lib.rs b/ghostscope-protocol/src/lib.rs index 1ac8180..594a485 100644 --- a/ghostscope-protocol/src/lib.rs +++ b/ghostscope-protocol/src/lib.rs @@ -58,9 +58,10 @@ pub use bpf_abi::{ BACKTRACE_UNWIND_WORD_RBP_OFFSET, BACKTRACE_UNWIND_WORD_REGISTERS, PID_ALIAS_VALUE_PROC_PID_OFFSET, PID_ALIAS_VALUE_SIZE, PROC_MODULE_KEY_COOKIE_HI_OFFSET, PROC_MODULE_KEY_COOKIE_LO_OFFSET, PROC_MODULE_KEY_PAD_OFFSET, PROC_MODULE_KEY_PID_OFFSET, - PROC_MODULE_KEY_SIZE, PROC_MODULE_OFFSETS_VALUE_BSS_OFFSET, - PROC_MODULE_OFFSETS_VALUE_DATA_OFFSET, PROC_MODULE_OFFSETS_VALUE_RODATA_OFFSET, - PROC_MODULE_OFFSETS_VALUE_SIZE, PROC_MODULE_OFFSETS_VALUE_TEXT_OFFSET, + PROC_MODULE_KEY_SIZE, PROC_MODULE_OFFSETS_VALUE_BASE_OFFSET, + PROC_MODULE_OFFSETS_VALUE_BSS_OFFSET, PROC_MODULE_OFFSETS_VALUE_DATA_OFFSET, + PROC_MODULE_OFFSETS_VALUE_RODATA_OFFSET, PROC_MODULE_OFFSETS_VALUE_SIZE, + PROC_MODULE_OFFSETS_VALUE_SIZE_OFFSET, PROC_MODULE_OFFSETS_VALUE_TEXT_OFFSET, }; pub use trace_context::TraceContext; diff --git a/ghostscope/src/script/compiler.rs b/ghostscope/src/script/compiler.rs index 3a712d2..df92a4c 100644 --- a/ghostscope/src/script/compiler.rs +++ b/ghostscope/src/script/compiler.rs @@ -74,16 +74,25 @@ fn apply_cached_offsets_for_session_pid( .coordinator .lock() .expect("coordinator mutex poisoned"); - coordinator.cached_offsets_pairs_for_pid(proc_pid) + coordinator + .cached_offsets_with_paths_for_pid(proc_pid) + .map(|entries| entries.to_vec()) }; if let Some(items) = items { use ghostscope_process::pinned_bpf_maps::ProcModuleOffsetsValue; let adapted: Vec<(u64, ProcModuleOffsetsValue)> = items .iter() - .map(|(cookie, off)| { + .map(|entry| { ( - *cookie, - ProcModuleOffsetsValue::new(off.text, off.rodata, off.data, off.bss), + entry.cookie, + ProcModuleOffsetsValue::new( + entry.offsets.text, + entry.offsets.rodata, + entry.offsets.data, + entry.offsets.bss, + entry.base, + entry.size, + ), ) }) .collect(); @@ -214,10 +223,12 @@ async fn create_and_attach_loader( // Group by pid for efficient batch insert use std::collections::HashMap; let mut by_pid: HashMap> = HashMap::new(); - for (pid, cookie, off) in entries { + for (pid, cookie, off, base, size) in entries { by_pid.entry(pid).or_default().push(( cookie, - ProcModuleOffsetsValue::new(off.text, off.rodata, off.data, off.bss), + ProcModuleOffsetsValue::new( + off.text, off.rodata, off.data, off.bss, base, size, + ), )); } let mut total = 0usize;