diff --git a/ghostscope-compiler/src/ebpf/codegen/backtrace.rs b/ghostscope-compiler/src/ebpf/codegen/backtrace.rs index 1baa3f4..9a95803 100644 --- a/ghostscope-compiler/src/ebpf/codegen/backtrace.rs +++ b/ghostscope-compiler/src/ebpf/codegen/backtrace.rs @@ -16,6 +16,7 @@ const BPF_INLINE_BACKTRACE_FRAME_LIMIT: u8 = 5; const BPF_BACKTRACE_FRAMES_PER_TAIL_CALL: u8 = 4; const BPF_BACKTRACE_MAX_STEP_INVOCATIONS: u8 = 32; const BPF_BACKTRACE_STEP_PROG_INDEX: u32 = 0; +const BPF_BACKTRACE_MODULE_CANDIDATE_LIMIT: usize = 16; struct RuntimeBtUnwindRow<'ctx> { found: IntValue<'ctx>, @@ -29,6 +30,12 @@ struct RuntimeBtUnwindRow<'ctx> { rbp_offset: IntValue<'ctx>, } +struct BtFrameModule<'ctx> { + cookie: IntValue<'ctx>, + bias: IntValue<'ctx>, + found: IntValue<'ctx>, +} + struct RuntimeBtRowScratch<'ctx> { found_ptr: PointerValue<'ctx>, cfa_register_ptr: PointerValue<'ctx>, @@ -77,6 +84,7 @@ enum BacktraceUnwindRowForPc { impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { pub(crate) fn prepare_backtrace_unwind_rows(&mut self, statements: &[Statement]) { self.backtrace_unwind_rows.clear(); + self.backtrace_module_cookies.clear(); self.backtrace_unwind_rows_use_runtime_pcs = false; self.backtrace_tail_call_slots = 1; self.next_backtrace_tail_call_slot = 0; @@ -132,7 +140,7 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { } fn runtime_backtrace_unwind_rows( - &self, + &mut self, analyzer: &ghostscope_dwarf::DwarfAnalyzer, ) -> Vec { let started_at = Instant::now(); @@ -156,16 +164,31 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { wire.pc_end = wire.pc_end.checked_add(module_bias)?; Some(wire) })); + let bpf_rows = rows.len().saturating_sub(row_start); + if bpf_rows > 0 { + let module_path = module.module_path.to_string_lossy(); + let cookie = self.cookie_for_module_or_fallback(&module_path); + if !self.backtrace_module_cookies.contains(&cookie) { + self.backtrace_module_cookies.push(cookie); + } + } modules += 1; tracing::info!( module = %module.module_path.display(), compact_rows = table.rows.len(), - bpf_rows = rows.len().saturating_sub(row_start), + bpf_rows, elapsed_ms = module_started_at.elapsed().as_millis(), "Prepared bt unwind rows for module" ); } rows.sort_by_key(|row| (row.pc_start, row.pc_end)); + if self.backtrace_module_cookies.len() > BPF_BACKTRACE_MODULE_CANDIDATE_LIMIT { + tracing::warn!( + candidates = self.backtrace_module_cookies.len(), + limit = BPF_BACKTRACE_MODULE_CANDIDATE_LIMIT, + "Limiting bt frame module range lookup candidates" + ); + } tracing::info!( modules, rows = rows.len(), @@ -269,6 +292,7 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { }; let module_cookie = self.cookie_for_module_or_fallback(&compile_ctx.module_path); + let module_cookie_value = self.context.i64_type().const_int(module_cookie, false); let pt_regs = self.get_pt_regs_parameter()?; let raw_ip = self.load_dwarf_register_i64(X86_64_DWARF_RIP, pt_regs)?; let (module_bias, offsets_found) = self.generate_runtime_address_from_offsets( @@ -277,8 +301,20 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { module_cookie, )?; let normalized_pc = self.normalized_pc_from_raw(raw_ip, module_bias, offsets_found)?; + let caller_fallback_found = if self.backtrace_unwind_rows_use_runtime_pcs { + self.context.bool_type().const_zero() + } else { + offsets_found + }; - self.store_backtrace_frame(inst_buffer, 0, module_cookie, normalized_pc, raw_ip, 0)?; + self.store_backtrace_frame( + inst_buffer, + 0, + module_cookie_value, + normalized_pc, + raw_ip, + 0, + )?; if depth == 1 { let status = @@ -368,8 +404,16 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; self.builder.position_at_end(initial_store_block); - let next_pc = self.normalized_pc_from_raw(next.ip, module_bias, offsets_found)?; - self.store_backtrace_frame(inst_buffer, 1, module_cookie, next_pc, next.ip, 0)?; + let frame_module = self.resolve_backtrace_frame_module( + next.ip, + module_cookie_value, + module_bias, + caller_fallback_found, + "bt_initial_frame_module", + )?; + let next_pc = + self.normalized_pc_from_raw(next.ip, frame_module.bias, frame_module.found)?; + self.store_backtrace_frame(inst_buffer, 1, frame_module.cookie, next_pc, next.ip, 0)?; self.store_u8_const( inst_buffer, data_base + BACKTRACE_DATA_FRAME_COUNT_OFFSET, @@ -485,11 +529,19 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; self.builder.position_at_end(store_block); - let next_pc = self.normalized_pc_from_raw(next.ip, module_bias, offsets_found)?; + let frame_module = self.resolve_backtrace_frame_module( + next.ip, + module_cookie_value, + module_bias, + caller_fallback_found, + &format!("bt_frame_{frame_index}_module"), + )?; + let next_pc = + self.normalized_pc_from_raw(next.ip, frame_module.bias, frame_module.found)?; self.store_backtrace_frame( inst_buffer, frame_index as usize, - module_cookie, + frame_module.cookie, next_pc, next.ip, 0, @@ -619,6 +671,7 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { }; let module_cookie = self.cookie_for_module_or_fallback(&compile_ctx.module_path); + let module_cookie_value = self.context.i64_type().const_int(module_cookie, false); let pt_regs = self.get_pt_regs_parameter()?; let raw_ip = self.load_dwarf_register_i64(X86_64_DWARF_RIP, pt_regs)?; let initial_rsp = self.load_dwarf_register_i64(X86_64_DWARF_RSP, pt_regs)?; @@ -629,8 +682,20 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { module_cookie, )?; let normalized_pc = self.normalized_pc_from_raw(raw_ip, module_bias, offsets_found)?; + let caller_fallback_found = if self.backtrace_unwind_rows_use_runtime_pcs { + self.context.bool_type().const_zero() + } else { + offsets_found + }; - self.store_backtrace_frame(inst_buffer, 0, module_cookie, normalized_pc, raw_ip, 0)?; + self.store_backtrace_frame( + inst_buffer, + 0, + module_cookie_value, + normalized_pc, + raw_ip, + 0, + )?; if depth == 1 { let status = @@ -691,6 +756,15 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { let ip_ptr = self.build_entry_alloca(i64_type, "bt_tail_prefix_ip")?; let rsp_ptr = self.build_entry_alloca(i64_type, "bt_tail_prefix_rsp")?; let rbp_ptr = self.build_entry_alloca(i64_type, "bt_tail_prefix_rbp")?; + let module_bias_ptr = self.build_entry_alloca(i64_type, "bt_tail_prefix_module_bias")?; + let module_cookie_ptr = + self.build_entry_alloca(i64_type, "bt_tail_prefix_module_cookie")?; + self.builder + .build_store(module_bias_ptr, module_bias) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(module_cookie_ptr, module_cookie_value) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; let runtime_row = self.runtime_row_from_static(row); let state = BtRegisterState { @@ -733,8 +807,16 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; self.builder.position_at_end(initial_store_block); - let next_pc = self.normalized_pc_from_raw(next.ip, module_bias, offsets_found)?; - self.store_backtrace_frame(inst_buffer, 1, module_cookie, next_pc, next.ip, 0)?; + let frame_module = self.resolve_backtrace_frame_module( + next.ip, + module_cookie_value, + module_bias, + caller_fallback_found, + "bt_tail_initial_frame_module", + )?; + let next_pc = + self.normalized_pc_from_raw(next.ip, frame_module.bias, frame_module.found)?; + self.store_backtrace_frame(inst_buffer, 1, frame_module.cookie, next_pc, next.ip, 0)?; self.store_u8_const( inst_buffer, data_base + BACKTRACE_DATA_FRAME_COUNT_OFFSET, @@ -762,6 +844,12 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { self.builder .build_store(rbp_ptr, next.rbp) .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(module_bias_ptr, frame_module.bias) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(module_cookie_ptr, frame_module.cookie) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; if depth == 2 { self.builder @@ -774,9 +862,11 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { let prefix_depth = depth.min(BPF_INLINE_BACKTRACE_FRAME_LIMIT); for frame_index in 2..prefix_depth { let current_ip = self.load_i64(ip_ptr, "bt_tail_prefix_lookup_ip")?; + let current_module_bias = + self.load_i64(module_bias_ptr, "bt_tail_prefix_lookup_module_bias")?; let lookup_raw = self.add_signed_offset(current_ip, -1, "bt_tail_prefix_lookup_raw")?; let lookup_pc = - self.backtrace_lookup_pc_from_raw(lookup_raw, module_bias, offsets_found)?; + self.backtrace_lookup_pc_from_raw(lookup_raw, current_module_bias, offsets_found)?; let runtime_row = self.lookup_backtrace_unwind_row(lookup_pc, &scratch.row)?; let found_block = self .context @@ -842,11 +932,22 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; self.builder.position_at_end(store_block); - let next_pc = self.normalized_pc_from_raw(next.ip, module_bias, offsets_found)?; + let fallback_cookie = + self.load_i64(module_cookie_ptr, "bt_tail_prefix_module_cookie")?; + let fallback_bias = self.load_i64(module_bias_ptr, "bt_tail_prefix_module_bias")?; + let frame_module = self.resolve_backtrace_frame_module( + next.ip, + fallback_cookie, + fallback_bias, + caller_fallback_found, + &format!("bt_tail_prefix_frame_{frame_index}_module"), + )?; + let next_pc = + self.normalized_pc_from_raw(next.ip, frame_module.bias, frame_module.found)?; self.store_backtrace_frame( inst_buffer, frame_index as usize, - module_cookie, + frame_module.cookie, next_pc, next.ip, 0, @@ -878,6 +979,12 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { self.builder .build_store(rbp_ptr, next.rbp) .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(module_bias_ptr, frame_module.bias) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(module_cookie_ptr, frame_module.cookie) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; if frame_index + 1 == depth { self.builder @@ -903,7 +1010,6 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { self.pending_backtrace_tail_call = Some(crate::ebpf::context::PendingBacktraceTailCall { step_program_name, - module_cookie, depth, instruction_size, }); @@ -944,6 +1050,8 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { let tail_ip = self.load_i64(ip_ptr, "bt_tail_state_prefix_ip")?; let tail_rsp = self.load_i64(rsp_ptr, "bt_tail_state_prefix_rsp")?; let tail_rbp = self.load_i64(rbp_ptr, "bt_tail_state_prefix_rbp")?; + let tail_module_bias = self.load_i64(module_bias_ptr, "bt_tail_state_module_bias")?; + let tail_module_cookie = self.load_i64(module_cookie_ptr, "bt_tail_state_module_cookie")?; self.store_state_i64( state_ptr, crate::BACKTRACE_TAIL_STATE_CURRENT_IP_OFFSET, @@ -965,13 +1073,13 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { self.store_state_i64( state_ptr, crate::BACKTRACE_TAIL_STATE_MODULE_BIAS_OFFSET, - module_bias, + tail_module_bias, "bt_state_module_bias", )?; - self.store_u64_const( + self.store_u64_value( state_ptr, crate::BACKTRACE_TAIL_STATE_MODULE_COOKIE_OFFSET, - module_cookie, + tail_module_cookie, "bt_state_module_cookie", )?; self.store_state_i32( @@ -1274,7 +1382,6 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { for _ in 0..BPF_BACKTRACE_FRAMES_PER_TAIL_CALL { self.generate_backtrace_tail_call_step_iteration( plan.depth, - plan.module_cookie, state_ptr, inst_buffer, &scratch, @@ -1450,7 +1557,6 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { fn generate_backtrace_tail_call_step_iteration( &mut self, depth: u8, - module_cookie: u64, state_ptr: PointerValue<'ctx>, inst_buffer: PointerValue<'ctx>, scratch: &BtScratch<'ctx>, @@ -1513,6 +1619,11 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { crate::BACKTRACE_TAIL_STATE_MODULE_BIAS_OFFSET, "bt_step_module_bias", )?; + let module_cookie = self.load_row_i64( + state_ptr, + crate::BACKTRACE_TAIL_STATE_MODULE_COOKIE_OFFSET, + "bt_step_module_cookie", + )?; let offsets_found_u8 = self.load_row_i8( state_ptr, crate::BACKTRACE_TAIL_STATE_OFFSETS_FOUND_OFFSET, @@ -1527,6 +1638,11 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { "bt_step_offsets_found", ) .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let caller_fallback_found = if self.backtrace_unwind_rows_use_runtime_pcs { + self.context.bool_type().const_zero() + } else { + offsets_found + }; let is_first_unwind = self .builder .build_int_compare( @@ -1610,12 +1726,20 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; self.builder.position_at_end(store_block); - let next_pc = self.normalized_pc_from_raw(next.ip, module_bias, offsets_found)?; + let frame_module = self.resolve_backtrace_frame_module( + next.ip, + module_cookie, + module_bias, + caller_fallback_found, + "bt_step_frame_module", + )?; + let next_pc = + self.normalized_pc_from_raw(next.ip, frame_module.bias, frame_module.found)?; self.store_backtrace_frame_dynamic( inst_buffer, frame_count, depth.saturating_sub(1), - module_cookie, + frame_module.cookie, next_pc, next.ip, )?; @@ -1657,6 +1781,18 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { next.rbp, "bt_step_state_next_rbp", )?; + self.store_state_i64( + state_ptr, + crate::BACKTRACE_TAIL_STATE_MODULE_BIAS_OFFSET, + frame_module.bias, + "bt_step_state_next_module_bias", + )?; + self.store_state_i64( + state_ptr, + crate::BACKTRACE_TAIL_STATE_MODULE_COOKIE_OFFSET, + frame_module.cookie, + "bt_step_state_next_module_cookie", + )?; let reached_depth = self .builder @@ -3243,6 +3379,129 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { .map_err(|e| CodeGenError::LLVMError(e.to_string())) } + fn resolve_backtrace_frame_module( + &mut self, + raw_ip: IntValue<'ctx>, + fallback_cookie: IntValue<'ctx>, + fallback_bias: IntValue<'ctx>, + fallback_found: IntValue<'ctx>, + name_prefix: &str, + ) -> Result> { + if !self.backtrace_unwind_rows_use_runtime_pcs || self.backtrace_module_cookies.is_empty() { + return Ok(BtFrameModule { + cookie: fallback_cookie, + bias: fallback_bias, + found: fallback_found, + }); + } + + let i64_type = self.context.i64_type(); + let mut cookie = fallback_cookie; + let mut bias = fallback_bias; + let mut found = fallback_found; + let candidates = self.backtrace_module_cookies.clone(); + for (idx, candidate_cookie) in candidates + .into_iter() + .take(BPF_BACKTRACE_MODULE_CANDIDATE_LIMIT) + .enumerate() + { + let lookup = self.lookup_proc_module_offsets_value( + candidate_cookie, + &format!("{name_prefix}_{idx}"), + )?; + let end = self + .builder + .build_int_add( + lookup.base, + lookup.size, + &format!("{name_prefix}_{idx}_end"), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let non_empty = self + .builder + .build_int_compare( + inkwell::IntPredicate::NE, + lookup.size, + i64_type.const_zero(), + &format!("{name_prefix}_{idx}_non_empty"), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let at_or_after_base = self + .builder + .build_int_compare( + inkwell::IntPredicate::UGE, + raw_ip, + lookup.base, + &format!("{name_prefix}_{idx}_after_base"), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let before_end = self + .builder + .build_int_compare( + inkwell::IntPredicate::ULT, + raw_ip, + end, + &format!("{name_prefix}_{idx}_before_end"), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let in_bounds = self + .builder + .build_and( + at_or_after_base, + before_end, + &format!("{name_prefix}_{idx}_in_bounds"), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let valid_range = self + .builder + .build_and( + lookup.found, + non_empty, + &format!("{name_prefix}_{idx}_range_has_offsets"), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let in_range = self + .builder + .build_and( + valid_range, + in_bounds, + &format!("{name_prefix}_{idx}_in_range"), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + cookie = self + .builder + .build_select::, _>( + in_range, + i64_type.const_int(candidate_cookie, false).into(), + cookie.into(), + &format!("{name_prefix}_{idx}_cookie"), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))? + .into_int_value(); + bias = self + .builder + .build_select::, _>( + in_range, + lookup.text.into(), + bias.into(), + &format!("{name_prefix}_{idx}_bias"), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))? + .into_int_value(); + found = self + .builder + .build_or(found, in_range, &format!("{name_prefix}_{idx}_found")) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + } + + Ok(BtFrameModule { + cookie, + bias, + found, + }) + } + fn backtrace_lookup_pc_from_raw( &self, raw_ip: IntValue<'ctx>, @@ -3349,14 +3608,14 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { &self, inst_buffer: PointerValue<'ctx>, frame_index: usize, - module_cookie: u64, + module_cookie: IntValue<'ctx>, pc: IntValue<'ctx>, raw_ip: IntValue<'ctx>, flags: u16, ) -> Result<()> { let frame_base = INSTRUCTION_HEADER_SIZE + BACKTRACE_DATA_SIZE + frame_index * BACKTRACE_FRAME_DATA_SIZE; - self.store_u64_const( + self.store_u64_value( inst_buffer, frame_base + BACKTRACE_FRAME_MODULE_COOKIE_OFFSET, module_cookie, @@ -3387,7 +3646,7 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { inst_buffer: PointerValue<'ctx>, frame_index: IntValue<'ctx>, max_frame_index: u8, - module_cookie: u64, + module_cookie: IntValue<'ctx>, pc: IntValue<'ctx>, raw_ip: IntValue<'ctx>, ) -> Result<()> { @@ -3436,7 +3695,7 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { let frame_base = self.dynamic_byte_gep(inst_buffer, frame_base_offset, "bt_dynamic_frame")?; - self.store_u64_const( + self.store_u64_value( frame_base, BACKTRACE_FRAME_MODULE_COOKIE_OFFSET, module_cookie, @@ -3523,21 +3782,6 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { Ok(()) } - fn store_u64_const( - &self, - base: PointerValue<'ctx>, - offset: usize, - value: u64, - name: &str, - ) -> Result<()> { - self.store_u64_value( - base, - offset, - self.context.i64_type().const_int(value, false), - name, - ) - } - fn store_u64_value( &self, base: PointerValue<'ctx>, @@ -3755,7 +3999,9 @@ fn backtrace_flags(stmt: &BacktraceStatement) -> u8 { #[cfg(test)] mod tests { - use super::backtrace_row_binary_search_steps; + use super::*; + use crate::CompileOptions; + use inkwell::AddressSpace; #[test] fn binary_search_steps_cover_power_of_two_row_counts() { @@ -3772,4 +4018,63 @@ mod tests { assert_eq!(backtrace_row_binary_search_steps(5), 4); assert_eq!(backtrace_row_binary_search_steps(9), 5); } + + #[test] + fn runtime_backtrace_frame_module_resolution_generates_range_lookups() { + let context = inkwell::context::Context::create(); + let opts = CompileOptions::default(); + let mut ctx = + EbpfContext::new(&context, "bt_frame_module_test", Some(0), &opts).expect("ctx"); + let i64_type = context.i64_type(); + let map_global = ctx.module.add_global( + i64_type, + Some(AddressSpace::default()), + "proc_module_offsets", + ); + map_global.set_initializer(&i64_type.const_zero()); + + let fn_type = context.i32_type().fn_type(&[], false); + let function = ctx.module.add_function("bt_frame_module", fn_type, None); + let entry = context.append_basic_block(function, "entry"); + ctx.builder.position_at_end(entry); + let key_type = context.i32_type().array_type(4); + ctx.pm_key_alloca = Some( + ctx.builder + .build_alloca(key_type, "pm_key") + .expect("pm_key alloca"), + ); + ctx.backtrace_unwind_rows_use_runtime_pcs = true; + ctx.backtrace_module_cookies = vec![0x1111, 0x2222]; + + let frame_module = ctx + .resolve_backtrace_frame_module( + i64_type.const_int(0x7f00_1234, false), + i64_type.const_int(0x1111, false), + i64_type.const_zero(), + context.bool_type().const_zero(), + "test_bt_frame_module", + ) + .expect("resolve frame module"); + ctx.store_u64_value( + ctx.pm_key_alloca.expect("pm key"), + 0, + frame_module.cookie, + "selected_cookie", + ) + .expect("store selected cookie"); + + let ir = ctx.module.print_to_string().to_string(); + assert!( + ir.contains("test_bt_frame_module_0_base") + && ir.contains("test_bt_frame_module_0_size") + && ir.contains("test_bt_frame_module_1_base") + && ir.contains("test_bt_frame_module_1_size"), + "resolver should load base/size for every module candidate\nIR:\n{ir}" + ); + assert!( + ir.contains("test_bt_frame_module_0_cookie") + && ir.contains("test_bt_frame_module_1_cookie"), + "resolver should select a per-frame module cookie\nIR:\n{ir}" + ); + } } diff --git a/ghostscope-compiler/src/ebpf/context.rs b/ghostscope-compiler/src/ebpf/context.rs index b615a5a..5fbd526 100644 --- a/ghostscope-compiler/src/ebpf/context.rs +++ b/ghostscope-compiler/src/ebpf/context.rs @@ -34,7 +34,6 @@ pub struct BacktraceTailCallProgram { #[derive(Debug, Clone)] pub(crate) struct PendingBacktraceTailCall { pub step_program_name: String, - pub module_cookie: u64, pub depth: u8, pub instruction_size: usize, } @@ -162,6 +161,7 @@ pub struct EbpfContext<'ctx, 'dw> { // === DWARF compact unwind rows for bt === pub backtrace_unwind_rows: Vec, + pub(crate) backtrace_module_cookies: Vec, pub(crate) backtrace_unwind_rows_use_runtime_pcs: bool, pub(crate) backtrace_tail_call_slots: u8, pub(crate) next_backtrace_tail_call_slot: u8, @@ -273,6 +273,7 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { string_vars: HashMap::new(), // Backtrace compact unwind rows backtrace_unwind_rows: Vec::new(), + backtrace_module_cookies: Vec::new(), backtrace_unwind_rows_use_runtime_pcs: false, backtrace_tail_call_slots: 1, next_backtrace_tail_call_slot: 0, diff --git a/ghostscope-compiler/src/ebpf/helper_functions.rs b/ghostscope-compiler/src/ebpf/helper_functions.rs index b18ad69..2647334 100644 --- a/ghostscope-compiler/src/ebpf/helper_functions.rs +++ b/ghostscope-compiler/src/ebpf/helper_functions.rs @@ -22,6 +22,16 @@ struct ProbeReadResult<'ctx> { not_found: IntValue<'ctx>, } +pub(crate) struct ProcModuleOffsetsLookup<'ctx> { + pub(crate) found: IntValue<'ctx>, + pub(crate) text: IntValue<'ctx>, + pub(crate) rodata: IntValue<'ctx>, + pub(crate) data: IntValue<'ctx>, + pub(crate) bss: IntValue<'ctx>, + pub(crate) base: IntValue<'ctx>, + pub(crate) size: IntValue<'ctx>, +} + impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { fn get_or_create_tls_scratch_buffer(&mut self) -> Result> { if let Some(alloca) = self.tls_scratch_alloca { @@ -420,14 +430,11 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { .into_int_value(); Ok((buf_global, status, arr_ty)) } - /// Compute runtime address from link-time address using proc_module_offsets map - /// section_type: 0=text, 1=rodata, 2=data, 3=bss; other values fallback to data - pub fn generate_runtime_address_from_offsets( + pub(crate) fn lookup_proc_module_offsets_value( &mut self, - link_addr: IntValue<'ctx>, - section_type: u8, module_cookie: u64, - ) -> Result<(IntValue<'ctx>, IntValue<'ctx>)> { + name_prefix: &str, + ) -> Result> { const BPF_FUNC_GET_NS_CURRENT_PID_TGID: u64 = 120; const BPF_PIDNS_INFO_SIZE: u64 = 8; // struct { u32 pid; u32 tgid; } @@ -444,7 +451,7 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { let map_ptr = map_global.as_pointer_value(); let map_ptr_cast = self .builder - .build_bit_cast(map_ptr, ptr_type, "map_ptr") + .build_bit_cast(map_ptr, ptr_type, &format!("{name_prefix}_map_ptr")) .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; // Use per-invocation key buffer [4 x u32] pre-allocated in entry block @@ -461,11 +468,16 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { let helper_fn_type = i64_type.fn_type(&[], false); let helper_fn_ptr = self .builder - .build_int_to_ptr(helper_id, ptr_type, "get_pid_fn") + .build_int_to_ptr(helper_id, ptr_type, &format!("{name_prefix}_get_pid_fn")) .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; let pid_tgid = self .builder - .build_indirect_call(helper_fn_type, helper_fn_ptr, &[], "pid_tgid") + .build_indirect_call( + helper_fn_type, + helper_fn_ptr, + &[], + &format!("{name_prefix}_pid_tgid"), + ) .map_err(|e| CodeGenError::LLVMError(e.to_string()))? .try_as_basic_value() .left() @@ -476,10 +488,15 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { // pid = upper 32 bits let shifted = self .builder - .build_right_shift(v, i64_type.const_int(32, false), false, "pid_shift") + .build_right_shift( + v, + i64_type.const_int(32, false), + false, + &format!("{name_prefix}_pid_shift"), + ) .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; self.builder - .build_int_truncate(shifted, i32_type, "pid32") + .build_int_truncate(shifted, i32_type, &format!("{name_prefix}_pid32")) .map_err(|e| CodeGenError::LLVMError(e.to_string()))? } else { return Err(CodeGenError::LLVMError( @@ -499,7 +516,11 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; let pidns_info_ptr = self .builder - .build_bit_cast(key_alloca, ptr_type, "offset_pidns_info_ptr") + .build_bit_cast( + key_alloca, + ptr_type, + &format!("{name_prefix}_pidns_info_ptr"), + ) .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; let helper_args = [ i64_type.const_int(pid_ns_dev, false).into(), @@ -511,7 +532,7 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { BPF_FUNC_GET_NS_CURRENT_PID_TGID, &helper_args, i64_type.into(), - "offset_ns_pid_tgid_ret", + &format!("{name_prefix}_ns_pid_tgid_ret"), )?; let helper_ret = match helper_ret { BasicValueEnum::IntValue(v) => v, @@ -527,7 +548,7 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { inkwell::IntPredicate::EQ, helper_ret, i64_type.const_zero(), - "offset_ns_helper_ok", + &format!("{name_prefix}_ns_helper_ok"), ) .map_err(|e| CodeGenError::Builder(e.to_string()))?; // SAFETY: key_alloca temporarily holds the two-field pid namespace @@ -537,13 +558,13 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { key_arr_ty, key_alloca, &[i32_type.const_zero(), i32_type.const_int(1, false)], - "offset_ns_tgid_ptr", + &format!("{name_prefix}_ns_tgid_ptr"), ) } .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; let ns_tgid = self .builder - .build_load(i32_type, ns_tgid_ptr, "offset_ns_tgid") + .build_load(i32_type, ns_tgid_ptr, &format!("{name_prefix}_ns_tgid")) .map_err(|e| CodeGenError::LLVMError(e.to_string()))? .into_int_value(); // The proc_module_offsets map is populated from `/proc//maps`, @@ -551,13 +572,18 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { // those `/proc` reads. This is *not* necessarily the same namespace // used for `$pid`/`$tid` or NamespaceTgid filtering. self.builder - .build_select(helper_ok, ns_tgid, host_tgid, "offset_pid_key") + .build_select( + helper_ok, + ns_tgid, + host_tgid, + &format!("{name_prefix}_pid_key"), + ) .map_err(|e| CodeGenError::Builder(e.to_string()))? .into_int_value() } else { host_tgid }; - let pid = self.lookup_proc_pid_alias(runtime_pid, "offset_pid")?; + let pid = self.lookup_proc_pid_alias(runtime_pid, name_prefix)?; let store_key_u32 = |offset: usize, value: IntValue<'ctx>, @@ -581,13 +607,13 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { store_key_u32( ghostscope_protocol::PROC_MODULE_KEY_PID_OFFSET, pid, - "pm_key_pid_ptr", + &format!("{name_prefix}_pm_key_pid_ptr"), self, )?; store_key_u32( ghostscope_protocol::PROC_MODULE_KEY_PAD_OFFSET, self.context.i32_type().const_zero(), - "pm_key_pad_ptr", + &format!("{name_prefix}_pm_key_pad_ptr"), self, )?; @@ -596,13 +622,13 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { store_key_u32( ghostscope_protocol::PROC_MODULE_KEY_COOKIE_LO_OFFSET, cookie_lo, - "pm_key_cookie_lo_ptr", + &format!("{name_prefix}_pm_key_cookie_lo_ptr"), self, )?; store_key_u32( ghostscope_protocol::PROC_MODULE_KEY_COOKIE_HI_OFFSET, cookie_hi, - "pm_key_cookie_hi_ptr", + &format!("{name_prefix}_pm_key_cookie_hi_ptr"), self, )?; @@ -611,17 +637,22 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { let lookup_fn_type = ptr_type.fn_type(&[ptr_type.into(), ptr_type.into()], false); let lookup_fn_ptr = self .builder - .build_int_to_ptr(lookup_id, ptr_type, "lookup_fn") + .build_int_to_ptr(lookup_id, ptr_type, &format!("{name_prefix}_lookup_fn")) .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; // Pass pointer to the beginning of the key buffer (void*) let key_arg = self .builder - .build_bit_cast(key_alloca, ptr_type, "key_arg") + .build_bit_cast(key_alloca, ptr_type, &format!("{name_prefix}_key_arg")) .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; let args: Vec = vec![map_ptr_cast.into(), key_arg.into()]; let val_ptr_any = self .builder - .build_indirect_call(lookup_fn_type, lookup_fn_ptr, &args, "val_ptr_any") + .build_indirect_call( + lookup_fn_type, + lookup_fn_ptr, + &args, + &format!("{name_prefix}_val_ptr_any"), + ) .map_err(|e| CodeGenError::LLVMError(e.to_string()))? .try_as_basic_value() .left() @@ -629,9 +660,15 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { let null_ptr = ptr_type.const_null(); let current_fn = self.current_function("generate module offset lookup")?; - let found_block = self.context.append_basic_block(current_fn, "found_offsets"); - let miss_block = self.context.append_basic_block(current_fn, "miss_offsets"); - let cont_block = self.context.append_basic_block(current_fn, "cont_offsets"); + let found_block = self + .context + .append_basic_block(current_fn, &format!("{name_prefix}_found_offsets")); + let miss_block = self + .context + .append_basic_block(current_fn, &format!("{name_prefix}_miss_offsets")); + let cont_block = self + .context + .append_basic_block(current_fn, &format!("{name_prefix}_cont_offsets")); // Compare against NULL let val_ptr = if let BasicValueEnum::PointerValue(p) = val_ptr_any { @@ -644,21 +681,22 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { .build_int_compare( inkwell::IntPredicate::EQ, self.builder - .build_ptr_to_int(val_ptr, i64_type, "val_ptr_i64") + .build_ptr_to_int(val_ptr, i64_type, &format!("{name_prefix}_val_ptr_i64")) .map_err(|e| CodeGenError::LLVMError(e.to_string()))?, i64_type.const_zero(), - "is_null_offsets", + &format!("{name_prefix}_is_null_offsets"), ) .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; self.builder .build_conditional_branch(is_null, miss_block, found_block) .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - // Found: load the requested field from ProcModuleOffsetsValue. The + // Found: load fields from ProcModuleOffsetsValue. The // byte offsets are shared through ghostscope_protocol::bpf_abi because // this map is an ABI between generated eBPF and userspace. self.builder.position_at_end(found_block); let load_field = |offset: usize, + field_name: &str, ctx: &mut EbpfContext<'ctx, 'dw>, base: PointerValue<'ctx>| -> Result> { @@ -672,7 +710,7 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { }; let loaded = ctx .builder - .build_load(ctx.context.i64_type(), field_ptr, "loaded_offset") + .build_load(ctx.context.i64_type(), field_ptr, field_name) .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; if let BasicValueEnum::IntValue(iv) = loaded { Ok(iv) @@ -680,32 +718,103 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { Err(CodeGenError::LLVMError("offset load failed".to_string())) } }; - let st = section_type; let off_text = load_field( ghostscope_protocol::PROC_MODULE_OFFSETS_VALUE_TEXT_OFFSET, + &format!("{name_prefix}_text"), self, val_ptr, )?; let off_rodata = load_field( ghostscope_protocol::PROC_MODULE_OFFSETS_VALUE_RODATA_OFFSET, + &format!("{name_prefix}_rodata"), self, val_ptr, )?; let off_data = load_field( ghostscope_protocol::PROC_MODULE_OFFSETS_VALUE_DATA_OFFSET, + &format!("{name_prefix}_data"), self, val_ptr, )?; let off_bss = load_field( ghostscope_protocol::PROC_MODULE_OFFSETS_VALUE_BSS_OFFSET, + &format!("{name_prefix}_bss"), + self, + val_ptr, + )?; + let base = load_field( + ghostscope_protocol::PROC_MODULE_OFFSETS_VALUE_BASE_OFFSET, + &format!("{name_prefix}_base"), + self, + val_ptr, + )?; + let size = load_field( + ghostscope_protocol::PROC_MODULE_OFFSETS_VALUE_SIZE_OFFSET, + &format!("{name_prefix}_size"), self, val_ptr, )?; + self.builder + .build_unconditional_branch(cont_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let found_end = self.current_insert_block("finish proc offsets found block")?; + + self.builder.position_at_end(miss_block); + self.builder + .build_unconditional_branch(cont_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let miss_end = self.current_insert_block("finish proc offsets miss block")?; + + self.builder.position_at_end(cont_block); + let zero_i64 = i64_type.const_zero(); + let phi_i64 = |ctx: &mut EbpfContext<'ctx, 'dw>, + name: &str, + hit_value: IntValue<'ctx>| + -> Result> { + let phi = ctx + .builder + .build_phi(i64_type, name) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + phi.add_incoming(&[(&hit_value, found_end), (&zero_i64, miss_end)]); + Ok(phi.as_basic_value().into_int_value()) + }; + let found_type = self.context.bool_type(); + let found_phi = self + .builder + .build_phi(found_type, &format!("{name_prefix}_found_phi")) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + found_phi.add_incoming(&[ + (&found_type.const_int(1, false), found_end), + (&found_type.const_zero(), miss_end), + ]); + + Ok(ProcModuleOffsetsLookup { + found: found_phi.as_basic_value().into_int_value(), + text: phi_i64(self, &format!("{name_prefix}_text_phi"), off_text)?, + rodata: phi_i64(self, &format!("{name_prefix}_rodata_phi"), off_rodata)?, + data: phi_i64(self, &format!("{name_prefix}_data_phi"), off_data)?, + bss: phi_i64(self, &format!("{name_prefix}_bss_phi"), off_bss)?, + base: phi_i64(self, &format!("{name_prefix}_base_phi"), base)?, + size: phi_i64(self, &format!("{name_prefix}_size_phi"), size)?, + }) + } + + /// Compute runtime address from link-time address using proc_module_offsets map + /// section_type: 0=text, 1=rodata, 2=data, 3=bss; other values fallback to data + pub fn generate_runtime_address_from_offsets( + &mut self, + link_addr: IntValue<'ctx>, + section_type: u8, + module_cookie: u64, + ) -> Result<(IntValue<'ctx>, IntValue<'ctx>)> { + let i32_type = self.context.i32_type(); + let offsets = self.lookup_proc_module_offsets_value(module_cookie, "offset")?; + // Build a bottom-up cascade to preserve earlier choices: // tmp = (section==data) ? off_data : off_bss // tmp2 = (section==rodata) ? off_rodata : tmp // off = (section==text) ? off_text : tmp2 - let st_val = i32_type.const_int(st as u64, false); + let st_val = i32_type.const_int(section_type as u64, false); let eq_text = self .builder .build_int_compare( @@ -736,56 +845,37 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { let tmp_any = self .builder - .build_select(eq_da, off_data, off_bss, "sel_data_bss") + .build_select(eq_da, offsets.data, offsets.bss, "sel_data_bss") .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; let tmp = tmp_any.into_int_value(); let tmp2_any = self .builder - .build_select(eq_ro, off_rodata, tmp, "sel_rodata_else") + .build_select(eq_ro, offsets.rodata, tmp, "sel_rodata_else") .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; let tmp2 = tmp2_any.into_int_value(); let off_final_any = self .builder - .build_select(eq_text, off_text, tmp2, "sel_text_else") + .build_select(eq_text, offsets.text, tmp2, "sel_text_else") .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; let off_final = off_final_any.into_int_value(); let rt_addr = self .builder .build_int_add(link_addr, off_final, "runtime_addr") .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - self.builder - .build_unconditional_branch(cont_block) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - - // Miss: return link_addr as-is (will likely fault and set ReadError) - self.builder.position_at_end(miss_block); - self.builder - .build_unconditional_branch(cont_block) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - - // Phi to merge address - self.builder.position_at_end(cont_block); - let phi = self - .builder - .build_phi(i64_type, "addr_phi") - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - phi.add_incoming(&[(&rt_addr, found_block), (&link_addr, miss_block)]); - let final_addr = phi.as_basic_value().into_int_value(); - - // Phi to merge found-flag (i1): 1 on found, 0 on miss - let i1_type = self.context.bool_type(); - let one = i1_type.const_int(1, false); - let zero = i1_type.const_int(0, false); - let flag_phi = self + let final_addr = self .builder - .build_phi(i1_type, "off_found_phi") - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - flag_phi.add_incoming(&[(&one, found_block), (&zero, miss_block)]); - let found_flag = flag_phi.as_basic_value().into_int_value(); + .build_select::, _>( + offsets.found, + rt_addr.into(), + link_addr.into(), + "addr_or_link", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))? + .into_int_value(); - Ok((final_addr, found_flag)) + Ok((final_addr, offsets.found)) } /// Load a register value from pt_regs pub fn load_register_value( diff --git a/ghostscope-compiler/src/ebpf/maps.rs b/ghostscope-compiler/src/ebpf/maps.rs index dc6414e..4ef25e1 100644 --- a/ghostscope-compiler/src/ebpf/maps.rs +++ b/ghostscope-compiler/src/ebpf/maps.rs @@ -259,7 +259,7 @@ impl<'ctx> MapManager<'ctx> { max_entries: u64, ) -> Result<()> { // Key: {pid:u32, pad:u32, cookie:u64} => 16 bytes => 128 bits - // Value: {text, rodata, data, bss: u64} => 32 bytes => 256 bits + // Value: {text, rodata, data, bss, base, size: u64} => 48 bytes => 384 bits self.create_map_definition( module, di_builder, @@ -268,7 +268,7 @@ impl<'ctx> MapManager<'ctx> { BpfMapType::Hash, max_entries, SizedType::integer(128), - SizedType::integer(256), + SizedType::integer(384), ) } diff --git a/ghostscope-process/src/offsets.rs b/ghostscope-process/src/offsets.rs index f09f9b8..0dfef72 100644 --- a/ghostscope-process/src/offsets.rs +++ b/ghostscope-process/src/offsets.rs @@ -50,6 +50,8 @@ struct CachedEntry { pid: u32, cookie: u64, offsets: SectionOffsets, + base: u64, + size: u64, } #[derive(Debug, Clone, Default)] @@ -255,11 +257,13 @@ impl ProcessManager { 3, std::time::Duration::from_millis(75), ) { - Ok((cookie, offsets, _base, _size)) => { + Ok((cookie, offsets, base, size)) => { cached.push(CachedEntry { pid, cookie, offsets, + base, + size, }); new_count += 1; } @@ -276,10 +280,17 @@ impl ProcessManager { Ok(new_count) } - pub fn cached_offsets_for_module(&self, module_path: &str) -> Vec<(u32, u64, SectionOffsets)> { + pub fn cached_offsets_for_module( + &self, + module_path: &str, + ) -> Vec<(u32, u64, SectionOffsets, u64, u64)> { self.module_cache .get(module_path) - .map(|v| v.iter().map(|e| (e.pid, e.cookie, e.offsets)).collect()) + .map(|v| { + v.iter() + .map(|e| (e.pid, e.cookie, e.offsets, e.base, e.size)) + .collect() + }) .unwrap_or_default() } @@ -649,11 +660,15 @@ mod tests { pid: 42, cookie: 1, offsets: SectionOffsets::default(), + base: 0, + size: 0, }, CachedEntry { pid: 7, cookie: 2, offsets: SectionOffsets::default(), + base: 0, + size: 0, }, ], ); diff --git a/ghostscope-process/src/pinned_bpf_maps.rs b/ghostscope-process/src/pinned_bpf_maps.rs index e2fb0bc..5a05a1e 100644 --- a/ghostscope-process/src/pinned_bpf_maps.rs +++ b/ghostscope-process/src/pinned_bpf_maps.rs @@ -206,6 +206,19 @@ pub fn bpffs_mount_hint_for_pin_path(pin_path: &Path) -> Option { pub use ghostscope_protocol::{PidAliasValue, ProcModuleKey, ProcModuleOffsetsValue}; +fn proc_offsets_pin_layout_matches(map: &MapData) -> bool { + match map.info() { + Ok(info) => { + info.key_size() == ghostscope_protocol::PROC_MODULE_KEY_SIZE as u32 + && info.value_size() == ghostscope_protocol::PROC_MODULE_OFFSETS_VALUE_SIZE as u32 + } + Err(e) => { + warn!("Unable to inspect pinned proc_module_offsets map layout: {e}"); + false + } + } +} + fn ensure_pin_dir(path: &Path) -> std::io::Result<()> { if let Some(dir) = path.parent() { std::fs::create_dir_all(dir) @@ -232,17 +245,28 @@ pub fn ensure_pinned_proc_offsets_exists(max_entries: u32) -> anyhow::Result<()> ) })?; - // If pinned file already exists, try to reuse it directly (idempotent) + // If pinned file already exists, reuse it only when the ABI layout matches. if pin_path.exists() { - if MapData::from_pin(&pin_path).is_ok() { - info!( - "Reusing existing pinned map at {} (no recreate)", - pin_path.display() - ); - return Ok(()); - } else { - // Stale/corrupted pin path, remove and recreate - let _ = std::fs::remove_file(&pin_path); + match MapData::from_pin(&pin_path) { + Ok(map) if proc_offsets_pin_layout_matches(&map) => { + info!( + "Reusing existing pinned map at {} (layout ok)", + pin_path.display() + ); + return Ok(()); + } + Ok(_) => { + warn!( + "Pinned {} at {} has stale ABI layout; recreating", + PROC_OFFSETS_MAP_NAME, + pin_path.display() + ); + let _ = std::fs::remove_file(&pin_path); + } + Err(_) => { + // Stale/corrupted pin path, remove and recreate + let _ = std::fs::remove_file(&pin_path); + } } } @@ -280,7 +304,7 @@ pub fn ensure_pinned_proc_offsets_exists(max_entries: u32) -> anyhow::Result<()> Err(e) => { // If another thread/process pinned concurrently, reuse the existing pin match MapData::from_pin(&pin_path) { - Ok(_) => { + Ok(map) if proc_offsets_pin_layout_matches(&map) => { info!( "Pin path {} already exists; reusing existing map ({}).", pin_path.display(), @@ -288,7 +312,7 @@ pub fn ensure_pinned_proc_offsets_exists(max_entries: u32) -> anyhow::Result<()> ); Ok(()) } - Err(_) => { + Ok(_) | Err(_) => { // Best-effort cleanup and propagate error let _ = std::fs::remove_file(&pin_path); let hint = bpffs_mount_hint_for_pin_path(&pin_path) @@ -580,8 +604,8 @@ pub fn insert_offsets_for_pid( match map.insert(key, *off, 0) { Ok(()) => { tracing::debug!( - "proc_module_offsets insert ok: pid={} cookie=0x{:08x}{:08x} text=0x{:x} rodata=0x{:x} data=0x{:x} bss=0x{:x}", - pid, key.cookie_hi, key.cookie_lo, off.text, off.rodata, off.data, off.bss + "proc_module_offsets insert ok: pid={} cookie=0x{:08x}{:08x} text=0x{:x} rodata=0x{:x} data=0x{:x} bss=0x{:x} base=0x{:x} size=0x{:x}", + pid, key.cookie_hi, key.cookie_lo, off.text, off.rodata, off.data, off.bss, off.base, off.size ); inserted += 1 } diff --git a/ghostscope-process/src/sysmon.rs b/ghostscope-process/src/sysmon.rs index 6ca3b1e..268d868 100644 --- a/ghostscope-process/src/sysmon.rs +++ b/ghostscope-process/src/sysmon.rs @@ -634,10 +634,12 @@ fn run_sysmon_loop( use std::collections::HashMap; let mut by_pid: HashMap> = HashMap::new(); - for (pid, cookie, off) in entries { + for (pid, cookie, off, base, size) in entries { by_pid.entry(pid).or_default().push(( cookie, - ProcModuleOffsetsValue::new(off.text, off.rodata, off.data, off.bss), + ProcModuleOffsetsValue::new( + off.text, off.rodata, off.data, off.bss, base, size, + ), )); } let mut total = 0usize; @@ -954,10 +956,14 @@ fn prefill_offsets_for_pid( } let mut by_pid: HashMap> = HashMap::new(); - for (pid, cookie, off) in guard.cached_offsets_for_module(&module_path) { + for (pid, cookie, off, base, size) in + guard.cached_offsets_for_module(&module_path) + { by_pid.entry(pid).or_default().push(( cookie, - ProcModuleOffsetsValue::new(off.text, off.rodata, off.data, off.bss), + ProcModuleOffsetsValue::new( + off.text, off.rodata, off.data, off.bss, base, size, + ), )); } for (pid, items) in by_pid { @@ -1023,6 +1029,8 @@ fn prefill_offsets_for_pid( e.offsets.rodata, e.offsets.data, e.offsets.bss, + e.base, + e.size, ), ) }) @@ -1094,10 +1102,10 @@ fn refresh_target_module_offsets( ); return; } - for (pid, cookie, off) in guard.cached_offsets_for_module(&module_path) { + for (pid, cookie, off, base, size) in guard.cached_offsets_for_module(&module_path) { by_pid.entry(pid).or_default().push(( cookie, - ProcModuleOffsetsValue::new(off.text, off.rodata, off.data, off.bss), + ProcModuleOffsetsValue::new(off.text, off.rodata, off.data, off.bss, base, size), )); } } diff --git a/ghostscope-protocol/src/bpf_abi.rs b/ghostscope-protocol/src/bpf_abi.rs index 2ee0182..df3edfe 100644 --- a/ghostscope-protocol/src/bpf_abi.rs +++ b/ghostscope-protocol/src/bpf_abi.rs @@ -38,15 +38,19 @@ pub struct ProcModuleOffsetsValue { pub rodata: u64, pub data: u64, pub bss: u64, + pub base: u64, + pub size: u64, } impl ProcModuleOffsetsValue { - pub fn new(text: u64, rodata: u64, data: u64, bss: u64) -> Self { + pub fn new(text: u64, rodata: u64, data: u64, bss: u64, base: u64, size: u64) -> Self { Self { text, rodata, data, bss, + base, + size, } } } @@ -59,6 +63,10 @@ pub const PROC_MODULE_OFFSETS_VALUE_DATA_OFFSET: usize = std::mem::offset_of!(ProcModuleOffsetsValue, data); pub const PROC_MODULE_OFFSETS_VALUE_BSS_OFFSET: usize = std::mem::offset_of!(ProcModuleOffsetsValue, bss); +pub const PROC_MODULE_OFFSETS_VALUE_BASE_OFFSET: usize = + std::mem::offset_of!(ProcModuleOffsetsValue, base); +pub const PROC_MODULE_OFFSETS_VALUE_SIZE_OFFSET: usize = + std::mem::offset_of!(ProcModuleOffsetsValue, size); pub const PROC_MODULE_OFFSETS_VALUE_SIZE: usize = std::mem::size_of::(); /// Value for the pinned `pid_aliases` map. @@ -263,11 +271,13 @@ mod tests { assert_eq!(PROC_MODULE_KEY_COOKIE_LO_OFFSET, 8); assert_eq!(PROC_MODULE_KEY_COOKIE_HI_OFFSET, 12); - assert_eq!(PROC_MODULE_OFFSETS_VALUE_SIZE, 32); + assert_eq!(PROC_MODULE_OFFSETS_VALUE_SIZE, 48); assert_eq!(PROC_MODULE_OFFSETS_VALUE_TEXT_OFFSET, 0); assert_eq!(PROC_MODULE_OFFSETS_VALUE_RODATA_OFFSET, 8); assert_eq!(PROC_MODULE_OFFSETS_VALUE_DATA_OFFSET, 16); assert_eq!(PROC_MODULE_OFFSETS_VALUE_BSS_OFFSET, 24); + assert_eq!(PROC_MODULE_OFFSETS_VALUE_BASE_OFFSET, 32); + assert_eq!(PROC_MODULE_OFFSETS_VALUE_SIZE_OFFSET, 40); assert_eq!(PID_ALIAS_VALUE_SIZE, 4); assert_eq!(PID_ALIAS_VALUE_PROC_PID_OFFSET, 0); diff --git a/ghostscope-protocol/src/lib.rs b/ghostscope-protocol/src/lib.rs index 1ac8180..594a485 100644 --- a/ghostscope-protocol/src/lib.rs +++ b/ghostscope-protocol/src/lib.rs @@ -58,9 +58,10 @@ pub use bpf_abi::{ BACKTRACE_UNWIND_WORD_RBP_OFFSET, BACKTRACE_UNWIND_WORD_REGISTERS, PID_ALIAS_VALUE_PROC_PID_OFFSET, PID_ALIAS_VALUE_SIZE, PROC_MODULE_KEY_COOKIE_HI_OFFSET, PROC_MODULE_KEY_COOKIE_LO_OFFSET, PROC_MODULE_KEY_PAD_OFFSET, PROC_MODULE_KEY_PID_OFFSET, - PROC_MODULE_KEY_SIZE, PROC_MODULE_OFFSETS_VALUE_BSS_OFFSET, - PROC_MODULE_OFFSETS_VALUE_DATA_OFFSET, PROC_MODULE_OFFSETS_VALUE_RODATA_OFFSET, - PROC_MODULE_OFFSETS_VALUE_SIZE, PROC_MODULE_OFFSETS_VALUE_TEXT_OFFSET, + PROC_MODULE_KEY_SIZE, PROC_MODULE_OFFSETS_VALUE_BASE_OFFSET, + PROC_MODULE_OFFSETS_VALUE_BSS_OFFSET, PROC_MODULE_OFFSETS_VALUE_DATA_OFFSET, + PROC_MODULE_OFFSETS_VALUE_RODATA_OFFSET, PROC_MODULE_OFFSETS_VALUE_SIZE, + PROC_MODULE_OFFSETS_VALUE_SIZE_OFFSET, PROC_MODULE_OFFSETS_VALUE_TEXT_OFFSET, }; pub use trace_context::TraceContext; diff --git a/ghostscope/src/script/compiler.rs b/ghostscope/src/script/compiler.rs index 3a712d2..df92a4c 100644 --- a/ghostscope/src/script/compiler.rs +++ b/ghostscope/src/script/compiler.rs @@ -74,16 +74,25 @@ fn apply_cached_offsets_for_session_pid( .coordinator .lock() .expect("coordinator mutex poisoned"); - coordinator.cached_offsets_pairs_for_pid(proc_pid) + coordinator + .cached_offsets_with_paths_for_pid(proc_pid) + .map(|entries| entries.to_vec()) }; if let Some(items) = items { use ghostscope_process::pinned_bpf_maps::ProcModuleOffsetsValue; let adapted: Vec<(u64, ProcModuleOffsetsValue)> = items .iter() - .map(|(cookie, off)| { + .map(|entry| { ( - *cookie, - ProcModuleOffsetsValue::new(off.text, off.rodata, off.data, off.bss), + entry.cookie, + ProcModuleOffsetsValue::new( + entry.offsets.text, + entry.offsets.rodata, + entry.offsets.data, + entry.offsets.bss, + entry.base, + entry.size, + ), ) }) .collect(); @@ -214,10 +223,12 @@ async fn create_and_attach_loader( // Group by pid for efficient batch insert use std::collections::HashMap; let mut by_pid: HashMap> = HashMap::new(); - for (pid, cookie, off) in entries { + for (pid, cookie, off, base, size) in entries { by_pid.entry(pid).or_default().push(( cookie, - ProcModuleOffsetsValue::new(off.text, off.rodata, off.data, off.bss), + ProcModuleOffsetsValue::new( + off.text, off.rodata, off.data, off.bss, base, size, + ), )); } let mut total = 0usize;