From 40be7238703e3f48559961471150d7af39200588 Mon Sep 17 00:00:00 2001 From: Yuxuan Shui Date: Mon, 12 Feb 2024 18:08:59 +0000 Subject: [PATCH] add interceptor to send stack traces --- flake.nix | 1 + interceptor.c | 53 +++++++++++++++++++++++++++++ tracer/Cargo.lock | 29 ++++++++++++++++ tracer/Cargo.toml | 3 ++ tracer/src/bpf/uprobe.bpf.c | 67 +++++++++++++++++++++++-------------- tracer/src/main.rs | 41 ++++++++++++++++++++--- 6 files changed, 165 insertions(+), 29 deletions(-) create mode 100644 interceptor.c diff --git a/flake.nix b/flake.nix index 414cacc9..c100e517 100644 --- a/flake.nix +++ b/flake.nix @@ -33,6 +33,7 @@ buildInputs = defaultPackage.buildInputs ++ (with pkgs; [ clang-tools_17 llvmPackages_17.clang-unwrapped.python + libunwind ]); hardeningDisable = [ "fortify" ]; shellHook = '' diff --git a/interceptor.c b/interceptor.c new file mode 100644 index 00000000..2251cfc8 --- /dev/null +++ b/interceptor.c @@ -0,0 +1,53 @@ +#include +#include +#include +#include +#include +#include +#include + +static ssize_t (*orig_recvmsg)(int, struct msghdr *, int) = NULL; +thread_local char buffer[4096]; + +__attribute__((noinline)) void recvmsg_stack_trace_probe(const char *ptr, uint64_t size) { + __asm__ volatile("" : : "r"(ptr), "r"(size)); +} + +void backtrace() { + unw_cursor_t cursor; + unw_context_t context; + + unw_getcontext(&context); + unw_init_local(&cursor, &context); + + int n = 0; + size_t buffer_offset = 0; + while (unw_step(&cursor)) { + unw_word_t ip, sp, off; + + unw_get_reg(&cursor, UNW_REG_IP, &ip); + unw_get_reg(&cursor, UNW_REG_SP, &sp); + + char symbol[256] = {""}; + char *name = symbol; + + unw_get_proc_name(&cursor, symbol, sizeof(symbol), &off); + + size_t written = (size_t)snprintf( + buffer + buffer_offset, sizeof(buffer) - buffer_offset, + "#%-2d 0x%016" PRIxPTR " sp=0x%016" PRIxPTR " %s + 0x%" PRIxPTR "\n", + ++n, ip, sp, name, off); + if (written >= sizeof(buffer) - buffer_offset) { + break; + } + buffer_offset += written; + } + recvmsg_stack_trace_probe(buffer, buffer_offset); +} +ssize_t recvmsg(int socket, struct msghdr *message, int flags) { + if (!orig_recvmsg) { + orig_recvmsg = dlsym((void *)RTLD_NEXT, "recvmsg"); + } + backtrace(); + return orig_recvmsg(socket, message, flags); +} diff --git a/tracer/Cargo.lock b/tracer/Cargo.lock index f1d3398e..1bb06486 100644 --- a/tracer/Cargo.lock +++ b/tracer/Cargo.lock @@ -41,6 +41,26 @@ version = "2.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed570934406eb16438a4e976b1b4500774099c13b8cb96eec99f620f05090ddf" +[[package]] +name = "bytemuck" +version = "1.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2ef034f05691a48569bd920a96c81b9d91bbad1ab5ac7c4616c1f6ef36cb79f" +dependencies = [ + "bytemuck_derive", +] + +[[package]] +name = "bytemuck_derive" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "965ab7eb5f8f97d2a083c799f3a1b994fc397b2fe2da5d1da1626ce15a39f2b1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.48", +] + [[package]] name = "byteorder" version = "1.5.0" @@ -202,6 +222,12 @@ version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0c62115964e08cb8039170eb33c1d0e2388a256930279edca206fff675f82c3" +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + [[package]] name = "indexmap" version = "2.2.3" @@ -384,6 +410,9 @@ checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" name = "picom-tracer" version = "0.1.0" dependencies = [ + "bytemuck", + "byteorder", + "hex", "libbpf-cargo", "libbpf-rs", "object", diff --git a/tracer/Cargo.toml b/tracer/Cargo.toml index 3957259c..72bfd483 100644 --- a/tracer/Cargo.toml +++ b/tracer/Cargo.toml @@ -6,6 +6,9 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +bytemuck = { version = "1.14.3", features = ["derive"] } +byteorder = "1.5.0" +hex = "0.4.3" libbpf-rs = { version = "0.22.1", default-features = false, features = ["novendor"] } object = "0.32.2" diff --git a/tracer/src/bpf/uprobe.bpf.c b/tracer/src/bpf/uprobe.bpf.c index 70f56a63..cfd328e8 100644 --- a/tracer/src/bpf/uprobe.bpf.c +++ b/tracer/src/bpf/uprobe.bpf.c @@ -16,13 +16,12 @@ struct xcb_connection_t { void *setup; int fd; }; - struct { - __uint(type, BPF_MAP_TYPE_ARRAY); - __type(key, u32); - __type(value, u64); - __uint(max_entries, 256); -} my_map SEC(".maps"); + __uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY); + __uint(key_size, sizeof(int)); + __uint(value_size, sizeof(u32)); + __uint(max_entries, 2); +} events SEC(".maps"); struct event { u8 task[TASK_COMM_LEN]; @@ -31,39 +30,58 @@ struct event { }; struct event _event = {0}; +u64 pid = 0; +void *conn_ptr; +char last_stack[4096]; +u64 last_recv_stack_size; + +SEC("uprobe") +int BPF_KPROBE(uprobe_recvmsg, const char *trace, u64 size) { + if (pid != bpf_get_current_pid_tgid() >> 32) { + return 0; + } + last_recv_stack_size = 0; + if (size > 4096) { + bpf_printk("invalid stack size %u", size); + return 0; + } + if (bpf_probe_read_user(&last_stack[0], size, trace)) { + bpf_printk("cannot read"); + return 0; + } + last_recv_stack_size = size; +} SEC("uprobe") int BPF_KPROBE(uprobe_epoll_wait) { - u64 *curr_pid = bpf_map_lookup_elem(&my_map, (u32[]){0}); - u32 pid = bpf_get_current_pid_tgid() >> 32; - if (!curr_pid || pid != *curr_pid) { - return 0; - } - void **conn_ptr = bpf_map_lookup_elem(&my_map, (u32[]){1}); - if (!conn_ptr) { + if (pid != bpf_get_current_pid_tgid() >> 32) { return 0; } struct xcb_connection_t conn; - if (bpf_probe_read_user(&conn, sizeof(conn), *conn_ptr)) { + if (bpf_probe_read_user(&conn, sizeof(conn), conn_ptr)) { bpf_printk("cannot read"); return 0; } -#if 0 - u64 request_read; - if (bpf_probe_read_user(&request_read, sizeof(request_read), (*conn_ptr) + 4216)) { + u32 queue_len; + if (bpf_probe_read_user(&queue_len, sizeof(queue_len), conn_ptr + 4212)) { bpf_printk("cannot read"); return 0; } - bpf_printk("request read %x", request_read); -#endif u64 event_head; - if (bpf_probe_read_user(&event_head, sizeof(event_head), (*conn_ptr) + 4272)) { + if (bpf_probe_read_user(&event_head, sizeof(event_head), conn_ptr + 4272)) { bpf_printk("cannot read"); return 0; } - if (event_head != 0) { - bpf_printk("epoll_wait %d %p", conn.fd, event_head); + if (event_head != 0 || queue_len != 0) { + bpf_printk("epoll_wait %d %p %d", conn.fd, event_head, queue_len); + char data[16]; + *(u64 *)data = event_head; + *(u64 *)(data + 8) = (u64)queue_len; + bpf_perf_event_output(ctx, &events, 1, data, 16); + if (last_recv_stack_size <= 4096) { + bpf_perf_event_output(ctx, &events, 0, last_stack, last_recv_stack_size); + } } return 0; } @@ -71,14 +89,13 @@ int BPF_KPROBE(uprobe_epoll_wait) { SEC("uprobe") int BPF_KPROBE(uprobe_xcb_conn, void *ptr) { struct xcb_connection_t conn; - u64 pid = bpf_get_current_pid_tgid() >> 32; - bpf_map_update_elem(&my_map, (u32[]){0}, &pid, 0); + pid = bpf_get_current_pid_tgid() >> 32; bpf_printk("xcb connection is %p", ptr); if (bpf_probe_read_user(&conn, sizeof(conn), ptr)) { bpf_printk("cannot read"); } else { - bpf_map_update_elem(&my_map, (u32[]){1}, &ptr, 0); bpf_printk("fd is %d", conn.fd); + conn_ptr = ptr; } return 0; } diff --git a/tracer/src/main.rs b/tracer/src/main.rs index 7b4d91f3..daaefe02 100644 --- a/tracer/src/main.rs +++ b/tracer/src/main.rs @@ -1,6 +1,8 @@ use libbpf_rs::skel::{OpenSkel, Skel, SkelBuilder}; -use libbpf_rs::{PerfBufferBuilder, UprobeOpts}; +use libbpf_rs::{MapFlags, PerfBufferBuilder, UprobeOpts}; +use std::cell::RefCell; use std::os::unix::ffi::OsStrExt; +use byteorder::ByteOrder; use object::read::elf::{Dyn, FileHeader}; mod uprobe { @@ -8,15 +10,30 @@ mod uprobe { } use object::Object; use uprobe::*; +thread_local! { + static SKEL: RefCell>> = RefCell::new(None); +} fn handle_lost_events(cpu: i32, count: u64) { eprintln!("Lost {count} events on CPU {cpu}"); } +#[repr(C)] +#[derive(bytemuck::Pod, bytemuck::Zeroable, Copy, Clone)] +struct Event { + event_head: u64, + queue_len: u64, +} fn handle_event(cpu: i32, data: &[u8]) { - eprintln!("Got {} bytes of data on {cpu}", data.len()); + if cpu == 0 { + eprintln!("{}", String::from_utf8_lossy(data)); + } else if cpu == 1 { + let event: Event = bytemuck::pod_read_unaligned(&data[..16]); + eprintln!("queue_len: {}, event_head: {:#x}", event.queue_len, event.event_head); + } } fn main() { let mut builder = UprobeSkelBuilder::default(); let picom_path = std::env::args().nth(1).unwrap(); + let interceptor_path = std::env::args().nth(2).unwrap(); let data = std::fs::read(&picom_path).unwrap(); let file = object::read::elf::ElfFile64::<'_, object::NativeEndian, _>::parse(&*data).unwrap(); let header = file.raw_header(); @@ -57,6 +74,12 @@ fn main() { let open_skel = builder.open().unwrap(); let mut skel = open_skel.load().unwrap(); let obj = skel.object_mut(); + let recv_probe = obj.prog_mut("uprobe_recvmsg").unwrap(); + let _link0 = recv_probe.attach_uprobe_with_opts(-1, &interceptor_path, 0, UprobeOpts { + retprobe: false, + func_name: "recvmsg_stack_trace_probe".to_string(), + ..Default::default() + }).unwrap(); let xcb_probe = obj.prog_mut("uprobe_xcb_conn").unwrap(); let _link = xcb_probe.attach_uprobe_with_opts(-1, &picom_path, 0, UprobeOpts { retprobe: false, @@ -64,10 +87,20 @@ fn main() { ..Default::default() }).unwrap(); let epoll_probe = obj.prog_mut("uprobe_epoll_wait").unwrap(); - let _link2 = epoll_probe.attach_uprobe_with_opts(-1, libc_path, 0, UprobeOpts { + let _link2 = epoll_probe.attach_uprobe_with_opts(-1, &libc_path, 0, UprobeOpts { retprobe: false, func_name: "epoll_wait".to_string(), ..Default::default() }).unwrap(); - std::thread::park(); + + let perf = PerfBufferBuilder::new(skel.maps_mut().events()) + .sample_cb(handle_event) + .lost_cb(handle_lost_events) + .build().unwrap(); + + SKEL.with_borrow_mut(|s| *s = Some(skel)); + + loop { + perf.poll(std::time::Duration::from_millis(100)).unwrap(); + } }