🍳 bpftrace Recipes

Practical bpftrace scripts for common system administration and performance analysis tasks. These recipes solve real-world problems with eBPF-powered observability.

---

🚀 Performance Analysis Recipes

CPU Profiling by Process

#!/usr/bin/env bpftrace

// Profile CPU usage across all processes
profile:hz:997
{
  @cpu_time[comm] = count();
}

// Print top consumers every 30 seconds
interval:s:30
{
  printf("=== CPU Usage (last 30s) ===\n");
  print(@cpu_time);
  clear(@cpu_time);
}

// Cleanup on exit
END
{
  clear(@cpu_time);
}

Memory Allocation Analysis

#!/usr/bin/env bpftrace

// Track malloc calls and sizes
uprobe:/lib/x86_64-linux-gnu/libc.so.6:malloc
{
  @malloc_sizes = hist(arg0);
  @malloc_by_process[comm] = sum(arg0);
}

// Track memory leaks (allocations without frees)
uprobe:/lib/x86_64-linux-gnu/libc.so.6:malloc
{
  @allocs[tid, arg0] = 1;
}

uretprobe:/lib/x86_64-linux-gnu/libc.so.6:malloc
{
  $ptr = retval;
  @alloc_ptrs[$ptr] = arg0;  // Store size with pointer
}

uprobe:/lib/x86_64-linux-gnu/libc.so.6:free
{
  delete(@alloc_ptrs[arg0]);  // Remove when freed
}

// Report leaks periodically
interval:s:60
{
  printf("=== Potential Memory Leaks ===\n");
  print(@alloc_ptrs);
}

I/O Latency Monitoring

#!/usr/bin/env bpftrace

// Track file I/O latency
tracepoint:syscalls:sys_enter_open
{
  @start[tid] = nsecs;
}

tracepoint:syscalls:sys_exit_open
/@start[tid]/
{
  @open_latency = hist(nsecs - @start[tid]);
  delete(@start[tid]);
}

tracepoint:syscalls:sys_enter_read
{
  @read_start[tid] = nsecs;
}

tracepoint:syscalls:sys_exit_read
/@read_start[tid] && args->ret > 0/
{
  @read_latency = hist(nsecs - @read_start[tid]);
  @read_sizes = hist(args->ret);
  delete(@read_start[tid]);
}

// Report every 5 minutes
interval:s:300
{
  printf("=== I/O Latency Report ===\n");
  printf("Open latency distribution:\n");
  print(@open_latency);
  printf("Read latency distribution:\n");
  print(@read_latency);
  printf("Read size distribution:\n");
  print(@read_sizes);

  clear(@open_latency);
  clear(@read_latency);
  clear(@read_sizes);
}

---

🌐 Network Monitoring Recipes

TCP Connection Tracking

#!/usr/bin/env bpftrace

// Monitor TCP connection attempts
kprobe:tcp_v4_connect
{
  $addr = (struct sockaddr_in *)arg1;
  @connections[comm, ntop(AF_INET, $addr->sin_addr.s_addr), $addr->sin_port] = count();
}

// Track established connections
kprobe:tcp_set_state
/arg1 == 1/  // TCP_ESTABLISHED
{
  @established[comm] = count();
}

// Track closed connections
kprobe:tcp_set_state
/arg1 == 7/  // TCP_CLOSE
{
  @closed[comm] = count();
}

// Report every minute
interval:s:60
{
  printf("=== TCP Connection Report ===\n");
  printf("New connections:\n");
  print(@connections);
  printf("Established connections:\n");
  print(@established);
  printf("Closed connections:\n");
  print(@closed);

  clear(@connections);
  clear(@established);
  clear(@closed);
}

Network Throughput Analysis

#!/usr/bin/env bpftrace

// Monitor network send/receive throughput
kprobe:tcp_sendmsg
{
  @bytes_sent[comm] = sum(arg2);  // arg2 is size
}

kprobe:tcp_recvmsg
{
  @bytes_received[comm] = sum(arg2);  // arg2 is size
}

// Calculate throughput every 10 seconds
interval:s:10
{
  printf("=== Network Throughput (last 10s) ===\n");
  printf("Bytes sent by process:\n");
  print(@bytes_sent);
  printf("Bytes received by process:\n");
  print(@bytes_received);

  // Calculate MB/s
  $sent_mb = sum(@bytes_sent) / 1024 / 1024 / 10;
  $recv_mb = sum(@bytes_received) / 1024 / 1024 / 10;
  printf("Throughput: %.2f MB/s sent, %.2f MB/s received\n", $sent_mb, $recv_mb);

  clear(@bytes_sent);
  clear(@bytes_received);
}

DNS Query Monitoring

#!/usr/bin/env bpftrace

// Monitor DNS queries (UDP port 53)
tracepoint:syscalls:sys_enter_sendto
/arg2 == 13568/  // htons(53) for little-endian
{
  printf("DNS Query: %s (PID: %d)\n", comm, pid);
  @dns_queries[comm] = count();
}

// Monitor DNS responses
tracepoint:syscalls:sys_enter_recvfrom
/arg2 == 13568/
{
  @dns_responses[comm] = count();
}

// Report DNS activity
interval:s:30
{
  printf("=== DNS Activity (last 30s) ===\n");
  printf("Queries: %d\n", sum(@dns_queries));
  printf("Responses: %d\n", sum(@dns_responses));
  print(@dns_queries);

  clear(@dns_queries);
  clear(@dns_responses);
}

---

🔒 Security and Audit Recipes

File Access Monitoring

#!/usr/bin/env bpftrace

// Monitor access to sensitive files
tracepoint:syscalls:sys_enter_openat
{
  $filename = str(args->filename);

  // Monitor /etc/passwd access
  if (strncmp($filename, "/etc/passwd", 11) == 0) {
    printf("ALERT: %s (PID: %d) accessed /etc/passwd\n", comm, pid);
    @passwd_access[comm, uid] = count();
  }

  // Monitor SSH directory access
  if (strstr($filename, ".ssh/") != 0) {
    printf("SSH ACCESS: %s (PID: %d) accessed %s\n", comm, pid, $filename);
    @ssh_access[comm, $filename] = count();
  }

  // Monitor executable creation
  if (args->flags & 0x40) {  // O_CREAT
    @file_creations[comm, $filename] = count();
  }
}

// Report security events
interval:s:60
{
  if (sum(@passwd_access) > 0) {
    printf("=== Security Alert: /etc/passwd Access ===\n");
    print(@passwd_access);
  }

  if (sum(@ssh_access) > 0) {
    printf("=== SSH Access Detected ===\n");
    print(@ssh_access);
  }

  if (sum(@file_creations) > 0) {
    printf("=== File Creations ===\n");
    print(@file_creations);
  }

  clear(@passwd_access);
  clear(@ssh_access);
  clear(@file_creations);
}

Privilege Escalation Detection

#!/usr/bin/env bpftrace

// Monitor setuid/setgid calls
kprobe:sys_setuid
{
  printf("SETUID: %s (PID: %d) setting UID to %d\n", comm, pid, arg0);
  @uid_changes[comm, uid, arg0] = count();
}

kprobe:sys_setgid
{
  printf("SETGID: %s (PID: %d) setting GID to %d\n", comm, pid, arg0);
  @gid_changes[comm, gid, arg0] = count();
}

// Monitor capability changes
kprobe:cap_capable
{
  printf("CAPABILITY CHECK: %s (PID: %d) requesting CAP %d\n", comm, pid, arg2);
  @capability_requests[comm, arg2] = count();
}

// Alert on suspicious activity
interval:s:30
{
  if (sum(@uid_changes) > 0 || sum(@gid_changes) > 0) {
    printf("=== Privilege Changes Detected ===\n");
    print(@uid_changes);
    print(@gid_changes);
  }

  if (sum(@capability_requests) > 0) {
    printf("=== Capability Requests ===\n");
    print(@capability_requests);
  }

  clear(@uid_changes);
  clear(@gid_changes);
  clear(@capability_requests);
}

Process Execution Monitoring

#!/usr/bin/env bpftrace

// Monitor all process executions
tracepoint:syscalls:sys_enter_execve
{
  $cmdline = str(args->argv[0]);

  // Log all executions
  printf("EXEC: %s (PID: %d, PPID: %d) -> %s\n", comm, pid, ((struct task_struct*)curtask)->real_parent->pid, $cmdline);

  // Monitor suspicious commands
  if (strstr($cmdline, "nc ") != 0 || strstr($cmdline, "ncat ") != 0 || strstr($cmdline, "socat ") != 0) {
    printf("SUSPICIOUS: Network tool executed: %s\n", $cmdline);
    @suspicious_execs[comm, $cmdline] = count();
  }

  // Monitor shell spawning
  if (strstr($cmdline, "sh") != 0 || strstr($cmdline, "bash") != 0 || strstr($cmdline, "zsh") != 0) {
    @shell_spawns[comm] = count();
  }
}

// Report execution statistics
interval:s:60
{
  printf("=== Process Execution Summary ===\n");
  if (sum(@suspicious_execs) > 0) {
    printf("Suspicious executions detected:\n");
    print(@suspicious_execs);
  }

  if (sum(@shell_spawns) > 0) {
    printf("Shell spawns: %d\n", sum(@shell_spawns));
  }

  clear(@suspicious_execs);
  clear(@shell_spawns);
}

---

🛠️ System Health Recipes

Resource Exhaustion Detection

#!/usr/bin/env bpftrace

// Monitor file descriptor usage
tracepoint:syscalls:sys_enter_open
{
  @fds_per_process[pid, comm] = count();
}

tracepoint:syscalls:sys_exit_close
{
  if (@fds_per_process[pid, comm] > 0) {
    @fds_per_process[pid, comm]--;
  }
}

// Alert on high FD usage
interval:s:10
{
  foreach ([pid, comm] : @fds_per_process) {
    if (@fds_per_process[pid, comm] > 1000) {
      printf("WARNING: Process %s (PID: %d) has %d open file descriptors\n",
             comm, pid, @fds_per_process[pid, comm]);
    }
  }
}

System Call Rate Monitoring

#!/usr/bin/env bpftrace

// Count system calls by type
tracepoint:syscalls:sys_enter_*
{
  @syscall_rates[probe] = count();
  @total_syscalls = count();
}

// Calculate rates and alert on anomalies
interval:s:5
{
  $rate = @total_syscalls / 5;  // Calls per second

  if ($rate > 10000) {
    printf("HIGH SYSCALL RATE: %d calls/sec\n", $rate);
    print(@syscall_rates);
  }

  clear(@syscall_rates);
  @total_syscalls = 0;
}

---

🧾 Summary

✅ Production-ready monitoring solutions ✅ Low-overhead system observation ✅ Customizable alerts and metrics ✅ Security-focused audit capabilities ✅ Performance-oriented analysis tools

---

🧾 See Also

• bpftrace One-Liners
• bpftrace vs strace
• eBPF Intro for Shell Users
• Linux Debugging Tools
• Advanced: Debugging and Tracing