Mastering pwru: Deep Dive into Cilium’s eBPF Packet Tracing Tool

pwru

is a network packet tracing tool released by Cilium, built on eBPF to provide fine‑grained packet inspection. The article explains how to install, configure, and use pwru, and describes its core implementation.

Installation Requirements

pwru

requires a Linux kernel newer than 5.5; the --output-skb feature needs kernel 5.9 or later and the following kernel config options enabled:

CONFIG_DEBUG_INFO_BTF=y (available since >=5.3)

CONFIG_KPROBES=y

CONFIG_PERF_EVENTS=y

CONFIG_BPF=y

CONFIG_BPF_SYSCALL=y

Usage

Usage of ./pwru:
  --filter-dst-ip string        filter destination IP addr
  --filter-dst-port uint16      filter destination port
  --filter-func string         filter kernel functions to be probed by name (exact match, supports RE2 regular expression)
  --filter-mark uint32          filter skb mark
  --filter-netns uint32         filter netns inode
  --filter-proto string        filter L4 protocol (tcp, udp, icmp)
  --filter-src-ip string       filter source IP addr
  --filter-src-port uint16      filter source port
  --output-limit-lines uint     exit the program after the number of events has been received/printed
  --output-meta                 print skb metadata
  --output-relative-timestamp   print relative timestamp per skb
  --output-skb                  print skb
  --output-stack                print stack
  --output-tuple                print L4 tuple

Case Demonstration

The following example shows how pwru quickly identifies that a packet was dropped by an iptables rule.

Before adding any iptables rule:

After inserting an iptables DROP rule:

iptables -t filter -I OUTPUT 1 -m tcp --proto tcp --dst 1.1.1.1/32 -j DROP

The trace shows a change after the nf_hook_slow function, where the packet verdict becomes NF_DROP and kfree_skb is called.

int nf_hook_slow(struct sk_buff *skb, struct nf_hook_state *state,
    const struct nf_hook_entries *e, unsigned int s)
{
  unsigned int verdict;
  int ret;

  for (; s < e->num_hook_entries; s++) {
    verdict = nf_hook_entry_hookfn(&e->hooks[s], skb, state);
    switch (verdict & NF_VERDICT_MASK) {
    case NF_ACCEPT:
      break;
    case NF_DROP:
      kfree_skb(skb);
      ret = NF_DROP_GETERR(verdict);
      if (ret == 0)
        ret = -EPERM;
      return ret;
    case NF_QUEUE:
      ret = nf_queue(skb, state, s, verdict);
      if (ret == 1)
        continue;
      return ret;
    default:
      return 0;
    }
  }
  return 1;
}

Implementation Principles

pwru

registers kprobes that run eBPF code. Parameters passed to pwru update an eBPF map, which the probe reads to apply filtering conditions.

type FilterCfg struct {
    FilterMark uint32
    // L3 filters
    FilterIPv6  uint8
    FilterSrcIP [16]byte
    FilterDstIP [16]byte
    // L4 filters
    FilterProto   uint8
    FilterSrcPort uint16
    FilterDstPort uint16
    // Output options
    OutputRelativeTS uint8
    OutputMeta       uint8
    OutputTuple       uint8
    OutputSkb         uint8
    OutputStack       uint8
    Pad               byte
}

The Go helper populates this map based on command‑line flags:

func ConfigBPFMap(flags *Flags, cfgMap *ebpf.Map) {
    cfg := FilterCfg{FilterMark: flags.FilterMark}
    if flags.FilterSrcPort > 0 {
        cfg.FilterSrcPort = byteorder.HostToNetwork16(flags.FilterSrcPort)
    }
    if flags.FilterDstPort > 0 {
        cfg.FilterDstPort = byteorder.HostToNetwork16(flags.FilterDstPort)
    }
    switch strings.ToLower(flags.FilterProto) {
    case "tcp":
        cfg.FilterProto = syscall.IPPROTO_TCP
    case "udp":
        cfg.FilterProto = syscall.IPPROTO_UDP
    case "icmp":
        cfg.FilterProto = syscall.IPPROTO_ICMP
    case "icmp6":
        cfg.FilterProto = syscall.IPPROTO_ICMPV6
    }
    if err := cfgMap.Update(uint32(0), cfg, 0); err != nil {
        log.Fatalf("Failed to set filter map: %v", err)
    }
}

Inside the eBPF program, the map is looked up with bpf_map_lookup_elem, the packet is filtered, and the result is emitted via bpf_perf_event_output:

struct config {
    u32 mark;
    u8  ipv6;
    union addr saddr;
    union addr daddr;
    u8  l4_proto;
    u16 sport;
    u16 dport;
    u8  output_timestamp;
    u8  output_meta;
    u8  output_tuple;
    u8  output_skb;
    u8  output_stack;
    u8  pad;
} __attribute__((packed));

static __always_inline int handle_everything(struct sk_buff *skb, struct pt_regs *ctx) {
    struct event_t event = {};
    u32 index = 0;
    struct config *cfg = bpf_map_lookup_elem(&cfg_map, &index);
    if (cfg) {
        if (!filter(skb, cfg))
            return 0;
        set_output(ctx, skb, &event, cfg);
    }
    event.pid = bpf_get_current_pid_tgid();
    event.addr = PT_REGS_IP(ctx);
    event.skb_addr = (u64)skb;
    event.ts = bpf_ktime_get_ns();
    bpf_perf_event_output(ctx, &events, BPF_F_CURRENT_CPU, &event, sizeof(event));
    return 0;
}

On the user side, a perf event reader consumes these events and prints them:

rd, err := perf.NewReader(events, os.Getpagesize())
if err != nil {
    log.Fatalf("Creating perf event reader: %s", err)
}
defer rd.Close()
var event pwru.Event
for {
    record, err := rd.Read()
    if err != nil {
        if perf.IsClosed(err) { return }
        log.Printf("Reading from perf event reader: %s", err)
    }
    if record.LostSamples != 0 {
        log.Printf("Perf event ring buffer full, dropped %d samples", record.LostSamples)
        continue
    }
    if err := binary.Read(bytes.NewBuffer(record.RawSample), binary.LittleEndian, &event); err != nil {
        log.Printf("Parsing perf event: %s", err)
        continue
    }
    output.Print(&event)
    select {
    case <-ctx.Done():
        break
    default:
        continue
    }
}

In summary, pwru provides a powerful, configurable eBPF‑based mechanism for tracing network packets, allowing developers and operators to pinpoint where and why packets are dropped or processed in the Linux kernel.