Dive into NSM: IPv4 Fragmentation and reassembly

IPv4 分片与重组机制

由于 MTU 限制,IP 报文在传输数据时,目的主机会进行 IP 重组。网络中不进行重组要比进行重组更能减轻路由转发软件(或硬件)的负担;同一数据报的不同分片可能经由不同的路径到达相同的目的地。
重组的方法十分暴力,有几个关键点:

  1. Identification: 唯一地标识一个报文的所有分片

  2. Flags:

000 保留
^
010 禁止分片 DF
 ^
001 更多分片 MF
  ^
  1. Fragment Offset: 对于原始报文开头的偏移量

实验 (以 ICMP 为例)

  1. 使用 libpnet 编写没有 IP 分片重组能力的流量捕获实验代码:

由于没有 IP 分片重组,过长的 ICMP 不能正常识别成 request 或 reply。
[{}]: ICMP packet {} -> {} (type={:?})

// datalink 
fn handle_packet(interface_name: &str, ethernet: &EthernetPacket, ip_defrag_hash_map: &mut HashMap<u64, VecDeque<IpInfo>>) {
    match ethernet.get_ethertype() {
        EtherTypes::Ipv4 => handle_ipv4_packet(interface_name, ethernet, ip_defrag_hash_map),
        _ => {
            println!("[{}]: Unknown packet: {} > {}; ethertype: {:?} length: {}",
                     interface_name,
                     ethernet.get_source(),
                     ethernet.get_destination(),
                     ethernet.get_ethertype(),
                     ethernet.packet().len())
        }
    }

// IPv4
fn handle_ipv4_packet(interface_name: &str, ethernet: &EthernetPacket) {
    let header = Ipv4Packet::new(ethernet.payload());
    if let Some(header) = header {
        handle_transport_protocol(interface_name,
                                  IpAddr::V4(header.get_source()),
                                  IpAddr::V4(header.get_destination()),
                                  header.get_next_level_protocol(),
                                  header.payload());
    } else {
        println!("[{}]: Malformed IPv4 Packet", interface_name);
    }
}


fn handle_transport_protocol(interface_name: &str,
                             source: IpAddr,
                             destination: IpAddr,
                             protocol: IpNextHeaderProtocol,
                             packet: &[u8]) {
    match protocol {
        IpNextHeaderProtocols::Icmp => {
            handle_icmp_packet(interface_name, source, destination, packet)
        }
        _ => {
            println!("[{}]: Unknown {} packet: {} > {}; protocol: {:?} length: {}",
                     interface_name,
                     match source {
                         IpAddr::V4(..) => "IPv4",
                         _ => "IPv6",
                     },
                     source,
                     destination,
                     protocol,
                     packet.len())
        }

    }
}

// ICMP
fn handle_icmp_packet(interface_name: &str, source: IpAddr, destination: IpAddr, packet: &[u8]) {
    let icmp_packet = IcmpPacket::new(packet);
    if let Some(icmp_packet) = icmp_packet {
        match icmp_packet.get_icmp_type() {
            IcmpTypes::EchoReply => {
                let echo_reply_packet = echo_reply::EchoReplyPacket::new(packet).unwrap();
                println!("[{}]: ICMP echo reply {} -> {} (seq={:?}, id={:?})",
                         interface_name,
                         source,
                         destination,
                         echo_reply_packet.get_sequence_number(),
                         echo_reply_packet.get_identifier());
            }
            IcmpTypes::EchoRequest => {
                let echo_request_packet = echo_request::EchoRequestPacket::new(packet).unwrap();
                println!("[{}]: ICMP echo request {} -> {} (seq={:?}, id={:?})",
                         interface_name,
                         source,
                         destination,
                         echo_request_packet.get_sequence_number(),
                         echo_request_packet.get_identifier());
            }
            _ => {
                println!("[{}]: ICMP packet {} -> {} (type={:?})",
                         interface_name,
                         source,
                         destination,
                         icmp_packet.get_icmp_type())
            }
        }
    } else {
        println!("[{}]: Malformed ICMP Packet", interface_name);
    }
}


  1. 对 IPv4 部分开刀,添加分片跟踪所需的 hashmap

MTU 1500

let mut ip_defrag_hash_map: HashMap<u64, VecDeque<IpInfo>> = HashMap::new();

  1. 存储 IP 重组必要的 数据结构
#[derive(Hash)]
struct IpFragId {
    src: Ipv4Addr,
    dst: Ipv4Addr,
    id: u16,
}

struct IpInfo {
    offset: u16,
    total: u16,
    frag_id: IpFragId,
    payload: Vec<u8>,
}
  1. hash 关键字
fn calculate_hash<T: Hash>(t: &T) -> u64 {
    let mut s = DefaultHasher::new();
    t.hash(&mut s);
    s.finish()
}

...

            // 注册 src dest id
            let ip_id = IpFragId {
                src: header.get_source(),
                dst: header.get_destination(),
                id: header.get_identification(),
            };
            // 计算 hash
            let hash = calculate_hash(&ip_id);

...
  1. 每个分片进入队列进行跟踪

...

            let ip_info = IpInfo {
                offset: header.get_fragment_offset(),
                total: header.get_total_length(),
                frag_id: ip_id,
                payload: {
                    let len = header.payload().len();
                    //println!("len: {}", len);
                    let mut payloads: Vec<_> = vec![0; len];
                    payloads.clone_from_slice(header.payload());
                    payloads
                }
            };

            // 在 hashmap 中 查找是否有存在过
            let mut temp_defrag_queue: VecDeque<IpInfo>;
            if ip_defrag_hash_map.contains_key(&hash) {
                // 如果 存在 追加数据
                let mut defrag_queue: &mut VecDeque<IpInfo> = ip_defrag_hash_map.get_mut(&hash).unwrap();
                // 存入队列
                defrag_queue.push_back(ip_info);
                //println!("add frag");
            } else {
                // 如果 不存在 新建队列
                let mut defrag_queue: VecDeque<IpInfo> = VecDeque::new();
                // 存入队列
                defrag_queue.push_back(ip_info);
                // 存入 hashmap
                ip_defrag_hash_map.insert(hash, defrag_queue);
                //println!("new frag");
            };

...

  1. 跟踪结束,重组 IP 报文

...

            if header.get_flags() == 0 {
                //println!("frag done!");
                if ip_defrag_hash_map.contains_key(&hash) {
                    let mut defrag_queue: &mut VecDeque<IpInfo> = ip_defrag_hash_map.get_mut(&hash).unwrap();
                    let mut payloads: Vec<u8> = Vec::new();
                    for ip_info in defrag_queue.iter() {
                        //println!("offset: {} ", ip_info.offset);
                        //println!("payload {:?} ", ip_info.payload);
                        payloads.extend(ip_info.payload.iter());
                    }
                    //println!("payloads: {:?}", payloads);

                    handle_transport_protocol(interface_name,
                                              IpAddr::V4(header.get_source()),
                                              IpAddr::V4(header.get_destination()),
                                              header.get_next_level_protocol(),
                                              &payloads);
                }
                // 回收 hashmap 销毁 队列
                ip_defrag_hash_map.remove(&hash);
            }

...

效果

ping 192.168.213.1 -c 1 -s 1472
ping 192.168.213.1 -c 1 -s 1473

ping 192.168.213.1  -s 2000

思考

  1. 上面的实现方式是一个简单的实现,缺少了很多细节处理,比如:offset 校验、超时、报文长度最大限定判断、首部重新校验。

  2. 工程上看上缺少最大 HashMap 限定、和最大队列限定,如果生产中这么处理,这段 IPv4 重组将会成为内存杀手。

  3. 安全性上看,如果 NSM 设备(当然也包含其他网络设备) 没有正确的处理 IPv4 重组,可能面临的安全风险或 NSM 功能缺失有:

    1. IP 报文分片多频率快,导致 NSM 设备内存开销大。
    2. IP 报文无分片结束,如果没有超时机制导致内存长时间占用。
    3. 如果没有正确校验 offset, 导致乱序重组。

示例程序

需要依赖 libpnet 0.18.0

[dependencies.pnet]
version = "0.18.0"

rustc 1.18.0 通过

extern crate pnet;


use pnet::datalink::{self, NetworkInterface};

use pnet::packet::Packet;
use pnet::packet::arp::ArpPacket;
use pnet::packet::ethernet::{EtherTypes, EthernetPacket};
use pnet::packet::icmp::{IcmpPacket, IcmpTypes, echo_reply, echo_request};
use pnet::packet::ip::{IpNextHeaderProtocol, IpNextHeaderProtocols};
use pnet::packet::ipv4::Ipv4Packet;
use pnet::packet::ipv6::Ipv6Packet;
use pnet::packet::tcp::TcpPacket;
use pnet::packet::udp::UdpPacket;
use std::env;
use std::io::{self, Write};
use std::process;
use std::net::IpAddr;
use std::net::Ipv4Addr;


use std::collections::VecDeque;
use std::collections::HashMap;
use std::collections::hash_map::DefaultHasher;
use std::hash::{Hash, Hasher};

#[derive(Hash)]
struct IpFragId {
    src: Ipv4Addr,
    dst: Ipv4Addr,
    id: u16,
}

struct IpInfo {
    offset: u16,
    total: u16,
    frag_id: IpFragId,
    payload: Vec<u8>,
}

fn calculate_hash<T: Hash>(t: &T) -> u64 {
    let mut s = DefaultHasher::new();
    t.hash(&mut s);
    s.finish()
}


fn handle_udp_packet(interface_name: &str, source: IpAddr, destination: IpAddr, packet: &[u8]) {
    let udp = UdpPacket::new(packet);

    if let Some(udp) = udp {
        println!("[{}]: UDP Packet: {}:{} > {}:{}; length: {}",
                 interface_name,
                 source,
                 udp.get_source(),
                 destination,
                 udp.get_destination(),
                 udp.get_length());
    } else {
        println!("[{}]: Malformed UDP Packet", interface_name);
    }
}

fn handle_icmp_packet(interface_name: &str, source: IpAddr, destination: IpAddr, packet: &[u8]) {
    let icmp_packet = IcmpPacket::new(packet);
    if let Some(icmp_packet) = icmp_packet {
        match icmp_packet.get_icmp_type() {
            IcmpTypes::EchoReply => {
                let echo_reply_packet = echo_reply::EchoReplyPacket::new(packet).unwrap();
                println!("[{}]: ICMP echo reply {} -> {} (seq={:?}, id={:?})",
                         interface_name,
                         source,
                         destination,
                         echo_reply_packet.get_sequence_number(),
                         echo_reply_packet.get_identifier());
            }
            IcmpTypes::EchoRequest => {
                let echo_request_packet = echo_request::EchoRequestPacket::new(packet).unwrap();
                println!("[{}]: ICMP echo request {} -> {} (seq={:?}, id={:?})",
                         interface_name,
                         source,
                         destination,
                         echo_request_packet.get_sequence_number(),
                         echo_request_packet.get_identifier());
            }
            _ => {
                println!("[{}]: ICMP packet {} -> {} (type={:?})",
                         interface_name,
                         source,
                         destination,
                         icmp_packet.get_icmp_type())
            }
        }
    } else {
        println!("[{}]: Malformed ICMP Packet", interface_name);
    }
}



fn handle_tcp_packet(interface_name: &str, source: IpAddr, destination: IpAddr, packet: &[u8]) {
    let tcp = TcpPacket::new(packet);
    if let Some(tcp) = tcp {
        println!("[{}]: TCP Packet: {}:{} > {}:{}; length: {}",
                 interface_name,
                 source,
                 tcp.get_source(),
                 destination,
                 tcp.get_destination(),
                 packet.len());
    } else {
        println!("[{}]: Malformed TCP Packet", interface_name);
    }
}



fn handle_transport_protocol(interface_name: &str,
                             source: IpAddr,
                             destination: IpAddr,
                             protocol: IpNextHeaderProtocol,
                             packet: &[u8]) {
    match protocol {
        IpNextHeaderProtocols::Udp => {
            handle_udp_packet(interface_name, source, destination, packet)
        }
        IpNextHeaderProtocols::Tcp => {
            handle_tcp_packet(interface_name, source, destination, packet)
        }
        IpNextHeaderProtocols::Icmp => {
            handle_icmp_packet(interface_name, source, destination, packet)
        }
        _ => {
            println!("[{}]: Unknown {} packet: {} > {}; protocol: {:?} length: {}",
                     interface_name,
                     match source {
                         IpAddr::V4(..) => "IPv4",
                         _ => "IPv6",
                     },
                     source,
                     destination,
                     protocol,
                     packet.len())
        }
    }
}

fn handle_ipv4_packet(interface_name: &str, ethernet: &EthernetPacket, ip_defrag_hash_map: &mut HashMap<u64, VecDeque<IpInfo>>) {
    let header = Ipv4Packet::new(ethernet.payload());
    if let Some(header) = header {
        /*
        println!("IPv4: {:?} -> {:?}", header.get_source(), header.get_destination());
        println!("\t ip header length: {:?}", header.get_header_length());
        println!("\t ip total length: {:?}", header.get_total_length());
        println!("\t ip identification: {:?}", header.get_identification());
        // 000 fragment done
        // 010 don't fragment
        // 001 need fragment
        println!("\t ip flag: {:b}", header.get_flags());
        println!("\t ip fragment offset: {:?}", header.get_fragment_offset());
        println!("\t packet: ");
        //println!("\t {:?}", header.packet());
        println!("\t payload: ");
        println!("\t {:?}", header.payload());
        */
        // 检查是否需要分片
        if !check_flag(header.get_flags()) {
            handle_transport_protocol(interface_name,
                                      IpAddr::V4(header.get_source()),
                                      IpAddr::V4(header.get_destination()),
                                      header.get_next_level_protocol(),
                                      header.payload());
        } else {
            // 注册 src dest id
            let ip_id = IpFragId {
                src: header.get_source(),
                dst: header.get_destination(),
                id: header.get_identification(),
            };
            // 计算 hash
            let hash = calculate_hash(&ip_id);
            //println!("hash: {}", hash);
            let ip_info = IpInfo {
                offset: header.get_fragment_offset(),
                total: header.get_total_length(),
                frag_id: ip_id,
                payload: {
                    let len = header.payload().len();
                    //println!("len: {}", len);
                    let mut payloads: Vec<_> = vec![0; len];
                    payloads.clone_from_slice(header.payload());
                    payloads
                }
            };

            // 在 hashmap 中 查找是否有存在过
            let mut temp_defrag_queue: VecDeque<IpInfo>;
            if ip_defrag_hash_map.contains_key(&hash) {
                // 如果 存在 追加数据
                let mut defrag_queue: &mut VecDeque<IpInfo> = ip_defrag_hash_map.get_mut(&hash).unwrap();
                // 存入队列
                defrag_queue.push_back(ip_info);
                //println!("add frag");
            } else {
                // 如果 不存在 新建队列
                let mut defrag_queue: VecDeque<IpInfo> = VecDeque::new();
                // 存入队列
                defrag_queue.push_back(ip_info);
                // 存入 hashmap
                ip_defrag_hash_map.insert(hash, defrag_queue);
                //println!("new frag");
            };


            if header.get_flags() == 0 {
                println!("frag done!");
                if ip_defrag_hash_map.contains_key(&hash) {
                    let mut defrag_queue: &mut VecDeque<IpInfo> = ip_defrag_hash_map.get_mut(&hash).unwrap();
                    let mut payloads: Vec<u8> = Vec::new();
                    for ip_info in defrag_queue.iter() {
                        //println!("offset: {} ", ip_info.offset);
                        //println!("payload {:?} ", ip_info.payload);
                        payloads.extend(ip_info.payload.iter());
                    }
                    //println!("payloads: {:?}", payloads);

                    handle_transport_protocol(interface_name,
                                              IpAddr::V4(header.get_source()),
                                              IpAddr::V4(header.get_destination()),
                                              header.get_next_level_protocol(),
                                              &payloads);
                }
                // 回收 hashmap 销毁 队列
                ip_defrag_hash_map.remove(&hash);
            }
        }
    } else {
        println!("[{}]: Malformed IPv4 Packet", interface_name);
    }
}


fn check_flag(flags: u8) -> bool {
    let ret = if flags == 2 {
        false
    } else {
        true
    };
    ret
}


fn handle_packet(interface_name: &str, ethernet: &EthernetPacket, ip_defrag_hash_map: &mut HashMap<u64, VecDeque<IpInfo>>) {
    match ethernet.get_ethertype() {
        EtherTypes::Ipv4 => handle_ipv4_packet(interface_name, ethernet, ip_defrag_hash_map),
        _ => {
            println!("[{}]: Unknown packet: {} > {}; ethertype: {:?} length: {}",
                     interface_name,
                     ethernet.get_source(),
                     ethernet.get_destination(),
                     ethernet.get_ethertype(),
                     ethernet.packet().len())
        }
    }
}

fn main() {
    use pnet::datalink::Channel::Ethernet;

    let iface_name = match env::args().nth(1) {
        Some(n) => n,
        None => {
            writeln!(io::stderr(), "USAGE: packetdump <NETWORK INTERFACE>").unwrap();
            process::exit(1);
        }
    };
    let interface_names_match = |iface: &NetworkInterface| iface.name == iface_name;

    // Find the network interface with the provided name
    let interfaces = datalink::interfaces();
    let interface = interfaces.into_iter().filter(interface_names_match).next().unwrap();

    // Create a channel to receive on
    let (_, mut rx) = match datalink::channel(&interface, Default::default()) {
        Ok(Ethernet(tx, rx)) => (tx, rx),
        Ok(_) => panic!("packetdump: unhandled channel type: {}"),
        Err(e) => panic!("packetdump: unable to create channel: {}", e),
    };
    let mut ip_defrag_hash_map: HashMap<u64, VecDeque<IpInfo>> = HashMap::new();
    let mut iter = rx.iter();
    loop {
        match iter.next() {
            Ok(packet) => handle_packet(&interface.name[..], &packet, &mut ip_defrag_hash_map),
            Err(e) => panic!("packetdump: unable to receive packet: {}", e),
        }
    }
}