mirrors/opensnitch
1
0
Fork 0
mirror of https://github.com/evilsocket/opensnitch.git synced 2025-03-04 08:34:40 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

fixed ebpf error logging

Browse source 
+ split functionality into different files.
This commit is contained in:
Gustavo Iñiguez Goia 2021-04-05 12:19:00 +02:00
parent 9497cf8394
commit 8ea58ade19
4 changed files with 429 additions and 396 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

98
daemon/procmon/ebpf/debug.go Normal file
View file

@ -0,0 +1,98 @@
package ebpf
import (
"fmt"
"os/exec"
"strconv"
"syscall"
"unsafe"
"github.com/evilsocket/opensnitch/daemon/log"
daemonNetlink "github.com/evilsocket/opensnitch/daemon/netlink"
elf "github.com/iovisor/gobpf/elf"
)
// print map contents. used only for debugging
func dumpMap(bpfmap *elf.Map, isIPv6 bool) {
var lookupKey []byte
var nextKey []byte
var value []byte
if !isIPv6 {
lookupKey = make([]byte, 12)
nextKey = make([]byte, 12)
value = make([]byte, 24)
} else {
lookupKey = make([]byte, 36)
nextKey = make([]byte, 36)
value = make([]byte, 24)
}
firstrun := true
i := 0
for {
i++
ok, err := m.LookupNextElement(bpfmap, unsafe.Pointer(&lookupKey[0]),
unsafe.Pointer(&nextKey[0]), unsafe.Pointer(&value[0]))
if err != nil {
log.Error("eBPF LookupNextElement error: %v", err)
return
}
if firstrun {
// on first run lookupKey is a dummy, nothing to delete
firstrun = false
copy(lookupKey, nextKey)
continue
}
fmt.Println("key, value", lookupKey, value)
if !ok { //reached end of map
break
}
copy(lookupKey, nextKey)
}
}
//PrintEverything prints all the stats. used only for debugging
func PrintEverything() {
bash, _ := exec.LookPath("bash")
//get the number of the first map
out, err := exec.Command(bash, "-c", "bpftool map show | head -n 1 | cut -d ':' -f1").Output()
if err != nil {
fmt.Println("bpftool map dump name tcpMap ", err)
}
i, _ := strconv.Atoi(string(out[:len(out)-1]))
fmt.Println("i is", i)
//dump all maps for analysis
for j := i; j < i+14; j++ {
_, _ = exec.Command(bash, "-c", "bpftool map dump id "+strconv.Itoa(j)+" > dump"+strconv.Itoa(j)).Output()
}
for sock1, v := range alreadyEstablishedTCP {
fmt.Println(*sock1, v)
}
fmt.Println("---------------------")
for sock1, v := range alreadyEstablishedTCPv6 {
fmt.Println(*sock1, v)
}
fmt.Println("---------------------")
sockets, _ := daemonNetlink.SocketsDump(syscall.AF_INET, syscall.IPPROTO_TCP)
for idx := range sockets {
fmt.Println("socket tcp: ", sockets[idx])
}
fmt.Println("---------------------")
sockets, _ = daemonNetlink.SocketsDump(syscall.AF_INET6, syscall.IPPROTO_TCP)
for idx := range sockets {
fmt.Println("socket tcp6: ", sockets[idx])
}
fmt.Println("---------------------")
sockets, _ = daemonNetlink.SocketsDump(syscall.AF_INET, syscall.IPPROTO_UDP)
for idx := range sockets {
fmt.Println("socket udp: ", sockets[idx])
}
fmt.Println("---------------------")
sockets, _ = daemonNetlink.SocketsDump(syscall.AF_INET6, syscall.IPPROTO_UDP)
for idx := range sockets {
fmt.Println("socket udp6: ", sockets[idx])
}
}

415
daemon/procmon/ebpf/ebpf.go
View file

@ -4,18 +4,13 @@ import (
"encoding/binary"
"fmt"
"net"
"os/exec"
"strconv"
"sync"
"syscall"
"time"
"unsafe"
"github.com/evilsocket/opensnitch/daemon/log"
"github.com/evilsocket/opensnitch/daemon/procmon"
"github.com/vishvananda/netlink"
daemonNetlink "github.com/evilsocket/opensnitch/daemon/netlink"
"github.com/evilsocket/opensnitch/daemon/procmon"
elf "github.com/iovisor/gobpf/elf"
)
@ -26,6 +21,17 @@ type ebpfMapsForProto struct {
lastPurgedMax uint64 // max counter value up to and including which the map was purged on the last purge
}
//Not in use, ~4usec faster lookup compared to m.LookupElement()
//mimics union bpf_attr's anonymous struct used by BPF_MAP_*_ELEM commands
//from <linux_headers>/include/uapi/linux/bpf.h
type bpf_lookup_elem_t struct {
map_fd uint64 //even though in bpf.h its type is __u32, we must make it 8 bytes long
//because "key" is of type __aligned_u64, i.e. "key" must be aligned on an 8-byte boundary
key uintptr
value uintptr
}
var (
m *elf.Module
mapSize = 12000
@ -45,7 +51,7 @@ var (
func Start() error {
m = elf.NewModule("/etc/opensnitchd/opensnitch.o")
if err := m.Load(nil); err != nil {
log.Error("Failed to load /etc/opensnitchd/opensnitch.o", err)
log.Error("eBPF Failed to load /etc/opensnitchd/opensnitch.o: %v", err)
return err
}
@ -54,11 +60,11 @@ func Start() error {
if err := m.EnableKprobes(0); err != nil {
m.Close()
if err := m.Load(nil); err != nil {
log.Error("Failed to load /etc/opensnitchd/opensnitch.o", err)
log.Error("eBPF failed to load /etc/opensnitchd/opensnitch.o (2): %v", err)
return err
}
if err := m.EnableKprobes(0); err != nil {
log.Error("Error when enabling kprobes", err)
log.Error("eBPF error when enabling kprobes: %v", err)
return err
}
}
@ -69,7 +75,7 @@ func Start() error {
for _, name := range []string{"tcpcounter", "tcpv6counter", "udpcounter", "udpv6counter"} {
err := m.UpdateElement(m.Map(name), unsafe.Pointer(&zeroKey[0]), unsafe.Pointer(&zeroValue[0]), 0)
if err != nil {
log.Error("Could not init counters to zero", err)
log.Error("eBPF could not init counters to zero: %v", err)
return err
}
}
@ -104,7 +110,7 @@ func Start() error {
// save already established connections
socketListTCP, err := daemonNetlink.SocketsDump(uint8(syscall.AF_INET), uint8(syscall.IPPROTO_TCP))
if err != nil {
log.Error("Could not dump TCP sockets via netlink", err)
log.Error("eBPF could not dump TCP sockets via netlink: %v", err)
return err
}
for _, sock := range socketListTCP {
@ -116,7 +122,7 @@ func Start() error {
socketListTCPv6, err := daemonNetlink.SocketsDump(uint8(syscall.AF_INET6), uint8(syscall.IPPROTO_TCP))
if err != nil {
log.Error("Could not dump TCPv6 sockets via netlink", err)
log.Error("eBPF could not dump TCPv6 sockets via netlink: %v", err)
return err
}
for _, sock := range socketListTCPv6 {
@ -132,6 +138,7 @@ func Start() error {
return nil
}
// Stop stops monitoring connections using kprobes
func Stop() {
stop = true
if m != nil {
@ -139,305 +146,6 @@ func Stop() {
}
}
// delete all map's elements which have counter value <= maxToDelete
func deleteOld(bpfmap *elf.Map, isIPv6 bool, maxToDelete uint64) {
var lookupKey []byte
var nextKey []byte
var value []byte
if !isIPv6 {
lookupKey = make([]byte, 12)
nextKey = make([]byte, 12)
value = make([]byte, 24)
} else {
lookupKey = make([]byte, 36)
nextKey = make([]byte, 36)
value = make([]byte, 24)
}
firstrun := true
i := 0
for {
i++
if i > 12000 {
// there were more iterations than the max amount of elements in map
// TODO find out what causes the endless loop
// maybe because ebpf prog modified the map while we were iterating
log.Error("Breaking because endless loop was detected in deleteOld")
break
}
ok, err := m.LookupNextElement(bpfmap, unsafe.Pointer(&lookupKey[0]),
unsafe.Pointer(&nextKey[0]), unsafe.Pointer(&value[0]))
if err != nil {
log.Error("LookupNextElement error", err)
return
}
if firstrun {
// on first run lookupKey is a dummy, nothing to delete
firstrun = false
copy(lookupKey, nextKey)
continue
}
// last 8 bytes of value is counter value
counterValue := hostByteOrder.Uint64(value[16:24])
if counterValue > maxToDelete {
copy(lookupKey, nextKey)
continue
}
if err := m.DeleteElement(bpfmap, unsafe.Pointer(&lookupKey[0])); err != nil {
log.Error("DeleteElement error", err)
return
}
if !ok { //reached end of map
break
}
copy(lookupKey, nextKey)
}
}
// we need to manually remove old connections from a bpf map
// since when a bpf map is full it doesn't allow any more insertions
func monitorMaps() {
zeroKey := make([]byte, 4)
for {
time.Sleep(time.Second * 1)
if stop {
return
}
for name, ebpfMap := range ebpfMaps {
value := make([]byte, 8)
if err := m.LookupElement(ebpfMap.counterMap,
unsafe.Pointer(&zeroKey[0]), unsafe.Pointer(&value[0])); err != nil {
log.Error("m.LookupElement", err)
}
counterValue := hostByteOrder.Uint64(value)
if counterValue-ebpfMap.lastPurgedMax > 10000 {
ebpfMap.lastPurgedMax = counterValue - 5000
deleteOld(ebpfMap.bpfmap, name == "tcp6" || name == "udp6", ebpfMap.lastPurgedMax)
}
}
}
}
// GetPid looks up process pid in a bpf map. If not found there, then it searches
// already-established TCP connections.
func GetPid(proto string, srcPort uint, srcIP net.IP, dstIP net.IP, dstPort uint) (int, int, error) {
if hostByteOrder == nil {
return -1, -1, fmt.Errorf("eBPF monitoring method not initialized yet")
}
if pid, uid := getPidFromEbpf(proto, srcPort, srcIP, dstIP, dstPort); pid != -1 {
return pid, uid, nil
}
//check if it comes from already established TCP
if proto == "tcp" || proto == "tcp6" {
if pid, uid, err := findInAlreadyEstablishedTCP(proto, srcPort, srcIP, dstIP, dstPort); err == nil {
return pid, uid, nil
}
}
//using netlink.GetSocketInfo to check if UID is 0 (in-kernel connection)
if uid, _ := daemonNetlink.GetSocketInfo(proto, srcIP, srcPort, dstIP, dstPort); uid == 0 {
return -100, -100, nil
}
if !findAddressInLocalAddresses(srcIP) {
// systemd-resolved sometimes makes a TCP Fast Open connection to a DNS server (8.8.8.8 on my machine)
// and we get a packet here with **source** (not detination!!!) IP 8.8.8.8
// Maybe it's an in-kernel response with spoofed IP because wireshark does not show neither
// resolved's TCP Fast Open packet, nor the response
// Until this is better understood, we simply do not allow this machine to make connections with
// arbitrary source IPs
return -1, -1, fmt.Errorf("Packet with unknown source IP: %s", srcIP)
}
return -1, -1, nil
}
// getPidFromEbpf looks up a connection in bpf map and returns PID if found
// the lookup keys and values are defined in opensnitch.c , e.g.
//
// struct tcp_key_t {
// u16 sport;
// u32 daddr;
// u16 dport;
// u32 saddr;
// }__attribute__((packed));
// struct tcp_value_t{
// u64 pid;
// u64 uid;
// u64 counter;
// }__attribute__((packed));;
func getPidFromEbpf(proto string, srcPort uint, srcIP net.IP, dstIP net.IP, dstPort uint) (pid int, uid int) {
var key []byte
var value []byte
var isIP4 bool = (proto == "tcp") || (proto == "udp") || (proto == "udplite")
if isIP4 {
key = make([]byte, 12)
value = make([]byte, 24)
copy(key[2:6], dstIP)
binary.BigEndian.PutUint16(key[6:8], uint16(dstPort))
copy(key[8:12], srcIP)
} else { // IPv6
key = make([]byte, 36)
value = make([]byte, 24)
copy(key[2:18], dstIP)
binary.BigEndian.PutUint16(key[18:20], uint16(dstPort))
copy(key[20:36], srcIP)
}
hostByteOrder.PutUint16(key[0:2], uint16(srcPort))
err := m.LookupElement(ebpfMaps[proto].bpfmap, unsafe.Pointer(&key[0]), unsafe.Pointer(&value[0]))
if err != nil {
// key not found
// maybe srcIP is 0.0.0.0 Happens especially with UDP sendto()
// TODO: can this happen with TCP?
if isIP4 {
zeroes := make([]byte, 4)
copy(key[8:12], zeroes)
} else {
zeroes := make([]byte, 16)
copy(key[20:36], zeroes)
}
err = m.LookupElement(ebpfMaps[proto].bpfmap, unsafe.Pointer(&key[0]), unsafe.Pointer(&value[0]))
}
if err != nil && proto == "udp" && srcIP.String() == dstIP.String() {
// very rarely I see this connection. It has srcIP and dstIP == 0.0.0.0 in ebpf map
// it is a localhost to localhost connection
// srcIP was already set to 0, set dstIP to zero also
// TODO try to reproduce it and look for srcIP/dstIP in other kernel structures
zeroes := make([]byte, 4)
copy(key[2:6], zeroes)
err = m.LookupElement(ebpfMaps[proto].bpfmap, unsafe.Pointer(&key[0]), unsafe.Pointer(&value[0]))
}
if err != nil {
// key not found in bpf map
return -1, -1
}
pid = int(hostByteOrder.Uint32(value[0:4]))
uid = int(hostByteOrder.Uint32(value[8:12]))
return pid, uid
}
// FindInAlreadyEstablishedTCP searches those TCP connections which were already established at the time
// when opensnitch started
func findInAlreadyEstablishedTCP(proto string, srcPort uint, srcIP net.IP, dstIP net.IP, dstPort uint) (int, int, error) {
var alreadyEstablished map[*daemonNetlink.Socket]int
if proto == "tcp" {
alreadyEstablished = alreadyEstablishedTCP
} else if proto == "tcp6" {
alreadyEstablished = alreadyEstablishedTCPv6
}
for sock, v := range alreadyEstablished {
if (*sock).ID.SourcePort == uint16(srcPort) && (*sock).ID.Source.Equal(srcIP) &&
(*sock).ID.Destination.Equal(dstIP) && (*sock).ID.DestinationPort == uint16(dstPort) {
return v, int((*sock).UID), nil
}
}
return -1, -1, fmt.Errorf("Inode not found")
}
//returns true if addr is in the list of this machine's addresses
func findAddressInLocalAddresses(addr net.IP) bool {
localAddressesLock.Lock()
defer localAddressesLock.Unlock()
for _, a := range localAddresses {
if addr.String() == a.String() {
return true
}
}
return false
}
// maintains a list of this machine's local addresses
func monitorLocalAddresses() {
for {
addr, err := netlink.AddrList(nil, netlink.FAMILY_ALL)
if err != nil {
log.Error("Error looking up this machine's addresses via netlink", err)
continue
}
localAddressesLock.Lock()
localAddresses = nil
for _, a := range addr {
localAddresses = append(localAddresses, a.IP)
}
localAddressesLock.Unlock()
time.Sleep(time.Second * 1)
if stop {
return
}
}
}
// monitorAlreadyEstablished makes sure that when an already-established connection is closed
// it will be removed from alreadyEstablished. If we don't do this and keep the alreadyEstablished entry forever,
// then after the genuine process quits,a malicious process may reuse PID-srcPort-srcIP-dstPort-dstIP
func monitorAlreadyEstablished() {
for {
time.Sleep(time.Second * 1)
if stop {
return
}
socketListTCP, err := daemonNetlink.SocketsDump(uint8(syscall.AF_INET), uint8(syscall.IPPROTO_TCP))
if err != nil {
log.Error("Error in dumping TCP sockets via netlink")
continue
}
for aesock := range alreadyEstablishedTCP {
found := false
for _, sock := range socketListTCP {
if (*aesock).INode == (*sock).INode &&
//inodes are unique enough, so the matches below will never have to be checked
(*aesock).ID.SourcePort == (*sock).ID.SourcePort &&
(*aesock).ID.Source.Equal((*sock).ID.Source) &&
(*aesock).ID.Destination.Equal((*sock).ID.Destination) &&
(*aesock).ID.DestinationPort == (*sock).ID.DestinationPort &&
(*aesock).UID == (*sock).UID {
found = true
break
}
}
if !found {
delete(alreadyEstablishedTCP, aesock)
}
}
socketListTCPv6, err := daemonNetlink.SocketsDump(uint8(syscall.AF_INET6), uint8(syscall.IPPROTO_TCP))
if err != nil {
log.Error("Error in dumping TCPv6 sockets via netlink")
continue
}
for aesock := range alreadyEstablishedTCPv6 {
found := false
for _, sock := range socketListTCPv6 {
if (*aesock).INode == (*sock).INode &&
//inodes are unique enough, so the matches below will never have to be checked
(*aesock).ID.SourcePort == (*sock).ID.SourcePort &&
(*aesock).ID.Source.Equal((*sock).ID.Source) &&
(*aesock).ID.Destination.Equal((*sock).ID.Destination) &&
(*aesock).ID.DestinationPort == (*sock).ID.DestinationPort &&
(*aesock).UID == (*sock).UID {
found = true
break
}
}
if !found {
delete(alreadyEstablishedTCPv6, aesock)
}
}
}
}
//Not in use, ~4usec faster lookup compared to m.LookupElement()
//mimics union bpf_attr's anonymous struct used by BPF_MAP_*_ELEM commands
//from <linux_headers>/include/uapi/linux/bpf.h
type bpf_lookup_elem_t struct {
map_fd uint64 //even though in bpf.h its type is __u32, we must make it 8 bytes long
//because "key" is of type __aligned_u64, i.e. "key" must be aligned on an 8-byte boundary
key uintptr
value uintptr
}
//make bpf() syscall with bpf_lookup prepared by the caller
func makeBpfSyscall(bpf_lookup *bpf_lookup_elem_t) uintptr {
BPF_MAP_LOOKUP_ELEM := 1 //cmd number
@ -448,88 +156,3 @@ func makeBpfSyscall(bpf_lookup *bpf_lookup_elem_t) uintptr {
uintptr(unsafe.Pointer(bpf_lookup)), uintptr(sizeOfStruct))
return r1
}
// print map contents. used only for debugging
func dumpMap(bpfmap *elf.Map, isIPv6 bool) {
var lookupKey []byte
var nextKey []byte
var value []byte
if !isIPv6 {
lookupKey = make([]byte, 12)
nextKey = make([]byte, 12)
value = make([]byte, 24)
} else {
lookupKey = make([]byte, 36)
nextKey = make([]byte, 36)
value = make([]byte, 24)
}
firstrun := true
i := 0
for {
i++
ok, err := m.LookupNextElement(bpfmap, unsafe.Pointer(&lookupKey[0]),
unsafe.Pointer(&nextKey[0]), unsafe.Pointer(&value[0]))
if err != nil {
log.Error("LookupNextElement error", err)
return
}
if firstrun {
// on first run lookupKey is a dummy, nothing to delete
firstrun = false
copy(lookupKey, nextKey)
continue
}
fmt.Println("key, value", lookupKey, value)
if !ok { //reached end of map
break
}
copy(lookupKey, nextKey)
}
}
//PrintEverything prints all the stats. used only for debugging
func PrintEverything() {
bash, _ := exec.LookPath("bash")
//get the number of the first map
out, err := exec.Command(bash, "-c", "bpftool map show | head -n 1 | cut -d ':' -f1").Output()
if err != nil {
fmt.Println("bpftool map dump name tcpMap ", err)
}
i, _ := strconv.Atoi(string(out[:len(out)-1]))
fmt.Println("i is", i)
//dump all maps for analysis
for j := i; j < i+14; j++ {
_, _ = exec.Command(bash, "-c", "bpftool map dump id "+strconv.Itoa(j)+" > dump"+strconv.Itoa(j)).Output()
}
for sock1, v := range alreadyEstablishedTCP {
fmt.Println(*sock1, v)
}
fmt.Println("---------------------")
for sock1, v := range alreadyEstablishedTCPv6 {
fmt.Println(*sock1, v)
}
fmt.Println("---------------------")
sockets, _ := daemonNetlink.SocketsDump(syscall.AF_INET, syscall.IPPROTO_TCP)
for idx := range sockets {
fmt.Println("socket tcp: ", sockets[idx])
}
fmt.Println("---------------------")
sockets, _ = daemonNetlink.SocketsDump(syscall.AF_INET6, syscall.IPPROTO_TCP)
for idx := range sockets {
fmt.Println("socket tcp6: ", sockets[idx])
}
fmt.Println("---------------------")
sockets, _ = daemonNetlink.SocketsDump(syscall.AF_INET, syscall.IPPROTO_UDP)
for idx := range sockets {
fmt.Println("socket udp: ", sockets[idx])
}
fmt.Println("---------------------")
sockets, _ = daemonNetlink.SocketsDump(syscall.AF_INET6, syscall.IPPROTO_UDP)
for idx := range sockets {
fmt.Println("socket udp6: ", sockets[idx])
}
}

142
daemon/procmon/ebpf/find.go Normal file
View file

@ -0,0 +1,142 @@
package ebpf
import (
"encoding/binary"
"fmt"
"net"
"unsafe"
daemonNetlink "github.com/evilsocket/opensnitch/daemon/netlink"
)
// we need to manually remove old connections from a bpf map
// GetPid looks up process pid in a bpf map. If not found there, then it searches
// already-established TCP connections.
func GetPid(proto string, srcPort uint, srcIP net.IP, dstIP net.IP, dstPort uint) (int, int, error) {
if hostByteOrder == nil {
return -1, -1, fmt.Errorf("eBPF monitoring method not initialized yet")
}
if pid, uid := getPidFromEbpf(proto, srcPort, srcIP, dstIP, dstPort); pid != -1 {
return pid, uid, nil
}
//check if it comes from already established TCP
if proto == "tcp" || proto == "tcp6" {
if pid, uid, err := findInAlreadyEstablishedTCP(proto, srcPort, srcIP, dstIP, dstPort); err == nil {
return pid, uid, nil
}
}
//using netlink.GetSocketInfo to check if UID is 0 (in-kernel connection)
if uid, _ := daemonNetlink.GetSocketInfo(proto, srcIP, srcPort, dstIP, dstPort); uid == 0 {
return -100, -100, nil
}
if !findAddressInLocalAddresses(srcIP) {
// systemd-resolved sometimes makes a TCP Fast Open connection to a DNS server (8.8.8.8 on my machine)
// and we get a packet here with **source** (not detination!!!) IP 8.8.8.8
// Maybe it's an in-kernel response with spoofed IP because wireshark does not show neither
// resolved's TCP Fast Open packet, nor the response
// Until this is better understood, we simply do not allow this machine to make connections with
// arbitrary source IPs
return -1, -1, fmt.Errorf("eBPF packet with unknown source IP: %s", srcIP)
}
return -1, -1, nil
}
// getPidFromEbpf looks up a connection in bpf map and returns PID if found
// the lookup keys and values are defined in opensnitch.c , e.g.
//
// struct tcp_key_t {
// u16 sport;
// u32 daddr;
// u16 dport;
// u32 saddr;
// }__attribute__((packed));
// struct tcp_value_t{
// u64 pid;
// u64 uid;
// u64 counter;
// }__attribute__((packed));;
func getPidFromEbpf(proto string, srcPort uint, srcIP net.IP, dstIP net.IP, dstPort uint) (pid int, uid int) {
var key []byte
var value []byte
var isIP4 bool = (proto == "tcp") || (proto == "udp") || (proto == "udplite")
if isIP4 {
key = make([]byte, 12)
value = make([]byte, 24)
copy(key[2:6], dstIP)
binary.BigEndian.PutUint16(key[6:8], uint16(dstPort))
copy(key[8:12], srcIP)
} else { // IPv6
key = make([]byte, 36)
value = make([]byte, 24)
copy(key[2:18], dstIP)
binary.BigEndian.PutUint16(key[18:20], uint16(dstPort))
copy(key[20:36], srcIP)
}
hostByteOrder.PutUint16(key[0:2], uint16(srcPort))
err := m.LookupElement(ebpfMaps[proto].bpfmap, unsafe.Pointer(&key[0]), unsafe.Pointer(&value[0]))
if err != nil {
// key not found
// maybe srcIP is 0.0.0.0 Happens especially with UDP sendto()
// TODO: can this happen with TCP?
if isIP4 {
zeroes := make([]byte, 4)
copy(key[8:12], zeroes)
} else {
zeroes := make([]byte, 16)
copy(key[20:36], zeroes)
}
err = m.LookupElement(ebpfMaps[proto].bpfmap, unsafe.Pointer(&key[0]), unsafe.Pointer(&value[0]))
}
if err != nil && proto == "udp" && srcIP.String() == dstIP.String() {
// very rarely I see this connection. It has srcIP and dstIP == 0.0.0.0 in ebpf map
// it is a localhost to localhost connection
// srcIP was already set to 0, set dstIP to zero also
// TODO try to reproduce it and look for srcIP/dstIP in other kernel structures
zeroes := make([]byte, 4)
copy(key[2:6], zeroes)
err = m.LookupElement(ebpfMaps[proto].bpfmap, unsafe.Pointer(&key[0]), unsafe.Pointer(&value[0]))
}
if err != nil {
// key not found in bpf map
return -1, -1
}
pid = int(hostByteOrder.Uint32(value[0:4]))
uid = int(hostByteOrder.Uint32(value[8:12]))
return pid, uid
}
// FindInAlreadyEstablishedTCP searches those TCP connections which were already established at the time
// when opensnitch started
func findInAlreadyEstablishedTCP(proto string, srcPort uint, srcIP net.IP, dstIP net.IP, dstPort uint) (int, int, error) {
var alreadyEstablished map[*daemonNetlink.Socket]int
if proto == "tcp" {
alreadyEstablished = alreadyEstablishedTCP
} else if proto == "tcp6" {
alreadyEstablished = alreadyEstablishedTCPv6
}
for sock, v := range alreadyEstablished {
if (*sock).ID.SourcePort == uint16(srcPort) && (*sock).ID.Source.Equal(srcIP) &&
(*sock).ID.Destination.Equal(dstIP) && (*sock).ID.DestinationPort == uint16(dstPort) {
return v, int((*sock).UID), nil
}
}
return -1, -1, fmt.Errorf("eBPF inode not found")
}
//returns true if addr is in the list of this machine's addresses
func findAddressInLocalAddresses(addr net.IP) bool {
localAddressesLock.Lock()
defer localAddressesLock.Unlock()
for _, a := range localAddresses {
if addr.String() == a.String() {
return true
}
}
return false
}

170
daemon/procmon/ebpf/monitor.go Normal file
View file

@ -0,0 +1,170 @@
package ebpf
import (
"syscall"
"time"
"unsafe"
"github.com/evilsocket/opensnitch/daemon/log"
daemonNetlink "github.com/evilsocket/opensnitch/daemon/netlink"
elf "github.com/iovisor/gobpf/elf"
"github.com/vishvananda/netlink"
)
// we need to manually remove old connections from a bpf map
// since when a bpf map is full it doesn't allow any more insertions
func monitorMaps() {
zeroKey := make([]byte, 4)
for {
time.Sleep(time.Second * 1)
if stop {
return
}
for name, ebpfMap := range ebpfMaps {
value := make([]byte, 8)
if err := m.LookupElement(ebpfMap.counterMap,
unsafe.Pointer(&zeroKey[0]), unsafe.Pointer(&value[0])); err != nil {
log.Error("eBPF m.LookupElement error: %v", err)
}
counterValue := hostByteOrder.Uint64(value)
if counterValue-ebpfMap.lastPurgedMax > 10000 {
ebpfMap.lastPurgedMax = counterValue - 5000
deleteOld(ebpfMap.bpfmap, name == "tcp6" || name == "udp6", ebpfMap.lastPurgedMax)
}
}
}
}
// maintains a list of this machine's local addresses
func monitorLocalAddresses() {
for {
addr, err := netlink.AddrList(nil, netlink.FAMILY_ALL)
if err != nil {
log.Error("eBPF error looking up this machine's addresses via netlink: %v", err)
continue
}
localAddressesLock.Lock()
localAddresses = nil
for _, a := range addr {
localAddresses = append(localAddresses, a.IP)
}
localAddressesLock.Unlock()
time.Sleep(time.Second * 1)
if stop {
return
}
}
}
// monitorAlreadyEstablished makes sure that when an already-established connection is closed
// it will be removed from alreadyEstablished. If we don't do this and keep the alreadyEstablished entry forever,
// then after the genuine process quits,a malicious process may reuse PID-srcPort-srcIP-dstPort-dstIP
func monitorAlreadyEstablished() {
for {
time.Sleep(time.Second * 1)
if stop {
return
}
socketListTCP, err := daemonNetlink.SocketsDump(uint8(syscall.AF_INET), uint8(syscall.IPPROTO_TCP))
if err != nil {
log.Error("eBPF error in dumping TCP sockets via netlink")
continue
}
for aesock := range alreadyEstablishedTCP {
found := false
for _, sock := range socketListTCP {
if (*aesock).INode == (*sock).INode &&
//inodes are unique enough, so the matches below will never have to be checked
(*aesock).ID.SourcePort == (*sock).ID.SourcePort &&
(*aesock).ID.Source.Equal((*sock).ID.Source) &&
(*aesock).ID.Destination.Equal((*sock).ID.Destination) &&
(*aesock).ID.DestinationPort == (*sock).ID.DestinationPort &&
(*aesock).UID == (*sock).UID {
found = true
break
}
}
if !found {
delete(alreadyEstablishedTCP, aesock)
}
}
socketListTCPv6, err := daemonNetlink.SocketsDump(uint8(syscall.AF_INET6), uint8(syscall.IPPROTO_TCP))
if err != nil {
log.Error("eBPF error in dumping TCPv6 sockets via netlink")
continue
}
for aesock := range alreadyEstablishedTCPv6 {
found := false
for _, sock := range socketListTCPv6 {
if (*aesock).INode == (*sock).INode &&
//inodes are unique enough, so the matches below will never have to be checked
(*aesock).ID.SourcePort == (*sock).ID.SourcePort &&
(*aesock).ID.Source.Equal((*sock).ID.Source) &&
(*aesock).ID.Destination.Equal((*sock).ID.Destination) &&
(*aesock).ID.DestinationPort == (*sock).ID.DestinationPort &&
(*aesock).UID == (*sock).UID {
found = true
break
}
}
if !found {
delete(alreadyEstablishedTCPv6, aesock)
}
}
}
}
// delete all map's elements which have counter value <= maxToDelete
func deleteOld(bpfmap *elf.Map, isIPv6 bool, maxToDelete uint64) {
var lookupKey []byte
var nextKey []byte
var value []byte
if !isIPv6 {
lookupKey = make([]byte, 12)
nextKey = make([]byte, 12)
value = make([]byte, 24)
} else {
lookupKey = make([]byte, 36)
nextKey = make([]byte, 36)
value = make([]byte, 24)
}
firstrun := true
i := 0
for {
i++
if i > 12000 {
// there were more iterations than the max amount of elements in map
// TODO find out what causes the endless loop
// maybe because ebpf prog modified the map while we were iterating
log.Error("Breaking because endless loop was detected in deleteOld")
break
}
ok, err := m.LookupNextElement(bpfmap, unsafe.Pointer(&lookupKey[0]),
unsafe.Pointer(&nextKey[0]), unsafe.Pointer(&value[0]))
if err != nil {
log.Error("eBPF LookupNextElement error: %v", err)
return
}
if firstrun {
// on first run lookupKey is a dummy, nothing to delete
firstrun = false
copy(lookupKey, nextKey)
continue
}
// last 8 bytes of value is counter value
counterValue := hostByteOrder.Uint64(value[16:24])
if counterValue > maxToDelete {
copy(lookupKey, nextKey)
continue
}
if err := m.DeleteElement(bpfmap, unsafe.Pointer(&lookupKey[0])); err != nil {
log.Error("eBPF DeleteElement error: %v", err)
return
}
if !ok { //reached end of map
break
}
copy(lookupKey, nextKey)
}
}