diff --git a/daemon/procmon/details.go b/daemon/procmon/details.go index d7352637..8bba259c 100644 --- a/daemon/procmon/details.go +++ b/daemon/procmon/details.go @@ -170,7 +170,16 @@ func (p *Process) ReadCmdline() { p.Args = append(p.Args, arg) } } + } + p.CleanArgs() +} +// CleanArgs applies fixes on the cmdline arguments. +// - AppImages cmdline reports the execuable launched as /proc/self/exe, +// instead of the actual path to the binary. +func (p *Process) CleanArgs() { + if len(p.Args) > 0 && p.Args[0] == "/proc/self/exe" { + p.Args[0] = p.Path } } @@ -259,9 +268,11 @@ func (p *Process) readStatus() { } } -// CleanPath removes extra characters from the link that it points to. -// When a running process is deleted, the symlink has the bytes " (deleted") -// appended to the link. +// CleanPath applies fixes on the path to the binary: +// - Remove extra characters from the link that it points to. +// When a running process is deleted, the symlink has the bytes " (deleted") +// appended to the link. +// - If the path is /proc/self/exe, resolve the symlink that it points to. func (p *Process) CleanPath() { // Sometimes the path to the binary reported is the symbolic link of the process itself. @@ -273,7 +284,6 @@ func (p *Process) CleanPath() { p.Path = link return } - // link read failed if len(p.Args) > 0 && p.Args[0] != "" { p.Path = p.Args[0] diff --git a/daemon/procmon/ebpf/ebpf.go b/daemon/procmon/ebpf/ebpf.go index 77b9dac8..3a3baf31 100644 --- a/daemon/procmon/ebpf/ebpf.go +++ b/daemon/procmon/ebpf/ebpf.go @@ -47,9 +47,7 @@ var ( TCP: make(map[*daemonNetlink.Socket]int), TCPv6: make(map[*daemonNetlink.Socket]int), } - - //stop == true is a signal for all goroutines to stop - stop = false + stopMonitors = make(chan bool) // list of local addresses of this machine localAddresses []net.IP @@ -60,6 +58,7 @@ var ( //Start installs ebpf kprobes func Start() error { if err := mountDebugFS(); err != nil { + log.Error("ebpf.Start -> mount debugfs error. Report on github please: %s", err) return err } @@ -85,20 +84,7 @@ func Start() error { return err } } - - lock.Lock() - //determine host byte order - buf := [2]byte{} - *(*uint16)(unsafe.Pointer(&buf[0])) = uint16(0xABCD) - switch buf { - case [2]byte{0xCD, 0xAB}: - hostByteOrder = binary.LittleEndian - case [2]byte{0xAB, 0xCD}: - hostByteOrder = binary.BigEndian - default: - log.Error("Could not determine host byte order.") - } - lock.Unlock() + determineHostByteOrder() ebpfCache = NewEbpfCache() ebpfMaps = map[string]*ebpfMapsForProto{ @@ -147,35 +133,30 @@ func saveEstablishedConnections(commDomain uint8) error { // Stop stops monitoring connections using kprobes func Stop() { - lock.Lock() - stop = true - lock.Unlock() + for i := 0; i < 4; i++ { + stopMonitors <- true + } ebpfCache.clear() for i := 0; i < eventWorkers; i++ { stopStreamEvents <- true } + + if m != nil { + m.Close() + } + for pm := range perfMapList { if pm != nil { pm.PollStop() } } - for _, mod := range perfMapList { + for k, mod := range perfMapList { if mod != nil { mod.Close() + delete(perfMapList, k) } } - - if m != nil { - m.Close() - } -} - -func isStopped() bool { - lock.RLock() - defer lock.RUnlock() - - return stop } //make bpf() syscall with bpf_lookup prepared by the caller diff --git a/daemon/procmon/ebpf/events.go b/daemon/procmon/ebpf/events.go index f301c416..00ff5543 100644 --- a/daemon/procmon/ebpf/events.go +++ b/daemon/procmon/ebpf/events.go @@ -21,20 +21,21 @@ const MaxArgs = 20 // MaxArgLen defines the maximum length of each argument. // NOTE: this value is 131072 (PAGE_SIZE * 32) // https://elixir.bootlin.com/linux/latest/source/include/uapi/linux/binfmts.h#L16 -const MaxArgLen = 512 +const MaxArgLen = 256 // TaskCommLen is the maximum num of characters of the comm field const TaskCommLen = 16 type execEvent struct { - Type uint64 - PID uint64 - PPID uint64 - UID uint64 - //ArgsCount uint64 - Filename [MaxPathLen]byte - //Args [MaxArgs][MaxArgLen]byte - Comm [TaskCommLen]byte + Type uint64 + PID uint64 + PPID uint64 + UID uint64 + ArgsCount uint64 + ArgsPartial uint64 + Filename [MaxPathLen]byte + Args [MaxArgs][MaxArgLen]byte + Comm [TaskCommLen]byte } // Struct that holds the metadata of a connection. @@ -106,6 +107,7 @@ func initEventsStreamer() { <-sig }(sig) + eventWorkers = 0 initPerfMap(mp) } @@ -121,7 +123,7 @@ func initPerfMap(mod *elf.Module) { perfMapList[perfMap] = mod eventWorkers += 4 - for i := 0; i < 4; i++ { + for i := 0; i < eventWorkers; i++ { go streamEventsWorker(i, channel, lostEvents, execEvents) } perfMap.PollStart() @@ -145,29 +147,10 @@ func streamEventsWorker(id int, chn chan []byte, lost chan uint64, execEvents *e if _, found := execEvents.isInStore(event.PID); found { continue } - proc := procmon.NewProcess(int(event.PID), byteArrayToString(event.Comm[:])) - // trust process path received from kernel - path := byteArrayToString(event.Filename[:]) - if path != "" { - proc.SetPath(path) - } else { - if proc.ReadPath() != nil { - continue - } + proc := event2process(&event) + if proc == nil { + continue } - proc.ReadCmdline() - proc.ReadCwd() - proc.ReadEnv() - proc.UID = int(event.UID) - - log.Debug("[eBPF exec event] ppid: %d, pid: %d, %s -> %s", event.PPID, event.PID, proc.Path, proc.Args) - /*args := make([]string, 0) - for i := 0; i < int(event.ArgsCount); i++ { - args = append(args, byteArrayToString(event.Args[i][:])) - } - proc.Args = args - log.Warning("[eBPF exec args] %s, %s", strings.Join(args, " "), proc.Args) - */ execEvents.add(event.PID, event, *proc) case EV_TYPE_SCHED_EXIT: @@ -185,3 +168,32 @@ func streamEventsWorker(id int, chn chan []byte, lost chan uint64, execEvents *e Exit: log.Debug("perfMap goroutine exited #%d", id) } + +func event2process(event *execEvent) (proc *procmon.Process) { + + proc = procmon.NewProcess(int(event.PID), byteArrayToString(event.Comm[:])) + // trust process path received from kernel + path := byteArrayToString(event.Filename[:]) + if path != "" { + proc.SetPath(path) + } else { + if proc.ReadPath() != nil { + return nil + } + } + proc.ReadCwd() + proc.ReadEnv() + proc.UID = int(event.UID) + + if event.ArgsPartial == 0 { + for i := 0; i < int(event.ArgsCount); i++ { + proc.Args = append(proc.Args, byteArrayToString(event.Args[i][:])) + } + proc.CleanArgs() + } else { + proc.ReadCmdline() + } + log.Debug("[eBPF exec event] ppid: %d, pid: %d, %s -> %s", event.PPID, event.PID, proc.Path, proc.Args) + + return +} diff --git a/daemon/procmon/ebpf/monitor.go b/daemon/procmon/ebpf/monitor.go index e2898b44..7b4cff08 100644 --- a/daemon/procmon/ebpf/monitor.go +++ b/daemon/procmon/ebpf/monitor.go @@ -14,53 +14,59 @@ import ( // since when a bpf map is full it doesn't allow any more insertions func monitorMaps() { for { - if isStopped() { - return - } - time.Sleep(time.Second * 5) - for name := range ebpfMaps { - // using a pointer to the map doesn't delete the items. - // bpftool still counts them. - if items := getItems(name, name == "tcp6" || name == "udp6"); items > 500 { - deleted := deleteOldItems(name, name == "tcp6" || name == "udp6", items/2) - log.Debug("[ebpf] old items deleted: %d", deleted) + select { + case <-stopMonitors: + goto Exit + default: + time.Sleep(time.Second * 5) + for name := range ebpfMaps { + // using a pointer to the map doesn't delete the items. + // bpftool still counts them. + if items := getItems(name, name == "tcp6" || name == "udp6"); items > 500 { + deleted := deleteOldItems(name, name == "tcp6" || name == "udp6", items/2) + log.Debug("[ebpf] old items deleted: %d", deleted) + } } } } +Exit: } func monitorCache() { for { select { + case <-stopMonitors: + goto Exit case <-ebpfCacheTicker.C: - if isStopped() { - return - } ebpfCache.DeleteOldItems() } } +Exit: } // maintains a list of this machine's local addresses // TODO: use netlink.AddrSubscribeWithOptions() func monitorLocalAddresses() { for { - addr, err := netlink.AddrList(nil, netlink.FAMILY_ALL) - if err != nil { - log.Error("eBPF error looking up this machine's addresses via netlink: %v", err) - continue - } - lock.Lock() - localAddresses = nil - for _, a := range addr { - localAddresses = append(localAddresses, a.IP) - } - lock.Unlock() - time.Sleep(time.Second * 1) - if isStopped() { - return + select { + case <-stopMonitors: + goto Exit + default: + addr, err := netlink.AddrList(nil, netlink.FAMILY_ALL) + if err != nil { + log.Error("eBPF error looking up this machine's addresses via netlink: %v", err) + continue + } + lock.Lock() + localAddresses = nil + for _, a := range addr { + localAddresses = append(localAddresses, a.IP) + } + lock.Unlock() + time.Sleep(time.Second * 1) } } +Exit: } // monitorAlreadyEstablished makes sure that when an already-established connection is closed @@ -68,52 +74,55 @@ func monitorLocalAddresses() { // then after the genuine process quits,a malicious process may reuse PID-srcPort-srcIP-dstPort-dstIP func monitorAlreadyEstablished() { for { - time.Sleep(time.Second * 1) - if isStopped() { - return - } - socketListTCP, err := daemonNetlink.SocketsDump(uint8(syscall.AF_INET), uint8(syscall.IPPROTO_TCP)) - if err != nil { - log.Debug("eBPF error in dumping TCP sockets via netlink") - continue - } - alreadyEstablished.Lock() - for aesock := range alreadyEstablished.TCP { - found := false - for _, sock := range socketListTCP { - if socketsAreEqual(aesock, sock) { - found = true - break - } - } - if !found { - delete(alreadyEstablished.TCP, aesock) - } - } - alreadyEstablished.Unlock() - - if core.IPv6Enabled { - socketListTCPv6, err := daemonNetlink.SocketsDump(uint8(syscall.AF_INET6), uint8(syscall.IPPROTO_TCP)) + select { + case <-stopMonitors: + goto Exit + default: + time.Sleep(time.Second * 1) + socketListTCP, err := daemonNetlink.SocketsDump(uint8(syscall.AF_INET), uint8(syscall.IPPROTO_TCP)) if err != nil { - log.Debug("eBPF error in dumping TCPv6 sockets via netlink: %s", err) + log.Debug("eBPF error in dumping TCP sockets via netlink") continue } alreadyEstablished.Lock() - for aesock := range alreadyEstablished.TCPv6 { + for aesock := range alreadyEstablished.TCP { found := false - for _, sock := range socketListTCPv6 { + for _, sock := range socketListTCP { if socketsAreEqual(aesock, sock) { found = true break } } if !found { - delete(alreadyEstablished.TCPv6, aesock) + delete(alreadyEstablished.TCP, aesock) } } alreadyEstablished.Unlock() + + if core.IPv6Enabled { + socketListTCPv6, err := daemonNetlink.SocketsDump(uint8(syscall.AF_INET6), uint8(syscall.IPPROTO_TCP)) + if err != nil { + log.Debug("eBPF error in dumping TCPv6 sockets via netlink: %s", err) + continue + } + alreadyEstablished.Lock() + for aesock := range alreadyEstablished.TCPv6 { + found := false + for _, sock := range socketListTCPv6 { + if socketsAreEqual(aesock, sock) { + found = true + break + } + } + if !found { + delete(alreadyEstablished.TCPv6, aesock) + } + } + alreadyEstablished.Unlock() + } } } +Exit: } func socketsAreEqual(aSocket, bSocket *daemonNetlink.Socket) bool { diff --git a/daemon/procmon/ebpf/utils.go b/daemon/procmon/ebpf/utils.go index 4225dd81..29f89a74 100644 --- a/daemon/procmon/ebpf/utils.go +++ b/daemon/procmon/ebpf/utils.go @@ -2,6 +2,7 @@ package ebpf import ( "bytes" + "encoding/binary" "fmt" "unsafe" @@ -9,6 +10,22 @@ import ( "github.com/evilsocket/opensnitch/daemon/log" ) +func determineHostByteOrder() { + lock.Lock() + //determine host byte order + buf := [2]byte{} + *(*uint16)(unsafe.Pointer(&buf[0])) = uint16(0xABCD) + switch buf { + case [2]byte{0xCD, 0xAB}: + hostByteOrder = binary.LittleEndian + case [2]byte{0xAB, 0xCD}: + hostByteOrder = binary.BigEndian + default: + log.Error("Could not determine host byte order.") + } + lock.Unlock() +} + func mountDebugFS() error { debugfsPath := "/sys/kernel/debug/" kprobesPath := fmt.Sprint(debugfsPath, "tracing/kprobe_events") diff --git a/ebpf_prog/common.h b/ebpf_prog/common.h index 142d9ddc..5c055e4e 100644 --- a/ebpf_prog/common.h +++ b/ebpf_prog/common.h @@ -15,8 +15,15 @@ //https://elixir.bootlin.com/linux/latest/source/include/uapi/linux/binfmts.h#L16 #define MAX_CMDLINE_LEN 4096 +// max args that I've been able to use before hitting the error: +// "dereference of modified ctx ptr disallowed" #define MAX_ARGS 20 -#define MAX_ARG_SIZE 512 +#define MAX_ARG_SIZE 256 + +// flags to indicate if we were able to read all the cmdline arguments, +// or if one of the arguments is >= MAX_ARG_SIZE, or there more than MAX_ARGS +#define COMPLETE_ARGS 0 +#define INCOMPLETE_ARGS 1 #define MAPSIZE 12000 @@ -64,9 +71,10 @@ struct data_t { u64 pid; // PID as in the userspace term (i.e. task->tgid in kernel) u64 ppid; // Parent PID as in the userspace term (i.e task->real_parent->tgid in kernel) u64 uid; - //u64 args_count; + u64 args_count; + u64 args_partial; char filename[MAX_PATH_LEN]; - //char args[MAX_ARGS][MAX_ARG_SIZE]; + char args[MAX_ARGS][MAX_ARG_SIZE]; char comm[TASK_COMM_LEN]; }__attribute__((packed)); diff --git a/ebpf_prog/opensnitch-procs.c b/ebpf_prog/opensnitch-procs.c index 3e3b71c7..d73060b9 100644 --- a/ebpf_prog/opensnitch-procs.c +++ b/ebpf_prog/opensnitch-procs.c @@ -74,21 +74,28 @@ int tracepoint__syscalls_sys_enter_execve(struct trace_sys_enter_execve* ctx) new_event(data); data->type = EVENT_EXEC; + // bpf_probe_read_user* helpers were introduced in kernel 5.5 + // Since the args can be overwritten anyway, maybe we could get them from + // mm_struct instead for a wider kernel version support range? bpf_probe_read_user_str(&data->filename, sizeof(data->filename), (const char *)ctx->filename); - /* if we get the args, we'd have to be sure that we get the whole cmdline, - * either by allocating the whole cmdline, or by sending each arg to userspace. const char *argp={0}; data->args_count = 0; - #pragma unroll (full) + data->args_partial = INCOMPLETE_ARGS; + #pragma unroll for (int i = 0; i < MAX_ARGS; i++) { - bpf_probe_read_user(&argp, sizeof(argp), &ctx->argv[i]); - if (!argp){ break; } + bpf_probe_read_user(&argp, sizeof(argp), &ctx->argv[i]); + if (!argp){ data->args_partial = COMPLETE_ARGS; break; } - bpf_probe_read_user_str(&data->args[i], MAX_ARG_SIZE, argp); - data->args_count++; - }*/ + if (bpf_probe_read_user_str(&data->args[i], MAX_ARG_SIZE, argp) >= MAX_ARG_SIZE){ + break; + } + data->args_count++; + } + // With some commands, this helper fails with error -28 (ENOSPC). Misleading error? cmd failed maybe? + // BUG: after coming back from suspend state, this helper fails with error -95 (EOPNOTSUPP) + // Possible workaround: count -95 errors, and from userspace reinitialize the streamer if errors >= n-errors bpf_perf_event_output(ctx, &events, BPF_F_CURRENT_CPU, data, sizeof(*data)); return 0; };