cache: delay the deletion time of an process

Sometimes we may receive a connection event after the exit of a
process:

[exec] /bin/xxx, pid 1234
[exit] /bin/xxx, pid 1234
[new conn] pid 1234 -> process unknown (on exec event and no /proc entry)

In these scenarios, we delay the deletion from cache a little
bit, to keep the PID available for a longer time.

daemon/procmon/cache_events.go

@@ -12,10 +12,18 @@ var (
 	EventsCache       *EventsStore
 	eventsCacheTicker *time.Ticker
 
-	// When we receive an Exit event, we'll delete it from cache.
+	// When we receive an Exit event, we'll delete it from cache if the PID is not alive.
 	// This TTL defines how much time we retain a PID on cache, before we receive
 	// an Exit event.
 	pidTTL = 20 // seconds
+
+	// Delay the deletion time of an item.
+	// Sometimes we may receive a connection event AFTER the Exit of the process.
+	// In these scenarios, we need to delay the deletion from cache a little bit.
+	// [exec] /bin/xxx, pid 1234
+	// [exit] /bin/xxx, pid 1234
+	// [new conn] pid 1234 -> process unknown (no /proc entry)
+	exitDelay, _ = time.ParseDuration("2s")
 )
 
 func init() {
@@ -245,15 +253,19 @@ func (e *EventsStore) Len() int {
 // Delete schedules an item to be deleted from cache.
 func (e *EventsStore) Delete(key int) {
 	e.mu.Lock()
-	defer e.mu.Unlock()
-
 	ev, found := e.eventByPID[key]
+	e.mu.Unlock()
+
 	if !found {
 		return
 	}
-	if !ev.Proc.IsAlive() {
-		delete(e.eventByPID, key)
-	}
+	time.AfterFunc(exitDelay, func() {
+		e.mu.Lock()
+		defer e.mu.Unlock()
+		if !ev.Proc.IsAlive() {
+			delete(e.eventByPID, key)
+		}
+	})
 }
 
 // DeleteOldItems deletes items that have exited and exceeded the TTL.

daemon/procmon/cache_events_test.go

@@ -20,7 +20,10 @@ func createNewProc(pid int) *Process {
 	return proc
 }
 
-// Test regular use.
+// Test regular use:
+// - New exec   -> add to cache
+// - Query      -> get it from cache
+// - Proc alive -> keep it in cache
 func TestCacheEvents(t *testing.T) {
 	evtsCache := NewEventsStore()
 	proc := createNewProc(ourPid)
@@ -73,10 +76,13 @@ func TestCacheEvents2(t *testing.T) {
 		}
 	})
 
-	// this process does not exist, so it should be removed from cache
+	// This process does not exist, so it should be removed from cache
+	// inmediately. We wait a couple of seconds, before deleting it.
+	// See exitDelay description for more info.
 	t.Run("Delete() !isAlive()", func(t *testing.T) {
 		evtsCache.Delete(fakePid)
-		if _, _, found := evtsCache.IsInStore(ourPid, nil); found {
+		<-time.After(3 * time.Second)
+		if _, _, found := evtsCache.IsInStore(fakePid, nil); found {
 			t.Error("PID not deleted from cache.")
 		}
 	})