Improved performance and details view

- Added a label to let the user know that an item in a column can be
clicked to view details about that item (process, host, user, ...).
- Improved performance by only adding the new items, or items that has
changed, instead of all the stats.
- Search General statistics by any column.
This commit is contained in:
Gustavo Iñiguez Goia 2019-11-17 00:57:08 +01:00
parent d38505650c
commit 918433a1dd
4 changed files with 171 additions and 71 deletions

View file

@ -1,6 +1,5 @@
from PyQt5 import QtWidgets, QtGui, QtCore
from PyQt5.QtSql import QSqlDatabase, QSqlDatabase, QSqlQueryModel, QSqlQuery
import faulthandler
import threading
class Database:
@ -15,7 +14,6 @@ class Database:
def __init__(self):
self._lock = threading.Lock()
self.db = None
faulthandler.enable()
self.initialize()
def initialize(self):
@ -31,18 +29,13 @@ class Database:
def _create_tables(self):
# https://www.sqlite.org/wal.html
#q = QSqlQuery("PRAGMA journal_mode=WAL;", self.db)
#q.exec_()
q = QSqlQuery("create table if not exists general (" \
"Time text, "\
"Action text, " \
"Process text, " \
"Destination text, " \
"DstPort text, " \
"Protocol text, " \
"Rule text, UNIQUE(Time,Action,Process,Destination,Rule))", self.db)
q = QSqlQuery("PRAGMA journal_mode = OFF", self.db)
q.exec_()
q = QSqlQuery("PRAGMA synchronous = OFF", self.db)
q.exec_()
q = QSqlQuery("create table if not exists connections (" \
"time text, " \
"action text, " \
"protocol text, " \
"src_ip text, " \
"src_port text, " \
@ -52,6 +45,7 @@ class Database:
"uid text, " \
"process text, " \
"process_args text, " \
"rule text, " \
"UNIQUE(protocol, src_ip, src_port, dst_ip, dst_port, uid, process, process_args))", self.db)
q.exec_()
q = QSqlQuery("create table if not exists rules (" \
@ -77,9 +71,19 @@ class Database:
q = QSqlQuery(".dump", self.db)
q.exec_()
def transaction(self):
self.db.transaction()
def commit(self):
self.db.commit()
def rollback(self):
self.db.rollback()
def _insert(self, query_str, columns):
try:
with self._lock:
q = QSqlQuery(self.db)
q.prepare(query_str)
for idx, v in enumerate(columns):
@ -90,15 +94,14 @@ class Database:
except Exception as e:
print("_insert exception", e)
finally:
q.clear()
q.finish()
def insert(self, table, fields, columns, update_field=None, update_value=None, action_on_conflict="REPLACE"):
action = "OR REPLACE"
if update_field != None:
action = ""
action_on_conflict = ""
qstr = "INSERT " + action + " INTO " + table + " " + fields + " VALUES("
qstr = "INSERT OR " + action_on_conflict + " INTO " + table + " " + fields + " VALUES("
update_fields=""
for col in columns:
qstr += "?,"
@ -124,7 +127,7 @@ class Database:
if not q.execBatch():
print(query_str)
print(q.lastError().driverText())
q.clear()
q.finish()
except Exception as e:
print("_insert_batch() exception:", e)
@ -149,8 +152,5 @@ class Database:
q = QSqlQuery(".dump", db=self.db)
q.exec_()
def get_query_details(self, table):
return "SELECT g.Time, g.Action, g.Destination, COUNT(g.Destination) as hits FROM " + table + " as p,general as g"
def get_query(self, table):
return "SELECT * FROM " + table
def get_query(self, table, fields):
return "SELECT " + fields + " FROM " + table

View file

@ -12,6 +12,7 @@ from PyQt5.QtSql import QSqlDatabase, QSqlDatabase, QSqlQueryModel, QSqlQuery, Q
import ui_pb2
from database import Database
from customqsqlquerymodel import CustomQSqlQueryModel
from config import Config
from version import version
@ -26,46 +27,60 @@ class StatsDialog(QtWidgets.QDialog, uic.loadUiType(DIALOG_UI_PATH)[0]):
LAST_ORDER_BY = 1
TABLES = {
0: {
"name": "general",
"name": "connections",
"label": None,
"tipLabel": None,
"cmd": None,
"view": None
"view": None,
"display_fields": "time as Time, action as Action, dst_host || ' -> ' || dst_port as Destination, protocol as Protocol, process as Process, rule as Rule"
},
1: {
"name": "rules",
"label": None,
"tipLabel": None,
"cmd": None,
"view": None
"view": None,
"display_fields": "*"
},
2: {
"name": "hosts",
"label": None,
"tipLabel": None,
"cmd": None,
"view": None
"view": None,
"display_fields": "*"
},
3: {
"name": "procs",
"label": None,
"tipLabel": None,
"cmd": None,
"view": None
"view": None,
"display_fields": "*"
},
4: {
"name": "addrs",
"label": None,
"tipLabel": None,
"cmd": None,
"view": None
"view": None,
"display_fields": "*"
},
5: {
"name": "ports",
"label": None,
"tipLabel": None,
"cmd": None,
"view": None
"view": None,
"display_fields": "*"
},
6: {
"name": "users",
"label": None,
"tipLabel": None,
"cmd": None,
"view": None
"view": None,
"display_fields": "*"
}
}
@ -104,20 +119,26 @@ class StatsDialog(QtWidgets.QDialog, uic.loadUiType(DIALOG_UI_PATH)[0]):
self._events_filter_line = self.findChild(QtWidgets.QLineEdit, "filterLine")
self._events_filter_line.textChanged.connect(self._cb_events_filter_line_changed)
self.TABLES[0]['view'] = self._setup_table(QtWidgets.QTreeView, "eventsTable", "general")
self.TABLES[1]['view'] = self._setup_table(QtWidgets.QTableView, "rulesTable", "rules")
self.TABLES[0]['view'] = self._setup_table(QtWidgets.QTreeView, "eventsTable", "connections", self.TABLES[0]['display_fields'], order_by="1")
self.TABLES[1]['view'] = self._setup_table(QtWidgets.QTableView, "rulesTable", "rules", order_by="1")
self.TABLES[2]['view'] = self._setup_table(QtWidgets.QTableView, "hostsTable", "hosts")
self.TABLES[3]['view'] = self._setup_table(QtWidgets.QTableView, "procsTable", "procs")
self.TABLES[4]['view'] = self._setup_table(QtWidgets.QTableView, "addrTable", "addrs")
self.TABLES[4]['view'] = self._setup_table(QtWidgets.QTableView, "addrTable", "addrs")
self.TABLES[5]['view'] = self._setup_table(QtWidgets.QTableView, "portsTable", "ports")
self.TABLES[6]['view'] = self._setup_table(QtWidgets.QTableView, "usersTable", "users")
self.TABLES[1]['label'] = self.findChild(QtWidgets.QLabel, "ruleLabel")
self.TABLES[1]['tipLabel'] = self.findChild(QtWidgets.QLabel, "tipRulesLabel")
self.TABLES[2]['label'] = self.findChild(QtWidgets.QLabel, "hostsLabel")
self.TABLES[2]['tipLabel'] = self.findChild(QtWidgets.QLabel, "tipHostsLabel")
self.TABLES[3]['label'] = self.findChild(QtWidgets.QLabel, "procsLabel")
self.TABLES[3]['tipLabel'] = self.findChild(QtWidgets.QLabel, "tipProcsLabel")
self.TABLES[4]['label'] = self.findChild(QtWidgets.QLabel, "addrsLabel")
self.TABLES[4]['tipLabel'] = self.findChild(QtWidgets.QLabel, "tipAddrsLabel")
self.TABLES[5]['label'] = self.findChild(QtWidgets.QLabel, "portsLabel")
self.TABLES[5]['tipLabel'] = self.findChild(QtWidgets.QLabel, "tipPortsLabel")
self.TABLES[6]['label'] = self.findChild(QtWidgets.QLabel, "usersLabel")
self.TABLES[6]['tipLabel'] = self.findChild(QtWidgets.QLabel, "tipUsersLabel")
self.TABLES[1]['cmd'] = self.findChild(QtWidgets.QPushButton, "cmdRulesBack")
self.TABLES[2]['cmd'] = self.findChild(QtWidgets.QPushButton, "cmdHostsBack")
@ -184,20 +205,24 @@ class StatsDialog(QtWidgets.QDialog, uic.loadUiType(DIALOG_UI_PATH)[0]):
def _cb_events_filter_line_changed(self, text):
model = self.TABLES[0]['view'].model()
if text != "":
qstr = self._db.get_query( self.TABLES[0]['name'] ) + " WHERE " + text + self._get_order()
qstr = self._db.get_query( self.TABLES[0]['name'], self.TABLES[0]['display_fields'] ) + " WHERE " + \
" Time = \"" + text + "\" OR Action = \"" + text + "\"" + \
" OR Protocol = \"" +text + "\" OR Destination LIKE '%" + text + "%'" + \
" OR Process = \"" + text + "\" OR Rule LIKE '%" + text + "%'" + \
self._get_order()
self.setQuery(model, qstr)
else:
self.setQuery(model, self._db.get_query("general") + self._get_order())
self.setQuery(model, self._db.get_query("connections", self.TABLES[0]['display_fields']) + self._get_order())
self._cfg.setSettings("statsDialog/general_filter_text", text)
def _cb_combo_action_changed(self, idx):
model = self.TABLES[0]['view'].model()
if self._combo_action.currentText() == "-":
self.setQuery(model, self._db.get_query("general") + self._get_order())
self.setQuery(model, self._db.get_query("connections", self.TABLES[0]['display_fields']) + self._get_order())
else:
action = "Action = '" + self._combo_action.currentText().lower() + "'"
qstr = self._db.get_query( self.TABLES[0]['name'] ) + " WHERE " + action + self._get_order()
qstr = self._db.get_query( self.TABLES[0]['name'], self.TABLES[0]['display_fields'] ) + " WHERE " + action + self._get_order()
self.setQuery(model, qstr)
self._cfg.setSettings("statsDialog/general_filter_action", idx)
@ -205,14 +230,19 @@ class StatsDialog(QtWidgets.QDialog, uic.loadUiType(DIALOG_UI_PATH)[0]):
def _cb_cmd_back_clicked(self, idx):
cur_idx = self._tabs.currentIndex()
self.TABLES[cur_idx]['label'].setVisible(False)
self.TABLES[cur_idx]['tipLabel'].setVisible(True)
self.TABLES[cur_idx]['cmd'].setVisible(False)
model = self._get_active_table().model()
if self.LAST_ORDER_BY > 2:
self.LAST_ORDER_BY = 1
self.setQuery(model, self._db.get_query(self.TABLES[cur_idx]['name']) + self._get_order())
self.setQuery(model, self._db.get_query(self.TABLES[cur_idx]['name'], self.TABLES[cur_idx]['display_fields']) + self._get_order())
def _cb_table_double_clicked(self, row):
cur_idx = self._tabs.currentIndex()
if cur_idx == 1 and row.column() != 1:
return
self.TABLES[cur_idx]['tipLabel'].setVisible(False)
self.TABLES[cur_idx]['label'].setVisible(True)
self.TABLES[cur_idx]['cmd'].setVisible(True)
self.TABLES[cur_idx]['label'].setText("<b>" + str(row.data()) + "</b>")
@ -221,7 +251,7 @@ class StatsDialog(QtWidgets.QDialog, uic.loadUiType(DIALOG_UI_PATH)[0]):
data = row.data()
if cur_idx == 1:
self.setQuery(model, "SELECT " \
"g.Time as Time, " \
"c.time as Time, " \
"r.name as RuleName, " \
"c.uid as UserID, " \
"c.protocol as Protocol, " \
@ -230,8 +260,8 @@ class StatsDialog(QtWidgets.QDialog, uic.loadUiType(DIALOG_UI_PATH)[0]):
"c.process as Process, " \
"c.process_args as Args, " \
"count(c.process) as Hits " \
"FROM rules as r, general as g, connections as c " \
"WHERE r.Name = '%s' AND r.Name = g.Rule AND c.process = g.Process GROUP BY c.process,c.dst_host %s" % (data, self._get_order()))
"FROM rules as r, connections as c " \
"WHERE r.Name = '%s' AND r.Name = c.rule GROUP BY c.process,c.dst_host %s" % (data, self._get_order()))
elif cur_idx == 2:
self.setQuery(model, "SELECT " \
"c.uid as UserID, " \
@ -245,15 +275,15 @@ class StatsDialog(QtWidgets.QDialog, uic.loadUiType(DIALOG_UI_PATH)[0]):
"WHERE c.dst_host = h.what AND h.what = '%s' GROUP BY c.process %s" % (data, self._get_order()))
elif cur_idx == 3:
self.setQuery(model, "SELECT " \
"g.Time, " \
"g.Destination, " \
"c.time as Time, " \
"c.dst_host as Destination, " \
"c.uid as UserID, " \
"g.Action, " \
"g.Process, " \
"c.action as Action, " \
"c.process as Process, " \
"c.process_args as Args, " \
"count(g.Destination) as Hits " \
"FROM procs as p,general as g, connections as c " \
"WHERE c.process = p.what AND p.what = g.Process AND p.what = '%s' GROUP BY g.Destination " % data)
"count(c.dst_host) as Hits " \
"FROM procs as p, connections as c " \
"WHERE p.what = c.process AND p.what = '%s' GROUP BY c.dst_host " % data)
elif cur_idx == 4:
self.setQuery(model, "SELECT " \
"c.uid as UserID, " \
@ -284,7 +314,7 @@ class StatsDialog(QtWidgets.QDialog, uic.loadUiType(DIALOG_UI_PATH)[0]):
"c.process_args as Args, " \
"count(c.dst_ip) as Hits " \
"FROM users as u, connections as c " \
"WHERE '%s' LIKE '%%' || c.uid || '%%' GROUP BY c.dst_ip" % data)
"WHERE u.what = '%s' AND u.what LIKE '%%(' || c.uid || ')' GROUP BY c.dst_ip" % data)
def _get_order(self):
return " ORDER BY %d %s" % (self.LAST_ORDER_BY, self.SORT_ORDER[self.LAST_ORDER_TO])
@ -341,11 +371,11 @@ class StatsDialog(QtWidgets.QDialog, uic.loadUiType(DIALOG_UI_PATH)[0]):
values.append(table.model().index(row, col).data())
w.writerow(values)
def _setup_table(self, widget, name, table_name):
def _setup_table(self, widget, name, table_name, fields="*", order_by="2"):
table = self.findChild(widget, name)
table.setSortingEnabled(True)
model = QSqlQueryModel()
self.setQuery(model, "SELECT * FROM " + table_name + " ORDER BY 1")
self.setQuery(model, "SELECT " + fields + " FROM " + table_name + " ORDER BY " + order_by + " DESC")
table.setModel(model)
try:
@ -357,6 +387,7 @@ class StatsDialog(QtWidgets.QDialog, uic.loadUiType(DIALOG_UI_PATH)[0]):
header.setSectionResizeMode(0, QtWidgets.QHeaderView.ResizeToContents)
header.sortIndicatorChanged.connect(self._cb_table_header_clicked)
#for col_idx, _ in enumerate(model.cols()):
# header.setSectionResizeMode(col_idx, \
# QtWidgets.QHeaderView.Stretch if col_idx == 0 else QtWidgets.QHeaderView.ResizeToContents)

View file

@ -408,7 +408,7 @@
<bool>true</bool>
</property>
<property name="placeholderText">
<string>Process LIKE '%firefox'</string>
<string>Ex.: firefox</string>
</property>
</widget>
</item>
@ -453,7 +453,7 @@
<string>Rules</string>
</attribute>
<layout class="QGridLayout" name="gridLayout">
<item row="1" column="0">
<item row="2" column="0">
<widget class="QTableView" name="rulesTable">
<property name="autoScroll">
<bool>false</bool>
@ -503,6 +503,16 @@
</item>
</layout>
</item>
<item row="1" column="0">
<widget class="QLabel" name="tipRulesLabel">
<property name="text">
<string>&lt;html&gt;&lt;head/&gt;&lt;body&gt;&lt;p&gt;&lt;span style=&quot; font-size:7pt;&quot;&gt;(double click on the Name column to view details of an item)&lt;/span&gt;&lt;/p&gt;&lt;/body&gt;&lt;/html&gt;</string>
</property>
<property name="textFormat">
<enum>Qt::RichText</enum>
</property>
</widget>
</item>
</layout>
</widget>
<widget class="QWidget" name="tab_4">
@ -561,6 +571,16 @@
</attribute>
</widget>
</item>
<item row="1" column="0">
<widget class="QLabel" name="tipHostsLabel">
<property name="text">
<string>&lt;html&gt;&lt;head/&gt;&lt;body&gt;&lt;p&gt;&lt;span style=&quot; font-size:7pt;&quot;&gt;(double click to view details of an item)&lt;/span&gt;&lt;/p&gt;&lt;/body&gt;&lt;/html&gt;</string>
</property>
<property name="textFormat">
<enum>Qt::AutoText</enum>
</property>
</widget>
</item>
</layout>
</widget>
<widget class="QWidget" name="tab_7">
@ -625,6 +645,13 @@
</item>
</layout>
</item>
<item row="1" column="0">
<widget class="QLabel" name="tipProcsLabel">
<property name="text">
<string>&lt;html&gt;&lt;head/&gt;&lt;body&gt;&lt;p&gt;&lt;span style=&quot; font-size:7pt;&quot;&gt;(double click to view details of an item)&lt;/span&gt;&lt;/p&gt;&lt;/body&gt;&lt;/html&gt;</string>
</property>
</widget>
</item>
</layout>
</widget>
<widget class="QWidget" name="tab_2">
@ -632,7 +659,7 @@
<string>Addresses</string>
</attribute>
<layout class="QGridLayout" name="gridLayout_7">
<item row="1" column="0">
<item row="2" column="0">
<widget class="QTableView" name="addrTable">
<property name="sortingEnabled">
<bool>false</bool>
@ -683,6 +710,13 @@
</item>
</layout>
</item>
<item row="1" column="0">
<widget class="QLabel" name="tipAddrsLabel">
<property name="text">
<string>&lt;html&gt;&lt;head/&gt;&lt;body&gt;&lt;p&gt;&lt;span style=&quot; font-size:7pt;&quot;&gt;(double click to view details of an item)&lt;/span&gt;&lt;/p&gt;&lt;/body&gt;&lt;/html&gt;</string>
</property>
</widget>
</item>
</layout>
</widget>
<widget class="QWidget" name="tab_5">
@ -690,7 +724,7 @@
<string>Ports</string>
</attribute>
<layout class="QGridLayout" name="gridLayout_8">
<item row="1" column="0">
<item row="2" column="0">
<widget class="QTableView" name="portsTable">
<property name="sortingEnabled">
<bool>false</bool>
@ -741,6 +775,13 @@
</item>
</layout>
</item>
<item row="1" column="0">
<widget class="QLabel" name="tipPortsLabel_2">
<property name="text">
<string>&lt;html&gt;&lt;head/&gt;&lt;body&gt;&lt;p&gt;&lt;span style=&quot; font-size:7pt;&quot;&gt;(double click to view details of an item)&lt;/span&gt;&lt;/p&gt;&lt;/body&gt;&lt;/html&gt;</string>
</property>
</widget>
</item>
</layout>
</widget>
<widget class="QWidget" name="tab_6">
@ -748,7 +789,7 @@
<string>Users</string>
</attribute>
<layout class="QGridLayout" name="gridLayout_9">
<item row="1" column="0">
<item row="2" column="0">
<widget class="QTableView" name="usersTable">
<property name="sortingEnabled">
<bool>false</bool>
@ -799,6 +840,13 @@
</item>
</layout>
</item>
<item row="1" column="0">
<widget class="QLabel" name="tipUsersLabel">
<property name="text">
<string>&lt;html&gt;&lt;head/&gt;&lt;body&gt;&lt;p&gt;&lt;span style=&quot; font-size:7pt;&quot;&gt;(double click to view details of an item)&lt;/span&gt;&lt;/p&gt;&lt;/body&gt;&lt;/html&gt;</string>
</property>
</widget>
</item>
</layout>
</widget>
</widget>

View file

@ -62,6 +62,8 @@ class UIService(ui_pb2_grpc.UIServicer, QtWidgets.QGraphicsObject):
self.check_thread.daemon = True
self.check_thread.start()
self.last_stats = None
# https://gist.github.com/pklaus/289646
def _setup_interfaces(self):
max_possible = 128 # arbitrary. raise if needed.
@ -185,19 +187,16 @@ class UIService(ui_pb2_grpc.UIServicer, QtWidgets.QGraphicsObject):
def _populate_stats(self, db, stats):
fields = []
values = []
for row, event in enumerate(stats.events):
db.insert("general",
"(Time, Action, Process, Destination, DstPort, Protocol, Rule)",
(event.time, event.rule.action, event.connection.process_path,
event.connection.dst_host, str(event.connection.dst_port), event.connection.protocol, event.rule.name),
action_on_conflict="IGNORE"
)
for row, event in enumerate(stats.events):
if self.last_stats != None and event in self.last_stats.events:
continue
db.insert("connections",
"(protocol, src_ip, src_port, dst_ip, dst_host, dst_port, uid, process, process_args)",
(event.connection.protocol, event.connection.src_ip, str(event.connection.src_port),
"(time, action, protocol, src_ip, src_port, dst_ip, dst_host, dst_port, uid, process, process_args, rule)",
(event.time, event.rule.action, event.connection.protocol, event.connection.src_ip, str(event.connection.src_port),
event.connection.dst_ip, event.connection.dst_host, str(event.connection.dst_port),
str(event.connection.user_id), event.connection.process_path, " ".join(event.connection.process_args)),
str(event.connection.user_id), event.connection.process_path, " ".join(event.connection.process_args),
event.rule.name),
action_on_conflict="IGNORE"
)
db.insert("rules",
@ -209,7 +208,11 @@ class UIService(ui_pb2_grpc.UIServicer, QtWidgets.QGraphicsObject):
fields = []
values = []
for row, event in enumerate(stats.by_host.items()):
items = stats.by_host.items()
last_items = self.last_stats.by_host.items() if self.last_stats != None else ''
for row, event in enumerate(items):
if self.last_stats != None and event in last_items:
continue
what, hits = event
fields.append(what)
values.append(int(hits))
@ -217,7 +220,11 @@ class UIService(ui_pb2_grpc.UIServicer, QtWidgets.QGraphicsObject):
fields = []
values = []
for row, event in enumerate(stats.by_executable.items()):
items = stats.by_executable.items()
last_items = self.last_stats.by_executable.items() if self.last_stats != None else ''
for row, event in enumerate(items):
if self.last_stats != None and event in last_items:
continue
what, hits = event
fields.append(what)
values.append(int(hits))
@ -225,7 +232,11 @@ class UIService(ui_pb2_grpc.UIServicer, QtWidgets.QGraphicsObject):
fields = []
values = []
for row, event in enumerate(stats.by_address.items()):
items = stats.by_address.items()
last_items = self.last_stats.by_address.items() if self.last_stats != None else ''
for row, event in enumerate(items):
if self.last_stats != None and event in last_items:
continue
what, hits = event
fields.append(what)
values.append(int(hits))
@ -233,7 +244,11 @@ class UIService(ui_pb2_grpc.UIServicer, QtWidgets.QGraphicsObject):
fields = []
values = []
for row, event in enumerate(stats.by_port.items()):
items = stats.by_port.items()
last_items = self.last_stats.by_port.items() if self.last_stats != None else ''
for row, event in enumerate(items):
if self.last_stats != None and event in last_items:
continue
what, hits = event
fields.append(what)
values.append(int(hits))
@ -241,7 +256,11 @@ class UIService(ui_pb2_grpc.UIServicer, QtWidgets.QGraphicsObject):
fields = []
values = []
for row, event in enumerate(stats.by_uid.items()):
items = stats.by_uid.items()
last_items = self.last_stats.by_uid.items() if self.last_stats != None else ''
for row, event in enumerate(items):
if self.last_stats != None and event in last_items:
continue
what, hits = event
pw_name = what
try:
@ -252,6 +271,8 @@ class UIService(ui_pb2_grpc.UIServicer, QtWidgets.QGraphicsObject):
values.append(int(hits))
db.insert_batch("users", "(what, hits)", (1,2), fields, values)
self.last_stats = stats
def Ping(self, request, context):
if self._is_local_request(context):
self._last_ping = datetime.now()