mirrors/opensnitch
1
0
Fork 0
mirror of https://github.com/evilsocket/opensnitch.git synced 2025-03-04 08:34:40 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

update_adlists.sh updated

Browse source 
- TOP_EU_US_Ads_Trackers_HOST list is no longer available, and the new
  list is not in hosts format.
- curben malware list moved to malware-filter.gitlab.io
- some improvements

Closes #695
This commit is contained in:
Gustavo Iñiguez Goia 2022-07-07 13:18:15 +02:00
parent 868974f593
commit c3a356bdb8
1 changed files with 24 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

33
utils/scripts/ads/update_adlists.sh
View file

@ -1,5 +1,5 @@
#!/bin/bash
# opensnitch - 2021
# opensnitch - 2021-2022
#
# https://github.com/evilsocket/opensnitch/wiki/block-lists
#
@ -18,20 +18,18 @@
# Use any directory you want to save the lists.
# If you use /etc/opensnitchd, give write permissions to blocklists/* for your user (chown -R /etc/opensnitchd/blocklists/).
# or use a directory from your user's home.
adsDir="/etc/opensnitchd/blocklists/domains/"
adsDir="/tmp/d/"
# If you add new urls, remember to add the corresponding filename where it'll be save on disk.
adsList=(
"https://curben.gitlab.io/malware-filter/urlhaus-filter-hosts.txt"
"https://raw.githubusercontent.com/Kees1958/W3C_annual_most_used_survey_blocklist/master/TOP_EU_US_Ads_Trackers_HOST"
"https://malware-filter.gitlab.io/malware-filter/urlhaus-filter-hosts.txt"
"https://hostfiles.frogeye.fr/multiparty-trackers-hosts.txt"
"https://hostfiles.frogeye.fr/firstparty-trackers-hosts.txt"
"https://www.github.developerdan.com/hosts/lists/tracking-aggressive-extended.txt"
"https://adaway.org/hosts.txt"
"https://pgl.yoyo.org/adservers/serverlist.php?hostformat=hosts&showintro=0&mimetype=plaintex")
"https://pgl.yoyo.org/adservers/serverlist.php?hostformat=hosts&showintro=0&mimetype=plaintext")
adsListNames=(
"urlhaus-filter-hosts.txt"
"top_eu_us_ads_trackers.txt"
"multiparty-trackers-hosts.txt"
"firstparty-trackers-hosts.txt"
"tracking-aggressive-extended.txt"
@ -57,6 +55,13 @@ function download_cname_trackers
fi
}
function download_list
{
echo -n "[+] downloading new ads list... $1 -> $2 ($3, $4)"
curl --silent "$1" -o $2
[ $? -eq 0 ] && echo " OK" || echo " FAIL"
}
function download_ads_list
{
reload=0
@ -64,18 +69,26 @@ function download_ads_list
do
echo "[+] Checking list ${adsList[$idx]}, ${adsListNames[$idx]}"
remoteSize=$(curl --silent -I ${adsList[$idx]}|awk '/content-length:/ {print $2}'|tr -d '\r')
localSize=$(stat -c %s $adsDir/${adsListNames[$idx]})
localSize=$(stat -c %s $adsDir/${adsListNames[$idx]} 2>/dev/null)
if [ ! -z $remoteSize ]; then
if [ "$remoteSize" != "$localSize" ]; then
echo "[+] downloading new ads list... ${adsList[$idx]}, $remoteSize, $localSize"
curl --silent "${adsList[$idx]}" -o $adsDir/${adsListNames[$idx]}
download_list "${adsList[$idx]}" "$adsDir/${adsListNames[$idx]}" $remoteSize $localSize
reload=1
else
echo "[-] ads list not updated yet: $remoteSize, $localSize - ${adsList[$idx]}"
fi
else
echo "[!] No content-length header found: ${adsList[$idx]}"
echo "[.] Trying with Last-Modidifed"
lastMod=$(date +%s -d "$(curl --silent -I ${adsList[$idx]}|grep Last-Modified|cut -d: -f 2)")
localMod=$(stat -c %Y $adsDir/${adsListNames[$idx]} 2>/dev/null)
[ $? -eq 1 ] && localMod=0
if [ ! -z $lastMod -a $lastMod -gt $localMod ]; then
download_list "${adsList[$idx]}" "$adsDir/${adsListNames[$idx]}" $remoteSize $localSize
else
echo "[-] ads list not updated yet: $lastMod, $localMod - ${adsList[$idx]}"
fi
fi
done
}
@ -89,3 +102,5 @@ cd $adsDir
download_ads_list
download_cname_trackers
echo "[~] Done"