uthor: Matthew Myrick
# Date Created: 2010-11-03
# Date Modified: 2011-03-03
# Purpose: To harvest intel out of FireEye 7000 Series Max updates
# Filename: feharvest.sh


# Files that don't lend themselves well to being scraped:
# /data/snort/rules/fireeyecnc.rules
# /data/snort/rules/fireeyefqc.rules
# /data/snort/rules/fireeyefqcopt.rules

#Files we would like scrape:
File1="/data/snort/rules/fireeyeinfect.rules"
File2="/data/snort/rules/fireeyefqc.noip.rules"
File3="/data/snort/rules/fireeyefqcopt.noip.rules"

File4="/data/blat/rules/fireeyekrnldns.rules"

#This is the normal way to do things 
#grep 'content\:\"Host\\grep 'content\:\"Host\\\: ' $File1 $File2 $File3 | cut -d ' ' -f 17 | cut -d '|' -f 1 | cut -d '\' -f 1 | uniq | while read url; do /bin/echo "$url"; done 

#We have to use this until FireEye fixes their malformed rules
grep 'content\:\"Host\\\: ' $File1 $File2 $File3 | cut -d ' ' -f 17 | grep -v "content\:" | cut -d '|' -f 1 | cut -d '\' -f 1 | grep -v "www.gstatic.com" | grep -v "feeds.feedburner.com" | grep -v "download.support.xerox.com" | grep -v "code.jquery.com" | grep -v "bit.ly" | grep -v "www.marriott.com" | grep -v "www.maxmind.com" | grep -v "www.amazon.com" | grep -v ".images-amazon.com" |uniq | while read url; do /var/home/root/bcsubmit.sh "$url"; done 
#Uncomment this view and not block
#grep 'content\:\"Host\\\: ' $File1 $File2 $File3 | cut -d ' ' -f 17 | grep -v "content\:" | cut -d '|' -f 1 | cut -d '\' -f 1 | grep -v "www.gstatic.com" | grep -v "feeds.feedburner.com" | grep -v "download.support.xerox.com" | grep -v "code.jquery.com" | grep -v "bit.ly" | grep -v "www.marriott.com" | grep -v "www.maxmind.com" | grep -v "www.amazon.com" | grep -v ".images-amazon.com"| uniq

#File 4 is formatted differently so we have to modify our scrap params
grep 'alert(sid' $File4 | grep -v 'fedeploymentcheck.dns.fireeye.com' | grep -v '#alert(sid' | cut -d '"' -f 2 | uniq | while read url; do /var/home/root/bcsubmit.sh "$url"; done
#Uncomment this view and not block
#grep 'alert(sid' $File4 | grep -v 'fedeploymentcheck.dns.fireeye.com' | grep -v '#alert(sid' | cut -d '"' -f 2 | uniq 

#This way is for testing
#grep 'content\:\"Host\\\: ' $File1 $File2 $File3 | cut -d ' ' -f 17 | more | cut -d '|' -f 1 | cut -d '\' -f 1 | uniq | while read url; do
#echo ${url##*.}
#echo $url
#done