#!/usr/bin/ruby # mel@hackinthebox.org # In PostgreSQL 8.2, set standard_conforming_strings in postgresql.conf to ignore warnings regarding \\ characters require 'dbi' dbh = DBI.connect('DBI:Pg:bro', '','') def usage puts "Usage: #{$0} "; exit 1 end def notice(lfile, sid, dbh) log = File.new(lfile, "r") count = 0 lno = 0 log.each_line{ |line| lno = lno + 1 line = line.gsub(/\\/, "") if (line =~ /^t=(\d{10}\.\d{6}) no=(AddressDropIgnored) na=(\w+) sa=(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}) msg=(.*)/) time = $1 notice_type = $2 notice_action = $3 src_addr = $4 msg = $5 elsif (line =~ /^t=(\d{10}\.\d{6}) no=(AddressScan) na=(\w+) sa=(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}) sp=(\d+\/\w+) da=(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}) dp=(\d+\/\w+) msg=(.*) tag=(@\d+)/) time = $1 notice_type = $2 notice_action = $3 src_addr = $4 src_port = $5 dst_addr = $6 dst_port = $7 msg = $8 tag = $9 elsif (line =~ /^t=(\d{10}\.\d{6}) no=(DroppedPackets|ResourceStats|WeirdActivity) na=(\w+) msg=(.*)/) time = $1 notice_type = $2 notice_action = $3 msg = $4 elsif (line =~ /^t=(\d{10}\.\d{6}) no=(HTTP_SensitiveURI) na=(\w+) sa=(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}) sp=(\d+\/\w+) da=(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}) dp=(\d+\/\w+) method=(\w+) url=(.*) num=(\d+) msg=(.*) tag=(@\d+)/) time = $1 notice_type = $2 notice_action = $3 src_addr = $4 src_port = $5 dst_addr = $6 dst_port = $7 method = $8 url = $9 num = $10 msg = $11 tag = $12 elsif (line =~ /^t=(\d{10}\.\d{6}) no=(PortScan|ProtocolViolation|IrcBotClientFound) na=(\w+) sa=(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}) sp=(\d+\/\w+) da=(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}) dp=(\d+\/\w+) msg=(.*) tag=()/) time = $1 notice_type = $2 notice_action = $3 src_addr = $4 src_port = $5 dst_addr = $6 dst_port = $7 msg = $8 tag = $9 elsif (line =~ /^t=(\d{10}\.\d{6}) no=(IrcBotServerFound) na=(\w+) sa=(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}) da=(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}) dp=(\d+\/\w+) msg=(.*) tag=(@\d+)/) time = $1 notice_type = $2 notice_action = $3 src_addr = $4 dst_addr = $5 dst_port = $6 msg = $7 tag = $8 elsif (line =~ /^t=(\d{10}\.\d{6}) no=(ScanSummary) na=(\w+) sa=(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}) num=(\d+) msg=(.*)/) time = $1 notice_type = $2 notice_action = $3 src_addr = $4 num = $5 msg = $6 elsif (line =~ /^t=(\d{10}\.\d{6}) no=SensitiveSignature na=(\w+) sa=(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}) da=(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}) file=(\w+-\w+-\w+) msg=(.*) sub=(.*) tag=(@\d+)/) time = $1 notice_type = "SensitiveSignature" notice_action = $2 src_addr = $3 dst_addr = $4 file = $5 msg = $6 sub = $7 tag = $8 elsif (line =~ /^t=(\d{10}\.\d{6}) no=SensitiveSignature na=(\w+) sa=(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}) sp=(\d+\/\w+) da=(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}) dp=(\d+\/\w+) file=(\w+-\w+-\w+) msg=(.*) sub=(.*) tag=(@\d+)/) time = $1 notice_type = "SensitiveSignature" notice_action = $2 src_addr = $3 src_port = $4 dst_addr = $5 dst_port = $6 file = $7 msg = $8 sub = $9 tag = $10 elsif (line =~ /^t=(\d{10}\.\d{6}) no=ResourceSummary na=(\w+) msg=(.*)/) time = $1 notice_type = "ResourceSummary" notice_action = $2 msg = $3 elsif (line =~ /^t=(\d{10}\.\d{6}) no=(MultipleSignatures) na=(\w+) sa=(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}) da=(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}) file=(\w+-\w+-\w+) num=(\d+) msg=(.*) sub=(.*)/) time = $1 notice_type = "MultipleSignatures" notice_action = $2 src_addr = $3 dst_addr = $4 file = $5 num = $6 msg = $7 sub = $8 elsif (line =~ /^t=(\d{10}\.\d{6}) no=MultipleSigResponders na=(\w+) sa=(\d+\.\d+\.\d+\.\d+) file=(\w+-\w+-\w+) num=(\d+) msg=(.*) sub=(.*)/) time = $1 notice_type = "MultipleSigResponders" notice_action = $2 src_addr = $3 file = $4 num = $5 msg = $6 sub = $7 else puts line count = count - 1 end dbh.do("INSERT INTO notice (network_time,notice_type,notice_action,message,src_addr,src_port, dst_addr,dst_port,tag,method,url,num,sensor_id,file,sub) VALUES (TIMESTAMPTZ 'epoch' + ? * '1 second'::interval,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", time, notice_type, notice_action, msg, src_addr,src_port, dst_addr,dst_port,tag,method,url,num,sid,file,sub) count = count + 1 } puts "#{count} of #{lno} records inserted" end def alarm(lfile, sid, dbh) log = File.new(lfile, "r") count = 0 lno = 0 log.each_line{ |line| lno = lno + 1 line = line.gsub(/\\/, "") # AddressScan, HTTP_SensitiveURI, IrcBotClientFound, IrcBotServerFound, PortScan, ProtocolViolation, ScanSummary if (line =~ /^t=(\d{10}\.\d{6})\sno=(AddressScan)\sna=(\w+)\ssa=(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})\ssp=(\d+\/\w+)\sda=(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})\sdp=(\d+\/\w+)\smsg=(.*)\stag=(@\d+)/) time = $1 notice_type = $2 notice_action = $3 src_addr = $4 src_port = $5 dst_addr = $6 dst_port = $7 msg = $8 tag = $9 elsif (line =~ /t=(\d{10}\.\d{6}) no=(HTTP_SensitiveURI) na=(\w+) sa=(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}) sp=(\d+\/\w+) da=(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}) dp=(\d+\/\w+) method=(\w+) url=(.*) num=(\d+) msg=(.*) tag=(@\d+)/) # NOTE: the http_tag (%some_number) may need to be extracted from the msg= in the future # format: addr/port > addr/port %n sometimes: addr/port > addr/port %n @n time = $1 notice_type = $2 notice_action = $3 src_addr = $4 src_port = $5 dst_addr = $6 dst_port = $7 method = $8 url = $9 num = $10 msg = $11 tag = $12 elsif (line =~ /^t=(\d{10}\.\d{6}) no=(IrcBotServerFound) na=(\w+) sa=(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}) da=(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}) dp=(\d+\/\w+) msg=(.*) tag=(@\d+)/) time = $1 notice_type = $2 notice_action = $3 src_addr = $4 dst_addr = $5 dst_port = $6 msg = $7 tag = $8 elsif (line =~ /^t=(\d{10}\.\d{6}) no=(PortScan|ProtocolViolation|IrcBotClientFound) na=(\w+) sa=(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}) sp=(\d+\/\w+) da=(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}) dp=(\d+\/\w+) msg=(.*) tag=()/) time = $1 notice_type = $2 notice_action = $3 src_addr = $4 src_port = $5 dst_addr = $6 dst_port = $7 msg = $8 tag = $9 elsif (line =~ /^t=(\d{10}\.\d{6}) no=(ScanSummary) na=(\w+) sa=(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}) num=(\d+) msg=(.*)/) time = $1 notice_type = $2 notice_action = $3 src_addr = $4 num = $5 msg = $6 elsif (line =~ /^t=(\d{10}\.\d{6}) no=(MultipleSignatures) na=(\w+) sa=(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}) da=(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}) file=(\w+-\w+-\w+) num=(\d+) msg=(.*) sub=(.*)/) time = $1 notice_type = $2 notice_action = $3 src_addr = $4 dst_addr = $5 file = $6 num = $7 msg = $8 sub = $9 elsif (line =~ /^t=(\d{10}\.\d{6}) no=SensitiveSignature na=(\w+) sa=(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}) da=(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}) file=(\w+-\w+-\w+) msg=(.*) sub=(.*) tag=(@\d+)/) time = $1 notice_type = "SensitiveSignature" notice_action = $2 src_addr = $3 dst_addr = $4 file = $5 msg = $6 sub = $7 tag = $8 elsif (line =~ /^t=(\d{10}\.\d{6}) no=SensitiveSignature na=(\w+) sa=(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}) sp=(\d+\/\w+) da=(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}) dp=(\d+\/\w+) file=(\w+-\w+-\w+) msg=(.*) sub=(.*) tag=(@\d+)/) time = $1 notice_type = "SensitiveSignature" notice_action = $2 src_addr = $3 src_port = $4 dst_addr = $5 dst_port = $6 file = $7 msg = $8 sub = $9 tag = $10 elsif (line =~ /^t=(\d{10}\.\d{6}) no=MultipleSigResponders na=(\w+) sa=(\d+\.\d+\.\d+\.\d+) file=(\w+-\w+-\w+) num=(\d+) msg=(.*) sub=(.*)/) time = $1 notice_type = "MultipleSigResponders" notice_action = $2 src_addr = $3 file = $4 num = $5 msg = $6 sub = $7 else puts line count = count - 1 end dbh.do("INSERT INTO alarm (network_time,notice_type,notice_action,message,src_addr,src_port, dst_addr,dst_port,tag,method,url,num,file,sub,sensor_id) VALUES (TIMESTAMPTZ 'epoch' + ? * '1 second'::interval,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", time, notice_type, notice_action, msg, src_addr,src_port, dst_addr,dst_port,tag,method,url,num,file,sub,sid) count = count + 1 } puts "#{count} of #{lno} records inserted" end def irc(lfile, sid, dbh) log = File.new(lfile, "r") count = 0 lno = 0 log.each_line{ |line| lno = lno + 1 if (line =~ /(^\d{10}\.\d{6})\s(\#\d+)\s(.*)/) time = $1 tag = $2 msg = $3 dbh.do("INSERT INTO irc(network_time, irc_tag_no, message, sensor_id) VALUES (TIMESTAMPTZ 'epoch' + ? * '1 second'::interval,?,?,?)", time, tag, msg, sid) count = count + 1 else puts line end } puts "#{count} of #{lno} records inserted" end def irc_detailed(lfile, sid, dbh) log = File.new(lfile, "r") count = 0 lno = 0 log.each_line{ |line| lno = lno + 1 if (line =~ /(^\d{10}\.\d{6})\s(\d+\.\d+\.\d+\.\d+)\/(\d+)\s>\s(\d+\.\d+\.\d+\.\d+)\/(\d+|\w+)\s(<|>)\s(.*)/) time = $1 src_addr = $2 src_port = $3 dst_addr = $4 dst_port = $5 dir = $6 msg = $7 dbh.do("INSERT INTO irc_detailed (network_time, src_addr, src_port, dst_addr, dst_port, direction, irc_message, sensor_id) VALUES (TIMESTAMPTZ 'epoch' + ? * '1 second'::interval,?,?,?,?,?,?,?)", time, src_addr, src_port, dst_addr, dst_port, dir, msg, sid ) count = count + 1 else puts line end } puts "#{count} of #{lno} records inserted" end def irc_bot(lfile, sid, dbh) log = File.new(lfile, "r") count = 0 lno = 0 log.each_line{ |line| lno = lno + 1 if (line =~ /(^\d{10}\.\d{6})\s(\d+\.\d+\.\d+\.\d+):(\d+)\s>\s(\d+\.\d+\.\d+\.\d+):(\d+)\s(\#\d+)\s(@\d+)\s(.*)/) time = $1 src_addr = $2 src_port = $3 dst_addr = $4 dst_port = $5 irc_tag_no = $6 conn_tag_no = $7 msg = $8 elsif (line =~ /(^\d{10}\.\d{6})\s(\d+\.\d+\.\d+\.\d+):(\d+)\s>\s(\d+\.\d+\.\d+\.\d+):(\d+)\s(\#\d+)\s(.*)/) time = $1 src_addr = $2 src_port = $3 dst_addr = $4 dst_port = $5 irc_tag_no = $6 msg = $7 else puts line end dbh.do("INSERT INTO irc_bot(network_time, src_addr, src_port, dst_addr, dst_port, irc_tag_no, conn_tag_no, msg, sensor_id) VALUES (TIMESTAMPTZ 'epoch' + ? * '1 second'::interval,?,?,?,?,?,?,?,?)", time, src_addr, src_port, dst_addr, dst_port, irc_tag_no, conn_tag_no, msg, sid ) count = count + 1 } puts "#{count} of #{lno} records inserted" end def http(lfile, sid, dbh) log = File.new(lfile, "r") count = 0 lno = 0 log.each_line{ |line| lno = lno + 1 if (line =~ /(^\d{10}\.\d{6})\s(%\d+)\s((GET|POST|start)\s.*)/) time = $1 tag = $2 msg = $3 dbh.do("INSERT INTO http (network_time, http_tag, message, sensor_id) VALUES (TIMESTAMPTZ 'epoch' + ? * '1 second'::interval,?,?,?)", time, tag, msg, sid) count = count + 1 else puts line end } puts "#{count} of #{lno} records inserted" end def smtp(lfile, sid, dbh) log = File.new(lfile, "r") count = 0 lno = 0 log.each_line{ |line| lno = lno + 1 if (line =~ /(^\d{10}\.\d{6})\s(\#\d+)\s(.*)/) time = $1 num = $2 msg = $3 dbh.do("INSERT INTO smtp (network_time, session_num, message, sensor_id) VALUES (TIMESTAMPTZ 'epoch' + ? * '1 second'::interval,?,?,?)", time, num, msg, sid) count = count + 1 else puts line end } puts "#{count} of #{lno} records inserted" end def weird(lfile, sid, dbh) log = File.new(lfile, "r") count = 0 lno = 0 log.each_line{ |line| lno = lno + 1 l = line.split(/\s/) time = l[0] src_addr = (l[1].split(/\//))[0] src_port = (l[1].split(/\//))[1] dst_addr = (l[3].split(/\//))[0] dst_port = (l[3].split(/\//))[1].gsub(/:/,"") msg = l[4] dbh.do("INSERT INTO weird(network_time, src_addr, src_port, dst_addr, dst_port, msg, sensor_id) VALUES (TIMESTAMPTZ 'epoch' + ? * '1 second'::interval,?,?,?,?,?,?)", time, src_addr, src_port, dst_addr, dst_port, msg, sid ) count = count + 1 } puts "#{count} of #{lno} records inserted" end def ssh(lfile, sid, dbh) log = File.new(lfile, "r") count = 0 lno = 0 log.each_line{ |line| lno = lno + 1 if (line =~ /(^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})\s(S|C)\s"(.*)"/) host = $1 type = $2 version = $3 dbh.do("INSERT INTO ssh (addr,type,version,sensor_id) VALUES (?,?,?,?)", host, type, version, sid) else puts line end count = count + 1 } puts "#{count} of #{lno}records inserted" end def ftp(lfile, sid, dbh) log = File.new(lfile, "r") count = 0 lno = 0 log.each_line{ |line| lno = lno + 1 if (line =~ /(^\d{10}\.\d{6})\s(\#\d+)\s(.*)/) time = $1 num = $2 msg = $3 dbh.do("INSERT INTO ftp (start_time, session_num, message, sensor_id) VALUES (TIMESTAMPTZ 'epoch' + ? * '1 second'::interval,?,?,?)", time, num, msg, sid) count = count + 1 else puts line end } puts "#{count} of #{lno}records inserted" end def signature(lfile, sid, dbh) log = File.new(lfile, "r") count = 0 lno = 0 log.each_line{ |line| lno = lno + 1 # need to check if the signature data (payload) is escaped # see signature.bro for reference line = line.gsub(/\\:/, "888888"); l = line.split(/:/) network_time = l[0] signature_note = l[1] src_addr = l[2] src_port = l[3] dst_addr = l[4] dst_port = l[5] sigid = l[6] if ((signature_note == "MultipleSigResponders") || (signature_note == "MultipleSignatures")) event_msg = l[7] sub_msg = l[8] sig_count = l[9] host_count = l[10] else event_msg = l[7] + ": " + l[8] sub_msg = l[9] sig_count = l[10] host_count = l[11] end sub_msg = sub_msg.gsub(/888888/, "\\:") if (sig_count.empty?) sig_count = "null" end if (host_count !~ /\d+/) host_count = "null" end # puts "#{line}" # puts "--------------------------" # puts "1 network_time #{network_time}" # puts "2 signature_note #{signature_note}" # puts "3 src_addr #{src_addr}" # puts "4 src_port #{src_port}" # puts "5 dst_addr #{dst_addr}" # puts "6 dst_port #{dst_port}" # puts "7 sigid #{sigid}" # puts "8 event_msg #{event_msg}" # puts "9 sub_msg #{sub_msg}" # puts "10 sig_count #{sig_count}" # puts "11 host_count #{host_count}" # puts "" begin dbh.do("INSERT INTO signature ( network_time, signature_note, src_addr, dst_addr, src_port, dst_port, sigid, event_msg, sub_msg, sig_count, host_count, sensor_id) VALUES ( TIMESTAMPTZ 'epoch' + ? * '1 second'::interval,?, ?,?,?,?, ?,?, ?,nullif(?,'null')::integer,nullif(?,'null')::integer,?)", network_time, signature_note, src_addr, dst_addr, src_port, dst_port, sigid, event_msg, sub_msg, sig_count, host_count, sid ) count = count + 1 rescue DBI::DatabaseError => e puts "Error code: #{e.err}" puts "Error mesg: #{e.errstr}" end } puts "#{count} of #{lno} data inserted" end def software(lfile, sid, dbh) log = File.new(lfile, "r") count = 0 lno = 0 log.each_line{ |line| lno = lno + 1 if (line =~ /^(\d{10}\.\d{6}) (\d+\.\d+\.\d+\.\d+) (server|client): (.*)/) time = $1 addr = $2 type = $3 name = $4 name = name.gsub(/\[/, ""); name = name.gsub(/\]/, "") dbh.do("INSERT INTO software (network_time,addr,type, name ,sensor_id) VALUES (TIMESTAMPTZ 'epoch' + ? * '1 second'::interval,?,?,?,?)", time,addr,type, name ,sid) count = count + 1 else puts "#{line}" end } puts "#{count} of #{lno} data inserted" end def conn(lfile, sid, dbh) log = File.new(lfile, "r") count=0 lno = 0 log.each_line{ |line| lno = lno + 1 l = line.split(/\s+/) s = l.size network_time = l[0] duration = l[1] src_addr = l[2] dst_addr = l[3] service = l[4] src_port = l[5] dst_port = l[6] proto = l[7] src_s = l[8] dst_s = l[9] state = l[10] origin = l[11] if (s == 14) addl = l[12] + " " + l[13] elsif (s == 13) addl = l[12] else addl = "" end if (src_s == "?") src_s = "null" end if (dst_s == "?") dst_s = "null" end if (duration == "?") duration = "null" end dbh.do("INSERT INTO conn( start_time,duration,orig_h,resp_h, service,orig_p,resp_p,proto, orig_s,resp_s,state,direction, tag,sensor_id) VALUES (TIMESTAMPTZ 'epoch' + ? * '1 second'::interval,nullif(?,'null')::numeric,?,?, ?,?,?,?, nullif(?,'null')::integer, nullif(?,'null')::integer,?,?, ?,?)", network_time, duration, src_addr, dst_addr, service, src_port, dst_port, proto, src_s, dst_s, state, origin, addl, sid ) count = count + 1 } puts "#{count} of #{lno} data inserted" end if (ARGV.length < 3) usage end lfile = ARGV[0] type = ARGV[1] sid = ARGV[2] puts "[+] processing log file: #{lfile}" case type when "conn" conn(lfile, sid, dbh) when "signature" signature(lfile, sid, dbh) when "ftp" ftp(lfile, sid, dbh) when "ssh" ssh(lfile, sid, dbh) when "weird" weird(lfile, sid, dbh) when "smtp" smtp(lfile, sid, dbh) when "http" http(lfile, sid, dbh) when "notice" notice(lfile, sid, dbh) when "irc-bot" irc_bot(lfile, sid, dbh) when "irc-detailed" irc_detailed(lfile, sid, dbh) when "irc" irc(lfile, sid, dbh) when "alarm" alarm(lfile, sid, dbh) when "notice" notice(lfile, sid, dbh) when "signature" signature(lfile, sid, dbh) when "software" software(lfile, sid, dbh) end