import sys
import json
import os
import pandas as pd
import tldextract
# import numpy as np
medata_field = '''
3 = sourceIP
4 = destIP
5 = sourcePort
6 = destPort
7 = protocol
12 = flowStartSeconds
13 = flowEndSecond
54 = DNSReplyCode
55 = DNSQueryName
56 = DNSRequestRRType
57 = DNSRRClass
58 = DNSDelay
59 = DNSReplyTTL
60 = DNSReplyIPv4
61 = DNSReplyIPv6
62 = DNSReplyRRType
77 = DNSReplyName
81 = payload
88 = DNSRequestLength
89 = DNSRequestErrLength
90 = DNSReplyLength
91 = DNSReplyErrLength
'''medata_field_num = []
medata_field_info = []
for l in medata_field.split("\n"):if len(l) == 0: continuenum, info = l.split(" = ")medata_field_num.append(int(num)-1)medata_field_info.append(info)
print medata_field_num
print medata_field_infodef extract_domain(domain):try:ext = tldextract.extract(domain)subdomain = ext.subdomainif ext.domain == "":mdomain = ext.suffixelse:mdomain = ".".join(ext[1:])return mdomainexcept Exception,e:print "extract_domain error:", ereturn "unknown"def parse_metadata(path):df = pd.read_csv(path, sep="^", header=None)dns_df = df.iloc[:, medata_field_num].copy()dns_df.columns = medata_field_info# print dns_df.tail()
dns_df["mdomain"] = dns_df["DNSQueryName"].apply(extract_domain)# print dns_df.groupby('mdomain').describe()# print dns_df.groupby('mdomain').groupsreturn dns_df.groupby('mdomain')def get_data_dist(df, col="sourceIP"):# group count by ip distgrouped = df.groupby(col)# print grouped.head(10)[col]print type(grouped.size())size = grouped.size()print sizeprint "-----------top 10-------------"print size.nlargest(10)def get_ipv4_dist(df, col="DNSReplyLength"):# group count by ip distdf2 = df[df[col] > 0]print "filter before length:", len(df), "filter after length:", len(df2)grouped = df2.groupby(by="DNSReplyIPv4")# print grouped.head(10)[col]size = grouped.size()print sizeprint "-----------top 10-------------"print size.nlargest(10)def move_to(srcpath, domain, dst_path):with open(dst_path, "w") as w:with open(srcpath) as r:for line in r:if extract_domain(line.split("^")[55-1]) == domain:w.write(line)def main():history_op = {}if os.path.exists("history_op.json"):with open("history_op.json") as h:history_op = json.load(h)print history_opfor day in range(24, 27):for hour in range(0, 24):path = "/home/bonelee/latest_metadata_sample/sampled/unknown_sample/debugdogcom-medata_wanted-2017-09-%d-%d.txt" % (day, hour)if not os.path.exists(path) or os.path.getsize(path) == 0:print path, "passed, file not exists or empty file."continueprint path, "running..."try:domains_info = parse_metadata(path)except IOError, e:print econtinuefor domain, group in domains_info:print "***************************************"print "domain:", domain, "flow count:", len(group)print "***************************************"# print type(group) #<class 'pandas.core.frame.DataFrame'>print "------------srcip-----------------"print group["sourceIP"].describe()print "--------------destip---------------"print group["destIP"].describe()print "----------------------------------------"print "ipv4 address return dist:"get_ipv4_dist(group)print "----------------------------------------"has_judged = Falseneed_break = Falsewhile True:print "-------------choose one--------------"print "sub domain: DNSQueryName(N)"print "ip: srcip(S) or dstip(D)"print "length: DNSRequestLength(R1) or DNSReplyLength(R2)"print "length too: DNSRequestErrLength(R3) or DNSReplyErrLength(R4)"print "port: sourcePort(P1) or destPort(P2) or DNSReplyTTL(T)"print "code: DNSReplyCode(C2) or DNSRequestRRType(C1)"print "other: DNSRRClass(RR) or DNSReplyIPv4(V)"dist_dict = {"R1": "DNSRequestLength","R2": "DNSReplyLength","R3": "DNSRequestErrLength","R4": "DNSReplyErrLength","P1": "sourcePort","P2": "destPort","T": "DNSReplyTTL","C2": "DNSReplyCode","C1": "DNSRequestRRType","RR": "DNSRRClass","V": "DNSReplyIPv4","S": "sourceIP","D": "destIP","N": "DNSQueryName"}print "-------------label or quit------------"print "black(B) or white(W) or cdn(CDN) or ddos(DDOS) or mddos(M) or unknown(U) or white-like(L)"print "next(Q) or exit(E)?"domain = domain.lower()if "win" == domain[-len("win"):] or "site" == domain[-len("site"):] or "vip" == domain[-len("vip"):]:check = "U"need_break = Trueelif "lan" in domain or "local" in domain or "dhcp" in domain or "workgroup" in domain or "home" in domain:check = "DDOS"need_break = Trueelif "cdn" in domain:check = "CDN"need_break = Trueelse:if domain in history_op and not has_judged:print "found history op:", history_op[domain]if not raw_input("OK(Enter for Y)?"):check = history_op[domain]need_break = Trueelse:check = raw_input("Input:")else:check = raw_input("Input:")has_judged = Trueif check == "Q":print path, "next OK!"breakelif check == "E":print path, "Exit!"with open("history_op.json", "w") as f:json.dump(history_op, f)print "saved history_op.json"sys.exit()elif check == "B":move_to(path, domain, "/home/bonelee/latest_metadata_sample/labeled_black/2017-8-%d-%d-%s.txt" % (day, hour, domain))history_op[domain] = "B"print "Saved OK!"if need_break: breakelif check == "W":move_to(path, domain, "/home/bonelee/latest_metadata_sample/labeled_white/2017-8-%d-%d-%s.txt" % (day, hour, domain))history_op[domain] = "W"print "Saved OK!"if need_break: breakelif check == "L":move_to(path, domain, "/home/bonelee/latest_metadata_sample/labeled_white_like/2017-8-%d-%d-%s.txt" % (day, hour, domain))history_op[domain] = "L"print "Saved OK!"if need_break: breakelif check == "CDN":move_to(path, domain, "/home/bonelee/latest_metadata_sample/labeled_cdn/2017-8-%d-%d-%s.txt" % (day, hour, domain))history_op[domain] = "CDN"print "Saved OK!"if need_break: breakelif check == "DDOS":move_to(path, domain, "/home/bonelee/latest_metadata_sample/labeled_ddos/2017-8-%d-%d-%s.txt" % (day, hour, domain))history_op[domain] = "DDOS"print "Saved OK!"if need_break: breakelif check == "M":move_to(path, domain, "/home/bonelee/latest_metadata_sample/labeled_mddos/2017-8-%d-%d-%s.txt" % (day, hour, domain))history_op[domain] = "M"print "Saved OK!"if need_break: breakelif check == "U":move_to(path, domain, "/home/bonelee/latest_metadata_sample/labeled_unknown/2017-8-%d-%d-%s.txt" % (day, hour, domain))history_op[domain] = "U"print "Saved OK!"if need_break: breakelse:if check in dist_dict:get_data_dist(group, dist_dict[check])else:print "unknown input!Choose the following one:"print "*******************************"print path, "check over..."print "*******************************"if __name__ == "__main__":main()