[4] | 1 | #!/usr/bin/env python3 |
---|
| 2 | #-*- mode: Python;-*- |
---|
| 3 | |
---|
| 4 | import sys |
---|
| 5 | import os |
---|
| 6 | import time |
---|
| 7 | import random |
---|
| 8 | import tempfile |
---|
| 9 | import argparse |
---|
| 10 | import socket |
---|
| 11 | import json |
---|
| 12 | try: |
---|
| 13 | import requests |
---|
| 14 | except: |
---|
| 15 | sys.stderr.write('ERROR: Could not import requests module. Ensure it is installed.\n') |
---|
| 16 | sys.stderr.write(' Under Debian, the package name is "python3-requests"\n.') |
---|
| 17 | sys.exit(1) |
---|
| 18 | |
---|
| 19 | VERSION = "{DEVELOPMENT}" |
---|
| 20 | if VERSION == "{DEVELOPMENT}": |
---|
| 21 | script_dir = '.' |
---|
| 22 | try: |
---|
| 23 | script_dir = os.path.dirname(os.path.realpath(__file__)) |
---|
| 24 | except: |
---|
| 25 | try: |
---|
| 26 | script_dir = os.path.dirname(os.path.abspath(sys.argv[0])) |
---|
| 27 | except: |
---|
| 28 | pass |
---|
| 29 | sys.path.append("%s/../lib" % script_dir) |
---|
| 30 | |
---|
| 31 | from nanownlib import * |
---|
| 32 | import nanownlib.storage |
---|
| 33 | |
---|
| 34 | parser = argparse.ArgumentParser( |
---|
| 35 | description="") |
---|
| 36 | parser.add_argument('-c', dest='cases', type=str, default='{"short":10000,"long":13000}', |
---|
| 37 | help='JSON representation of echo timing cases.') |
---|
[5] | 38 | parser.add_argument('--no-tcpts', action='store_true', help='Disbale TCP timestamp profiling') |
---|
[9] | 39 | parser.add_argument('--no-control', action='store_true', help='Do not collect separate control data. Instead, synthesize it from test and train data.') |
---|
[4] | 40 | parser.add_argument('session_name', default=None, |
---|
| 41 | help='Name for the sampler session (used in output filenames)') |
---|
| 42 | parser.add_argument('sample_count', type=int, default=None, |
---|
| 43 | help='Number of samples to collect') |
---|
| 44 | parser.add_argument('host', default=None, |
---|
| 45 | help='IP address or host name of server') |
---|
| 46 | parser.add_argument('port', nargs='?', type=int, default=80, |
---|
| 47 | help='TCP port number of HTTP service (default: 80)') |
---|
| 48 | options = parser.parse_args() |
---|
| 49 | |
---|
| 50 | |
---|
| 51 | session_name = options.session_name |
---|
| 52 | num_samples=options.sample_count |
---|
| 53 | hostname = options.host |
---|
| 54 | port = options.port |
---|
| 55 | protocol = 'http' |
---|
| 56 | |
---|
| 57 | cases = json.loads(options.cases) |
---|
| 58 | |
---|
| 59 | def extractReportedRuntime(headers, body): |
---|
| 60 | if body.startswith("waited: "): |
---|
| 61 | return int(body.split("waited: ")[1], 10) |
---|
| 62 | |
---|
| 63 | |
---|
| 64 | def sendRequest(data=None): |
---|
| 65 | method = 'GET' |
---|
| 66 | path = '/data/clamav-audit-results.txt?t=' + data |
---|
| 67 | url = "%s://%s:%d%s" % (protocol,hostname,port,path) |
---|
| 68 | headers = {} |
---|
| 69 | body = (b'') |
---|
| 70 | req = requests.Request('GET', url).prepare() |
---|
| 71 | |
---|
| 72 | retry = True |
---|
| 73 | while retry: |
---|
| 74 | try: |
---|
| 75 | session = requests.Session() |
---|
| 76 | response = session.send(req) |
---|
| 77 | #print(repr(response.raw._original_response.local_address)) |
---|
| 78 | reported = extractReportedRuntime(response.headers, response.text) |
---|
| 79 | retry = False |
---|
| 80 | except Exception as e: |
---|
| 81 | sys.stderr.write("ERROR: HTTP request problem: %s\n" % repr(e)) |
---|
| 82 | time.sleep(1.0) |
---|
| 83 | sys.stderr.write("ERROR: retrying...\n") |
---|
| 84 | |
---|
| 85 | return {'userspace_rtt':response.elapsed.microseconds*1000, |
---|
| 86 | 'reported':reported, |
---|
| 87 | 'local_port':response.raw._original_response.local_address[1]} |
---|
| 88 | |
---|
| 89 | |
---|
| 90 | def fetch(probedata, data): |
---|
| 91 | # http://docs.python-requests.org/en/latest/api/#requests.Response |
---|
| 92 | result = sendRequest("%d" % data) |
---|
| 93 | result.update(probedata) |
---|
| 94 | |
---|
| 95 | return result |
---|
| 96 | |
---|
| 97 | |
---|
| 98 | setCPUAffinity() |
---|
| 99 | setTCPTimestamps() |
---|
| 100 | host_ip = socket.gethostbyname(hostname) #XXX: what about multiple A records? |
---|
| 101 | |
---|
| 102 | meta = {'tcpts_mean':None,'tcpts_stddev':None,'tcpts_slopes':None} |
---|
[5] | 103 | if not options.no_tcpts: |
---|
[4] | 104 | sys.stderr.write("INFO: Probing target for TCP timestamp precision...\n") |
---|
| 105 | sniffer_fp = tempfile.NamedTemporaryFile('w+t') |
---|
| 106 | sniffer = startSniffer(host_ip, port, sniffer_fp.name) |
---|
| 107 | time.sleep(1.0) |
---|
| 108 | ports = runTimestampProbes(host_ip, port, hostname, 12) |
---|
| 109 | time.sleep(1.0) |
---|
| 110 | stopSniffer(sniffer) |
---|
| 111 | sniffer_fp.seek(0) |
---|
| 112 | mean,stddev,slopes = computeTimestampPrecision(sniffer_fp, ports) |
---|
| 113 | meta = {'tcpts_mean':mean,'tcpts_stddev':stddev,'tcpts_slopes':json.dumps(slopes)} |
---|
| 114 | |
---|
| 115 | if meta['tcpts_mean'] == None: |
---|
| 116 | sys.stderr.write("INFO: TCP timestamps not supported.\n") |
---|
| 117 | elif meta['tcpts_stddev'] == None: |
---|
| 118 | sys.stderr.write("INFO: Not enough TCP timestamp samples.\n") |
---|
| 119 | elif meta['tcpts_stddev']/meta['tcpts_mean'] > 0.05: |
---|
| 120 | sys.stderr.write("INFO: TCP timestamp precision is inconsistent.\n") |
---|
| 121 | sys.stderr.write("INFO: mean=%(tcpts_mean)f,stddev=%(tcpts_stddev)f,samples=%(tcpts_slopes)s\n" |
---|
| 122 | % meta) |
---|
| 123 | else: |
---|
| 124 | sys.stderr.write("INFO: Estimated TCP timestamp precision: %f (stddev: %f, %f%%)\n" |
---|
| 125 | % (meta['tcpts_mean'], meta['tcpts_stddev'], 100*meta['tcpts_stddev']/meta['tcpts_mean'])) |
---|
| 126 | |
---|
| 127 | |
---|
| 128 | sniffer_fp = tempfile.NamedTemporaryFile('w+t') |
---|
| 129 | db_file = "%s.db" % session_name |
---|
| 130 | |
---|
| 131 | sniffer = startSniffer(host_ip, port, sniffer_fp.name) |
---|
| 132 | db = nanownlib.storage.db(db_file) |
---|
| 133 | db.addMeta(meta) |
---|
| 134 | time.sleep(0.5) # ensure sniffer is fully ready and our process is migrated |
---|
| 135 | |
---|
[9] | 136 | if options.no_control: |
---|
| 137 | num_control = 0 |
---|
| 138 | else: |
---|
| 139 | num_control = int(num_samples*2/5) |
---|
| 140 | |
---|
| 141 | num_train = int((num_samples-num_control)/3) |
---|
| 142 | num_test = num_samples-num_train-num_control |
---|
| 143 | |
---|
| 144 | sample_types = [('train',num_train), |
---|
| 145 | ('train_null',num_control), |
---|
| 146 | ('test',num_test)] |
---|
| 147 | |
---|
[4] | 148 | sid = 0 |
---|
| 149 | report_interval = 20 |
---|
| 150 | start = time.time() |
---|
| 151 | next_report = start+report_interval |
---|
| 152 | for st,count in sample_types: |
---|
| 153 | if sniffer.poll() != None: |
---|
| 154 | sys.stderr.write('ERROR: Sniffer quit, sender exiting...\n') |
---|
| 155 | break |
---|
| 156 | |
---|
| 157 | for k in range(0,count): |
---|
| 158 | sample_order = list(cases.items()) |
---|
| 159 | random.shuffle(sample_order) |
---|
| 160 | if st.endswith('null'): |
---|
| 161 | for i in range(1,len(sample_order)): |
---|
| 162 | sample_order[i] = (sample_order[i][0],sample_order[0][1]) |
---|
| 163 | random.shuffle(sample_order) |
---|
| 164 | #print('after', sample_order) |
---|
| 165 | |
---|
| 166 | results = [] |
---|
| 167 | now = int(time.time()*1000000000) |
---|
| 168 | for i in range(len(sample_order)): |
---|
| 169 | results.append(fetch({'sample':sid, 'test_case':sample_order[i][0], |
---|
| 170 | 'type':st, 'tc_order':i, 'time_of_day':now}, |
---|
| 171 | sample_order[i][1])) |
---|
| 172 | |
---|
| 173 | #print(results) |
---|
| 174 | db.addProbes(results) |
---|
| 175 | db.conn.commit() |
---|
| 176 | sid += 1 |
---|
| 177 | |
---|
| 178 | if (time.time() > next_report): |
---|
| 179 | #s = time.time() |
---|
| 180 | reportProgress(db, sample_types, start) |
---|
| 181 | #print("reportProgress time:", time.time()-s) |
---|
| 182 | next_report += report_interval |
---|
| 183 | |
---|
| 184 | print("probes complete in %f" % (time.time()-start)) |
---|
| 185 | time.sleep(2.0) # Give sniffer a chance to collect remaining packets |
---|
| 186 | stopSniffer(sniffer) |
---|
| 187 | |
---|
| 188 | start = time.time() |
---|
| 189 | associatePackets(sniffer_fp, db) |
---|
| 190 | sniffer_fp.close() |
---|
| 191 | end = time.time() |
---|
| 192 | print("associate time:", end-start) |
---|
| 193 | |
---|
[9] | 194 | if options.no_control: |
---|
| 195 | print("TODO: implement control synthesizing!") |
---|
| 196 | |
---|
[4] | 197 | start = time.time() |
---|
| 198 | num_probes = analyzeProbes(db) |
---|
| 199 | end = time.time() |
---|
| 200 | print("analyzed %d probes' packets in: %f" % (num_probes, end-start)) |
---|