| 1 | |
|---|
| 2 | import time |
|---|
| 3 | import statistics |
|---|
| 4 | import functools |
|---|
| 5 | import pprint |
|---|
| 6 | import json |
|---|
| 7 | |
|---|
| 8 | from .stats import * |
|---|
| 9 | from .parallel import WorkerThreads |
|---|
| 10 | |
|---|
| 11 | def trainBoxTest(db, unusual_case, greater, num_observations): |
|---|
| 12 | db.resetOffsets() |
|---|
| 13 | |
|---|
| 14 | def trainAux(low,high,num_trials): |
|---|
| 15 | estimator = functools.partial(multiBoxTest, {'low':low, 'high':high}, greater) |
|---|
| 16 | estimates = bootstrap3(estimator, db, 'train', unusual_case, num_observations, num_trials) |
|---|
| 17 | null_estimates = bootstrap3(estimator, db, 'train_null', unusual_case, num_observations, num_trials) |
|---|
| 18 | |
|---|
| 19 | bad_estimates = len([e for e in estimates if e != 1]) |
|---|
| 20 | bad_null_estimates = len([e for e in null_estimates if e != 0]) |
|---|
| 21 | |
|---|
| 22 | false_negatives = 100.0*bad_estimates/num_trials |
|---|
| 23 | false_positives = 100.0*bad_null_estimates/num_trials |
|---|
| 24 | return false_positives,false_negatives |
|---|
| 25 | |
|---|
| 26 | #start = time.time() |
|---|
| 27 | wt = WorkerThreads(2, trainAux) |
|---|
| 28 | |
|---|
| 29 | num_trials = 200 |
|---|
| 30 | width = 1.0 |
|---|
| 31 | performance = [] |
|---|
| 32 | for low in range(0,50): |
|---|
| 33 | wt.addJob(low, (low,low+width,num_trials)) |
|---|
| 34 | wt.wait() |
|---|
| 35 | while not wt.resultq.empty(): |
|---|
| 36 | job_id,errors = wt.resultq.get() |
|---|
| 37 | fp,fn = errors |
|---|
| 38 | performance.append(((fp+fn)/2.0, job_id, fn, fp)) |
|---|
| 39 | performance.sort() |
|---|
| 40 | #pprint.pprint(performance) |
|---|
| 41 | #print(time.time()-start) |
|---|
| 42 | |
|---|
| 43 | num_trials = 200 |
|---|
| 44 | lows = [p[1] for p in performance[0:5]] |
|---|
| 45 | widths = [w/10.0 for w in range(5,155,10)] |
|---|
| 46 | performance = [] |
|---|
| 47 | for width in widths: |
|---|
| 48 | false_positives = [] |
|---|
| 49 | false_negatives = [] |
|---|
| 50 | for low in lows: |
|---|
| 51 | wt.addJob(low,(low,low+width,num_trials)) |
|---|
| 52 | wt.wait() |
|---|
| 53 | while not wt.resultq.empty(): |
|---|
| 54 | job_id,errors = wt.resultq.get() |
|---|
| 55 | fp,fn = errors |
|---|
| 56 | false_negatives.append(fn) |
|---|
| 57 | false_positives.append(fp) |
|---|
| 58 | |
|---|
| 59 | #print(width, false_negatives) |
|---|
| 60 | #print(width, false_positives) |
|---|
| 61 | #performance.append(((statistics.mean(false_positives)+statistics.mean(false_negatives))/2.0, |
|---|
| 62 | # width, statistics.mean(false_negatives), statistics.mean(false_positives))) |
|---|
| 63 | performance.append((abs(statistics.mean(false_positives)-statistics.mean(false_negatives)), |
|---|
| 64 | width, statistics.mean(false_negatives), statistics.mean(false_positives))) |
|---|
| 65 | performance.sort() |
|---|
| 66 | #pprint.pprint(performance) |
|---|
| 67 | good_width = performance[0][1] |
|---|
| 68 | #print("good_width:",good_width) |
|---|
| 69 | |
|---|
| 70 | |
|---|
| 71 | num_trials = 500 |
|---|
| 72 | performance = [] |
|---|
| 73 | for low in lows: |
|---|
| 74 | wt.addJob(low, (low,low+good_width,num_trials)) |
|---|
| 75 | wt.wait() |
|---|
| 76 | while not wt.resultq.empty(): |
|---|
| 77 | job_id,errors = wt.resultq.get() |
|---|
| 78 | fp,fn = errors |
|---|
| 79 | performance.append(((fp+fn)/2.0, job_id, fn, fp)) |
|---|
| 80 | performance.sort() |
|---|
| 81 | #pprint.pprint(performance) |
|---|
| 82 | best_low = performance[0][1] |
|---|
| 83 | #print("best_low:", best_low) |
|---|
| 84 | |
|---|
| 85 | |
|---|
| 86 | num_trials = 500 |
|---|
| 87 | widths = [good_width+(x/100.0) for x in range(-120,125,5) if good_width+(x/100.0) > 0.0] |
|---|
| 88 | performance = [] |
|---|
| 89 | for width in widths: |
|---|
| 90 | wt.addJob(width, (best_low,best_low+width,num_trials)) |
|---|
| 91 | wt.wait() |
|---|
| 92 | while not wt.resultq.empty(): |
|---|
| 93 | job_id,errors = wt.resultq.get() |
|---|
| 94 | fp,fn = errors |
|---|
| 95 | #performance.append(((fp+fn)/2.0, job_id, fn, fp)) |
|---|
| 96 | performance.append((abs(fp-fn), job_id, fn, fp)) |
|---|
| 97 | performance.sort() |
|---|
| 98 | #pprint.pprint(performance) |
|---|
| 99 | best_width=performance[0][1] |
|---|
| 100 | #print("best_width:",best_width) |
|---|
| 101 | #print("final_performance:", performance[0][0]) |
|---|
| 102 | |
|---|
| 103 | wt.stop() |
|---|
| 104 | params = json.dumps({"low":best_low,"high":best_low+best_width}, sort_keys=True) |
|---|
| 105 | return {'trial_type':"train", |
|---|
| 106 | 'num_observations':num_observations, |
|---|
| 107 | 'num_trials':num_trials, |
|---|
| 108 | 'params':params, |
|---|
| 109 | 'false_positives':performance[0][3], |
|---|
| 110 | 'false_negatives':performance[0][2]} |
|---|
| 111 | |
|---|
| 112 | |
|---|
| 113 | def trainSummary(summaryFunc, db, unusual_case, greater, num_observations): |
|---|
| 114 | db.resetOffsets() |
|---|
| 115 | stest = functools.partial(summaryTest, summaryFunc) |
|---|
| 116 | |
|---|
| 117 | def trainAux(distance, threshold, num_trials): |
|---|
| 118 | estimator = functools.partial(stest, {'distance':distance,'threshold':threshold}, greater) |
|---|
| 119 | estimates = bootstrap3(estimator, db, 'train', unusual_case, num_observations, num_trials) |
|---|
| 120 | null_estimates = bootstrap3(estimator, db, 'train_null', unusual_case, num_observations, num_trials) |
|---|
| 121 | |
|---|
| 122 | bad_estimates = len([e for e in estimates if e != 1]) |
|---|
| 123 | bad_null_estimates = len([e for e in null_estimates if e != 0]) |
|---|
| 124 | |
|---|
| 125 | false_negatives = 100.0*bad_estimates/num_trials |
|---|
| 126 | false_positives = 100.0*bad_null_estimates/num_trials |
|---|
| 127 | return false_positives,false_negatives |
|---|
| 128 | |
|---|
| 129 | #determine expected delta based on differences |
|---|
| 130 | mean_diffs = [s['unusual_packet']-s['other_packet'] for s in db.subseries('train', unusual_case)] |
|---|
| 131 | threshold = summaryFunc(mean_diffs)/2.0 |
|---|
| 132 | #print("init_threshold:", threshold) |
|---|
| 133 | |
|---|
| 134 | wt = WorkerThreads(2, trainAux) |
|---|
| 135 | |
|---|
| 136 | num_trials = 500 |
|---|
| 137 | performance = [] |
|---|
| 138 | for distance in range(1,50): |
|---|
| 139 | wt.addJob(distance, (distance,threshold,num_trials)) |
|---|
| 140 | wt.wait() |
|---|
| 141 | while not wt.resultq.empty(): |
|---|
| 142 | job_id,errors = wt.resultq.get() |
|---|
| 143 | fp,fn = errors |
|---|
| 144 | performance.append(((fp+fn)/2.0, job_id, fn, fp)) |
|---|
| 145 | |
|---|
| 146 | performance.sort() |
|---|
| 147 | #pprint.pprint(performance) |
|---|
| 148 | good_distance = performance[0][1] |
|---|
| 149 | #print("good_distance:",good_distance) |
|---|
| 150 | |
|---|
| 151 | |
|---|
| 152 | num_trials = 500 |
|---|
| 153 | performance = [] |
|---|
| 154 | for t in range(80,122,2): |
|---|
| 155 | wt.addJob(threshold*(t/100.0), (good_distance,threshold*(t/100.0),num_trials)) |
|---|
| 156 | wt.wait() |
|---|
| 157 | while not wt.resultq.empty(): |
|---|
| 158 | job_id,errors = wt.resultq.get() |
|---|
| 159 | fp,fn = errors |
|---|
| 160 | #performance.append(((fp+fn)/2.0, job_id, fn, fp)) |
|---|
| 161 | performance.append((abs(fp-fn), job_id, fn, fp)) |
|---|
| 162 | performance.sort() |
|---|
| 163 | #pprint.pprint(performance) |
|---|
| 164 | good_threshold = performance[0][1] |
|---|
| 165 | #print("good_threshold:", good_threshold) |
|---|
| 166 | |
|---|
| 167 | |
|---|
| 168 | num_trials = 500 |
|---|
| 169 | performance = [] |
|---|
| 170 | for d in [good_distance+s for s in range(-4,5) |
|---|
| 171 | if good_distance+s > -1 and good_distance+s < 51]: |
|---|
| 172 | wt.addJob(d, (d,good_threshold,num_trials)) |
|---|
| 173 | wt.wait() |
|---|
| 174 | while not wt.resultq.empty(): |
|---|
| 175 | job_id,errors = wt.resultq.get() |
|---|
| 176 | fp,fn = errors |
|---|
| 177 | performance.append(((fp+fn)/2.0, job_id, fn, fp)) |
|---|
| 178 | performance.sort() |
|---|
| 179 | #pprint.pprint(performance) |
|---|
| 180 | best_distance = performance[0][1] |
|---|
| 181 | #print("best_distance:",best_distance) |
|---|
| 182 | |
|---|
| 183 | |
|---|
| 184 | num_trials = 500 |
|---|
| 185 | performance = [] |
|---|
| 186 | for t in range(90,111): |
|---|
| 187 | wt.addJob(good_threshold*(t/100.0), (best_distance,good_threshold*(t/100.0),num_trials)) |
|---|
| 188 | wt.wait() |
|---|
| 189 | while not wt.resultq.empty(): |
|---|
| 190 | job_id,errors = wt.resultq.get() |
|---|
| 191 | fp,fn = errors |
|---|
| 192 | #performance.append(((fp+fn)/2.0, job_id, fn, fp)) |
|---|
| 193 | performance.append((abs(fp-fn), job_id, fn, fp)) |
|---|
| 194 | performance.sort() |
|---|
| 195 | #pprint.pprint(performance) |
|---|
| 196 | best_threshold = performance[0][1] |
|---|
| 197 | #print("best_threshold:", best_threshold) |
|---|
| 198 | |
|---|
| 199 | wt.stop() |
|---|
| 200 | params = json.dumps({'distance':best_distance,'threshold':best_threshold}, sort_keys=True) |
|---|
| 201 | return {'trial_type':"train", |
|---|
| 202 | 'num_observations':num_observations, |
|---|
| 203 | 'num_trials':num_trials, |
|---|
| 204 | 'params':params, |
|---|
| 205 | 'false_positives':performance[0][3], |
|---|
| 206 | 'false_negatives':performance[0][2]} |
|---|
| 207 | |
|---|
| 208 | |
|---|
| 209 | def trainKalman(db, unusual_case, greater, num_observations): |
|---|
| 210 | db.resetOffsets() |
|---|
| 211 | |
|---|
| 212 | def trainAux(params, num_trials): |
|---|
| 213 | estimator = functools.partial(kalmanTest, params, greater) |
|---|
| 214 | estimates = bootstrap3(estimator, db, 'train', unusual_case, num_observations, num_trials) |
|---|
| 215 | null_estimates = bootstrap3(estimator, db, 'train_null', unusual_case, num_observations, num_trials) |
|---|
| 216 | |
|---|
| 217 | bad_estimates = len([e for e in estimates if e != 1]) |
|---|
| 218 | bad_null_estimates = len([e for e in null_estimates if e != 0]) |
|---|
| 219 | |
|---|
| 220 | false_negatives = 100.0*bad_estimates/num_trials |
|---|
| 221 | false_positives = 100.0*bad_null_estimates/num_trials |
|---|
| 222 | return false_positives,false_negatives |
|---|
| 223 | |
|---|
| 224 | mean_diffs = [s['unusual_packet']-s['other_packet'] for s in db.subseries('train', unusual_case)] |
|---|
| 225 | good_threshold = kfilter({},mean_diffs)['est'][-1]/2.0 |
|---|
| 226 | |
|---|
| 227 | wt = WorkerThreads(2, trainAux) |
|---|
| 228 | num_trials = 200 |
|---|
| 229 | performance = [] |
|---|
| 230 | for t in range(90,111): |
|---|
| 231 | params = {'threshold':good_threshold*(t/100.0)} |
|---|
| 232 | wt.addJob(good_threshold*(t/100.0), (params,num_trials)) |
|---|
| 233 | wt.wait() |
|---|
| 234 | while not wt.resultq.empty(): |
|---|
| 235 | job_id,errors = wt.resultq.get() |
|---|
| 236 | fp,fn = errors |
|---|
| 237 | #performance.append(((fp+fn)/2.0, job_id, fn, fp)) |
|---|
| 238 | performance.append((abs(fp-fn), job_id, fn, fp)) |
|---|
| 239 | performance.sort() |
|---|
| 240 | #pprint.pprint(performance) |
|---|
| 241 | best_threshold = performance[0][1] |
|---|
| 242 | #print("best_threshold:", best_threshold) |
|---|
| 243 | params = {'threshold':best_threshold} |
|---|
| 244 | |
|---|
| 245 | wt.stop() |
|---|
| 246 | |
|---|
| 247 | return {'trial_type':"train", |
|---|
| 248 | 'num_observations':num_observations, |
|---|
| 249 | 'num_trials':num_trials, |
|---|
| 250 | 'params':json.dumps(params, sort_keys=True), |
|---|
| 251 | 'false_positives':performance[0][3], |
|---|
| 252 | 'false_negatives':performance[0][2]} |
|---|
| 253 | |
|---|
| 254 | |
|---|
| 255 | def trainTsval(db, unusual_case, greater, num_observations): |
|---|
| 256 | db.resetOffsets() |
|---|
| 257 | |
|---|
| 258 | def trainAux(params, num_trials): |
|---|
| 259 | estimator = functools.partial(tsvalwmeanTest, params, greater) |
|---|
| 260 | estimates = bootstrap3(estimator, db, 'train', unusual_case, num_observations, num_trials) |
|---|
| 261 | null_estimates = bootstrap3(estimator, db, 'train_null', unusual_case, num_observations, num_trials) |
|---|
| 262 | |
|---|
| 263 | bad_estimates = len([e for e in estimates if e != 1]) |
|---|
| 264 | bad_null_estimates = len([e for e in null_estimates if e != 0]) |
|---|
| 265 | |
|---|
| 266 | false_negatives = 100.0*bad_estimates/num_trials |
|---|
| 267 | false_positives = 100.0*bad_null_estimates/num_trials |
|---|
| 268 | return false_positives,false_negatives |
|---|
| 269 | |
|---|
| 270 | train = db.subseries('train', unusual_case) |
|---|
| 271 | null = db.subseries('train_null', unusual_case) |
|---|
| 272 | good_threshold = (tsvalwmean(train)+tsvalwmean(null))/2.0 |
|---|
| 273 | |
|---|
| 274 | wt = WorkerThreads(2, trainAux) |
|---|
| 275 | num_trials = 200 |
|---|
| 276 | performance = [] |
|---|
| 277 | for t in range(90,111): |
|---|
| 278 | params = {'threshold':good_threshold*(t/100.0)} |
|---|
| 279 | wt.addJob(good_threshold*(t/100.0), (params,num_trials)) |
|---|
| 280 | wt.wait() |
|---|
| 281 | while not wt.resultq.empty(): |
|---|
| 282 | job_id,errors = wt.resultq.get() |
|---|
| 283 | fp,fn = errors |
|---|
| 284 | #performance.append(((fp+fn)/2.0, job_id, fn, fp)) |
|---|
| 285 | performance.append((abs(fp-fn), job_id, fn, fp)) |
|---|
| 286 | performance.sort() |
|---|
| 287 | #pprint.pprint(performance) |
|---|
| 288 | best_threshold = performance[0][1] |
|---|
| 289 | #print("best_threshold:", best_threshold) |
|---|
| 290 | params = {'threshold':best_threshold} |
|---|
| 291 | |
|---|
| 292 | wt.stop() |
|---|
| 293 | |
|---|
| 294 | return {'trial_type':"train", |
|---|
| 295 | 'num_observations':num_observations, |
|---|
| 296 | 'num_trials':num_trials, |
|---|
| 297 | 'params':json.dumps(params, sort_keys=True), |
|---|
| 298 | 'false_positives':performance[0][3], |
|---|
| 299 | 'false_negatives':performance[0][2]} |
|---|
| 300 | |
|---|
| 301 | |
|---|
| 302 | #from pykalman import KalmanFilter |
|---|
| 303 | _pykalman4d_params = None |
|---|
| 304 | _pykalman4d_params = {'observation_covariance': [[11960180434.411114, 4760272534.795976, 8797551081.431936, 6908794128.927051], [4760272534.795962, 12383598172.428213, 5470747537.2599745, 11252625555.297853], [8797551081.431955, 5470747537.2601185, 1466222848395.7058, 72565713883.12643], [6908794128.927095, 11252625555.297981, 72565713883.12654, 1519760903943.507]], 'transition_offsets': [592.5708159274, 583.3804671015271, 414.4187239098291, 562.166786712371], 'observation_offsets': [165.2279084503762, 157.76807691937614, 168.4235495099334, 225.33433430227353], 'initial_state_covariance': [[33599047.5, -18251285.25, 3242535690.59375, -8560730487.84375], [-18251285.25, 9914252.3125, -1761372688.59375, 4650260880.1875], [3242535690.59375, -1761372688.59375, 312926663745.03125, -826168494791.7188], [-8560730487.84375, 4650260880.1875, -826168494791.7188, 2181195982530.4688]], 'initial_state_mean': [12939012.5625, 12934563.71875, 13134751.608, 13138990.9985], 'transition_covariance': [[2515479496.145993, -401423541.70620924, 1409951418.1627903, 255932902.74454522], [-401423541.706214, 2744353887.676857, 1162316.2019491254, 1857251491.3987627], [1409951418.1628358, 1162316.2020361447, 543279068599.8229, -39399311190.5746], [255932902.74459982, 1857251491.398838, -39399311190.574585, 537826124257.5266]], 'observation_matrices': [[1.4255288693095167, -0.4254638445329988, 0.0003406844036817347, -0.0005475021956726778], [-0.46467270827589857, 1.4654311778340343, -0.0003321330280128265, -0.0002853945703691352], [-0.2644570970067974, -0.33955835481495455, 1.7494161615202275, -0.15394117603733548], [-0.3419097544041847, -0.23992883666045373, -0.15587790880447727, 1.7292393175137022]], 'transition_matrices': [[0.52163952865412, 0.47872618354122665, -0.0004322286766109684, 0.00017293351811531466], [0.5167436693545113, 0.48319044922845933, 7.765428142114672e-05, -0.00021518950285326355], [0.2091705950622469, 0.41051399729482796, 0.19341113299389256, 0.19562916616052917], [0.368592004009912, 0.22263632461118732, 0.20756792378812872, 0.20977025833570906]]} |
|---|
| 305 | _pykalman4d_good_threshold = 2009.25853272 |
|---|
| 306 | _pykalman4d_params = None |
|---|
| 307 | |
|---|
| 308 | _pykalman4d_params = {'observation_covariance': [[32932883342.63772, 18054300398.442295, 27538911550.824535, 17152378956.778696], [18054300398.446983, 436546443436.5115, 37327644533.69647, 424485386677.31274], [27538911550.838238, 37327644533.706024, 3276324705772.982, 456017515263.88715], [17152378956.788027, 424485386677.317, 456017515263.88245, 3767844180658.1724]], 'observation_matrices': [[1.025773112769464, -0.028755990114063934, 0.0003540921897382532, 0.0025748564713126143], [-0.8595457826320256, 1.8607522167556567, -0.003520779053701517, 0.002309145982167138], [-0.5806427858959466, 0.22466075141448982, 1.6247192012813798, -0.27363797512617793], [-0.5853369461874607, 0.262177909212312, -0.28415108658843735, 1.6020343138710018]], 'initial_state_mean': [0.0, 0.0, 0.0, 0.0], 'observation_offsets': [549.4498515668686, 484.2106453284049, 648.556719142234, 380.10978090584763], 'transition_covariance': [[4147844406.7768326, -1308763245.5992138, 2920744388.523955, 860096280.797968], [-1308763245.5998695, 171190325905.83395, 3557618712.218984, 165332873663.83142], [2920744388.532502, 3557618712.2283373, 1054894349089.0673, -117551209299.73402], [860096280.805706, 165332873663.83963, -117551209299.73474, 1223605046475.7324]], 'transition_offsets': [1156.9264087977374, 1150.752680207601, 1312.2595286459816, 1267.4069537452415], 'initial_state_covariance': [[667999273207241.0, 669330484615232.1, 713726904326576.2, 731731206363217.4], [669330484615390.9, 670664348906228.8, 715149243295271.9, 733189424910272.2], [713726904326843.4, 715149243295370.6, 762584802695960.9, 781821582244358.5], [731731206363417.0, 733189424910299.0, 781821582244278.6, 801543624134758.0]], 'transition_matrices': [[0.9680677036616316, 0.03260717171917804, 0.0005279411071512641, -0.0012363486571871363], [0.9555219601128613, 0.03851351491891819, 0.00411268796118236, 0.0017357967358293536], [0.622254432930994, -0.2583795512595657, 0.31745705251401546, 0.32357126976364725], [0.6644076824932768, -0.33545285094373867, 0.3295778964272671, 0.34682391469482354]]} |
|---|
| 309 | _pykalman4d_good_threshold = -253.849393803 |
|---|
| 310 | def trainPyKalman4D(db, unusual_case, greater, num_observations): |
|---|
| 311 | global _pykalman4d_params |
|---|
| 312 | global _pykalman4d_good_threshold |
|---|
| 313 | db.resetOffsets() |
|---|
| 314 | |
|---|
| 315 | if _pykalman4d_params == None: |
|---|
| 316 | train = db.subseries('train',unusual_case, offset=0) |
|---|
| 317 | null = db.subseries('train_null',unusual_case, offset=0) |
|---|
| 318 | train_array = numpy.asarray([(s['unusual_packet'],s['other_packet'],s['unusual_tsval'],s['other_tsval']) |
|---|
| 319 | for s in train]) |
|---|
| 320 | null_array = numpy.asarray([(s['unusual_packet'],s['other_packet'],s['unusual_tsval'],s['other_tsval']) |
|---|
| 321 | for s in null]) |
|---|
| 322 | kf = KalmanFilter(n_dim_obs=4, n_dim_state=4) |
|---|
| 323 | #initial_state_mean=[quadsummary([s['unusual_packet'] for s in train]), |
|---|
| 324 | # quadsummary([s['other_packet'] for s in train]), |
|---|
| 325 | # numpy.mean([s['unusual_tsval'] for s in train]), |
|---|
| 326 | # numpy.mean([s['other_tsval'] for s in train])]) |
|---|
| 327 | |
|---|
| 328 | kf = kf.em(train_array+null_array[0:50000], n_iter=10, |
|---|
| 329 | em_vars=('transition_matrices', |
|---|
| 330 | 'observation_matrices', |
|---|
| 331 | 'transition_offsets', |
|---|
| 332 | 'observation_offsets', |
|---|
| 333 | 'transition_covariance', |
|---|
| 334 | 'observation_covariance', |
|---|
| 335 | 'initial_state_covariance')) |
|---|
| 336 | _pykalman4d_params = {'transition_matrices': kf.transition_matrices.tolist(), |
|---|
| 337 | 'observation_matrices': kf.observation_matrices.tolist(), |
|---|
| 338 | 'transition_offsets': kf.transition_offsets.tolist(), |
|---|
| 339 | 'observation_offsets': kf.observation_offsets.tolist(), |
|---|
| 340 | 'transition_covariance': kf.transition_covariance.tolist(), |
|---|
| 341 | 'observation_covariance': kf.observation_covariance.tolist(), |
|---|
| 342 | 'initial_state_mean': kf.initial_state_mean.tolist(), |
|---|
| 343 | 'initial_state_covariance': kf.initial_state_covariance.tolist()} |
|---|
| 344 | print(_pykalman4d_params) |
|---|
| 345 | |
|---|
| 346 | kf = KalmanFilter(n_dim_obs=4, n_dim_state=4, **_pykalman4d_params) |
|---|
| 347 | smoothed,covariance = kf.smooth(train_array) |
|---|
| 348 | null_smoothed,covariance = kf.smooth(null_array) |
|---|
| 349 | |
|---|
| 350 | kp = _pykalman4d_params.copy() |
|---|
| 351 | #kp['initial_state_mean']=[quadsummary([s['unusual_packet'] for s in train]), |
|---|
| 352 | # quadsummary([s['other_packet'] for s in train]), |
|---|
| 353 | # numpy.mean([s['unusual_tsval'] for s in train]), |
|---|
| 354 | # numpy.mean([s['other_tsval'] for s in train])] |
|---|
| 355 | #kf = KalmanFilter(n_dim_obs=4, n_dim_state=4, **kp) |
|---|
| 356 | #null_smoothed,covariance = kf.smooth(null_array) |
|---|
| 357 | |
|---|
| 358 | _pykalman4d_good_threshold = (numpy.mean([m[0]-m[1] for m in smoothed])+numpy.mean([m[0]-m[1] for m in null_smoothed]))/2.0 |
|---|
| 359 | print(_pykalman4d_good_threshold) |
|---|
| 360 | |
|---|
| 361 | |
|---|
| 362 | def trainAux(params, num_trials): |
|---|
| 363 | estimator = functools.partial(pyKalman4DTest, params, greater) |
|---|
| 364 | estimates = bootstrap3(estimator, db, 'train', unusual_case, num_observations, num_trials) |
|---|
| 365 | null_estimates = bootstrap3(estimator, db, 'train_null', unusual_case, num_observations, num_trials) |
|---|
| 366 | |
|---|
| 367 | bad_estimates = len([e for e in estimates if e != 1]) |
|---|
| 368 | bad_null_estimates = len([e for e in null_estimates if e != 0]) |
|---|
| 369 | |
|---|
| 370 | false_negatives = 100.0*bad_estimates/num_trials |
|---|
| 371 | false_positives = 100.0*bad_null_estimates/num_trials |
|---|
| 372 | return false_positives,false_negatives |
|---|
| 373 | |
|---|
| 374 | params = {'threshold':_pykalman4d_good_threshold, 'kparams':_pykalman4d_params} |
|---|
| 375 | |
|---|
| 376 | wt = WorkerThreads(2, trainAux) |
|---|
| 377 | num_trials = 50 |
|---|
| 378 | performance = [] |
|---|
| 379 | for t in range(-80,100,20): |
|---|
| 380 | thresh = _pykalman4d_good_threshold + abs(_pykalman4d_good_threshold)*(t/100.0) |
|---|
| 381 | params['threshold'] = thresh |
|---|
| 382 | wt.addJob(thresh, (params.copy(),num_trials)) |
|---|
| 383 | wt.wait() |
|---|
| 384 | while not wt.resultq.empty(): |
|---|
| 385 | job_id,errors = wt.resultq.get() |
|---|
| 386 | fp,fn = errors |
|---|
| 387 | #performance.append(((fp+fn)/2.0, job_id, fn, fp)) |
|---|
| 388 | performance.append((abs(fp-fn), job_id, fn, fp)) |
|---|
| 389 | performance.sort() |
|---|
| 390 | #pprint.pprint(performance) |
|---|
| 391 | best_threshold = performance[0][1] |
|---|
| 392 | #print("best_threshold:", best_threshold) |
|---|
| 393 | params['threshold']=best_threshold |
|---|
| 394 | |
|---|
| 395 | wt.stop() |
|---|
| 396 | |
|---|
| 397 | return {'trial_type':"train", |
|---|
| 398 | 'num_observations':num_observations, |
|---|
| 399 | 'num_trials':num_trials, |
|---|
| 400 | 'params':json.dumps(params, sort_keys=True), |
|---|
| 401 | 'false_positives':performance[0][3], |
|---|
| 402 | 'false_negatives':performance[0][2]} |
|---|
| 403 | |
|---|
| 404 | |
|---|
| 405 | |
|---|
| 406 | classifiers = {'boxtest':{'train':trainBoxTest, 'test':multiBoxTest, 'train_results':[]}, |
|---|
| 407 | 'midsummary':{'train':functools.partial(trainSummary, midsummary), 'test':midsummaryTest, 'train_results':[]}, |
|---|
| 408 | #'ubersummary':{'train':functools.partial(trainSummary, ubersummary), 'test':ubersummaryTest, 'train_results':[]}, |
|---|
| 409 | 'quadsummary':{'train':functools.partial(trainSummary, quadsummary), 'test':quadsummaryTest, 'train_results':[]}, |
|---|
| 410 | 'septasummary':{'train':functools.partial(trainSummary, septasummary), 'test':septasummaryTest, 'train_results':[]}, |
|---|
| 411 | #'pykalman4d':{'train':trainPyKalman4D, 'test':pyKalman4DTest, 'train_results':[]}, |
|---|
| 412 | #'tsvalwmean':{'train':trainTsval, 'test':tsvalwmeanTest, 'train_results':[]}, |
|---|
| 413 | #'kalman':{'train':trainKalman, 'test':kalmanTest, 'train_results':[]}, |
|---|
| 414 | #'_trimean':{'train':None, 'test':trimeanTest, 'train_results':[]}, |
|---|
| 415 | } |
|---|
