Changeset 16 for trunk/bin


Ignore:
Timestamp:
08/01/15 19:01:31 (9 years ago)
Author:
tim
Message:

.

Location:
trunk/bin
Files:
3 edited

Legend:

Unmodified
Added
Removed

  • trunk/bin/analyze_packets

    r10 r16  
    1010import socket
    1111import json
    12 try:
    13     import requests
    14 except:
    15     sys.stderr.write('ERROR: Could not import requests module.  Ensure it is installed.\n')
    16     sys.stderr.write('       Under Debian, the package name is "python3-requests"\n.')
    17     sys.exit(1)
     12
    1813
    1914VERSION = "{DEVELOPMENT}"
    … …  
    4035
    4136
    42 
    4337db = nanownlib.storage.db(options.db_file)
    4438
    … …  
    4640import cProfile
    4741#cProfile.run('num_probes = analyzeProbes(db)')
    48 num_probes = analyzeProbes(db)
     42num_probes = analyzeProbes(db, recompute=True)
    4943end = time.time()
    5044print("analyzed %d probes' packets in: %f" % (num_probes, end-start))

  • trunk/bin/graph

    r14 r16  
    3636parser.add_argument('db_file', default=None,
    3737                    help='')
     38parser.add_argument('unusual_case', nargs='?', type=str, default=None,
     39                    help='The test case that is most unusual from the others. (default: auto detect)')
    3840options = parser.parse_args()
    3941db = nanownlib.storage.db(options.db_file)
     42if options.unusual_case == None:
     43    unusual_case,delta = findUnusualTestCase(db)
    4044
    4145
    … …  
    6973#print('(test): %f' % weightedMean(derived,weights))
    7074
    71 diffs = list(differences(db, 'long'))
    72 reported_diffs = list(differences(db, 'long', 'reported'))
     75diffs = list(differences(db, unusual_case))
     76reported_diffs = list(differences(db, unusual_case, 'reported'))
    7377#shorts = [s['packet_rtt'] for s in samples.values() if s['test_case']=='short']
    7478#longs = [s['packet_rtt'] for s in samples.values() if s['test_case']=='long']
    … …  
    7680
    7781def basicStatistics():
    78     print('packet_rtt diff mean: %f' % statistics.mean(diffs))
    79     print('packet_rtt diff median: %f' % statistics.median(diffs))
    80     print('packet_rtt diff midhinge: %f' % midsummary(diffs))
    81     print('packet_rtt diff trimean: %f' % trimean(diffs))
    82     print('packet_rtt diff quadsummary: %f' % quadsummary(diffs))
    83     print('packet_rtt diff ubersummary: %f' % ubersummary(diffs))
    84     print('packet_rtt diff septasummary: %f' % septasummary(diffs))
    85     print('packet_rtt diff MAD: %f' % mad(diffs))
     82    print('packet_rtt diff midhinge:     %10.2f' % midsummary(diffs))
     83    print('packet_rtt diff quadsummary:  %10.2f' % quadsummary(diffs))
     84    print('packet_rtt diff septasummary: %10.2f' % septasummary(diffs))
     85    print('packet_rtt diff MAD:          %10.2f' % mad(diffs))
    8686    try:
    87         print('reported diff trimean: %f' % trimean(reported_diffs))
    88         print('reported diff quadsummary: %f' % quadsummary(reported_diffs))
    89         print('reported diff ubersummary: %f' % ubersummary(reported_diffs))
    90         print('reported diff septasummary: %f' % septasummary(reported_diffs))
    91         print('reported diff MAD: %f' % mad(reported_diffs))
     87        print('reported diff midhinge:       %10.2f' % midsummary(reported_diffs))
     88        print('reported diff quadsummary:    %10.2f' % quadsummary(reported_diffs))
     89        print('reported diff septasummary:   %10.2f' % septasummary(reported_diffs))
     90        print('reported diff MAD:            %10.2f' % mad(reported_diffs))
    9291
    9392        #import cProfile
    … …  
    108107    #print('tsval null diff weighted mean: %f' % tsvalwmean(db.subseries('train_null','long')))
    109108
    110 
     109basicStatistics()
    111110
    112111def exampleBoxTestHistogram(low,high):
    113112    num_bins = 300
    114     all = db.subseries('train','long')+db.subseries('test','long')
     113    all = db.subseries('train',unusual_case)+db.subseries('test',unusual_case)
    115114    s   = [s['other_packet'] for s in all]
    116115    l   = [s['unusual_packet'] for s in all]
    … …  
    362361    #plt.savefig('paper/graphs/dists-vs-dist-of-diffs2.svg')
    363362
    364 #tsFilteredHistogram()
     363tsFilteredHistogram()
    365364
    366365
    … …  
    459458
    460459    chartname = "/home/tim/blindspot/research/timing-analysis/paper/figures/results/%s.svg" % (basename)
    461     print(chartname)
     460    #print(chartname)
    462461   
    463462    plt.clf()
    … …  
    469468    color_id = 0
    470469
    471     cursor = db.conn.cursor()
    472     query = """
    473       SELECT classifier FROM classifier_results GROUP BY classifier ORDER BY classifier;
    474     """
    475     cursor.execute(query)
    476     classifiers = []
    477     for c in cursor:
    478         classifiers.append(c[0])
    479 
    480     best_obs = []
    481     best_error = []
     470    best_obs,best_error = evaluateTestResults(db)
     471    best_obs =   sorted(best_obs,   key=lambda x: x['num_observations'])
     472    best_error = sorted(best_error, key=lambda x: x['error'])
     473    winner = None
     474    for bo in best_obs:
     475        sys.stdout.write("%(num_observations)d obs / %(classifier)s / %(params)s" % bo)
     476        if winner == None:
     477            sys.stdout.write(" (winner)")
     478            winner = bo
     479        print()
     480       
     481    for be in best_error:
     482        sys.stdout.write("%(error)f%% error / %(classifier)s / %(params)s" % be)
     483        if winner == None:
     484            sys.stdout.write(" (winner)")
     485            winner = be
     486        print()
     487
     488    all = sorted(best_obs+best_error, key=lambda x: x['classifier'])
    482489    max_obs = 0
    483     for classifier in classifiers:
    484         query="""
    485         SELECT params,num_observations FROM classifier_results
    486         WHERE trial_type='test'
    487          AND classifier=:classifier
    488          AND (false_positives+false_negatives)/2.0 < 5.0
    489         ORDER BY num_observations,(false_positives+false_negatives)
    490         LIMIT 1
    491         """
    492         cursor.execute(query, {'classifier':classifier})
    493         row = cursor.fetchone()
    494         if row == None:
    495             query="""
    496             SELECT params,(false_positives+false_negatives)/2 FROM classifier_results
    497             WHERE trial_type='test' and classifier=:classifier
    498             ORDER BY (false_positives+false_negatives),num_observations
    499             LIMIT 1
    500             """
    501             cursor.execute(query, {'classifier':classifier})
    502             row = cursor.fetchone()
    503             if row == None:
    504                 sys.stderr.write("WARN: couldn't find test results for classifier '%s'.\n" % classifier)
    505                 continue
    506 
    507             best_error.append((row[1], classifier))
    508         else:
    509             best_obs.append((row[1], classifier))
    510 
    511         best_params = row[0]
     490    for result in all:
    512491        query="""
    513492        SELECT num_observations,(false_positives+false_negatives)/2.0 FROM classifier_results
    … …  
    517496        ORDER BY num_observations
    518497        """
    519         cursor.execute(query, {'classifier':classifier,'params':best_params})
     498        cursor = db.conn.cursor()
     499        cursor.execute(query, {'classifier':result['classifier'],'params':result['params']})
    520500
    521501        num_obs = []
    … …  
    528508        path = plt.scatter(num_obs, performance, color=colors[color_id], s=4, alpha=0.8, linewidths=3.0)
    529509        plt.plot(num_obs, performance, color=colors[color_id], alpha=0.8)
    530         legend.append((classifier,path))
     510        legend.append((result['classifier'],path))
    531511        color_id = (color_id+1) % len(colors)
    532512
    533     best_obs.sort()
    534     best_error.sort()
    535     winner = None
    536     for bo in best_obs:
    537         sys.stdout.write("%d obs / %s" % bo)
    538         if winner == None:
    539             sys.stdout.write(" (winner)")
    540             winner = bo
    541         print()
    542        
    543     for be in best_error:
    544         sys.stdout.write("%f%% error / %s" % be)
    545         if winner == None:
    546             sys.stdout.write(" (winner)")
    547             winner = be
    548         print()
    549    
    550513    plt.legend([l[1] for l in legend], [l[0] for l in legend], scatterpoints=1, fontsize='x-small')
    551514    plt.plot([0, max_obs], [5.0, 5.0], "k--")
    552515    plt.xlabel('Number of Observations')
    553516    plt.ylabel('Error Rate')
    554     plt.savefig(chartname)
    555     #plt.show()
     517    #plt.savefig(chartname)
     518    plt.show()
    556519   
    557520graphTestResults()

  • trunk/bin/train

    r13 r16  
    6767            result = trainer(db,unusual_case,greater,num_obs)
    6868            result['classifier'] = classifier
    69             train_time = "%f" % (time.time()-start)
     69            train_time = "%8.2f" % (time.time()-start)
    7070           
    7171        error = statistics.mean([result['false_positives'],result['false_negatives']])
    72         print("number of observations: %d | error: %f | false_positives: %f | false_negatives: %f | train time: %s | params: %s"
     72        print("num. observations: %5d | error: %6.2f | fp: %6.2f | fn: %6.2f | train time: %s | params: %s"
    7373              % (num_obs, error, result['false_positives'],result['false_negatives'], train_time, result['params']))
    7474        db.addClassifierResult(result)
    … …  
    9999        false_negatives = 100.0*bad_estimates/num_trials
    100100        false_positives = 100.0*bad_null_estimates/num_trials
    101         print("testAux:", num_observations, false_positives, false_negatives, params)
    102101        return false_positives,false_negatives
    103102
    … …  
    107106        result = db.fetchClassifierResult(classifier, 'test', num_obs, jparams)
    108107        if result:
     108            test_time = '(stored)'
    109109            fp = result['false_positives']
    110110            fn = result['false_negatives']
    111111        else:
     112            start = time.time()
    112113            fp,fn = testAux(params, num_trials, num_obs)
    113114            result = {'classifier':classifier,
    … …  
    119120                      'false_negatives':fn}
    120121            db.addClassifierResult(result)
     122            test_time = '%8.2f' % (time.time()-start)
     123           
     124        print("num. observations: %5d | error: %6.2f | fp: %6.2f | fn: %6.2f | test time: %s"
     125              % (num_obs,(fp+fn)/2.0,fp,fn,test_time))
    121126        return ((fp+fn)/2.0,result)
    122127   
    … …  
    126131
    127132
    128     test_results = []
    129133    lte = math.log(target_error/100.0)
    130134    for tr in classifiers[classifier]['train_results']:
    … …  
    133137        num_obs = tr['num_observations']
    134138   
    135         print("initial test")
     139        print("parameters:", params)
    136140        error,result = getResult(classifier,params,num_obs,num_trials)
    137         print("walking up")
     141        #print("walking up")
    138142        while (error > target_error) and (num_obs < max_obs):
    139143            increase_factor = 1.5 * lte/math.log(error/100.0) # don't ask how I came up with this
    … …  
    142146            error,result = getResult(classifier,params,num_obs,num_trials)
    143147
    144         print("walking down")
     148        #print("walking down")
    145149        while (num_obs > 0):
    146             current_best = (error,result)
    147150            num_obs = int(0.95*num_obs)
    148151            error,result = getResult(classifier,params,num_obs,num_trials)
    149152            if error > target_error:
    150153                break
    151        
    152     return current_best
    153 
     154   
    154155
    155156if options.unusual_case != None:
    156157    unusual_case,greater = options.unusual_case.split(',')
    157158    greater = bool(int(greater))
     159    db.setUnusualCase(unusual_case,greater)
    158160else:
    159     start = time.time()
    160     unusual_case,unusual_diff = findUnusualTestCase(db)
    161     greater = (unusual_diff > 0)
    162     print("unusual_case:", unusual_case)
    163     print("unusual_diff:", unusual_diff)
    164     end = time.time()
    165     print(":", end-start)
     161    ucg = db.getUnusualCase()
     162    if ucg != None:
     163        unusual_case,greater = ucg
     164        print("Using cached unusual_case:", unusual_case)
     165    else:
     166        unusual_case,delta = findUnusualTestCase(db)
     167        greater = (delta > 0)
     168        print("Auto-detected unusual_case '%s' with delta: %d" %  (unusual_case,delta))
     169        db.setUnusualCase(unusual_case,greater)
    166170
    167171
    … …  
    172176    print("Training %s..." % c)
    173177    result = trainClassifier(db, unusual_case, greater, c, c in options.retrain)
    174     print("%s result:" % c)
    175     pprint.pprint(result)
    176     print("completed in:", time.time()-start)
     178    #print("%s result:" % c)
     179    #pprint.pprint(result)
     180    print("completed in: %8.2f\n"% (time.time()-start))
    177181
    178182db.clearCache()
    … …  
    181185    start = time.time()
    182186    print("Testing %s..." % c)
    183     error,result = testClassifier(db, unusual_case, greater, c, c in (options.retest+options.retrain))
    184     print("%s result:" % c)
    185     pprint.pprint(result)
    186     classifiers[c]['test_error'] = error
    187     print("completed in:", time.time()-start)
     187    testClassifier(db, unusual_case, greater, c, c in (options.retest+options.retrain))
     188    print("completed in: %8.2f\n"% (time.time()-start))
     189
     190
     191best_obs,best_error = evaluateTestResults(db)
     192best_obs =   sorted(best_obs,   key=lambda x: x['num_observations'])
     193best_error = sorted(best_error, key=lambda x: x['error'])
     194winner = None
     195for bo in best_obs:
     196    sys.stdout.write("%(num_observations)5d obs   | %(classifier)12s | %(params)s" % bo)
     197    if winner == None:
     198        sys.stdout.write(" (winner)")
     199        winner = bo
     200    print()
     201       
     202for be in best_error:
     203    sys.stdout.write("%(error)3.2f%% error | %(classifier)12s | %(params)s" % be)
     204    if winner == None:
     205        sys.stdout.write(" (winner)")
     206        winner = be
     207    print()
Note: See TracChangeset for help on using the changeset viewer.