Index: trunk/bin/analyze_packets
===================================================================
--- trunk/bin/analyze_packets	(revision 14)
+++ trunk/bin/analyze_packets	(revision 16)
@@ -10,10 +10,5 @@
 import socket
 import json
-try:
-    import requests
-except:
-    sys.stderr.write('ERROR: Could not import requests module.  Ensure it is installed.\n')
-    sys.stderr.write('       Under Debian, the package name is "python3-requests"\n.')
-    sys.exit(1)
+
 
 VERSION = "{DEVELOPMENT}"
@@ -40,5 +35,4 @@
 
 
-
 db = nanownlib.storage.db(options.db_file)
 
@@ -46,5 +40,5 @@
 import cProfile
 #cProfile.run('num_probes = analyzeProbes(db)')
-num_probes = analyzeProbes(db)
+num_probes = analyzeProbes(db, recompute=True)
 end = time.time()
 print("analyzed %d probes' packets in: %f" % (num_probes, end-start))
Index: trunk/bin/graph
===================================================================
--- trunk/bin/graph	(revision 14)
+++ trunk/bin/graph	(revision 16)
@@ -36,6 +36,10 @@
 parser.add_argument('db_file', default=None,
                     help='')
+parser.add_argument('unusual_case', nargs='?', type=str, default=None,
+                    help='The test case that is most unusual from the others. (default: auto detect)')
 options = parser.parse_args()
 db = nanownlib.storage.db(options.db_file)
+if options.unusual_case == None:
+    unusual_case,delta = findUnusualTestCase(db)
 
 
@@ -69,6 +73,6 @@
 #print('(test): %f' % weightedMean(derived,weights))
 
-diffs = list(differences(db, 'long'))
-reported_diffs = list(differences(db, 'long', 'reported'))
+diffs = list(differences(db, unusual_case))
+reported_diffs = list(differences(db, unusual_case, 'reported'))
 #shorts = [s['packet_rtt'] for s in samples.values() if s['test_case']=='short']
 #longs = [s['packet_rtt'] for s in samples.values() if s['test_case']=='long']
@@ -76,18 +80,13 @@
 
 def basicStatistics():
-    print('packet_rtt diff mean: %f' % statistics.mean(diffs))
-    print('packet_rtt diff median: %f' % statistics.median(diffs))
-    print('packet_rtt diff midhinge: %f' % midsummary(diffs))
-    print('packet_rtt diff trimean: %f' % trimean(diffs))
-    print('packet_rtt diff quadsummary: %f' % quadsummary(diffs))
-    print('packet_rtt diff ubersummary: %f' % ubersummary(diffs))
-    print('packet_rtt diff septasummary: %f' % septasummary(diffs))
-    print('packet_rtt diff MAD: %f' % mad(diffs))
+    print('packet_rtt diff midhinge:     %10.2f' % midsummary(diffs))
+    print('packet_rtt diff quadsummary:  %10.2f' % quadsummary(diffs))
+    print('packet_rtt diff septasummary: %10.2f' % septasummary(diffs))
+    print('packet_rtt diff MAD:          %10.2f' % mad(diffs))
     try:
-        print('reported diff trimean: %f' % trimean(reported_diffs))
-        print('reported diff quadsummary: %f' % quadsummary(reported_diffs))
-        print('reported diff ubersummary: %f' % ubersummary(reported_diffs))
-        print('reported diff septasummary: %f' % septasummary(reported_diffs))
-        print('reported diff MAD: %f' % mad(reported_diffs))
+        print('reported diff midhinge:       %10.2f' % midsummary(reported_diffs))
+        print('reported diff quadsummary:    %10.2f' % quadsummary(reported_diffs))
+        print('reported diff septasummary:   %10.2f' % septasummary(reported_diffs))
+        print('reported diff MAD:            %10.2f' % mad(reported_diffs))
 
         #import cProfile
@@ -108,9 +107,9 @@
     #print('tsval null diff weighted mean: %f' % tsvalwmean(db.subseries('train_null','long')))
 
-
+basicStatistics()
 
 def exampleBoxTestHistogram(low,high):
     num_bins = 300
-    all = db.subseries('train','long')+db.subseries('test','long')
+    all = db.subseries('train',unusual_case)+db.subseries('test',unusual_case)
     s   = [s['other_packet'] for s in all]
     l   = [s['unusual_packet'] for s in all]
@@ -362,5 +361,5 @@
     #plt.savefig('paper/graphs/dists-vs-dist-of-diffs2.svg')
 
-#tsFilteredHistogram()
+tsFilteredHistogram()
 
 
@@ -459,5 +458,5 @@
 
     chartname = "/home/tim/blindspot/research/timing-analysis/paper/figures/results/%s.svg" % (basename)
-    print(chartname)
+    #print(chartname)
     
     plt.clf()
@@ -469,45 +468,25 @@
     color_id = 0
 
-    cursor = db.conn.cursor()
-    query = """
-      SELECT classifier FROM classifier_results GROUP BY classifier ORDER BY classifier;
-    """
-    cursor.execute(query)
-    classifiers = []
-    for c in cursor:
-        classifiers.append(c[0])
-
-    best_obs = []
-    best_error = []
+    best_obs,best_error = evaluateTestResults(db)
+    best_obs =   sorted(best_obs,   key=lambda x: x['num_observations'])
+    best_error = sorted(best_error, key=lambda x: x['error'])
+    winner = None
+    for bo in best_obs:
+        sys.stdout.write("%(num_observations)d obs / %(classifier)s / %(params)s" % bo)
+        if winner == None:
+            sys.stdout.write(" (winner)")
+            winner = bo
+        print()
+        
+    for be in best_error:
+        sys.stdout.write("%(error)f%% error / %(classifier)s / %(params)s" % be)
+        if winner == None:
+            sys.stdout.write(" (winner)")
+            winner = be
+        print()
+
+    all = sorted(best_obs+best_error, key=lambda x: x['classifier'])
     max_obs = 0
-    for classifier in classifiers:
-        query="""
-        SELECT params,num_observations FROM classifier_results 
-        WHERE trial_type='test'
-         AND classifier=:classifier
-         AND (false_positives+false_negatives)/2.0 < 5.0 
-        ORDER BY num_observations,(false_positives+false_negatives) 
-        LIMIT 1
-        """
-        cursor.execute(query, {'classifier':classifier})
-        row = cursor.fetchone()
-        if row == None:
-            query="""
-            SELECT params,(false_positives+false_negatives)/2 FROM classifier_results 
-            WHERE trial_type='test' and classifier=:classifier
-            ORDER BY (false_positives+false_negatives),num_observations
-            LIMIT 1
-            """
-            cursor.execute(query, {'classifier':classifier})
-            row = cursor.fetchone()
-            if row == None:
-                sys.stderr.write("WARN: couldn't find test results for classifier '%s'.\n" % classifier)
-                continue
-
-            best_error.append((row[1], classifier))
-        else:
-            best_obs.append((row[1], classifier))
-
-        best_params = row[0]
+    for result in all:
         query="""
         SELECT num_observations,(false_positives+false_negatives)/2.0 FROM classifier_results 
@@ -517,5 +496,6 @@
         ORDER BY num_observations
         """
-        cursor.execute(query, {'classifier':classifier,'params':best_params})
+        cursor = db.conn.cursor()
+        cursor.execute(query, {'classifier':result['classifier'],'params':result['params']})
 
         num_obs = []
@@ -528,30 +508,13 @@
         path = plt.scatter(num_obs, performance, color=colors[color_id], s=4, alpha=0.8, linewidths=3.0)
         plt.plot(num_obs, performance, color=colors[color_id], alpha=0.8)
-        legend.append((classifier,path))
+        legend.append((result['classifier'],path))
         color_id = (color_id+1) % len(colors)
 
-    best_obs.sort()
-    best_error.sort()
-    winner = None
-    for bo in best_obs:
-        sys.stdout.write("%d obs / %s" % bo)
-        if winner == None:
-            sys.stdout.write(" (winner)")
-            winner = bo
-        print()
-        
-    for be in best_error:
-        sys.stdout.write("%f%% error / %s" % be)
-        if winner == None:
-            sys.stdout.write(" (winner)")
-            winner = be
-        print()
-    
     plt.legend([l[1] for l in legend], [l[0] for l in legend], scatterpoints=1, fontsize='x-small')
     plt.plot([0, max_obs], [5.0, 5.0], "k--")
     plt.xlabel('Number of Observations')
     plt.ylabel('Error Rate')
-    plt.savefig(chartname)
-    #plt.show()
+    #plt.savefig(chartname)
+    plt.show()
     
 graphTestResults()
Index: trunk/bin/train
===================================================================
--- trunk/bin/train	(revision 14)
+++ trunk/bin/train	(revision 16)
@@ -67,8 +67,8 @@
             result = trainer(db,unusual_case,greater,num_obs)
             result['classifier'] = classifier
-            train_time = "%f" % (time.time()-start)
+            train_time = "%8.2f" % (time.time()-start)
             
         error = statistics.mean([result['false_positives'],result['false_negatives']])
-        print("number of observations: %d | error: %f | false_positives: %f | false_negatives: %f | train time: %s | params: %s"
+        print("num. observations: %5d | error: %6.2f | fp: %6.2f | fn: %6.2f | train time: %s | params: %s"
               % (num_obs, error, result['false_positives'],result['false_negatives'], train_time, result['params']))
         db.addClassifierResult(result)
@@ -99,5 +99,4 @@
         false_negatives = 100.0*bad_estimates/num_trials
         false_positives = 100.0*bad_null_estimates/num_trials
-        print("testAux:", num_observations, false_positives, false_negatives, params)
         return false_positives,false_negatives
 
@@ -107,7 +106,9 @@
         result = db.fetchClassifierResult(classifier, 'test', num_obs, jparams)
         if result:
+            test_time = '(stored)'
             fp = result['false_positives']
             fn = result['false_negatives']
         else:
+            start = time.time()
             fp,fn = testAux(params, num_trials, num_obs)
             result = {'classifier':classifier,
@@ -119,4 +120,8 @@
                       'false_negatives':fn}
             db.addClassifierResult(result)
+            test_time = '%8.2f' % (time.time()-start)
+            
+        print("num. observations: %5d | error: %6.2f | fp: %6.2f | fn: %6.2f | test time: %s"
+              % (num_obs,(fp+fn)/2.0,fp,fn,test_time))
         return ((fp+fn)/2.0,result)
     
@@ -126,5 +131,4 @@
 
 
-    test_results = []
     lte = math.log(target_error/100.0)
     for tr in classifiers[classifier]['train_results']:
@@ -133,7 +137,7 @@
         num_obs = tr['num_observations']
     
-        print("initial test")
+        print("parameters:", params)
         error,result = getResult(classifier,params,num_obs,num_trials)
-        print("walking up")
+        #print("walking up")
         while (error > target_error) and (num_obs < max_obs):
             increase_factor = 1.5 * lte/math.log(error/100.0) # don't ask how I came up with this
@@ -142,26 +146,26 @@
             error,result = getResult(classifier,params,num_obs,num_trials)
 
-        print("walking down")
+        #print("walking down")
         while (num_obs > 0):
-            current_best = (error,result)
             num_obs = int(0.95*num_obs)
             error,result = getResult(classifier,params,num_obs,num_trials)
             if error > target_error:
                 break
-        
-    return current_best
-
+    
 
 if options.unusual_case != None:
     unusual_case,greater = options.unusual_case.split(',')
     greater = bool(int(greater))
+    db.setUnusualCase(unusual_case,greater)
 else:
-    start = time.time()
-    unusual_case,unusual_diff = findUnusualTestCase(db)
-    greater = (unusual_diff > 0)
-    print("unusual_case:", unusual_case)
-    print("unusual_diff:", unusual_diff)
-    end = time.time()
-    print(":", end-start)
+    ucg = db.getUnusualCase()
+    if ucg != None:
+        unusual_case,greater = ucg
+        print("Using cached unusual_case:", unusual_case)
+    else:
+        unusual_case,delta = findUnusualTestCase(db)
+        greater = (delta > 0)
+        print("Auto-detected unusual_case '%s' with delta: %d" %  (unusual_case,delta))
+        db.setUnusualCase(unusual_case,greater)
 
 
@@ -172,7 +176,7 @@
     print("Training %s..." % c)
     result = trainClassifier(db, unusual_case, greater, c, c in options.retrain)
-    print("%s result:" % c)
-    pprint.pprint(result)
-    print("completed in:", time.time()-start)
+    #print("%s result:" % c)
+    #pprint.pprint(result)
+    print("completed in: %8.2f\n"% (time.time()-start))
 
 db.clearCache()
@@ -181,7 +185,23 @@
     start = time.time()
     print("Testing %s..." % c)
-    error,result = testClassifier(db, unusual_case, greater, c, c in (options.retest+options.retrain))
-    print("%s result:" % c)
-    pprint.pprint(result)
-    classifiers[c]['test_error'] = error
-    print("completed in:", time.time()-start)
+    testClassifier(db, unusual_case, greater, c, c in (options.retest+options.retrain))
+    print("completed in: %8.2f\n"% (time.time()-start))
+
+
+best_obs,best_error = evaluateTestResults(db)
+best_obs =   sorted(best_obs,   key=lambda x: x['num_observations'])
+best_error = sorted(best_error, key=lambda x: x['error'])
+winner = None
+for bo in best_obs:
+    sys.stdout.write("%(num_observations)5d obs   | %(classifier)12s | %(params)s" % bo)
+    if winner == None:
+        sys.stdout.write(" (winner)")
+        winner = bo
+    print()
+        
+for be in best_error:
+    sys.stdout.write("%(error)3.2f%% error | %(classifier)12s | %(params)s" % be)
+    if winner == None:
+        sys.stdout.write(" (winner)")
+        winner = be
+    print()
Index: trunk/lib/nanownlib/__init__.py
===================================================================
--- trunk/lib/nanownlib/__init__.py	(revision 14)
+++ trunk/lib/nanownlib/__init__.py	(revision 16)
@@ -18,6 +18,11 @@
 import gzip
 import statistics
-import numpy
-import netifaces
+try:
+    import numpy
+except:
+    sys.stderr.write('ERROR: Could not import numpy module.  Ensure it is installed.\n')
+    sys.stderr.write('       Under Debian, the package name is "python3-numpy"\n.')
+    sys.exit(1)
+
 try:
     import requests
@@ -39,4 +44,11 @@
 
 def getIfaceForIP(ip):
+    try:
+        import netifaces
+    except:
+        sys.stderr.write('ERROR: Could not import netifaces module.  Ensure it is installed.\n')
+        sys.stderr.write('       Try: pip3 install netifaces\n.')
+        sys.exit(1)
+    
     for iface in netifaces.interfaces():
         addrs = netifaces.ifaddresses(iface).get(netifaces.AF_INET, None)
@@ -176,5 +188,5 @@
     my_ip = getLocalIP(target_ip, target_port)
     my_iface = getIfaceForIP(my_ip)
-    return subprocess.Popen(['chrt', '-r', '99', 'nanown-csamp', my_iface, my_ip,
+    return subprocess.Popen(['chrt', '-r', '99', 'nanown-listen', my_iface, my_ip,
                              target_ip, "%d" % target_port, output_file, '0'])
 
@@ -303,5 +315,5 @@
 
 
-def analyzeProbes(db):
+def analyzeProbes(db, trim=None, recompute=False):
     db.conn.execute("CREATE INDEX IF NOT EXISTS packets_probe ON packets (probe_id)")
     db.conn.commit()
@@ -316,8 +328,12 @@
     pcursor.execute("DELETE FROM trim_analysis")
     db.conn.commit()
+    if recompute:
+        pcursor.execute("DELETE FROM analysis")
+        db.conn.commit()
 
     def loadPackets(db):
         cursor = db.conn.cursor()
-        cursor.execute("SELECT * FROM packets ORDER BY probe_id")
+        #cursor.execute("SELECT * FROM packets ORDER BY probe_id")
+        cursor.execute("SELECT * FROM packets WHERE probe_id NOT IN (SELECT probe_id FROM analysis) ORDER BY probe_id")
 
         probe_id = None
@@ -334,94 +350,81 @@
         ret_val.append((probe_id,entry))
         return ret_val
-    
-    start = time.time()
+
+    def processPackets(packet_cache, strim, rtrim):
+        sent_tally = []
+        rcvd_tally = []
+        analyses = []
+        for probe_id,packets in packet_cache:
+            try:
+                analysis,s,r = analyzePackets(packets, timestamp_precision)
+                analysis['probe_id'] = probe_id
+                analyses.append(analysis)
+                sent_tally.append(s)
+                rcvd_tally.append(r)
+            except Exception as e:
+                #traceback.print_exc()
+                sys.stderr.write("WARN: couldn't find enough packets for probe_id=%s\n" % probe_id)
+        db.addTrimAnalyses(analyses)
+        db.conn.commit()
+        return statistics.mode(sent_tally),statistics.mode(rcvd_tally)
+    
+    #start = time.time()
     packet_cache = loadPackets(db)
-    print("packets loaded in: %f" % (time.time()-start))
-    
-    count = 0
-    sent_tally = []
-    rcvd_tally = []
-    for probe_id,packets in packet_cache:
-        try:
-            analysis,s,r = analyzePackets(packets, timestamp_precision)
-            analysis['probe_id'] = probe_id
-            sent_tally.append(s)
-            rcvd_tally.append(r)
-            db.addTrimAnalyses([analysis])
-        except Exception as e:
-            #traceback.print_exc()
-            sys.stderr.write("WARN: couldn't find enough packets for probe_id=%s\n" % probe_id)
-        
-        #print(pid,analysis)
-        count += 1
+    #print("packets loaded in: %f" % (time.time()-start))
+
+    if trim != None:
+        best_strim,best_rtrim = trim
+        processPackets(packet_cache, best_strim, best_rtrim)
+    else:
+        num_sent,num_rcvd = processPackets(packet_cache, 0, 0)
+        print("num_sent: %d, num_rcvd: %d" % (num_sent,num_rcvd))
+    
+        for strim in range(0,num_sent):
+            for rtrim in range(0,num_rcvd):
+                #print(strim,rtrim)
+                if strim == 0 and rtrim == 0:
+                    continue # no point in doing 0,0 again
+                processPackets(packet_cache, strim, rtrim)
+
+    
+        unusual_case,delta = findUnusualTestCase(db, (0,0))
+        evaluations = {}
+        for strim in range(0,num_sent):
+            for rtrim in range(0,num_rcvd):
+                evaluations[(strim,rtrim)] = evaluateTrim(db, unusual_case, strim, rtrim)
+
+        import pprint
+        pprint.pprint(evaluations)
+
+        delta_margin = 0.15
+        best_strim = 0
+        best_rtrim = 0
+        good_delta,good_mad = evaluations[(0,0)]
+    
+        for strim in range(1,num_sent):
+            delta,mad = evaluations[(strim,0)]
+            if delta*good_delta > 0.0 and (abs(good_delta) - abs(delta)) < abs(delta_margin*good_delta) and mad < good_mad:
+                best_strim = strim
+            else:
+                break
+
+        good_delta,good_mad = evaluations[(best_strim,0)]
+        for rtrim in range(1,num_rcvd):
+            delta,mad = evaluations[(best_strim,rtrim)]
+            if delta*good_delta > 0.0 and (abs(good_delta) - abs(delta)) < abs(delta_margin*good_delta) and mad < good_mad:
+                best_rtrim = rtrim
+            else:
+                break
+
+        print("selected trim parameters:",(best_strim,best_rtrim))
+    
+    pcursor.execute("""INSERT OR IGNORE INTO analysis 
+                         SELECT id,probe_id,suspect,packet_rtt,tsval_rtt 
+                           FROM trim_analysis 
+                           WHERE sent_trimmed=? AND rcvd_trimmed=?""",
+                    (best_strim,best_rtrim))
     db.conn.commit()
-    num_sent = statistics.mode(sent_tally)
-    num_rcvd = statistics.mode(rcvd_tally)
-    sent_tally = None
-    rcvd_tally = None
-    print("num_sent: %d, num_rcvd: %d" % (num_sent,num_rcvd))
-    
-    for strim in range(0,num_sent):
-        for rtrim in range(0,num_rcvd):
-            #print(strim,rtrim)
-            if strim == 0 and rtrim == 0:
-                continue # no point in doing 0,0 again
-            for probe_id,packets in packet_cache:
-                try:
-                    analysis,s,r = analyzePackets(packets, timestamp_precision, strim, rtrim)
-                    analysis['probe_id'] = probe_id
-                except Exception as e:
-                    #traceback.print_exc()
-                    sys.stderr.write("WARN: couldn't find enough packets for probe_id=%s\n" % probe_id)
-                    
-                db.addTrimAnalyses([analysis])
-    db.conn.commit()
-
-    # Populate analysis table so findUnusualTestCase can give us a starting point
-    pcursor.execute("DELETE FROM analysis")
-    db.conn.commit()
-    pcursor.execute("INSERT INTO analysis SELECT id,probe_id,suspect,packet_rtt,tsval_rtt FROM trim_analysis WHERE sent_trimmed=0 AND rcvd_trimmed=0")
-    
-    unusual_case,delta = findUnusualTestCase(db)
-    evaluations = {}
-    for strim in range(0,num_sent):
-        for rtrim in range(0,num_rcvd):
-            evaluations[(strim,rtrim)] = evaluateTrim(db, unusual_case, strim, rtrim)
-
-    import pprint
-    pprint.pprint(evaluations)
-
-    delta_margin = 0.15
-    best_strim = 0
-    best_rtrim = 0
-    good_delta,good_mad = evaluations[(0,0)]
-    
-    for strim in range(1,num_sent):
-        delta,mad = evaluations[(strim,0)]
-        if delta*good_delta > 0.0 and (abs(good_delta) - abs(delta)) < abs(delta_margin*good_delta) and mad < good_mad:
-            best_strim = strim
-        else:
-            break
-
-    good_delta,good_mad = evaluations[(best_strim,0)]
-    for rtrim in range(1,num_rcvd):
-        delta,mad = evaluations[(best_strim,rtrim)]
-        if delta*good_delta > 0.0 and (abs(good_delta) - abs(delta)) < abs(delta_margin*good_delta) and mad < good_mad:
-            best_rtrim = rtrim
-        else:
-            break
-
-    print("selected trim parameters:",(best_strim,best_rtrim))
-    
-    if best_strim != 0 or best_rtrim !=0:
-        pcursor.execute("DELETE FROM analysis")
-        db.conn.commit()
-        pcursor.execute("INSERT INTO analysis SELECT id,probe_id,suspect,packet_rtt,tsval_rtt FROM trim_analysis WHERE sent_trimmed=? AND rcvd_trimmed=?",
-                        (best_strim,best_rtrim))
-
-    #pcursor.execute("DELETE FROM trim_analysis")
-    db.conn.commit()
-    
-    return count
+    
+    return len(packet_cache)
 
 
@@ -442,5 +445,5 @@
     ptimes = cursor.fetchone()
     window_size = 100*int((ptimes['end']-ptimes['start'])/ptimes['count'])
-    print("associate window_size:", window_size)
+    #print("associate window_size:", window_size)
 
     db.addPackets(parseJSONLines(sniffer_fp), window_size)
@@ -460,26 +463,47 @@
 
 
-def findUnusualTestCase(db):
+def findUnusualTestCase(db, trim=None):
     test_cases = enumStoredTestCases(db)
-
+    if trim != None:
+        params = {'strim':trim[0], 'rtrim':trim[1]}
+        qsuffix = " AND sent_trimmed=:strim AND rcvd_trimmed=:rtrim"
+        table = "trim_analysis"
+    else:
+        params = {}
+        qsuffix = ""
+        table = "analysis"
+    
     cursor = db.conn.cursor()
-    cursor.execute("SELECT packet_rtt FROM probes,analysis WHERE probes.id=analysis.probe_id AND probes.type in ('train','test')")
+    cursor.execute("SELECT packet_rtt FROM probes,"+table+" a WHERE probes.id=a.probe_id AND probes.type in ('train','test')"+qsuffix, params)
     global_tm = quadsummary([row['packet_rtt'] for row in cursor])
 
     tm_abs = []
     tm_map = {}
+
     # XXX: if more speed needed, percentile extension to sqlite might be handy...
     for tc in test_cases:
-        cursor.execute("SELECT packet_rtt FROM probes,analysis WHERE probes.id=analysis.probe_id AND probes.type in ('train','test') AND probes.test_case=?", (tc,))
+        params['test_case']=tc
+        query = """SELECT packet_rtt FROM probes,"""+table+""" a
+                   WHERE probes.id=a.probe_id AND probes.type in ('train','test') 
+                   AND probes.test_case=:test_case""" + qsuffix
+        cursor.execute(query, params)
         tm_map[tc] = quadsummary([row['packet_rtt'] for row in cursor])
         tm_abs.append((abs(tm_map[tc]-global_tm), tc))
 
     magnitude,tc = max(tm_abs)
-    cursor.execute("SELECT packet_rtt FROM probes,analysis WHERE probes.id=analysis.probe_id AND probes.type in ('train','test') AND probes.test_case<>?", (tc,))
+    params['test_case']=tc
+    query = """SELECT packet_rtt FROM probes,"""+table+""" a
+               WHERE probes.id=a.probe_id AND probes.type in ('train','test') 
+               AND probes.test_case<>:test_case""" + qsuffix
+    cursor.execute(query,params)
     remaining_tm = quadsummary([row['packet_rtt'] for row in cursor])
 
-    ret_val = (tc, tm_map[tc]-remaining_tm)
-    print("unusual_case: %s, delta: %f" % ret_val)
-    return ret_val
+    delta = tm_map[tc]-remaining_tm
+    # Hack to make the chosen unusual_case more intuitive to the user
+    if len(test_cases) == 2 and delta < 0.0:
+        tc = [t for t in test_cases if t != tc][0]
+        delta = abs(delta)
+
+    return tc,delta
 
 
@@ -492,10 +516,59 @@
         cursor.execute("SELECT count(id) c FROM (SELECT id FROM probes WHERE type=? AND time_of_day>? GROUP BY sample)", (st[0],int(start_time*1000000000)))
         count = cursor.fetchone()[0]
-        output += " | %s remaining: %d" % (st[0], st[1]-count)
+        output += " | %s remaining: %6d" % (st[0], st[1]-count)
         total_completed += count
         total_requested += st[1]
 
     rate = total_completed / (time.time() - start_time)
-    total_time = total_requested / rate        
+    total_time = total_requested / rate
     eta = datetime.datetime.fromtimestamp(start_time+total_time)
-    print("STATUS:",output[3:],"| est. total_time: %s | est. ETA: %s" % (str(datetime.timedelta(seconds=total_time)), str(eta)))
+    print("STATUS:",output[3:],"| est. total_time: %s | ETA: %s" % (str(datetime.timedelta(seconds=total_time)), eta.strftime("%Y-%m-%d %X")))
+
+
+
+def evaluateTestResults(db):
+    cursor = db.conn.cursor()
+    query = """
+      SELECT classifier FROM classifier_results GROUP BY classifier ORDER BY classifier;
+    """
+    cursor.execute(query)
+    classifiers = []
+    for c in cursor:
+        classifiers.append(c[0])
+
+    best_obs = []
+    best_error = []
+    max_obs = 0
+    for classifier in classifiers:
+        query="""
+        SELECT classifier,params,num_observations,(false_positives+false_negatives)/2 error 
+        FROM classifier_results 
+        WHERE trial_type='test'
+         AND classifier=:classifier
+         AND (false_positives+false_negatives)/2.0 < 5.0 
+        ORDER BY num_observations,(false_positives+false_negatives) 
+        LIMIT 1
+        """
+        cursor.execute(query, {'classifier':classifier})
+        row = cursor.fetchone()
+        if row == None:
+            query="""
+            SELECT classifier,params,num_observations,(false_positives+false_negatives)/2 error 
+            FROM classifier_results 
+            WHERE trial_type='test' and classifier=:classifier
+            ORDER BY (false_positives+false_negatives),num_observations
+            LIMIT 1
+            """
+            cursor.execute(query, {'classifier':classifier})
+            row = cursor.fetchone()
+            if row == None:
+                sys.stderr.write("WARN: couldn't find test results for classifier '%s'.\n" % classifier)
+                continue
+            row = dict(row)
+
+            best_error.append(dict(row))
+        else:
+            best_obs.append(dict(row))
+
+
+    return best_obs,best_error
Index: trunk/lib/nanownlib/parallel.py
===================================================================
--- trunk/lib/nanownlib/parallel.py	(revision 14)
+++ trunk/lib/nanownlib/parallel.py	(revision 16)
@@ -48,5 +48,12 @@
     
     def stop(self):
-        for i in range(0,len(self.workers)):
+        try:
+            while True:
+                self.workq.get(block=False)
+                self.workq.task_done()
+        except queue.Empty as e:
+            pass
+        
+        for i in range(len(self.workers)):
             self.workq.put(None)
         for w in self.workers:
Index: trunk/lib/nanownlib/stats.py
===================================================================
--- trunk/lib/nanownlib/stats.py	(revision 14)
+++ trunk/lib/nanownlib/stats.py	(revision 16)
@@ -7,6 +7,4 @@
 import gzip
 import random
-import scipy
-import scipy.stats
 import numpy
 
@@ -249,4 +247,5 @@
 
     mh = f(diffs, params['distance'])
+    #print("estimate:", mh)
     if greater:
         if mh > params['threshold']:
Index: trunk/lib/nanownlib/storage.py
===================================================================
--- trunk/lib/nanownlib/storage.py	(revision 14)
+++ trunk/lib/nanownlib/storage.py	(revision 16)
@@ -7,6 +7,11 @@
 import threading
 import sqlite3
-
-import numpy
+try:
+    import numpy
+except:
+    sys.stderr.write('ERROR: Could not import numpy module.  Ensure it is installed.\n')
+    sys.stderr.write('       Under Debian, the package name is "python3-numpy"\n.')
+    sys.exit(1)
+
 # Don't trust numpy's seeding
 numpy.random.seed(random.SystemRandom().randint(0,2**32-1))
@@ -39,5 +44,7 @@
                                       tcpts_mean REAL,
                                       tcpts_stddev REAL,
-                                      tcpts_slopes TEXT)
+                                      tcpts_slopes TEXT,
+                                      unusual_case TEXT,
+                                      greater INTEGER)
                 """)
 
@@ -196,5 +203,5 @@
     def addPackets(self, pkts, window_size):
         query = ("INSERT INTO packets (id,probe_id,sent,observed,tsval,payload_len,tcpseq,tcpack)"
-                 " VALUES(randomblob(16),"
+                 " VALUES(hex(randomblob(16)),"
                  "(SELECT id FROM probes WHERE local_port=:local_port AND :observed>time_of_day"
                  " AND :observed<time_of_day+userspace_rtt+%d" 
@@ -254,3 +261,33 @@
         self.conn.execute(query, params)
         self.conn.commit()
-        
+    
+    def setUnusualCase(self, unusual_case, greater):
+        query = """SELECT * FROM meta LIMIT 1"""
+        cursor = self.conn.cursor()
+        cursor.execute(query)
+        row = cursor.fetchone()
+        if row == None:
+            params = {"id":_newid()}
+        else:
+            params = dict(row)
+
+        params["unusual_case"]=unusual_case
+        params["greater"]=greater
+        
+        keys = params.keys()
+        columns = ','.join(keys)
+        placeholders = ':'+', :'.join(keys)
+        
+        query = """INSERT OR REPLACE INTO meta (%s) VALUES (%s)""" % (columns, placeholders)
+        cursor.execute(query, params)
+        
+        
+    def getUnusualCase(self):
+        query = """SELECT unusual_case,greater FROM meta LIMIT 1"""
+        cursor = self.conn.cursor()
+        cursor.execute(query)
+        row = cursor.fetchone()
+        if row == None or row[0] == None or row[1] == None:
+            return None
+        else:
+            return tuple(row)
Index: trunk/lib/nanownlib/train.py
===================================================================
--- trunk/lib/nanownlib/train.py	(revision 14)
+++ trunk/lib/nanownlib/train.py	(revision 16)
@@ -43,5 +43,5 @@
     num_trials = 200
     lows = [p[1] for p in performance[0:5]]
-    widths = [w/10.0 for w in range(5,65,5)]
+    widths = [w/10.0 for w in range(5,155,10)]
     performance = []
     for width in widths:
@@ -85,5 +85,5 @@
     
     num_trials = 500
-    widths = [good_width+(x/100.0) for x in range(-70,75,5) if good_width+(x/100.0) > 0.0]
+    widths = [good_width+(x/100.0) for x in range(-120,125,5) if good_width+(x/100.0) > 0.0]
     performance = []
     for width in widths:
Index: trunk/src/compile.sh
===================================================================
--- trunk/src/compile.sh	(revision 14)
+++ trunk/src/compile.sh	(revision 16)
@@ -1,3 +1,3 @@
 #!/bin/sh
 
-gcc -ggdb -Wl,-z,relro,-z,now -fstack-protector-strong -Wformat -Werror=format-security -D_FORTIFY_SOURCE=2 csamp.c -lpcap -o ../bin/csamp
+gcc -ggdb -Wl,-z,relro,-z,now -fstack-protector-strong -Wformat -Werror=format-security -D_FORTIFY_SOURCE=2 listen.c -lpcap -o ../bin/nanown-listen
Index: trunk/src/csamp.c
===================================================================
--- trunk/src/csamp.c	(revision 14)
+++ 	(revision )
@@ -1,392 +1,0 @@
-/*
-Portions are Copyright 2002 Tim Carstens. All rights reserved. 
-Redistribution and use, with or without modification, are permitted provided
-that the following conditions are met:
-
-- Redistribution must retain the above copyright notice and this list of 
-  conditions.
-
-- The name of Tim Carstens may not be used to endorse or promote products
-  derived from this document without specific prior written permission.
-*/
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <sys/types.h>
-#include <inttypes.h>
-#include <sys/stat.h>
-#include <fcntl.h>
-#include <errno.h>
-#include <netinet/in.h>
-#include <pcap.h>
-
-#define DEBUG 0
-
-/* ethernet headers are always exactly 14 bytes */
-/* XXX: ^^^ really? */
-#define SIZE_ETHERNET 14
-
-/* Ethernet addresses are 6 bytes */
-#define ETHER_ADDR_LEN	6
-
-/* Ethernet header */
-/*
-struct sniff_ethernet
-{
-  u_char ether_dhost[ETHER_ADDR_LEN]; / * Destination host address * /
-  u_char ether_shost[ETHER_ADDR_LEN]; / * Source host address * /
-  u_short ether_type; / * IP? ARP? RARP? etc * /
-};
-*/
-
-/* IP header */
-struct sniff_ip
-{
-  u_char ip_vhl;		/* version << 4 | header length >> 2 */
-  u_char ip_tos;		/* type of service */
-  u_short ip_len;		/* total length */
-  u_short ip_id;		/* identification */
-  u_short ip_off;		/* fragment offset field */
-#define IP_RF 0x8000		/* reserved fragment flag */
-#define IP_DF 0x4000		/* dont fragment flag */
-#define IP_MF 0x2000		/* more fragments flag */
-#define IP_OFFMASK 0x1fff	/* mask for fragmenting bits */
-  u_char ip_ttl;		/* time to live */
-  u_char ip_p;		/* protocol */
-  u_short ip_sum;		/* checksum */
-  struct in_addr ip_src,ip_dst; /* source and dest address */
-};
-#define IP_HL(ip)		(((ip)->ip_vhl) & 0x0f)
-#define IP_V(ip)		(((ip)->ip_vhl) >> 4)
-
-/* TCP header */
-typedef u_int tcp_seq;
-
-struct sniff_tcp
-{
-  u_short th_sport;	/* source port */
-  u_short th_dport;	/* destination port */
-  tcp_seq th_seq;		/* sequence number */
-  tcp_seq th_ack;		/* acknowledgement number */
-  u_char th_offx2;	/* data offset, rsvd */
-#define TH_OFF(th)	(((th)->th_offx2 & 0xf0) >> 4)
-  u_char th_flags;
-#define TH_FIN 0x01
-#define TH_SYN 0x02
-#define TH_RST 0x04
-#define TH_PUSH 0x08
-#define TH_ACK 0x10
-#define TH_URG 0x20
-#define TH_ECE 0x40
-#define TH_CWR 0x80
-#define TH_FLAGS (TH_FIN|TH_SYN|TH_RST|TH_ACK|TH_URG|TH_ECE|TH_CWR)
-  u_short th_win;		/* window */
-  u_short th_sum;		/* checksum */
-  u_short th_urp;		/* urgent pointer */
-};
-
-typedef struct {
-  uint8_t kind;
-  uint8_t size;
-} tcp_option_t;
-
-
-
-struct in_addr my_ip_raw;
-struct in_addr target_ip_raw;
-uint16_t target_port;
-
-FILE* output;
-
-struct useful_info
-{
-  uint32_t tcpseq;
-  uint32_t tcpack;
-  uint32_t tsval;
-  uint16_t local_port;
-  uint16_t payload_len;
-  uint8_t sent;
-};
-
-
-char* output_template = "{\"local_port\":%u,\"sent\":%u,\"payload_len\":%u,\"tcpseq\":%u,\"tcpack\":%u,\"observed\":%lu%06lu000,\"tsval\":%u}\n";
-uint8_t payloads_only = 1;
-
-
-pcap_t* create_listener(const char* dev, int snaplen, int promisc, int to_ms, char* errbuf)
-{
-  pcap_t* ret_val;
-  int status;
-  int tstcount, i, besttst;
-  int* tstypes;
-
-  ret_val = pcap_create(dev, errbuf);
-  if (ret_val == NULL)
-    return (NULL);
-  
-  status = pcap_set_snaplen(ret_val, snaplen);
-  if (status < 0)
-    goto fail;
-  
-  status = pcap_set_promisc(ret_val, promisc);
-  if (status < 0)
-    goto fail;
-  
-  status = pcap_set_timeout(ret_val, to_ms);
-  if (status < 0)
-    goto fail;
-
-
-  /* Try to select the best timestamp source for packet capture */
-  tstcount = pcap_list_tstamp_types(ret_val, &tstypes);
-  if(tstcount > 0)
-  {
-#if DEBUG
-    fprintf(stderr, "INFO: Available Packet Timers: ");
-#endif
-    besttst = -1;
-    for(i=0; i < tstcount; i++)
-    {
-#if DEBUG
-        fprintf(stderr, " %s", pcap_tstamp_type_val_to_name(tstypes[i]));
-#endif
-      switch (tstypes[i])
-      {
-      case PCAP_TSTAMP_HOST:
-        break;
-      case PCAP_TSTAMP_HOST_LOWPREC:
-        break;
-      case PCAP_TSTAMP_HOST_HIPREC:
-        {
-          if(besttst != PCAP_TSTAMP_ADAPTER_UNSYNCED
-             && besttst != PCAP_TSTAMP_ADAPTER)
-            besttst = PCAP_TSTAMP_HOST_HIPREC;
-          break;
-        }
-      case PCAP_TSTAMP_ADAPTER:
-        {
-          if(besttst != PCAP_TSTAMP_ADAPTER_UNSYNCED)
-            besttst = PCAP_TSTAMP_ADAPTER;
-          break;
-        }
-      case PCAP_TSTAMP_ADAPTER_UNSYNCED:
-          /*besttst = PCAP_TSTAMP_ADAPTER_UNSYNCED;*/
-          break;
-      default:
-#if DEBUG
-          fprintf(stderr, " unknown_type_%d", tstypes[i]);
-#endif
-        break;
-      }
-    }
-    pcap_free_tstamp_types(tstypes);
-#if DEBUG
-    fprintf(stderr, "\n");
-#endif
-    /*besttst = PCAP_TSTAMP_HOST;*/
-    if(besttst != -1)
-    {
-      fprintf(stderr, "INFO: Attempting to set the timestamp source to: %s\n",
-              pcap_tstamp_type_val_to_name(besttst));
-      if(pcap_set_tstamp_type(ret_val, besttst) != 0)
-        fprintf(stderr, "WARN: Failed to set preferred timestamp source.\n");
-    }
-  }
-  
-  /* Attempt to set nanosecond timestamp precision */    
-  if(pcap_set_tstamp_precision(ret_val, PCAP_TSTAMP_PRECISION_NANO) != 0)
-    fprintf(stderr, "INFO: Failed to set packet capture nanosecond precision.\n");
-  else
-    output_template = "{\"local_port\":%u,\"sent\":%u,\"payload_len\":%u,\"tcpseq\":%u,\"tcpack\":%u,\"observed\":%lu%09lu,\"tsval\":%u}\n";
-
-
-  status = pcap_activate(ret_val);
-  if (status < 0)
-    goto fail;
-  
-  return ret_val;
-
- fail:
-  pcap_close(ret_val);
-  return NULL;
-}
-
-
-static int extract_packet_fields(const struct pcap_pkthdr *header,
-                                 const u_char* packet,
-                                 struct useful_info* fields)
-{
-  const struct sniff_ip* ip;               /* The IP header */
-  const struct sniff_tcp* tcp;             /* The TCP header */
-  u_int iphdr_size;
-  u_int tcphdr_size;
-  uint16_t ip_len;
-  uint16_t tsval;
-  uint8_t* opt;
-  tcp_option_t* _opt;
-  
-  /*printf("snaplen: %u\n", header->caplen);*/
-  
-  ip = (struct sniff_ip*)(packet + SIZE_ETHERNET);
-  iphdr_size = IP_HL(ip)*4;
-  if (iphdr_size < 20)
-    return 0;
-  ip_len = ntohs(ip->ip_len);
-  
-  tcp = (struct sniff_tcp*)(packet + SIZE_ETHERNET + iphdr_size);
-  tcphdr_size = TH_OFF(tcp)*4;
-  if (tcphdr_size < 20)
-    return 0;
-
-  /*fprintf(stderr, "%d\n", (ip_len - iphdr_size - tcphdr_size));*/
-  if ((ip_len - iphdr_size - tcphdr_size < 0)
-      || (payloads_only && (ip_len - iphdr_size - tcphdr_size == 0)))
-  { return 0; }
-
-  fields->tcpseq = ntohl(tcp->th_seq);
-  fields->tcpack = ntohl(tcp->th_ack);
-  fields->payload_len = ip_len - iphdr_size - tcphdr_size;
-  if(ip->ip_src.s_addr == target_ip_raw.s_addr && ntohs(tcp->th_sport) == target_port)
-  {
-    fields->sent = 0;
-    fields->local_port = ntohs(tcp->th_dport);
-  }
-  else
-  {
-    fields->sent = 1;
-    fields->local_port = ntohs(tcp->th_sport);
-  }
-  /*printf("src: %lX / dst: %lX\n", ip->ip_src, ip->ip_dst);*/
-
-  fields->tsval = 0;
-  if(tcphdr_size > 20)
-  {
-    opt = (uint8_t*)(packet + SIZE_ETHERNET + iphdr_size + 20);
-    while((*opt != 0) && (opt - packet) < header->caplen)
-    {
-      _opt = (tcp_option_t*)opt;
-      if(_opt->kind == 1) /* NOP */
-      {
-        ++opt;  // NOP is one byte;
-        continue;
-      }
-      
-      if(_opt->kind == 8) /* timestamp */
-      {
-        fields->tsval = ntohl(*(uint32_t*)(opt+2));
-        break;
-      }
-      
-      opt += _opt->size;
-    }
-  }
-
-  return 1;
-}
-
-
-void process_packet(u_char *args, const struct pcap_pkthdr *header, const u_char *packet)
-{
-  struct useful_info fields;
-
-  if(extract_packet_fields(header, packet, &fields))
-  {
-    fprintf(output, output_template, fields.local_port, fields.sent,
-            fields.payload_len, fields.tcpseq, fields.tcpack,
-            header->ts.tv_sec, header->ts.tv_usec, fields.tsval);
-    fflush(output);
-  }
-}
-
-
-
-int main(int argc, char** argv)
-{
-  char* dev;                      /* The device to sniff on */
-  char* my_ip;
-  char* target_ip;
-  pcap_t* handle;                 /* Session handle */
-  char bpf[255];                  /* The filter expression (length of 168 should be enough)*/
-  char errbuf[PCAP_ERRBUF_SIZE];  /* Error string */
-  struct bpf_program fp;          /* The compiled filter */
-  bpf_u_int32 mask;               /* Our netmask */
-  bpf_u_int32 net;                /* Our IP */
-  struct pcap_pkthdr header;      /* The header that pcap gives us */
-  const u_char *packet;           /* The actual packet */
-    
-  if(argc < 6)
-  {
-    fprintf(stderr, "USAGE:\n  %s {interface} {my_ip} {target_ip} {target_port} {output_file} [{payloads_only?}]\n", argv[0]);
-    return 1;
-  }
-
-  dev = argv[1];
-  my_ip = argv[2];
-  target_ip = argv[3];
-  target_port = atoi(argv[4]);
-  if(argc == 7 && argv[6][0] == '0')
-    payloads_only = 0;
-  
-  if(!(output = fopen(argv[5], "w+")))
-  {
-    fprintf(stderr, "ERROR: could not open output file due to: %s\n", strerror(errno));
-    return 2;
-  }
-  
-  snprintf(bpf, 255, "(src host %s and dst host %s and tcp and src port %u) or (dst host %s and src host %s and tcp and dst port %u)",
-           target_ip, my_ip, target_port, target_ip, my_ip, target_port);
-
-
-  if(!inet_aton(my_ip, &my_ip_raw))
-  {
-    fprintf(stderr, "Couldn't parse my_ip.\n");
-    return 1;
-  }
-
-  if(!inet_aton(target_ip, &target_ip_raw))
-  {
-    fprintf(stderr, "Couldn't parse target_ip.\n");
-    return 1;
-  }
-
-  memset(errbuf, 0, PCAP_ERRBUF_SIZE);
-
-  /* Find the properties for the device */
-  if (pcap_lookupnet(dev, &net, &mask, errbuf) == -1)
-  {
-    fprintf(stderr, "Couldn't get netmask for device %s: %s\n", dev, errbuf);
-    net = 0;
-    mask = 0;
-  }
-  
-  /* Open the session in promiscuous mode */
-  /* XXX: does to_ms timeout (param 4) matter? */
-  handle = create_listener(dev, BUFSIZ, 0, 1000, errbuf);
-  if (handle == NULL)
-  {
-    fprintf(stderr, "Couldn't open device %s: %s\n", dev, errbuf);
-    return 2;
-  }
-
-  /* Compile and apply the filter */
-  if (pcap_compile(handle, &fp, bpf, 0, net) == -1)
-  {
-    fprintf(stderr, "Couldn't parse filter %s: %s\n", bpf, pcap_geterr(handle));
-    return 2;
-  }
-  
-  if (pcap_setfilter(handle, &fp) == -1)
-  {
-    fprintf(stderr, "Couldn't install filter %s: %s\n", bpf, pcap_geterr(handle));
-    return 2;
-  }
-
-  /* XXX: report errors */
-  pcap_loop(handle, 0, process_packet, NULL);
-  
-  /* And close the session */
-  pcap_close(handle);
-
-  return 0;
-}
Index: trunk/src/listen.c
===================================================================
--- trunk/src/listen.c	(revision 16)
+++ trunk/src/listen.c	(revision 16)
@@ -0,0 +1,394 @@
+/*
+Portions are Copyright 2002 Tim Carstens. All rights reserved. 
+Redistribution and use, with or without modification, are permitted provided
+that the following conditions are met:
+
+- Redistribution must retain the above copyright notice and this list of 
+  conditions.
+
+- The name of Tim Carstens may not be used to endorse or promote products
+  derived from this document without specific prior written permission.
+*/
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/types.h>
+#include <inttypes.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <errno.h>
+#include <netinet/in.h>
+#include <pcap.h>
+
+#define DEBUG 0
+
+/* ethernet headers are always exactly 14 bytes */
+/* XXX: ^^^ really? */
+#define SIZE_ETHERNET 14
+
+/* Ethernet addresses are 6 bytes */
+#define ETHER_ADDR_LEN	6
+
+/* Ethernet header */
+/*
+struct sniff_ethernet
+{
+  u_char ether_dhost[ETHER_ADDR_LEN]; / * Destination host address * /
+  u_char ether_shost[ETHER_ADDR_LEN]; / * Source host address * /
+  u_short ether_type; / * IP? ARP? RARP? etc * /
+};
+*/
+
+/* IP header */
+struct sniff_ip
+{
+  u_char ip_vhl;		/* version << 4 | header length >> 2 */
+  u_char ip_tos;		/* type of service */
+  u_short ip_len;		/* total length */
+  u_short ip_id;		/* identification */
+  u_short ip_off;		/* fragment offset field */
+#define IP_RF 0x8000		/* reserved fragment flag */
+#define IP_DF 0x4000		/* dont fragment flag */
+#define IP_MF 0x2000		/* more fragments flag */
+#define IP_OFFMASK 0x1fff	/* mask for fragmenting bits */
+  u_char ip_ttl;		/* time to live */
+  u_char ip_p;		/* protocol */
+  u_short ip_sum;		/* checksum */
+  struct in_addr ip_src,ip_dst; /* source and dest address */
+};
+#define IP_HL(ip)		(((ip)->ip_vhl) & 0x0f)
+#define IP_V(ip)		(((ip)->ip_vhl) >> 4)
+
+/* TCP header */
+typedef u_int tcp_seq;
+
+struct sniff_tcp
+{
+  u_short th_sport;	/* source port */
+  u_short th_dport;	/* destination port */
+  tcp_seq th_seq;		/* sequence number */
+  tcp_seq th_ack;		/* acknowledgement number */
+  u_char th_offx2;	/* data offset, rsvd */
+#define TH_OFF(th)	(((th)->th_offx2 & 0xf0) >> 4)
+  u_char th_flags;
+#define TH_FIN 0x01
+#define TH_SYN 0x02
+#define TH_RST 0x04
+#define TH_PUSH 0x08
+#define TH_ACK 0x10
+#define TH_URG 0x20
+#define TH_ECE 0x40
+#define TH_CWR 0x80
+#define TH_FLAGS (TH_FIN|TH_SYN|TH_RST|TH_ACK|TH_URG|TH_ECE|TH_CWR)
+  u_short th_win;		/* window */
+  u_short th_sum;		/* checksum */
+  u_short th_urp;		/* urgent pointer */
+};
+
+typedef struct {
+  uint8_t kind;
+  uint8_t size;
+} tcp_option_t;
+
+
+
+struct in_addr my_ip_raw;
+struct in_addr target_ip_raw;
+uint16_t target_port;
+
+FILE* output;
+
+struct useful_info
+{
+  uint32_t tcpseq;
+  uint32_t tcpack;
+  uint32_t tsval;
+  uint16_t local_port;
+  uint16_t payload_len;
+  uint8_t sent;
+};
+
+
+char* output_template = "{\"local_port\":%u,\"sent\":%u,\"payload_len\":%u,\"tcpseq\":%u,\"tcpack\":%u,\"observed\":%lu%06lu000,\"tsval\":%u}\n";
+uint8_t payloads_only = 1;
+
+
+pcap_t* create_listener(const char* dev, int snaplen, int promisc, int to_ms, char* errbuf)
+{
+  pcap_t* ret_val;
+  int status;
+  int tstcount, i, besttst;
+  int* tstypes;
+
+  ret_val = pcap_create(dev, errbuf);
+  if (ret_val == NULL)
+    return (NULL);
+  
+  status = pcap_set_snaplen(ret_val, snaplen);
+  if (status < 0)
+    goto fail;
+  
+  status = pcap_set_promisc(ret_val, promisc);
+  if (status < 0)
+    goto fail;
+  
+  status = pcap_set_timeout(ret_val, to_ms);
+  if (status < 0)
+    goto fail;
+
+
+  /* Try to select the best timestamp source for packet capture */
+  tstcount = pcap_list_tstamp_types(ret_val, &tstypes);
+  if(tstcount > 0)
+  {
+#if DEBUG
+    fprintf(stderr, "INFO: Available Packet Timers: ");
+#endif
+    besttst = -1;
+    for(i=0; i < tstcount; i++)
+    {
+#if DEBUG
+        fprintf(stderr, " %s", pcap_tstamp_type_val_to_name(tstypes[i]));
+#endif
+      switch (tstypes[i])
+      {
+      case PCAP_TSTAMP_HOST:
+        break;
+      case PCAP_TSTAMP_HOST_LOWPREC:
+        break;
+      case PCAP_TSTAMP_HOST_HIPREC:
+        {
+          if(besttst != PCAP_TSTAMP_ADAPTER_UNSYNCED
+             && besttst != PCAP_TSTAMP_ADAPTER)
+            besttst = PCAP_TSTAMP_HOST_HIPREC;
+          break;
+        }
+      case PCAP_TSTAMP_ADAPTER:
+        {
+          if(besttst != PCAP_TSTAMP_ADAPTER_UNSYNCED)
+            besttst = PCAP_TSTAMP_ADAPTER;
+          break;
+        }
+      case PCAP_TSTAMP_ADAPTER_UNSYNCED:
+          /*besttst = PCAP_TSTAMP_ADAPTER_UNSYNCED;*/
+          break;
+      default:
+#if DEBUG
+          fprintf(stderr, " unknown_type_%d", tstypes[i]);
+#endif
+        break;
+      }
+    }
+    pcap_free_tstamp_types(tstypes);
+#if DEBUG
+    fprintf(stderr, "\n");
+#endif
+    /*besttst = PCAP_TSTAMP_HOST;*/
+    if(besttst != -1)
+    {
+#if DEBUG
+      fprintf(stderr, "INFO: Attempting to set the timestamp source to: %s\n",
+              pcap_tstamp_type_val_to_name(besttst));
+#endif
+      if(pcap_set_tstamp_type(ret_val, besttst) != 0)
+        fprintf(stderr, "WARN: Failed to set preferred timestamp source.\n");
+    }
+  }
+  
+  /* Attempt to set nanosecond timestamp precision */    
+  if(pcap_set_tstamp_precision(ret_val, PCAP_TSTAMP_PRECISION_NANO) != 0)
+    fprintf(stderr, "INFO: Failed to set packet capture nanosecond precision.\n");
+  else
+    output_template = "{\"local_port\":%u,\"sent\":%u,\"payload_len\":%u,\"tcpseq\":%u,\"tcpack\":%u,\"observed\":%lu%09lu,\"tsval\":%u}\n";
+
+
+  status = pcap_activate(ret_val);
+  if (status < 0)
+    goto fail;
+  
+  return ret_val;
+
+ fail:
+  pcap_close(ret_val);
+  return NULL;
+}
+
+
+static int extract_packet_fields(const struct pcap_pkthdr *header,
+                                 const u_char* packet,
+                                 struct useful_info* fields)
+{
+  const struct sniff_ip* ip;               /* The IP header */
+  const struct sniff_tcp* tcp;             /* The TCP header */
+  u_int iphdr_size;
+  u_int tcphdr_size;
+  uint16_t ip_len;
+  uint16_t tsval;
+  uint8_t* opt;
+  tcp_option_t* _opt;
+  
+  /*printf("snaplen: %u\n", header->caplen);*/
+  
+  ip = (struct sniff_ip*)(packet + SIZE_ETHERNET);
+  iphdr_size = IP_HL(ip)*4;
+  if (iphdr_size < 20)
+    return 0;
+  ip_len = ntohs(ip->ip_len);
+  
+  tcp = (struct sniff_tcp*)(packet + SIZE_ETHERNET + iphdr_size);
+  tcphdr_size = TH_OFF(tcp)*4;
+  if (tcphdr_size < 20)
+    return 0;
+
+  /*fprintf(stderr, "%d\n", (ip_len - iphdr_size - tcphdr_size));*/
+  if ((ip_len - iphdr_size - tcphdr_size < 0)
+      || (payloads_only && (ip_len - iphdr_size - tcphdr_size == 0)))
+  { return 0; }
+
+  fields->tcpseq = ntohl(tcp->th_seq);
+  fields->tcpack = ntohl(tcp->th_ack);
+  fields->payload_len = ip_len - iphdr_size - tcphdr_size;
+  if(ip->ip_src.s_addr == target_ip_raw.s_addr && ntohs(tcp->th_sport) == target_port)
+  {
+    fields->sent = 0;
+    fields->local_port = ntohs(tcp->th_dport);
+  }
+  else
+  {
+    fields->sent = 1;
+    fields->local_port = ntohs(tcp->th_sport);
+  }
+  /*printf("src: %lX / dst: %lX\n", ip->ip_src, ip->ip_dst);*/
+
+  fields->tsval = 0;
+  if(tcphdr_size > 20)
+  {
+    opt = (uint8_t*)(packet + SIZE_ETHERNET + iphdr_size + 20);
+    while((*opt != 0) && (opt - packet) < header->caplen)
+    {
+      _opt = (tcp_option_t*)opt;
+      if(_opt->kind == 1) /* NOP */
+      {
+        ++opt;  // NOP is one byte;
+        continue;
+      }
+      
+      if(_opt->kind == 8) /* timestamp */
+      {
+        fields->tsval = ntohl(*(uint32_t*)(opt+2));
+        break;
+      }
+      
+      opt += _opt->size;
+    }
+  }
+
+  return 1;
+}
+
+
+void process_packet(u_char *args, const struct pcap_pkthdr *header, const u_char *packet)
+{
+  struct useful_info fields;
+
+  if(extract_packet_fields(header, packet, &fields))
+  {
+    fprintf(output, output_template, fields.local_port, fields.sent,
+            fields.payload_len, fields.tcpseq, fields.tcpack,
+            header->ts.tv_sec, header->ts.tv_usec, fields.tsval);
+    fflush(output);
+  }
+}
+
+
+
+int main(int argc, char** argv)
+{
+  char* dev;                      /* The device to sniff on */
+  char* my_ip;
+  char* target_ip;
+  pcap_t* handle;                 /* Session handle */
+  char bpf[255];                  /* The filter expression (length of 168 should be enough)*/
+  char errbuf[PCAP_ERRBUF_SIZE];  /* Error string */
+  struct bpf_program fp;          /* The compiled filter */
+  bpf_u_int32 mask;               /* Our netmask */
+  bpf_u_int32 net;                /* Our IP */
+  struct pcap_pkthdr header;      /* The header that pcap gives us */
+  const u_char *packet;           /* The actual packet */
+    
+  if(argc < 6)
+  {
+    fprintf(stderr, "USAGE:\n  %s {interface} {my_ip} {target_ip} {target_port} {output_file} [{payloads_only?}]\n", argv[0]);
+    return 1;
+  }
+
+  dev = argv[1];
+  my_ip = argv[2];
+  target_ip = argv[3];
+  target_port = atoi(argv[4]);
+  if(argc == 7 && argv[6][0] == '0')
+    payloads_only = 0;
+  
+  if(!(output = fopen(argv[5], "w+")))
+  {
+    fprintf(stderr, "ERROR: could not open output file due to: %s\n", strerror(errno));
+    return 2;
+  }
+  
+  snprintf(bpf, 255, "(src host %s and dst host %s and tcp and src port %u) or (dst host %s and src host %s and tcp and dst port %u)",
+           target_ip, my_ip, target_port, target_ip, my_ip, target_port);
+
+
+  if(!inet_aton(my_ip, &my_ip_raw))
+  {
+    fprintf(stderr, "Couldn't parse my_ip.\n");
+    return 1;
+  }
+
+  if(!inet_aton(target_ip, &target_ip_raw))
+  {
+    fprintf(stderr, "Couldn't parse target_ip.\n");
+    return 1;
+  }
+
+  memset(errbuf, 0, PCAP_ERRBUF_SIZE);
+
+  /* Find the properties for the device */
+  if (pcap_lookupnet(dev, &net, &mask, errbuf) == -1)
+  {
+    fprintf(stderr, "Couldn't get netmask for device %s: %s\n", dev, errbuf);
+    net = 0;
+    mask = 0;
+  }
+  
+  /* Open the session in promiscuous mode */
+  /* XXX: does to_ms timeout (param 4) matter? */
+  handle = create_listener(dev, BUFSIZ, 0, 1000, errbuf);
+  if (handle == NULL)
+  {
+    fprintf(stderr, "Couldn't open device %s: %s\n", dev, errbuf);
+    return 2;
+  }
+
+  /* Compile and apply the filter */
+  if (pcap_compile(handle, &fp, bpf, 0, net) == -1)
+  {
+    fprintf(stderr, "Couldn't parse filter %s: %s\n", bpf, pcap_geterr(handle));
+    return 2;
+  }
+  
+  if (pcap_setfilter(handle, &fp) == -1)
+  {
+    fprintf(stderr, "Couldn't install filter %s: %s\n", bpf, pcap_geterr(handle));
+    return 2;
+  }
+
+  /* XXX: report errors */
+  pcap_loop(handle, 0, process_packet, NULL);
+  
+  /* And close the session */
+  pcap_close(handle);
+
+  return 0;
+}
