Index: /trunk/bin/sampler
===================================================================
--- /trunk/bin/sampler	(revision 19)
+++ /trunk/bin/sampler	(revision 20)
@@ -30,4 +30,6 @@
 
 from nanownlib import *
+from nanownlib.platform import *
+from nanownlib.tcpts import *
 import nanownlib.storage
 
@@ -36,5 +38,5 @@
 parser.add_argument('-c', dest='cases', type=str, default='{"short":10000,"long":13000}',
                     help='JSON representation of echo timing cases.')
-parser.add_argument('--no-tcpts', action='store_true', help='Disbale TCP timestamp profiling')
+parser.add_argument('--no-tcpts', action='store_true', help='Disable TCP timestamp profiling')
 parser.add_argument('--no-control', action='store_true', help='Do not collect separate control data.  Instead, synthesize it from test and train data.')
 parser.add_argument('session_name', default=None,
@@ -49,11 +51,13 @@
 
 
-session_name = options.session_name
-num_samples=options.sample_count
-hostname = options.host
-port = options.port
-protocol = 'http'
+num_samples = options.sample_count
+hostname    = options.host
+port        = options.port
+protocol    = 'http'
 
 cases = json.loads(options.cases)
+db_file = "%s.db" % options.session_name
+db = nanownlib.storage.db(db_file)
+
 
 def extractReportedRuntime(headers, body):
@@ -64,5 +68,5 @@
 def sendRequest(data=None):
     method = 'GET'
-    path = '/data/clamav-audit-results.txt?t=' + data
+    path = '/?t=' + data
     url = "%s://%s:%d%s" % (protocol,hostname,port,path)
     headers = {}
@@ -74,6 +78,6 @@
         try:
             session = requests.Session()
-            response = session.send(req)
-            #print(repr(response.raw._original_response.local_address))
+            response = session.send(req, allow_redirects=False)
+            #print("sendRequest:", repr(response.raw._original_response.local_address))
             reported = extractReportedRuntime(response.headers, response.text)
             retry = False
@@ -82,5 +86,6 @@
             time.sleep(1.0)
             sys.stderr.write("ERROR: retrying...\n")
-    
+
+        
     return {'userspace_rtt':response.elapsed.microseconds*1000,
             'reported':reported,
@@ -97,6 +102,16 @@
 
 setCPUAffinity()
-setTCPTimestamps()
-host_ip = socket.gethostbyname(hostname) #XXX: what about multiple A records?
+time.sleep(0.25) # Give our process a chance to migrate to a different CPU if necessary
+setPowersave(False) # XXX: test this to see if it helps
+setLowLatency(True) # XXX: test this to see if it helps
+tcpts_previous = setTCPTimestamps(True)
+
+
+#XXX: what about multiple A records?
+#     perform this during a script generation step, measuring lowest latency server
+#     and hard-coding the IP and host name separately.  However, including all
+#     server IPs in comments
+host_ip = socket.gethostbyname(hostname) 
+
 
 meta = {'tcpts_mean':None,'tcpts_stddev':None,'tcpts_slopes':None}
@@ -125,42 +140,32 @@
                      % (meta['tcpts_mean'], meta['tcpts_stddev'], 100*meta['tcpts_stddev']/meta['tcpts_mean']))
 
-
-sniffer_fp = tempfile.NamedTemporaryFile('w+t')
-db_file = "%s.db" % session_name
-
-sniffer = startSniffer(host_ip, port, sniffer_fp.name)
-db = nanownlib.storage.db(db_file)
 db.addMeta(meta)
-time.sleep(0.5) # ensure sniffer is fully ready and our process is migrated
-
-if options.no_control:
-    num_control = 0
-else:
-    num_control = int(num_samples*2/5)
-
-num_train = int((num_samples-num_control)/3)
-num_test = num_samples-num_train-num_control
-
-sample_types = [('train',num_train),
-                ('train_null',num_control),
-                ('test',num_test)]
-
-sid = 0
-report_interval = 20
-start = time.time()
-next_report = start+report_interval
-for st,count in sample_types:
-    if sniffer.poll() != None:
-        sys.stderr.write('ERROR: Sniffer quit, sender exiting...\n')
-        break
+
+
+def findNextSampleID(db):
+    cursor = db.conn.cursor()
+    cursor.execute("SELECT max(sample) FROM probes")
+    row = cursor.fetchone()
+    if row != None and row[0] != None:
+        return row[0]+1
+
+    return 0
+
+
+def collectSamples(db, sample_type, count, sniffer):
+    sniffer.start()
+
+    if not sniffer.is_running():
+        sys.stderr.write('ERROR: Sniffer did not start...\n')
+        return
         
+    sid = findNextSampleID(db)
     for k in range(0,count):
         sample_order = list(cases.items())
         random.shuffle(sample_order)
-        if st.endswith('null'):
+        if sample_type.endswith('null'):
             for i in range(1,len(sample_order)):
                 sample_order[i] = (sample_order[i][0],sample_order[0][1])
             random.shuffle(sample_order)
-            #print('after', sample_order)
             
         results = []
@@ -168,28 +173,47 @@
         for i in range(len(sample_order)):
             results.append(fetch({'sample':sid, 'test_case':sample_order[i][0],
-                                  'type':st, 'tc_order':i, 'time_of_day':now},
+                                  'type':sample_type, 'tc_order':i, 'time_of_day':now},
                                  sample_order[i][1]))
 
-        #print(results)
+        print(results)
         db.addProbes(results)
         db.conn.commit()
         sid += 1
 
-        if (time.time() > next_report):
-            #s = time.time()
-            reportProgress(db, sample_types, start)
-            #print("reportProgress time:", time.time()-s)
-            next_report += report_interval
-
-print("probes complete in %f" % (time.time()-start))
-time.sleep(2.0) # Give sniffer a chance to collect remaining packets
-stopSniffer(sniffer)
-
-start = time.time()
-associatePackets(sniffer_fp, db)
-sniffer_fp.close()
-end = time.time()
-print("associate time:", end-start)
-
+    time.sleep(2.0) # Give sniffer a chance to collect remaining packets
+    sniffer.stop()
+    #print(sniffer.openPacketLog().read())
+    start = time.time()
+    associatePackets(sniffer.openPacketLog(), db)
+    end = time.time()
+    print("associate time:", end-start)
+    
+
+
+if options.no_control:
+    num_control = 0
+else:
+    num_control = int(num_samples*2/5)
+
+num_train = int((num_samples-num_control)/3)
+num_test = num_samples-num_train-num_control
+
+sample_types = [('train',num_train),
+                ('train_null',num_control),
+                ('test',num_test)]
+
+sniffer = snifferProcess(host_ip, port)
+for st,count in sample_types:
+    collectSamples(db, st,count,sniffer)
+
+
+#start = time.time()
+#report_interval = 20
+#next_report = start+report_interval
+#        if (time.time() > next_report):
+#            reportProgress(db, sample_types, start)
+#            next_report += report_interval
+
+    
 if options.no_control:
     print("TODO: implement control synthesizing!")
@@ -199,2 +223,7 @@
 end = time.time()
 print("analyzed %d probes' packets in: %f" % (num_probes, end-start))
+
+
+setPowersave(True) # XXX: test this to see if it actually helps
+setLowLatency(False) # XXX: test this to see if it actually helps
+setTCPTimestamps(tcpts_previous)
Index: /trunk/lib/nanownlib/__init__.py
===================================================================
--- /trunk/lib/nanownlib/__init__.py	(revision 19)
+++ /trunk/lib/nanownlib/__init__.py	(revision 20)
@@ -5,23 +5,12 @@
 import time
 import traceback
-import random
-import argparse
 import socket
 import datetime
 import http.client
-import threading
-import queue
 import subprocess
-import multiprocessing
-import csv
+import tempfile
 import json
 import gzip
 import statistics
-try:
-    import numpy
-except:
-    sys.stderr.write('ERROR: Could not import numpy module.  Ensure it is installed.\n')
-    sys.stderr.write('       Under Debian, the package name is "python3-numpy"\n.')
-    sys.exit(1)
 
 try:
@@ -59,130 +48,46 @@
 
 
-def setTCPTimestamps(enabled=True):
-    fh = open('/proc/sys/net/ipv4/tcp_timestamps', 'r+b')
-    ret_val = False
-    if fh.read(1) == b'1':
-        ret_val = True
-
-    fh.seek(0)
-    if enabled:
-        fh.write(b'1')
-    else:
-        fh.write(b'0')
-    fh.close()
-    
-    return ret_val
-
-
-def trickleHTTPRequest(ip,port,hostname):
-    my_port = None
-    try:
-        sock = socket.create_connection((ip, port))
-        my_port = sock.getsockname()[1]
+class snifferProcess(object):
+    my_ip = None
+    my_iface = None
+    target_ip = None
+    target_port = None
+    _proc = None
+    _spool = None
+    
+    def __init__(self, target_ip, target_port):
+        self.target_ip = target_ip
+        self.target_port = target_port
+        self.my_ip = getLocalIP(target_ip, target_port)
+        self.my_iface = getIfaceForIP(self.my_ip)
+        print(self.my_ip, self.my_iface)
+
+    def start(self):
+        self._spool = tempfile.NamedTemporaryFile('w+t')
+        self._proc = subprocess.Popen(['chrt', '-r', '99', 'nanown-listen',
+                                       self.my_iface, self.my_ip,
+                                       self.target_ip, "%d" % self.target_port,
+                                       self._spool.name, '0'])
+        time.sleep(0.25)
+
+    def openPacketLog(self):
+        return open(self._spool.name, 'rt')
         
-        #print('.')
-        sock.sendall(b'GET / HTTP/1.1\r\n')
-        time.sleep(0.5)
-        rest = b'''Host: '''+hostname.encode('utf-8')+b'''\r\nUser-Agent: Secret Agent Man\r\nX-Extra: extra read all about it!\r\nConnection: close\r\n'''
-        for r in rest:
-            sock.sendall(bytearray([r]))
-            time.sleep(0.05)
-
-        time.sleep(0.5)
-        sock.sendall('\r\n')
-
-        r = None
-        while r != b'':
-            r = sock.recv(16)
-
-        sock.close()
-    except Exception as e:
-        pass
-
-    return my_port
-
-
-def runTimestampProbes(host_ip, port, hostname, num_trials, concurrency=4): 
-    myq = queue.Queue()
-    def threadWrapper(*args):
-        try:
-            myq.put(trickleHTTPRequest(*args))
-        except Exception as e:
-            sys.stderr.write("ERROR from trickleHTTPRequest: %s\n" % repr(e))
-            myq.put(None)
-
-    threads = []
-    ports = []
-    for i in range(num_trials):
-        if len(threads) >= concurrency:
-            ports.append(myq.get())
-        t = threading.Thread(target=threadWrapper, args=(host_ip, port, hostname))
-        t.start()
-        threads.append(t)
-
-    for t in threads:
-        t.join()
-
-    while myq.qsize() > 0:
-        ports.append(myq.get())
-
-    return ports
-
-
-def computeTimestampPrecision(sniffer_fp, ports):
-    rcvd = []
-    for line in sniffer_fp:
-        p = json.loads(line)
-        if p['sent']==0:
-            rcvd.append((p['observed'],p['tsval'],int(p['local_port'])))
-
-    slopes = []
-    for port in ports:
-        trcvd = [tr for tr in rcvd if tr[2]==port and tr[1]!=0]
-
-        if len(trcvd) < 2:
-            sys.stderr.write("WARN: Inadequate data points.\n")
-            continue
-        
-        if trcvd[0][1] > trcvd[-1][1]:
-            sys.stderr.write("WARN: TSval wrap.\n")
-            continue
-
-        x = [tr[1] for tr in trcvd]
-        y = [tr[0] for tr in trcvd]
-
-        slope,intercept = OLSRegression(x, y)
-        slopes.append(slope)
-
-    if len(slopes) == 0:
-        return None,None,None
-
-    m = statistics.mean(slopes)
-    if len(slopes) == 1:
-        return (m, None, slopes)
-    else:
-        return (m, statistics.stdev(slopes), slopes)
-
-    
-def OLSRegression(x,y):
-    #print(x,y)
-    x = numpy.array(x)
-    y = numpy.array(y)
-    #A = numpy.vstack([x, numpy.ones(len(x))]).T
-    #m, c = numpy.linalg.lstsq(A, y)[0] # broken
-    #c,m = numpy.polynomial.polynomial.polyfit(x, y, 1) # less accurate
-    c,m = numpy.polynomial.Polynomial.fit(x,y,1).convert().coef
-
-    #print(m,c)
-
-    #import matplotlib.pyplot as plt
-    #plt.clf()
-    #plt.scatter(x, y)
-    #plt.plot(x, m*x + c, 'r', label='Fitted line')
-    #plt.show()
-    
-    return (m,c)
-
-
+    def stop(self):
+        if self._proc:
+            self._proc.terminate()
+            self._proc.wait(2)
+            if self._proc.poll() == None:
+                self._proc.kill()
+                self._proc.wait(1)
+            self._proc = None
+    
+    def is_running(self):
+        return (self._proc.poll() == None)
+            
+    def __del__(self):
+        self.stop()
+
+            
 def startSniffer(target_ip, target_port, output_file):
     my_ip = getLocalIP(target_ip, target_port)
@@ -198,14 +103,4 @@
         sniffer.wait(1)
 
-        
-def setCPUAffinity():
-    import ctypes
-    from ctypes import cdll,c_int,byref
-    cpus = multiprocessing.cpu_count()
-    
-    libc = cdll.LoadLibrary("libc.so.6")
-    #libc.sched_setaffinity(os.getpid(), 1, ctypes.byref(ctypes.c_int(0x01)))
-    return libc.sched_setaffinity(0, 4, byref(c_int(0x00000001<<(cpus-1))))
-
 
 # Monkey patching that instruments the HTTPResponse to collect connection source port info
@@ -215,4 +110,5 @@
     def __init__(self, sock, *args, **kwargs):
         self.local_address = sock.getsockname()
+        #print(self.local_address)
         super(MonitoredHTTPResponse, self).__init__(sock,*args,**kwargs)
             
Index: /trunk/lib/nanownlib/platform.py
===================================================================
--- /trunk/lib/nanownlib/platform.py	(revision 20)
+++ /trunk/lib/nanownlib/platform.py	(revision 20)
@@ -0,0 +1,63 @@
+
+import multiprocessing
+
+def setCPUAffinity():
+    from ctypes import cdll,c_int,byref
+    cpus = multiprocessing.cpu_count()
+    
+    libc = cdll.LoadLibrary("libc.so.6")
+    #libc.sched_setaffinity(os.getpid(), 1, ctypes.byref(ctypes.c_int(0x01)))
+    return libc.sched_setaffinity(0, 4, byref(c_int(0x00000001<<(cpus-1))))
+
+
+def setTCPTimestamps(enabled=True):
+    fh = open('/proc/sys/net/ipv4/tcp_timestamps', 'r+b')
+    ret_val = False
+    if fh.read(1) == b'1':
+        ret_val = True
+
+    fh.seek(0)
+    if enabled:
+        fh.write(b'1')
+    else:
+        fh.write(b'0')
+    fh.close()
+    
+    return ret_val
+
+
+previous_governors = None
+def setPowersave(enabled):
+    global previous_governors
+    cpus = multiprocessing.cpu_count()
+    if enabled:
+        if previous_governors == None:
+            previous_governors = [b"powersave"]*cpus
+        new_governors = previous_governors
+    else:
+        new_governors = [b"performance"]*cpus
+
+    previous_governors = []
+    for c in range(cpus):
+        fh = open('/sys/devices/system/cpu/cpu%d/cpufreq/scaling_governor' % c, 'r+b')
+        previous_governors.append(fh.read())
+        fh.seek(0)
+        fh.write(new_governors[c])
+        fh.close()
+        
+
+def setLowLatency(enabled):
+    fh = open('/proc/sys/net/ipv4/tcp_low_latency', 'r+b')
+    ret_val = False
+    if fh.read(1) == b'1':
+        ret_val = True
+
+    fh.seek(0)
+    if enabled:
+        fh.write(b'1')
+    else:
+        fh.write(b'0')
+    fh.close()
+    
+    return ret_val
+
Index: /trunk/lib/nanownlib/stats.py
===================================================================
--- /trunk/lib/nanownlib/stats.py	(revision 19)
+++ /trunk/lib/nanownlib/stats.py	(revision 20)
@@ -7,5 +7,10 @@
 import gzip
 import random
-import numpy
+try:
+    import numpy
+except:
+    sys.stderr.write('ERROR: Could not import numpy module.  Ensure it is installed.\n')
+    sys.stderr.write('       Under Debian, the package name is "python3-numpy"\n.')
+    sys.exit(1)
 
 # Don't trust numpy's seeding
@@ -31,4 +36,24 @@
 
     return statistics.mean(products)
+
+    
+def OLSRegression(x,y):
+    #print(x,y)
+    x = numpy.array(x)
+    y = numpy.array(y)
+    #A = numpy.vstack([x, numpy.ones(len(x))]).T
+    #m, c = numpy.linalg.lstsq(A, y)[0] # broken
+    #c,m = numpy.polynomial.polynomial.polyfit(x, y, 1) # less accurate
+    c,m = numpy.polynomial.Polynomial.fit(x,y,1).convert().coef
+
+    #print(m,c)
+
+    #import matplotlib.pyplot as plt
+    #plt.clf()
+    #plt.scatter(x, y)
+    #plt.plot(x, m*x + c, 'r', label='Fitted line')
+    #plt.show()
+    
+    return (m,c)
 
 
Index: /trunk/lib/nanownlib/tcpts.py
===================================================================
--- /trunk/lib/nanownlib/tcpts.py	(revision 20)
+++ /trunk/lib/nanownlib/tcpts.py	(revision 20)
@@ -0,0 +1,99 @@
+
+import sys
+import socket
+import queue
+import statistics
+import threading
+import json
+from .stats import OLSRegression
+
+
+def trickleHTTPRequest(ip,port,hostname):
+    my_port = None
+    try:
+        sock = socket.create_connection((ip, port))
+        my_port = sock.getsockname()[1]
+        
+        #print('.')
+        sock.sendall(b'GET / HTTP/1.1\r\n')
+        time.sleep(0.5)
+        rest = b'''Host: '''+hostname.encode('utf-8')+b'''\r\nUser-Agent: Secret Agent Man\r\nX-Extra: extra read all about it!\r\nConnection: close\r\n'''
+        for r in rest:
+            sock.sendall(bytearray([r]))
+            time.sleep(0.05)
+
+        time.sleep(0.5)
+        sock.sendall('\r\n')
+
+        r = None
+        while r != b'':
+            r = sock.recv(16)
+
+        sock.close()
+    except Exception as e:
+        pass
+
+    return my_port
+
+
+def runTimestampProbes(host_ip, port, hostname, num_trials, concurrency=4):
+    #XXX: can we use WorkerThreads for this parallel stuff?
+    myq = queue.Queue()
+    def threadWrapper(*args):
+        try:
+            myq.put(trickleHTTPRequest(*args))
+        except Exception as e:
+            sys.stderr.write("ERROR from trickleHTTPRequest: %s\n" % repr(e))
+            myq.put(None)
+
+    threads = []
+    ports = []
+    for i in range(num_trials):
+        if len(threads) >= concurrency:
+            ports.append(myq.get())
+        t = threading.Thread(target=threadWrapper, args=(host_ip, port, hostname))
+        t.start()
+        threads.append(t)
+
+    for t in threads:
+        t.join()
+
+    while myq.qsize() > 0:
+        ports.append(myq.get())
+
+    return ports
+
+
+def computeTimestampPrecision(sniffer_fp, ports):
+    rcvd = []
+    for line in sniffer_fp:
+        p = json.loads(line)
+        if p['sent']==0:
+            rcvd.append((p['observed'],p['tsval'],int(p['local_port'])))
+
+    slopes = []
+    for port in ports:
+        trcvd = [tr for tr in rcvd if tr[2]==port and tr[1]!=0]
+
+        if len(trcvd) < 2:
+            sys.stderr.write("WARN: Inadequate data points.\n")
+            continue
+        
+        if trcvd[0][1] > trcvd[-1][1]:
+            sys.stderr.write("WARN: TSval wrap.\n")
+            continue
+
+        x = [tr[1] for tr in trcvd]
+        y = [tr[0] for tr in trcvd]
+
+        slope,intercept = OLSRegression(x, y)
+        slopes.append(slope)
+
+    if len(slopes) == 0:
+        return None,None,None
+
+    m = statistics.mean(slopes)
+    if len(slopes) == 1:
+        return (m, None, slopes)
+    else:
+        return (m, statistics.stdev(slopes), slopes)
