Changeset 22 for trunk/lib

Timestamp:

11/09/15 20:37:05 (9 years ago)

Author:

tim

Message:

Fixed a bug in processPackets where trim parameters weren't passed through

Reorganized analyzeProbes to work as a O(max(S,R)) algorithm, rather than O(S*R)

Sped up addTrimAnalyses by using a new _insertMany primative

Location:

trunk/lib/nanownlib

Files:

• __init__.py (modified) (4 diffs)
• storage.py (modified) (2 diffs)

trunk/lib/nanownlib/__init__.py

@@ -206 +206 @@
     cursor.execute(query, params)
     differences = [row[0] for row in cursor]
-    
+
     return septasummary(differences),mad(differences)
 
@@ -253 +253 @@
         for probe_id,packets in packet_cache:
             try:
-                analysis,s,r = analyzePackets(packets, timestamp_precision)
+                analysis,s,r = analyzePackets(packets, timestamp_precision,strim,rtrim)
                 analysis['probe_id'] = probe_id
                 analyses.append(analysis)
@@ -261 +261 @@
                 #traceback.print_exc()
                 sys.stderr.write("WARN: couldn't find enough packets for probe_id=%s\n" % probe_id)
+
+        start = time.time()    
         db.addTrimAnalyses(analyses)
         db.conn.commit()
+        print("addTrimAnalyses: %f" % (time.time()-start))
+
         return statistics.mode(sent_tally),statistics.mode(rcvd_tally)
     
@@ -275 +279 @@
         num_sent,num_rcvd = processPackets(packet_cache, 0, 0)
         print("num_sent: %d, num_rcvd: %d" % (num_sent,num_rcvd))
-    
-        for strim in range(0,num_sent):
-            for rtrim in range(0,num_rcvd):
-                #print(strim,rtrim)
-                if strim == 0 and rtrim == 0:
-                    continue # no point in doing 0,0 again
-                processPackets(packet_cache, strim, rtrim)
-
-    
         unusual_case,delta = findUnusualTestCase(db, (0,0))
-        evaluations = {}
-        for strim in range(0,num_sent):
-            for rtrim in range(0,num_rcvd):
-                evaluations[(strim,rtrim)] = evaluateTrim(db, unusual_case, strim, rtrim)
-
-        import pprint
-        pprint.pprint(evaluations)
-
+        print("unusual_case: %s, delta: %f" % (unusual_case,delta))
+        
         delta_margin = 0.15
         best_strim = 0
         best_rtrim = 0
-        good_delta,good_mad = evaluations[(0,0)]
-    
+        
+        good_delta,good_mad = evaluateTrim(db, unusual_case, best_strim, best_rtrim)
+        print("trim (%d,%d): delta=%f, mad=%f" % (best_strim,best_rtrim, good_delta, good_mad))
+        
         for strim in range(1,num_sent):
-            delta,mad = evaluations[(strim,0)]
+            processPackets(packet_cache, strim, best_rtrim)
+            delta,mad = evaluateTrim(db, unusual_case, strim, best_rtrim)
+            print("trim (%d,%d): delta=%f, mad=%f" % (strim,best_rtrim, delta, mad))
             if delta*good_delta > 0.0 and (abs(good_delta) - abs(delta)) < abs(delta_margin*good_delta) and mad < good_mad:
                 best_strim = strim
+                good_delta,good_mad = delta,mad
             else:
                 break
 
-        good_delta,good_mad = evaluations[(best_strim,0)]
         for rtrim in range(1,num_rcvd):
-            delta,mad = evaluations[(best_strim,rtrim)]
+            processPackets(packet_cache, best_strim, rtrim)
+            delta,mad = evaluateTrim(db, unusual_case, best_strim, rtrim)
+            print("trim (%d,%d): delta=%f, mad=%f" % (best_strim, rtrim, delta, mad))            
             if delta*good_delta > 0.0 and (abs(good_delta) - abs(delta)) < abs(delta_margin*good_delta) and mad < good_mad:
                 best_rtrim = rtrim

trunk/lib/nanownlib/storage.py

@@ -189 +189 @@
         placeholders = ':'+', :'.join(keys)
         query = "INSERT INTO %s (id,%s) VALUES ('%s',%s)" % (table, columns, rid, placeholders)
-        #print(row)
+        #print(query,row)
         self.conn.execute(query, row)
         return rid
 
+    def _insertMany(self, table, rows):
+        if len(rows) < 1:
+            return
+        
+        keys = rows[0].keys()
+        columns = ','.join(keys)
+        placeholders = ':'+', :'.join(keys)
+        query = "INSERT INTO %s (id,%s) VALUES (hex(randomblob(16)),%s)" % (table, columns, placeholders)
+        #print(query,row)
+        self.conn.executemany(query, rows)
+    
     def addMeta(self, meta):
         ret_val = self._insert('meta', meta)
@@ -220 +231 @@
 
     def addTrimAnalyses(self, analyses):
-        return [self._insert('trim_analysis', row) for row in analyses]
+        self._insertMany('trim_analysis', analyses)
 
     def addClassifierResult(self, results):