Changeset 7 for trunk

Timestamp:

07/09/15 12:58:10 (9 years ago)

Author:

tim

Message:

.

Location:

trunk

Files:

• bin/train (modified) (2 diffs)
• lib/nanownlib/stats.py (modified) (2 diffs)
• lib/nanownlib/storage.py (modified) (3 diffs)

trunk/bin/train

@@ -323 +323 @@
     
     start = time.time()
-    wt = WorkerThreads(2, trainAux)
+    wt = WorkerThreads(4, trainAux)
     
     num_trials = 20
@@ -429 +429 @@
 
 start = time.time()
-#cProfile.run('results = trainMidhinge(db, unusual_case, greater, 100)')
-results = trainMidhinge(db, unusual_case, greater, 100)
+cProfile.run('results = trainMidhinge(db, unusual_case, greater, 100)')
+#results = trainMidhinge(db, unusual_case, greater, 100)
 #db.addClassifierResults(results)
 print("midhinge result:", results)

trunk/lib/nanownlib/stats.py

@@ -229 +229 @@
 
 def subseries(db, probe_type, unusual_case, size=None, offset=None, field='packet_rtt'):
-    cursor = db.conn.cursor()
-    cursor.execute("SELECT max(c) FROM (SELECT count(sample) c FROM probes WHERE type=? GROUP BY test_case)", (probe_type,))
-    population_size = cursor.fetchone()[0]
+    population_size = db.populationSize(probe_type)
 
     if size == None or size > population_size:
@@ -248 +246 @@
     
     params = {"probe_type":probe_type, "unusual_case":unusual_case, "offset":offset, "size":size}
+    cursor = db.conn.cursor()
     cursor.execute(query, params)
-    for row in cursor:
-        size -= 1
-        yield dict(row)
-
+    ret_val = [dict(row) for row in cursor.fetchall()]
+    #for row in cursor:
+    #    size -= 1
+    #    yield dict(row)
+
+    size -= len(ret_val)
     if size > 0:
         params['offset'] = 0
         params['size'] = size
         cursor.execute(query, params)
-        for row in cursor:
-            yield dict(row)
-    
+        ret_val += [dict(row) for row in cursor.fetchall()]
+        #for row in cursor:
+        #    yield dict(row)
+    
+    return ret_val
+
 
 # if test_cases=None, include all of them.  Otherwise, include only the specified test cases.

trunk/lib/nanownlib/storage.py

@@ -14 +14 @@
     conn = None
     cursor = None
+    _population_sizes = None
+    
     def __init__(self, path):
         exists = os.path.exists(path)
@@ -19 +21 @@
         self.conn.execute("PRAGMA foreign_keys = ON;")
         self.conn.row_factory = sqlite3.Row
+        self._population_sizes = {}
         
         if not exists:
@@ -86 +89 @@
             self.conn.close()
 
+    def populationSize(self, probe_type):
+        if probe_type in self._population_sizes:
+            return self._population_sizes[probe_type]
+
+        try:
+            cursor = self.conn.cursor()
+            cursor.execute("SELECT max(c) FROM (SELECT count(sample) c FROM probes WHERE type=? GROUP BY test_case)", (probe_type,))
+            self._population_sizes[probe_type] = cursor.fetchone()[0]
+            return self._population_sizes[probe_type]
+        except Exception as e:
+            print(e)
+            return 0
+        
     def _insert(self, table, row):
         rid = _newid()