Source Code for Module vsql_core.plan_generators.generate_ova_plan_general

 1  """ 
 2  This class is used to generate ova plan 
 3  """ 
 4   
 5  import sys_appends 
 6   
 7  from victor_plan_execution import PlanExecutor 
 8  import generate_plan 
 9  from multiprocessing import Pool 
10  import psycopg2 
11  import time 
12   
13  ######################## 
14  # Main working thread 
15 -def execute_work(work_tuple): 
16      """ 
17      Execute work function 
18      """ 
19      (plan_id, connect_str, setup_q, plan) = work_tuple 
20      plan_id = plan_id + 1 
21      pe = PlanExecutor(plan_id, connect_str, setup_q, plan) 
22      pe.go() 
23      pe.close() 
24      return plan_id 
25   
26   
27   
28  # General OVA plan 
29 -def regression_plan(connect_str, dataset,label_set, iname_table, base_mid, epochs, stepsize, diminish, dim, create_function,curried_grad, grad, obj_agg, obj_item,print_only=False, sparse_data=False, batch_plan=False, prox=None):    
30      """ 
31      Regression plan function 
32      """ 
33      work_list = [] 
34      mid       = base_mid         
35      setup_query = "DELETE FROM %s where mid=%d;\n" % (iname_table, mid)         
36      setup_query = setup_query + "INSERT INTO %s VALUES (%d, %f, %s(%d,%d));\n" % (iname_table, mid, stepsize, create_function, mid, dim) 
37      if sparse_data: 
38          data_query   = "SELECT label, k, v FROM %s" % (dataset)     
39      else: 
40          data_query   = "SELECT label, vec FROM %s" % (dataset)     
41      P = generate_plan.Python_Plan(mid, epochs, stepsize, diminish, iname_table, curried_grad, grad, data_query, prox, obj_agg, obj_item, sparse_data=sparse_data) 
42       
43      if batch_plan:         
44          plan = P.batchgradient_plan() 
45      else: 
46          plan = P.incremental_plan() 
47           
48      if print_only: 
49          print setup_query 
50          print data_query 
51          print plan 
52      else: 
53          execute_work((0, connect_str, setup_query, plan )) 
54   
55   
56  # General OVA plan 
57 -def ova_plan(connect_str, nprocesses, dataset,label_set, iname_table, base_mid, epochs, stepsize, diminish, dim, create_function,curried_grad, grad, obj_agg, obj_item,print_only=False, sparse_data=False, batch_plan=False, prox=None):    
58      """ 
59      Generates ova plan 
60      """ 
61      work_list = [] 
62      t0 = time.time() 
63      for i in label_set: 
64          # Convention is: we start at the base  mid and we make one model per class. This isn't ideal 
65          # and eventually we need a good (and probably heavier) mechansim 
66          mid       = base_mid + i         
67          setup_query = "DELETE FROM %s where mid=%d;\n" % (iname_table, mid)         
68          setup_query = setup_query + "INSERT INTO %s VALUES (%d, %f, %s(%d,%d));\n" % (iname_table, mid, stepsize, create_function, mid, dim) 
69          if sparse_data: 
70                  data_query   = "SELECT 2*CAST((label = %d) AS INT) - 1 as label, k, v FROM %s" % (i, dataset)     
71          else: 
72                  data_query   = "SELECT 2*CAST((label = %d) AS INT) - 1 as label, vec FROM %s" % (i, dataset)     
73          P    = generate_plan.Python_Plan(mid, epochs, stepsize, diminish, iname_table, curried_grad, grad, data_query, prox, obj_agg, obj_item, sparse_data=sparse_data) 
74   
75          # Generate the plan 
76          if batch_plan:         
77              plan = P.batchgradient_plan() 
78          else: 
79              plan = P.incremental_plan() 
80       
81          if print_only: 
82              print setup_query 
83              print data_query 
84              print plan 
85          else: 
86              work_list.append((i, connect_str, setup_query, plan)) 
87   
88      pool = Pool(processes=nprocesses)             
89      print pool.map(execute_work, work_list) 
90      tend = time.time() 
91      print "Completed execution of %d epochs in %f seconds" % (epochs, tend - t0) 
92