model_descriptions.linear

1 """ 2 This module contains functions for linear gradients 3 """ 4 5 import victor_utils 6 from math import * 7 8 # These are model descriptions for 9 # two different models. 10 # TODO: 11 # o FACTOR THESE INTO TWO SEPARATE FILES BASED AS DIFFERENT DESCRIPTIONS 12 # o NB: Some parts are common to many different model descriptions (e.g. hinge loss) 13 # These should go in a separate python module by type (e.g., "common objective functions") 14 # 15

16 -def ell2_ball_constraint_avg(w,w_hat, steps, x, y, mu, B, stepsize):

17 """ 18 Calculates and returns ell2 ball constraint avg 19 @type w: vector 20 @param w: model vector 21 @type w_hat: vector 22 @param w_hat: w_hat vector 23 @type steps: number 24 @param steps: number of steps 25 @type x: vector 26 @param x: feature vector of the example 27 @type y: number 28 @param y: label of the example 29 @type mu: double 30 @param mu: mu 31 @type B: double 32 @param B: B 33 @type stepsize: double 34 @param stepsize: step size 35 36 @rtype: tuple 37 @return: returns w, w_hat, # of steps, mu and B 38 """ 39 wx = victor_utils.dot(w, x) 40 err = (wx - y) 41 etd = - stepsize * err 42 victor_utils.scale_and_add_i(w, x, etd) 43 victor_utils.l2_project(w, B) 44 victor_utils.incremental_average(w_hat, w, steps) 45 return (w,w_hat, steps+1, mu, B)

46

47 -def hinge_loss(w, x, y):

48 """ 49 Calculates and returns hinge loss 50 @type w: vector 51 @param w: model vector 52 @type x: vector 53 @param x: feature vector of the example 54 @type y: number 55 @param y: label of the example 56 57 @rtype: double 58 @return: returns the hinge loss 59 """ 60 wx = victor_utils.dot(w, x) 61 return max(1 - y*wx,0)

62

63 -def sparse_hinge_loss(w,indexes,vecs, y):

64 """ 65 Calculates hinge lss for sparse vectors and returns hinge loss value 66 @type w: vector 67 @param w: model vector 68 @type indexes: vector 69 @param indexes: indexes of the feature vector of the example 70 @type vecs: vector 71 @param vecs: vector values of the feature vector of the example 72 @type y: number 73 @param y: label of the example 74 75 @rtype: double 76 @return: returns the hinge loss 77 """ 78 wx = victor_utils.dot_dss(w, indexes, vecs) 79 return max(1 - y*wx,0)

80

81 -def mse_loss(w, x, y):

82 """ 83 Calculates and returns mse loss 84 @type w: vector 85 @param w: model vector 86 @type x: vector 87 @param x: feature vector of the example 88 @type y: number 89 @param y: label of the example 90 91 @rtype: double 92 @return: returns the mse loss 93 """ 94 wx = victor_utils.dot(w, x) 95 v = wx - y 96 return v*v

97

98 -def logit_loss_ss(w, index, vecs, y):

99 """ 100 Calculates logit loss for sparse vectors and returns the value 101 @type w: vector 102 @param w: model vector 103 @type index: vector 104 @param index: indexes of the feature vector of the example 105 @type vecs: vector 106 @param vecs: vector values of the feature vector of the example 107 @type y: number 108 @param y: label of the example 109 110 @rtype: double 111 @return: returns the logit loss 112 """ 113 wx = victor_utils.dot_dss(w, index, vecs) 114 err = log( 1+ exp( - y*wx ) ) 115 return err

116

117 -def test():

118 """ 119 Tests ell2 ball constraint and hinge loss functions 120 """ 121 ell2_ball_constraint_avg([1.0,2.0,3.0], [0.0,1.0,2.0], 3, [2.0,3.0,4.0], 1, 0.1, 1.5, 0.9) 122 hinge_loss([1,2,3], [1,1,1], -1)

123 124 ######################################### 125 # NAIVELY ASSUMES CONSTANT STEP SIZE!!! 126 # THESE ARE HACKS TO MAKE L1 PROX FASTER!!! 127 ##########################################

128 -def lazy_shrink(w, current_time, timestamps, indexes, u):

129 """ 130 Lazy shrink function 131 @type w: vector 132 @param w: model vector 133 @type current_time: double 134 @param current_time: current time 135 @type timestamps: vector 136 @param timestamps: time stamps vector 137 @type indexes: vector 138 @param indexes: indexes of the example 139 @type u: double 140 @param u: u 141 """ 142 for i in indexes: 143 delta = current_time - timestamps[i] 144 ud = delta* u 145 if w[i] > ud: 146 w[i] = w[i] - ud 147 else: 148 if w[i] < -ud: 149 w[i] = w[i] + ud 150 else: 151 w[i] = 0.0

152

153 -def incremental_average_sparse(w_hat, w, steps, indexes):

154 """ 155 Incremental average sparse function 156 @type w_hat: vector 157 @param w_hat: w hat vector 158 @type w: vector 159 @param w: model vector 160 @type steps: double 161 @param steps: steps of the incremental stage 162 @type indexes: vector 163 @param indexes: indexes 164 """ 165 if steps == 0: 166 for i in range(len(w_hat)): 167 w_hat[i] = w[i] 168 else: 169 steps = float(steps) 170 for i in indexes: 171 w_hat[i] = w[i] * 1.0/steps + steps/(steps + 1) * w_hat[i]

172 173 174 ### LOGISTIC!!

175 -def l1_prox_lazy(w, what, timestamps, current_time, steps, stepsize, mu, B, indexes, vecs, y):

176 """ 177 l1 prox lazy function 178 @type w: vector 179 @param w: model vector 180 @type what: vector 181 @param what: w hat 182 @type timestamps: vector 183 @param timestamps: time stamps vector 184 @type current_time: double 185 @param current_time: current time 186 @type steps: double 187 @param steps: steps of the incremental stage 188 @type stepsize: number 189 @param stepsize: step size 190 @type mu: double 191 @param mu: mu 192 @type B: double 193 @param B: B 194 @type indexes: vector 195 @param indexes: indexes vector of the example's feature vector 196 @type vecs: vector 197 @param vecs: vector values of the example's feature vector 198 @type y: number 199 @param y: label of the example 200 """ 201 lazy_shrink(w, current_time, timestamps, indexes, stepsize*mu) 202 #lazy_average_update(w, what, timestamps, current_time, indexes) 203 # update the timestamps 204 for i in indexes: 205 timestamps[i] = current_time 206 wx = victor_utils.dot_dss(w, indexes, vecs) 207 sig = victor_utils.sigma(- wx*y) 208 victor_utils.scale_and_add_dss(w, indexes, vecs, stepsize*y*sig) 209 incremental_average_sparse(what, w, steps, indexes)

210 211

212 -def svm_l1_prox_lazy(w, what, timestamps, current_time, steps, stepsize, mu, B, indexes, vecs, y):

213 """ 214 svm l1 prox lazy function 215 @type w: vector 216 @param w: model vector 217 @type what: vector 218 @param what: w hat 219 @type timestamps: vector 220 @param timestamps: time stamps vector 221 @type current_time: double 222 @param current_time: current time 223 @type steps: double 224 @param steps: steps of the incremental stage 225 @type stepsize: number 226 @param stepsize: step size 227 @type mu: double 228 @param mu: mu 229 @type B: double 230 @param B: B 231 @type indexes: vector 232 @param indexes: indexes vector of the example's feature vector 233 @type vecs: vector 234 @param vecs: vector values of the example's feature vector 235 @type y: number 236 @param y: label of the example 237 """ 238 lazy_shrink(w, current_time, timestamps, indexes, stepsize*mu) 239 wx = victor_utils.dot_dss(w, indexes, vecs) 240 if(y * wx < 1): 241 victor_utils.scale_and_add_dss(w, indexes, vecs, stepsize*y) 242 incremental_average_sparse(what, w, steps, indexes)

243 244

245 -def svm_l2_prox_lazy(w, what, wscale, steps, stepsize, mu, B, indexes, vecs, y):

246 """ 247 svm l2 prox lazy function 248 @type w: vector 249 @param w: model vector 250 @type what: vector 251 @param what: w hat 252 @type wscale: double 253 @param wscale: used to scale model vector(w) 254 @type steps: double 255 @param steps: steps of the incremental stage 256 @type stepsize: number 257 @param stepsize: step size 258 @type mu: double 259 @param mu: mu 260 @type B: double 261 @param B: B 262 @type indexes: vector 263 @param indexes: indexes vector of the example's feature vector 264 @type vecs: vector 265 @param vecs: vector values of the example's feature vector 266 @type y: number 267 @param y: label of the example 268 """ 269 wscale = 1/(1+stepsize*mu) 270 if(wscale < 1e-10): 271 victor_utils.scale(w,wscale) 272 wscale = 1.0 273 274 wx = victor_utils.dot_dss(w, indexes, vecs) 275 if(y * wx < 1): 276 victor_utils.scale_and_add_dss(w, indexes, vecs, stepsize*y) 277 incremental_average_sparse(what, w, steps, indexes)

278

Source Code for Module model_descriptions.linear_gradients