%PDF-1.3 1 0 obj << /Kids [ 3 0 R 4 0 R 5 0 R 6 0 R 7 0 R 8 0 R 9 0 R 10 0 R 11 0 R 12 0 R ] /Type /Pages /Count 10 >> endobj 2 0 obj << /Title (Geometry Aware Constrained Optimization Techniques for Deep Learning) /Producer (PyPDF2) /Author (Soumava Kumar Roy\054 Zakaria Mhammedi\054 Mehrtash Harandi) /Subject (2018 IEEE Conference on Computer Vision and Pattern Recognition) >> endobj 3 0 obj << /Parent 1 0 R /Rotate 0 /Type /Page /Contents 14 0 R /Resources << /XObject << /x8 15 0 R /x6 18 0 R /x12 21 0 R /x10 24 0 R >> /ExtGState << /s9 27 0 R /s11 30 0 R /a0 << /CA 1 /ca 1 >> /R20 33 0 R /s5 34 0 R /s7 37 0 R >> /Font << /F2 40 0 R /R47 41 0 R /R45 44 0 R /R43 48 0 R /R41 51 0 R /R29 55 0 R /F1 59 0 R /R39 60 0 R /R21 65 0 R /R37 69 0 R /R23 74 0 R /R35 79 0 R /R25 84 0 R /R33 88 0 R /R27 92 0 R /R31 96 0 R >> /ProcSet [ /ImageC /Text /PDF /ImageI /ImageB ] >> /MediaBox [ 0 0 612 792 ] /Annots [ 100 0 R 101 0 R 102 0 R 103 0 R 104 0 R 105 0 R 106 0 R 107 0 R 108 0 R 109 0 R 110 0 R ] >> endobj 4 0 obj << /Parent 1 0 R /Rotate 0 /Type /Page /Contents 111 0 R /Resources << /XObject << /R67 112 0 R /R68 113 0 R >> /ExtGState << /R20 33 0 R >> /Font << /F2 114 0 R /R79 115 0 R /R73 119 0 R /R71 122 0 R /R77 125 0 R /R75 129 0 R /R29 55 0 R /F1 132 0 R /R21 65 0 R /R81 133 0 R /R69 136 0 R /R25 84 0 R /R27 92 0 R >> /ProcSet [ /Text /ImageC /ImageB /PDF /ImageI ] >> /MediaBox [ 0 0 612 792 ] /Annots [ 140 0 R 141 0 R 142 0 R 143 0 R 144 0 R 145 0 R 146 0 R 147 0 R 148 0 R 149 0 R 150 0 R 151 0 R 152 0 R ] >> endobj 5 0 obj << /Parent 1 0 R /Rotate 0 /Type /Page /Contents 153 0 R /Resources << /ExtGState << /R20 33 0 R >> /Font << /R47 41 0 R /R94 154 0 R /R45 44 0 R /R43 48 0 R /R41 51 0 R /F2 157 0 R /F1 158 0 R /R39 60 0 R /R21 65 0 R /R37 69 0 R /R23 74 0 R /R35 79 0 R /R25 84 0 R /R33 88 0 R /R27 92 0 R /R31 96 0 R >> /ProcSet [ /ImageC /Text /PDF /ImageI /ImageB ] >> /MediaBox [ 0 0 612 792 ] /Annots [ 159 0 R 160 0 R 161 0 R 162 0 R 163 0 R ] >> endobj 6 0 obj << /Parent 1 0 R /Rotate 0 /Type /Page /Contents 164 0 R /Resources << /ExtGState << /R20 33 0 R >> /Font << /R47 41 0 R /R45 44 0 R /R43 48 0 R /R41 51 0 R /F2 165 0 R /F1 166 0 R /R94 154 0 R /R118 167 0 R /R116 170 0 R /R114 173 0 R /R39 60 0 R /R21 65 0 R /R37 69 0 R /R23 74 0 R /R35 79 0 R /R25 84 0 R /R33 88 0 R /R27 92 0 R /R31 96 0 R >> /ProcSet [ /ImageC /Text /PDF /ImageI /ImageB ] >> /MediaBox [ 0 0 612 792 ] /Annots [ 177 0 R 178 0 R 179 0 R 180 0 R 181 0 R 182 0 R 183 0 R 184 0 R 185 0 R 186 0 R 187 0 R 188 0 R 189 0 R ] >> endobj 7 0 obj << /Parent 1 0 R /Rotate 0 /Type /Page /Contents 190 0 R /Resources << /ExtGState << /R20 33 0 R >> /Font << /R47 41 0 R /R45 44 0 R /R43 48 0 R /R41 51 0 R /F2 191 0 R /F1 192 0 R /R135 193 0 R /R137 197 0 R /R94 154 0 R /R118 167 0 R /R116 170 0 R /R39 60 0 R /R21 65 0 R /R37 69 0 R /R23 74 0 R /R35 79 0 R /R25 84 0 R /R33 88 0 R /R27 92 0 R /R31 96 0 R >> /ProcSet [ /ImageC /Text /PDF /ImageI /ImageB ] >> /MediaBox [ 0 0 612 792 ] /Annots [ 201 0 R 202 0 R 203 0 R 204 0 R 205 0 R 206 0 R 207 0 R 208 0 R 209 0 R 210 0 R ] >> endobj 8 0 obj << /Parent 1 0 R /Rotate 0 /Type /Page /Contents 211 0 R /Resources << /ColorSpace << /R158 212 0 R >> /ExtGState << /R20 33 0 R >> /Font << /R181 214 0 R /R45 44 0 R /R43 48 0 R /R77 125 0 R /R41 51 0 R /F2 218 0 R /F1 219 0 R /R159 220 0 R /R163 224 0 R /R161 228 0 R /R155 231 0 R /R167 235 0 R /R179 239 0 R /R165 243 0 R /R175 247 0 R /R169 251 0 R /R177 255 0 R /R171 259 0 R /R173 263 0 R /R39 60 0 R /R21 65 0 R /R37 69 0 R /R23 74 0 R /R35 79 0 R /R25 84 0 R /R33 88 0 R /R27 92 0 R /R31 96 0 R >> /ProcSet [ /ImageC /Text /PDF /ImageI /ImageB ] >> /MediaBox [ 0 0 612 792 ] /Annots [ 267 0 R 268 0 R 269 0 R 270 0 R 271 0 R 272 0 R 273 0 R 274 0 R 275 0 R 276 0 R 277 0 R ] >> endobj 9 0 obj << /Parent 1 0 R /Rotate 0 /Type /Page /Contents 278 0 R /Resources << /ExtGState << /R20 33 0 R >> /Font << /R47 41 0 R /R43 48 0 R /R45 44 0 R /R116 170 0 R /R41 51 0 R /F2 279 0 R /F1 280 0 R /R39 60 0 R /R21 65 0 R /R37 69 0 R /R23 74 0 R /R35 79 0 R /R25 84 0 R /R33 88 0 R /R27 92 0 R /R31 96 0 R >> /ProcSet [ /ImageC /Text /PDF /ImageI /ImageB ] >> /MediaBox [ 0 0 612 792 ] /Annots [ 281 0 R 282 0 R 283 0 R 284 0 R 285 0 R 286 0 R 287 0 R 288 0 R 289 0 R 290 0 R 291 0 R 292 0 R 293 0 R 294 0 R 295 0 R 296 0 R 297 0 R 298 0 R 299 0 R 300 0 R 301 0 R ] >> endobj 10 0 obj << /Parent 1 0 R /Rotate 0 /Type /Page /Contents 302 0 R /Resources << /ExtGState << /R20 33 0 R >> /Font << /R238 303 0 R /F2 306 0 R /F1 307 0 R /R21 65 0 R /R23 74 0 R /R25 84 0 R /R27 92 0 R >> /ProcSet [ /ImageC /Text /PDF /ImageI /ImageB ] >> /MediaBox [ 0 0 612 792 ] /Annots [ 308 0 R 309 0 R 310 0 R 311 0 R 312 0 R 313 0 R 314 0 R 315 0 R 316 0 R 317 0 R 318 0 R 319 0 R 320 0 R 321 0 R 322 0 R 323 0 R 324 0 R 325 0 R 326 0 R 327 0 R 328 0 R 329 0 R 330 0 R ] >> endobj 11 0 obj << /Parent 1 0 R /Rotate 0 /Type /Page /Contents 331 0 R /Resources << /ExtGState << /R20 33 0 R >> /Font << /R77 125 0 R /F2 332 0 R /F1 333 0 R /R21 65 0 R /R25 84 0 R /R27 92 0 R >> /ProcSet [ /ImageC /Text /PDF /ImageI /ImageB ] >> /MediaBox [ 0 0 612 792 ] /Annots [ 334 0 R 335 0 R 336 0 R 337 0 R 338 0 R 339 0 R 340 0 R 341 0 R 342 0 R 343 0 R 344 0 R 345 0 R 346 0 R 347 0 R 348 0 R 349 0 R 350 0 R 351 0 R 352 0 R 353 0 R 354 0 R 355 0 R 356 0 R 357 0 R 358 0 R 359 0 R 360 0 R 361 0 R 362 0 R 363 0 R 364 0 R 365 0 R 366 0 R 367 0 R 368 0 R 369 0 R 370 0 R 371 0 R 372 0 R 373 0 R 374 0 R 375 0 R 376 0 R 377 0 R 378 0 R ] >> endobj 12 0 obj << /Parent 1 0 R /Rotate 0 /Type /Page /Contents 379 0 R /Resources << /ExtGState << /R20 33 0 R >> /Font << /F2 380 0 R /F1 381 0 R /R25 84 0 R /R27 92 0 R >> /ProcSet [ /ImageC /Text /PDF /ImageI /ImageB ] >> /MediaBox [ 0 0 612 792 ] /Annots [ 382 0 R 383 0 R 384 0 R 385 0 R 386 0 R 387 0 R 388 0 R 389 0 R 390 0 R 391 0 R 392 0 R 393 0 R 394 0 R 395 0 R 396 0 R 397 0 R 398 0 R ] >> endobj 13 0 obj << /Type /Catalog /Pages 1 0 R >> endobj 14 0 obj << /Length 19681 >> stream q q q 0.1 0 0 0.1 0 0 cm /R20 gs 0 g q 10 0 0 10 0 0 cm BT /R21 14.3462 Tf 1 0 0 1 69.216 675.067 Tm [ (Geometry) -249.996 (A) 90 (war) 17.9972 (e) -249.999 (Constrained) -250.005 (Optimization) -249.99 (T) 91.9897 (echniques) -250.012 (f) 24.9923 (or) -249.995 (Deep) -250.008 (Lear) 14.9893 (ning) ] TJ /R23 9.9626 Tf 456.792 5.20586 Td <03> Tj /R25 11.9552 Tf -399.222 -53.0262 Td [ (Souma) 20.0149 (v) 24.9844 (a) -249.997 (K) 14.9984 (umar) -249.991 (Ro) 10.0105 (y) ] TJ /R25 7.9701 Tf 102.766 4.33828 Td (1) Tj /R25 11.9552 Tf 4.4832 -4.33828 Td [ (\054) -250.012 (Zakaria) -250.002 (Mhammedi) ] TJ /R25 7.9701 Tf 100.602 4.33828 Td (1\0542) Tj /R25 11.9552 Tf 10.4609 -4.33828 Td [ (\054) -250.012 (and) -249.987 (Mehrtash) -250.014 (Harandi) ] TJ /R25 7.9701 Tf 112.879 4.33828 Td (1\0542) Tj -285.492 -23.9102 Td (1) Tj /R25 11.9552 Tf 4.48281 -4.33906 Td [ (Australian) -249.997 (National) -250 (Uni) 24.9957 (v) 14.9851 (ersity) 64.9887 (\054) -250.012 (Canberra\054) -249.995 (Australia) ] TJ /R25 7.9701 Tf 30.9871 -9.60898 Td (2) Tj /R25 11.9552 Tf 4.48281 -4.33789 Td [ (Data61\054) -250.006 (CSIR) 40.0135 (O\054) -250.01 (Canberra\054) -249.995 (Australia) ] TJ /R25 7.9701 Tf -83.716 -9.61016 Td (1) Tj /R23 11.9552 Tf 4.48398 -4.33789 Td (f) Tj /R25 11.9552 Tf 5.97695 0 Td [ (souma) 20.0118 (v) 24.9844 (a\056kumarro) 9.99418 (y) 65.0091 (\054) -250.012 (zak\056mhammedi\054) -249.997 (mehrtash\056harandi) ] TJ /R23 11.9552 Tf 258.541 0 Td (g) Tj /R25 11.9552 Tf 5.97695 0 Td (\100anu\056edu\056au) Tj /R21 11.9552 Tf -257.706 -49.8129 Td (Abstract) Tj /R27 9.9626 Tf -83.9281 -24.3512 Td [ (In) -271.992 (this) -272.007 (paper) 111.018 (\054) -277.01 (we) -271.985 (g) 10.0032 (ener) 15.0196 (alize) -272.012 (the) ] TJ /R25 9.9626 Tf 128.128 0 Td [ (Stochastic) -271.994 (Gradient) -272.018 (De\055) ] TJ -140.083 -11.9547 Td (scent) Tj /R27 9.9626 Tf 22.9129 0 Td [ (\050SGD\051) -245.014 (and) -245.003 (RMSPr) 44.9937 (op) -245.008 (algorithms) -245 (to) -245.006 (the) -245.011 (setting) -244.984 (of) -245.006 (Rie\055) ] TJ -22.9129 -11.9563 Td [ (mannian) -295.014 (optimization\056) -444.991 (SGD) -294.999 (is) -294.996 (a) -295.005 (popular) -295.01 (method) -294.99 (for) -294.992 (lar) 36.9914 (g) 10.0032 (e) ] TJ 11.9551 TL T* [ (scale) -312.017 (optimization\056) -494.982 (In) -311.986 (particular) 111.011 (\054) -327.002 (it) -312.012 (is) -310.993 (widely) -312.002 (used) -312.014 (to) -312.014 (tr) 14.9901 (ain) ] TJ T* [ (the) -414.982 (weights) -414.009 (of) ] TJ /R25 9.9626 Tf 64.2199 0 Td [ (Deep) -414.991 (Neural) -414.006 (Netw) 10.0069 (orks) ] TJ /R27 9.9626 Tf 95.0281 0 Td [ (\056) -803.981 (Howe) 14.995 (ver) 110.999 (\054) -456 (gr) 14.9901 (adi\055) ] TJ -159.248 -11.9551 Td [ (ents) -405.997 (computed) -405.01 (using) -405.982 (standar) 37.0134 (d) -404.986 (SGD) -406 (can) -404.986 (have) -406.008 (lar) 36.9914 (g) 10.0032 (e) -406.016 (vari\055) ] TJ T* [ (ance) 10 (\054) -348.998 (whic) 14.9987 (h) -329 (is) -328.99 (detrimental) -330.018 (for) -328.986 (the) -328.997 (con) 39.9982 (ver) 37.011 (g) 10.0032 (ence) -329.011 (r) 14.9828 (ate) -328.997 (of) -330.011 (the) ] TJ T* [ (algorithm\056) -707.986 (Other) -383.017 (methods) -383.007 (suc) 14.9852 (h) -382.01 (as) -382.984 (RMSPr) 44.9925 (op) -382.985 (and) -383.019 (AD) 34.9819 (AM) ] TJ 11.9559 TL T* [ (addr) 36.9951 (ess) -330.996 (this) -331.017 (issue) 14.9975 (\056) -553.984 (Ne) 14.9865 (vertheless\054) -351.016 (these) -330.994 (methods) -331.996 (cannot) -330.982 (be) ] TJ 11.9551 TL T* [ (dir) 36.9926 (ectly) -360.997 (applied) -361.007 (to) -362.006 (constr) 15.0024 (ained) -360.98 (optimization) -361.008 (pr) 44.9851 (oblems\056) -643.986 (In) ] TJ T* [ (this) -274.007 (pap) -1.00412 (e) 1.01515 (r) 111.007 (\054) -281.009 (we) -275.004 (e) 19.9918 (xtend) -273.985 (some) -275.003 (popular) -273.995 (optimization) -274.984 (algorithm) ] TJ T* [ (to) -378.003 (the) -378.988 (Riemannian) -378.013 (\050constr) 15 (ained\051) -378 (setting) 15.0122 (\056) -695.019 (W) 91.9871 (e) -379 (substantiate) ] TJ T* [ (our) -286.015 (pr) 44.9839 (oposed) -286.989 (e) 19.9918 (xtensions) -285.998 (with) -285.998 (a) -287.006 (r) 14.984 (ang) 10.013 (e) -285.996 (of) -286.018 (r) 37.0183 (ele) 15.0147 (vant) -286.999 (pr) 44.9851 (oblems) ] TJ 11.9559 TL T* [ (in) -390.001 (mac) 14.9816 (hine) -388.981 (learning) -390.017 (suc) 14.9852 (h) -388.989 (as) -390.002 (incr) 36.9877 (emental) -389.014 (Principal) -390.011 (Com\055) ] TJ 11.9551 TL T* [ (ponent) -304.985 (Analysis\054) -317.986 (computating) -304.997 (the) -305 (Riemannian) -305.005 (centr) 44.9827 (oids) -304.986 (of) ] TJ T* [ (SPD) -213.992 (matrices\054) -220.994 (and) -213.009 (Deep) -214.005 (Metric) -213.01 (Learning) 14.9828 (\056) -297.987 (W) 91.9871 (e) -214.008 (ac) 15.0171 (hie) 14.9852 (ve) -214.013 (com\055) ] TJ T* [ (petitive) -320.984 (r) 37.0196 (esults) -319.998 (a) 10.0032 (gainst) -321.009 (the) -320.998 (state) -320.984 (of) -320.013 (the) -320.998 (art) -320.988 (for) -320.988 <026e652d6772> 14.9926 (ained) ] TJ T* [ (object) -249.982 (r) 37.0196 (eco) 9.99466 (gnition) -249.995 (datasets\056) ] TJ /R21 11.9552 Tf 37.1871 TL T* [ (1\056) -249.99 (Intr) 18.0146 (oduction) ] TJ /R25 9.9626 Tf 11.9547 -19.3699 Td [ (The) -418.985 <02656c64> -417.987 (of) -418.988 (machine) -419.012 (learning) -419.011 (and) -418.004 (computer) -419.006 (vision) -418.994 (is) ] TJ -11.9547 -11.9551 Td [ (ab) 19.9973 (undant) -403.008 (with) -403.009 (problems) -402.998 (that) -401.997 (can) -402.995 <62656e650274> -403.009 (from) -402.982 (the) -402.984 (use) -402.984 (of) ] TJ 11.9551 TL T* [ (constraints) -382.997 (to) -382.983 (obtain) -383.01 (optimal) -384.014 (solutions\056) -709.002 (F) 14.9926 (or) -382.992 (e) 15.0122 (xample\054) -417.006 (or) 20 (\055) ] TJ 11.9559 TL T* [ (thogonality) -253.982 (constraints) -255.019 (ha) 19.9979 (v) 14.9828 (e) -254.982 (sho) 24.9922 (wn) -253.989 (to) -255.004 (be) -253.997 (v) 14.9828 (ery) -255.019 (useful) -253.982 (in) -255.004 (lin\055) ] TJ 11.9551 TL T* [ (ear) -341.014 (dimensionality) -341.99 (reduction) -341.019 (algorithms) -342.014 (such) -340.99 (as) ] TJ /R27 9.9626 Tf 198.612 0 Td (Principal) Tj -198.612 -11.9551 Td [ (Component) -315.982 (Analysis) ] TJ /R25 9.9626 Tf 85.9957 0 Td [ (\050PCA\051) -631.988 (\133) ] TJ ET Q 0 1 0 rg q 10 0 0 10 0 0 cm BT /R25 9.9626 Tf 1 0 0 1 171.734 181.706 Tm (4) Tj ET Q 0 g q 10 0 0 10 0 0 cm BT /R25 9.9626 Tf 1 0 0 1 176.715 181.706 Tm [ (\135\056) -507.995 (Ov) 14.9852 (er) -315.994 (the) -316.018 (past) -316.011 (fe) 25.0081 (w) -315.984 (years\054) ] TJ -126.603 -11.9551 Td [ (deep) -283.01 (learning) -282.014 (has) -283.004 (led) -283.004 (to) -283 (tremendous) -281.998 (impro) 15.0048 (v) 14.9828 (ement) -282.997 (in) -283 (man) 14.9901 (y) ] TJ 11.9551 TL T* [ (applications) -333.981 (such) -332.991 (as) -333.982 (image) -334.007 (and) -332.997 (object) -334.007 (recognition\054) -355.002 (speech) ] TJ T* [ (recognition\054) -226.003 (natural) -219.01 (language) -219.988 (processing\054) -226.011 (a) 1.01454 (n) -1.01454 (d) -219.017 (content) -219.983 (based) ] TJ 11.9563 TL T* [ (retrie) 25.0148 (v) 24.9811 (al) -281.989 (systems) -283.013 (\133) ] TJ ET Q 0 1 0 rg q 10 0 0 10 0 0 cm BT /R25 9.9626 Tf 1 0 0 1 123.307 133.885 Tm (33) Tj ET Q 0 g q 10 0 0 10 0 0 cm BT /R25 9.9626 Tf 1 0 0 1 133.27 133.885 Tm (\135\056) Tj /R27 9.9626 Tf 9.86289 0 Td [ (Deep) -281.993 (Metric) -282.998 (Learning) ] TJ /R25 9.9626 Tf 92.5551 0 Td [ (\050DML\051) -281.98 (\133) ] TJ ET Q 0 1 0 rg q 10 0 0 10 0 0 cm BT /R25 9.9626 Tf 1 0 0 1 270.587 133.885 Tm (27) Tj ET Q 0 g q 10 0 0 10 0 0 cm BT /R25 9.9626 Tf 1 0 0 1 280.549 133.885 Tm (\135\054) Tj -230.437 -11.9551 Td [ (for) -285.993 (e) 15.0128 (xample\054) -296.006 (has) -285.984 (recei) 25.0069 (v) 14.9828 (ed) -287.01 (a) -285.996 <7369676e690263616e74> -287.002 (amount) -286.001 (of) -287.008 (attention) ] TJ 11.9547 TL T* [ (lately) 64.9873 (\054) -259.993 (and) -257.01 (it) -257.982 (is) -257.002 (well) -257.984 (kno) 24.9909 (wn) -257.989 (that) -257.001 (learning) -258.018 (a) -256.982 (metric) -257.981 (can) -257.999 (bene\055) ] TJ ET Q 3.98 w 0 G 501.121 998.789 m 1446.11 998.789 l S q 10 0 0 10 0 0 cm BT /R29 5.9776 Tf 1 0 0 1 60.141 93.277 Tm [ <03> -0.90058 ] TJ /R25 7.9701 Tf 4.3168 -2.81211 Td [ (This) -413.982 (w) 10.0129 (ork) -414.008 (w) 10.0121 (as) -413.995 (supported) -413.988 (under) -412.986 (the) -413.997 (Australian) -414.008 (Research) -414.014 (Council\140s) ] TJ -14.3457 -9.46484 Td [ (Disco) 15.0105 (v) 14.9844 (ery) -249.996 (Projects) -250.01 (funding) -250 (scheme) -250.008 (\050project) -250.019 (DP150104645\051) ] TJ /R25 9.9626 Tf 258.75 444.628 Td [ <0274> -254.014 (from) -253.987 (the) -253.989 (use) -253.989 (of) -253.994 (orthogonality) -254.021 (constraints) -253.997 (\133) ] TJ ET Q 0 1 0 rg q 10 0 0 10 0 0 cm BT /R25 9.9626 Tf 1 0 0 1 488.179 525.628 Tm (41) Tj ET Q 0 g q 10 0 0 10 0 0 cm BT /R25 9.9626 Tf 1 0 0 1 498.141 525.628 Tm [ (\135\056) -321.985 (Ho) 24.986 (we) 25.0154 (v) 14.9828 (er) 39.9835 (\054) ] TJ -189.279 -11.9563 Td [ (most) -323.995 (popular) -322.985 (optimizers) -323.988 (which) -324.01 (use) -324.019 (a) -323.01 (v) 24.9811 (ariant) -324.015 (of) ] TJ /R27 9.9626 Tf 195.443 0 Td [ (Stoc) 14.9926 (hastic) ] TJ -195.443 -11.9547 Td [ (Gr) 14.9877 (adient) -256.011 (Descent) ] TJ /R25 9.9626 Tf 73.1969 0 Td [ (\050SGD\051) -255.982 (cannot) -255.982 (directly) -256.016 (be) -255.996 (applied) -256.016 (to) -257.006 (con\055) ] TJ -73.1969 -11.9551 Td [ (strained) -262.995 (optimization\056) -349.998 (W) 79.9866 (e) -263.02 (attempt) -262.986 (to) -263.005 (bridge) -263 (this) -262.99 (g) 4.98446 (ap) -263.995 (by) -263.005 (e) 15.0122 (x\055) ] TJ 11.9551 TL T* [ (tending) -258.006 (certain) -259.013 (popular) -258.016 (optimization) -258.016 (algorithms) -257.986 (to) -259.006 (enhance) ] TJ T* [ (their) -443.992 (capability) -442.992 (in) -443.992 (handling) -444.009 (constraints) -442.985 (on) -443.995 (the) -443.995 (parameter) ] TJ T* [ (space\056) -577.002 (W) 79.9866 (e) -339.007 (be) 15.0171 (gin) -338.985 (our) -338.995 (formulation) -338.997 (by) -340.012 (studying) -338.997 (the) -338.992 (popular) ] TJ 11.9563 TL T* [ (class) -360.009 (of) -360.016 (empirical) -361.016 (risk) -360.004 (minimization) -359.989 (problems) -359.984 (in) -360.009 (the) -360.989 (Rie\055) ] TJ 11.9551 TL T* [ (mannian) -240.994 (setting\054) -242.982 (where) -240.984 (the) -241.989 (objecti) 25 (v) 14.9828 (e) -240.984 (can) -240.984 (tak) 10.0081 (e) -240.984 (the) -241.989 (follo) 24.9958 (wing) ] TJ (form) ' /R31 9.9626 Tf 58.8039 -28.473 Td [ (ar) -0.69841 (g) -180 (m) -1.29145 (i) -0.79889 (n) -0.60039 ] TJ /R33 6.9738 Tf 6.45117 -8.44102 Td [ <12> -0.70017 ] TJ /R35 6.9738 Tf 4.63164 0 Td [ (2) -0.80169 (M) -0.70017 ] TJ /R31 9.9626 Tf 24.5402 15.1809 Td (1) Tj ET Q 4027.89 3920.49 m 4087.69 3920.49 l S q 10 0 0 10 0 0 cm BT /R37 9.9626 Tf 1 0 0 1 402.789 382.724 Tm [ (n) -0.1985 ] TJ /R39 6.9738 Tf 13.5691 19.2871 Td [ (n) -0.20305 ] TJ /R41 9.9626 Tf -4.7332 -2.98789 Td [ (X) -0.49992 ] TJ /R39 6.9738 Tf 0.74297 -21.2199 Td [ (i) -0.19605 ] TJ /R43 6.9738 Tf 2.81797 0 Td [ (\0751) -0.49712 ] TJ /R37 9.9626 Tf 12.4902 11.7547 Td [ (J) -0.49992 ] TJ /R31 9.9626 Tf 6.48281 0 Td [ (\050) -0.90181 ] TJ /R45 9.9626 Tf 3.87422 0 Td [ (y) -0.29897 ] TJ /R39 6.9738 Tf 6.25 -2.43477 Td [ (i) -0.19605 ] TJ /R37 9.9626 Tf 3.3168 2.43477 Td [ (\073) -167.791 (f) -0.59794 ] TJ /R33 6.9738 Tf 9.30508 -1.49375 Td [ <12> -0.70017 ] TJ /R31 9.9626 Tf 5.13008 1.49375 Td [ (\050) -0.90181 ] TJ /R45 9.9626 Tf 3.875 0 Td (x) Tj /R39 6.9738 Tf 6.56484 -1.49375 Td [ (i) -0.19605 ] TJ /R31 9.9626 Tf 3.3168 1.49375 Td [ (\051) -0.90181 (\051) -0.90181 ] TJ /R37 9.9626 Tf 7.74922 0 Td [ (\073) -0.80379 ] TJ /R25 9.9626 Tf 49.9551 0 Td (\0501\051) Tj -224.634 -33.1148 Td (with) Tj /R31 9.9626 Tf 20.3418 0 Td [ (\050) -0.90181 ] TJ /R45 9.9626 Tf 3.875 0 Td (x) Tj /R39 6.9738 Tf 6.56523 -1.49492 Td [ (i) -0.19955 ] TJ /R37 9.9626 Tf 3.3168 1.49492 Td [ (\073) -0.79889 ] TJ /R45 9.9626 Tf 4.42812 0 Td [ (y) -0.30142 ] TJ /R39 6.9738 Tf 6.25 -2.43594 Td [ (i) -0.19955 ] TJ /R31 9.9626 Tf 3.3168 2.43594 Td [ (\051) -0.90181 ] TJ /R23 9.9626 Tf 6.89805 0 Td [ (2) -0.70086 ] TJ /R47 9.9626 Tf 9.66523 0 Td [ (R) -0.20095 ] TJ /R39 6.9738 Tf 7.1957 3.61484 Td [ (d) -0.70017 ] TJ /R23 9.9626 Tf 6.96211 -3.61484 Td [ <02> -0.79889 ] TJ /R47 9.9626 Tf 10.0652 0 Td [ (R) -0.20095 ] TJ /R39 6.9738 Tf 7.19492 3.61484 Td [ (\140) -0.19955 ] TJ /R25 9.9626 Tf 6.46211 -3.61484 Td [ (refers) -264.01 (to) -263.985 (the) ] TJ /R37 9.9626 Tf 50.4867 0 Td [ (i) -0.49992 ] TJ /R39 6.9738 Tf 3.43203 3.61484 Td [ (t) -0.60214 (h) -0.79819 ] TJ /R25 9.9626 Tf 10.802 -3.61484 Td [ (data) -263.99 (instance\054) ] TJ /R23 9.9626 Tf 57.0281 0 Td [ (M) -0.89691 ] TJ /R25 9.9626 Tf -224.286 -11.9559 Td [ (is) -278.02 (a) -279.017 (Riemannian) -277.99 (manifold\054) ] TJ /R37 9.9626 Tf 108.847 0 Td [ (f) -0.59794 ] TJ /R33 6.9738 Tf 4.87695 -1.49414 Td [ <12> -0.70017 ] TJ /R31 9.9626 Tf 8.42109 1.49414 Td [ (\072) -0.80379 ] TJ /R47 9.9626 Tf 6.05781 0 Td [ (R) -0.19605 ] TJ /R39 6.9738 Tf 7.19609 3.61602 Td [ (d) -0.70017 ] TJ /R23 9.9626 Tf 7.93594 -3.61602 Td (\041) Tj /R47 9.9626 Tf 13.2543 0 Td [ (R) -0.19605 ] TJ /R39 6.9738 Tf 7.19492 3.61602 Td [ (\140) -0.19605 ] TJ /R25 9.9626 Tf 6.60586 -3.61602 Td [ (is) -278.017 (a) -279.017 (\223prediction\224) ] TJ -170.39 -11.9551 Td [ (function) -252.99 (\050) ] TJ /R27 9.9626 Tf 39.0418 0 Td [ (e) 15.0122 (\056g) ] TJ /R25 9.9626 Tf 11.7461 0 Td [ (\056) -253.014 (output) -252.982 (of) -251.995 (a) -252.982 (neural) -253.012 (netw) 10.0081 (ork\051) -253.012 (parameterized) -253.012 (by) ] TJ /R45 9.9626 Tf -50.7879 -11.9551 Td [ <12> -0.80134 ] TJ /R25 9.9626 Tf 5.91484 0 Td [ (\054) -222 (and) ] TJ /R37 9.9626 Tf 21.2391 0 Td [ (J) -0.49992 ] TJ /R25 9.9626 Tf 8.62891 0 Td [ (a) -214.989 (positi) 25.013 (v) 14.9828 (e) -216.008 (real\055v) 25.013 (alued) -215.013 (objecti) 25 (v) 14.9828 (e) -216.008 (function) -214.998 (to) -215.993 (be) -214.984 (min\055) ] TJ -35.7828 -11.9551 Td (imized\056) Tj /R27 9.9626 Tf 38.0098 0 Td [ (Gr) 14.9877 (adient) -427.983 (Descent) ] TJ /R25 9.9626 Tf 76.6184 0 Td [ (\050GD\051) -428.007 (is) -427.012 (a) -428.012 (standard) -427.992 (technique) ] TJ -114.628 -11.9547 Td [ (used) -340.009 (to) -338.99 (optimize) -339.987 (Equation) -339.007 (\050) ] TJ ET Q 1 0 0 rg q 10 0 0 10 0 0 cm BT /R25 9.9626 Tf 1 0 0 1 422.565 296.667 Tm (1) Tj ET Q 0 g q 10 0 0 10 0 0 cm BT /R25 9.9626 Tf 1 0 0 1 427.546 296.667 Tm [ (\051) -339.987 (by) -340.012 (updating) -339.007 (the) -340.012 (parameters) ] TJ -118.684 -11.9551 Td [ (of) -307.985 (the) -307 (model) -308.003 (along) -306.99 (ne) 15.0171 (g) 4.98446 (ati) 24.986 (v) 14.9828 (e) -307.993 (gradient) -306.988 (directions\056) -482.983 (Ho) 24.986 (we) 25.0154 (v) 14.9828 (er) 39.9835 (\054) ] TJ 11.9559 TL T* [ (this) -294.005 (approach) -293.985 (can) -295.012 (be) -293.99 (fragile) -293.985 (when) -293.98 (the) -293.98 (objecti) 25 (v) 14.9828 (e) -295.014 (functi) 0.99493 (o) -1.01454 (n) -293.985 (is) ] TJ 11.9551 TL T* [ (highly) -250.002 (non\055con) 40.0105 (v) 14.9828 (e) 15.0122 (x\056) -312.019 (In) -249.995 (this) -250.992 (case\054) -250.002 (the) -251.007 (standard) -249.983 (GD) -251.012 (algorithm) ] TJ T* [ (can) -273.996 (get) -273.986 (stuck) -274.013 (in) -275 (a) -273.998 (bad) -273.989 (local) -273.984 (minima) -273.993 (leading) -274.013 (to) -275.003 (poor) -273.984 (perfor) 19.9918 (\055) ] TJ T* [ (mance\056) -293.003 (The) -196.982 (SGD) -395.993 (\133) ] TJ ET Q 0 1 0 rg q 10 0 0 10 0 0 cm BT /R25 9.9626 Tf 1 0 0 1 384.916 236.891 Tm (10) Tj ET Q 0 g q 10 0 0 10 0 0 cm BT /R25 9.9626 Tf 1 0 0 1 394.879 236.891 Tm [ (\135\054) -208.005 (the) -197.996 (stochastic) -198.016 (v) 24.9811 (ariant) -197.996 (of) -198.006 (GD\054) -197.996 (w) 10.0032 (as) -198.006 (in\055) ] TJ -86.0168 -11.9551 Td [ (troduced) -243.984 (to) -243.006 (o) 14.9828 (v) 14.9828 (ercome) -243.999 (this) -242.994 (dra) 15.0122 (wback\056) -307.983 (SGD) -244.018 (trains) -242.999 (the) -243.989 (model) ] TJ 11.9559 TL T* [ (on) -243.989 (mini\055batches) -245.018 (of) -243.996 (the) -243.991 (training) -244.008 (data\054) -245.993 (resulting) -244.003 (in) -243.989 (f) 9.99343 (aster) -245.018 (con\055) ] TJ 11.9551 TL T* [ (v) 14.9828 (er) 17.9896 (gence) -250 (rates) -249.995 (compared) -249.988 (to) -249.988 (GD\056) ] TJ 11.9551 -12.4289 Td [ (The) -321.981 (Computed) -322.995 (gradients) -322 (during) -322.98 (SGD) -322.005 (can) -321.99 (display) -323.015 (lar) 17.997 (ge) ] TJ -11.9551 -11.9551 Td [ (v) 24.9811 (ariances) -428.009 (due) -426.983 (to) -427.995 (the) -428 (random) -427.007 (generation) -427.987 (of) -428.007 (mini\055batches\056) ] TJ T* [ (This) -345.018 (can) -345.004 (lead) -344.999 (to) -344.989 (undesirable) -345.016 (oscillations) -344.987 (of) -346.021 (the) -344.991 (model) -345.016 (pa\055) ] TJ T* [ (rameters) -362.993 (around) -362.986 (optimal) -362.998 (v) 24.9811 (alues) -362.008 (\133) ] TJ ET Q 0 1 0 rg q 10 0 0 10 0 0 cm BT /R25 9.9626 Tf 1 0 0 1 444.254 152.731 Tm (29) Tj ET Q 0 g q 10 0 0 10 0 0 cm BT /R25 9.9626 Tf 1 0 0 1 454.216 152.731 Tm [ (\135\056) -648.99 (Ensuring) -363.003 (that) -362.984 (these) ] TJ -145.354 -11.9551 Td [ (oscillations) -366.003 (are) -365.01 (dampened) -365.998 (has) -366.012 (been) -366.017 (the) -364.988 (subject) -365.998 (of) -366.017 (se) 25.0154 (v) 14.9828 (eral) ] TJ 11.9551 TL T* [ (w) 10 (orks) -247.005 (lately) -247 (\133) ] TJ ET Q 0 1 0 rg q 10 0 0 10 0 0 cm BT /R25 9.9626 Tf 1 0 0 1 363.487 128.821 Tm (30) Tj ET Q 0 g q 10 0 0 10 0 0 cm BT /R25 9.9626 Tf 1 0 0 1 373.45 128.821 Tm (\054) Tj ET Q 0 1 0 rg q 10 0 0 10 0 0 cm BT /R25 9.9626 Tf 1 0 0 1 378.401 128.821 Tm (17) Tj ET Q 0 g q 10 0 0 10 0 0 cm BT /R25 9.9626 Tf 1 0 0 1 388.364 128.821 Tm [ (\135\056) -309.007 (T) 79.9916 (w) 10.0032 (o) -247.013 (prominent) -247.998 (algori) 0.99493 (thms) -248.008 (which) -247.003 (ad\055) ] TJ -79.5016 -11.9559 Td [ (dress) -259.006 (this) -258.991 (issue) -258.996 (are) -258.008 (the) ] TJ /R27 9.9626 Tf 92.027 0 Td [ (Momentum) -258.991 (SGD) ] TJ /R25 9.9626 Tf 69.3469 0 Td [ (\050SGD\055M\051) -258.996 (\133) ] TJ ET Q 0 1 0 rg q 10 0 0 10 0 0 cm BT /R25 9.9626 Tf 1 0 0 1 514.868 116.865 Tm (29) Tj ET Q 0 g q 10 0 0 10 0 0 cm BT /R25 9.9626 Tf 1 0 0 1 524.831 116.865 Tm [ (\135) -259.021 (and) ] TJ -215.969 -11.9551 Td (the) Tj /R27 9.9626 Tf 14.082 0 Td [ (Stoc) 14.9901 (hastic) -191.98 (V) 111.006 (ariance) -191.007 (Reduced) -192.002 (Gr) 14.9877 (adient) ] TJ /R25 9.9626 Tf 154.163 0 Td [ (\050SVRG\051) -191.002 (\133) ] TJ ET Q 0 1 0 rg q 10 0 0 10 0 0 cm BT /R25 9.9626 Tf 1 0 0 1 515.533 104.91 Tm (17) Tj ET Q 0 g q 10 0 0 10 0 0 cm BT /R25 9.9626 Tf 1 0 0 1 525.495 104.91 Tm [ (\135) -192.012 (and) ] TJ -216.633 -11.9551 Td [ (the) 14.9852 (y) -247.993 (ha) 19.9967 (v) 14.9828 (e) -246.983 (been) -247.995 (quite) -246.998 (successful) -248.017 (in) -247.988 (achie) 25.0154 (vi) 0.99003 (ng) -247.988 (higher) -247.983 (con) 39.9982 (v) 14.9828 (er) 19.9869 (\055) ] TJ 11.9551 TL T* [ (gence) -250 (rates) -249.995 (o) 14.9828 (v) 14.9828 (er) -250.005 (SGD) -250.017 (and) -249.993 (GD\056) ] TJ -13.741 -29.8879 Td (1) Tj ET Q Q Q q q 1 1 1 rg /a0 gs 48.406 786.422 515.188 -52.699 re f q /s5 gs /x6 Do Q q /s7 gs /x8 Do Q q /s9 gs /x10 Do Q q /s11 gs /x12 Do Q Q Q Q q 1 0 0 1 0 0 cm BT /F1 12 Tf 14.4 TL ET 1 1 1 rg n 270 47 72 14 re f* 0.5 0.5 0.5 rg BT /F2 9 Tf 10.8 TL ET BT 1 0 0 1 297 50 Tm (4460) Tj T* ET Q endstream endobj 15 0 obj << /Filter /FlateDecode /Resources << /ExtGState << /a0 << /CA 1 /ca 1 >> >> /XObject << /x18 16 0 R >> >> /Length 28 /Group << /Type /Group /S /Transparency /CS /DeviceRGB /I true >> /BBox [ 78 746 96 765 ] /Type /XObject /Subtype /Form >> stream x+O4PH/VЯ0Pp 0 endstream endobj 16 0 obj << /Filter /FlateDecode /Resources 17 0 R /Length 107 /Type /XObject /BBox [ 78 746 96 765 ] /Subtype /Form >> stream xe AC̬wʠ =p,?]%+H-
Jc "82w8VSnGW;"
endstream
endobj
17 0 obj
<<
/ExtGState <<
/a0 <<
/CA 1
/ca 1
>>
>>
>>
endobj
18 0 obj
<<
/Filter /FlateDecode
/Resources <<
/ExtGState <<
/a0 <<
/CA 1
/ca 1
>>
>>
/XObject <<
/x15 19 0 R
>>
>>
/Length 28
/Group <<
/Type /Group
/S /Transparency
/CS /DeviceRGB
/I true
>>
/BBox [ 67 752 84 775 ]
/Type /XObject
/Subtype /Form
>>
stream
x+O4PH/VЯ04Up
0
endstream
endobj
19 0 obj
<<
/Filter /FlateDecode
/Resources 20 0 R
/Length 228
/Type /XObject
/BBox [ 67 752 84 775 ]
/Subtype /Form
>>
stream
xeQKn!s ?FPav6R٪TS.
b];15YyR
{7QL.\:Rv/x9l+L7h%1!}i/AI(kz"U&,YO![R hg{3}4/GyYF:!w}Gn+'xJcO9i뽼_-:`
endstream
endobj
20 0 obj
<<
/ExtGState <<
/a0 <<
/CA 1
/ca 1
>>
>>
>>
endobj
21 0 obj
<<
/Filter /FlateDecode
/Resources <<
/ExtGState <<
/a0 <<
/CA 1
/ca 1
>>
>>
/XObject <<
/x24 22 0 R
>>
>>
/Length 28
/Group <<
/Type /Group
/S /Transparency
/CS /DeviceRGB
/I true
>>
/BBox [ 132 751 480 772 ]
/Type /XObject
/Subtype /Form
>>
stream
x+O4PH/VЯ02Qp
0
endstream
endobj
22 0 obj
<<
/Filter /FlateDecode
/Resources 23 0 R
/Length 53223
/Type /XObject
/BBox [ 132 751 480 772 ]
/Subtype /Form
>>
stream
xtI:6%Q㨈?7rA= u%6 ?Y(WbWo{B>9
x`Znϳ|8{3?0x*z ǃ|,@:w>`c|*ϻⳅKO3`g
:_|}}><.6`Z{{3]#<_o"~:ͺgk7/Ұ@|K yp ]03ʷCmş8˽Y?>(3!Bwqs.Z8,~~=rMT̩y+/*w: uBZ_`ߵp`%M?ɝ1ɳw=vDۉy&xb4Q>d@ sg~lA