%PDF-1.7
%
1 0 obj
<< /Metadata 3 0 R /Pages 4 0 R /Type /Catalog >>
endobj
2 0 obj
<< /Author (Chirag Agarwal; Daniel D'souza; Sara Hooker) /Producer (pikepdf 5.1.3) /Subject (IEEE Conference on Computer Vision and Pattern Recognition) /Title (Estimating Example Difficulty Using Variance of Gradients) >>
endobj
3 0 obj
<< /Subtype /XML /Type /Metadata /Length 1142 >>
stream
Estimating Example Difficulty Using Variance of GradientsChirag Agarwal Daniel D'souza Sara HookerIEEE Conference on Computer Vision and Pattern Recognition
endstream
endobj
4 0 obj
<< /Count 11 /Kids [ 5 0 R 6 0 R 7 0 R 8 0 R 9 0 R 10 0 R 11 0 R 12 0 R 13 0 R 14 0 R 15 0 R ] /Type /Pages >>
endobj
5 0 obj
<< /Contents 16 0 R /CropBox [ 0 0 612 792 ] /MediaBox [ 0 0 612 792 ] /Parent 4 0 R /Resources 17 0 R /Type /Page >>
endobj
6 0 obj
<< /Contents 18 0 R /CropBox [ 0 0 612 792 ] /MediaBox [ 0 0 612 792 ] /Parent 4 0 R /Resources 19 0 R /Type /Page >>
endobj
7 0 obj
<< /Contents 20 0 R /CropBox [ 0 0 612 792 ] /Group 21 0 R /MediaBox [ 0 0 612 792 ] /Parent 4 0 R /Resources 22 0 R /Type /Page >>
endobj
8 0 obj
<< /Contents 23 0 R /CropBox [ 0 0 612 792 ] /MediaBox [ 0 0 612 792 ] /Parent 4 0 R /Resources 24 0 R /Type /Page >>
endobj
9 0 obj
<< /Contents 25 0 R /CropBox [ 0 0 612 792 ] /Group 26 0 R /MediaBox [ 0 0 612 792 ] /Parent 4 0 R /Resources 27 0 R /Type /Page >>
endobj
10 0 obj
<< /Contents 28 0 R /CropBox [ 0 0 612 792 ] /Group 29 0 R /MediaBox [ 0 0 612 792 ] /Parent 4 0 R /Resources 30 0 R /Type /Page >>
endobj
11 0 obj
<< /Contents 31 0 R /CropBox [ 0 0 612 792 ] /Group 32 0 R /MediaBox [ 0 0 612 792 ] /Parent 4 0 R /Resources 33 0 R /Type /Page >>
endobj
12 0 obj
<< /Contents 34 0 R /CropBox [ 0 0 612 792 ] /MediaBox [ 0 0 612 792 ] /Parent 4 0 R /Resources 35 0 R /Type /Page >>
endobj
13 0 obj
<< /Contents 36 0 R /CropBox [ 0 0 612 792 ] /MediaBox [ 0 0 612 792 ] /Parent 4 0 R /Resources 37 0 R /Type /Page >>
endobj
14 0 obj
<< /Contents 38 0 R /CropBox [ 0 0 612 792 ] /MediaBox [ 0 0 612 792 ] /Parent 4 0 R /Resources 39 0 R /Type /Page >>
endobj
15 0 obj
<< /Contents 40 0 R /CropBox [ 0 0 612 792 ] /MediaBox [ 0 0 612 792 ] /Parent 4 0 R /Resources 41 0 R /Type /Page >>
endobj
16 0 obj
<< /Length 11502 >>
stream
q
q
q
q
0 g 0 G
0 g 0 G
0 g 0 G
0 g 0 G
0 g 0 G
BT
/F57 14.3462 Tf 117.347 675.067 Td [(Estimating)-250(Example)-250(Dif\002culty)-250(using)-250(V)92(ariance)-250(of)-250(Gradients)]TJ/F58 11.9552 Tf 14.897 -37.858 Td [(Chirag)-250(Ag)5(arw)10(al)]TJ -8.399 -13.947 Td [(MDSR)-250(Lab,)-250(Adobe)]TJ/F59 8.9664 Tf -23.283 -13.948 Td [(chiragagarwall12@gmail.com)]TJ/F58 11.9552 Tf 180.056 27.895 Td [(Daniel)-250(D')55(souza)]TJ 2.224 -13.947 Td [(ML)-250(Collecti)25(v)15(e)]TJ/F59 8.9664 Tf -10.934 -13.948 Td [(ddsouza@umich.edu)]TJ/F58 11.9552 Tf 142.857 27.895 Td [(Sara)-250(Hook)10(er)]TJ -11.016 -13.947 Td [(Google)-250(Research)]TJ/F59 8.9664 Tf -8.913 -13.948 Td [(shooker@google.com)]TJ
0 g 0 G
/F57 11.9552 Tf -248.841 -41.046 Td [(Abstract)]TJ/F61 9.9626 Tf -83.928 -24.258 Td [(In)-328(mac)15(hine)-329(learning)10(,)-348(a)-328(question)-328(of)-329(gr)37(eat)-328(inter)37(est)-328(is)-329(un-)]TJ -11.955 -11.956 Td [(der)10(standing)-301(what)-301(e)20(xamples)-301(ar)37(e)-301(c)15(hallenging)-301(for)-301(a)-301(model)-301(to)]TJ 0 -11.955 Td [(classify)55(.)-418(Identifying)-286(atypical)-286(e)20(xamples)-286(ensur)37(es)-287(the)-286(safe)-286(de-)]TJ 0 -11.955 Td [(ployment)-382(of)-381(models,)-415(isolates)-382(samples)-381(that)-382(r)37(equir)37(e)-382(further)]TJ 0 -11.955 Td [(human)-329(inspection)-329(and)-328(pr)45(o)10(vides)-329(interpr)37(etability)-329(into)-329(model)]TJ 0 -11.955 Td [(behavior)111(.)-552(In)-330(this)-331(work,)-350(we)-331(pr)45(opose)-331(V)111(ariance)-330(of)-331(Gr)15(adients)]TJ 0 -11.956 Td [(\(V)111(oG\))-397(as)-397(a)-397(valuable)-398(and)-397(ef)18(\002cient)-397(metric)-397(to)-397(r)15(ank)-397(data)-398(by)]TJ 0 -11.955 Td [(dif)18(\002culty)-248(and)-249(to)-248(surface)-249(a)-248(tr)15(actable)-249(subset)-248(of)-249(the)-248(most)-249(c)15(hal-)]TJ 0 -11.955 Td [(lenging)-246(e)20(xamples)-245(for)-246(human-in-the-loop)-246(audit)1(ing)15(.)-309(W)92(e)-246(show)]TJ 0 -11.955 Td [(that)-267(data)-268(points)-267(with)-268(high)-267(V)111(oG)-268(scor)37(es)-267(ar)37(e)-268(far)-267(mor)37(e)-268(dif)18(\002cult)]TJ 0 -11.955 Td [(for)-221(the)-221(model)-221(to)-222(learn)-221(and)-221(o)10(ver)20(-inde)20(x)-221(on)-221(corrupted)-222(or)-221(mem-)]TJ 0 -11.955 Td [(orized)-320(e)20(xamples.)-518(Further)111(,)-337(r)37(estricting)-320(the)-320(e)15(valuat)1(ion)-320(to)-320(the)]TJ 0 -11.956 Td [(test)-257(set)-256(instances)-257(with)-257(t)1(he)-257(lowest)-257(V)111(oG)-256(impr)45(o)10(ves)-257(the)-257(model')40(s)]TJ 0 -11.955 Td [(g)10(ener)15(alization)-237(performance)15(.)-305(F)45(inally)55(,)-239(we)-237(show)-237(that)-236(V)111(oG)-237(is)-237(a)]TJ 0 -11.955 Td [(valuable)-258(and)-257(ef)18(\002cient)-258(r)15(anking)-258(for)-257(out-of-distrib)20(ution)-258(detec-)]TJ 0 -11.955 Td [(tion.)]TJ/F57 11.9552 Tf 0 -36.909 Td [(1.)-250(Intr)18(oduction)]TJ/F58 9.9626 Tf 11.955 -19.277 Td [(Ov)15(er)-546(the)-546(past)-546(decade,)-620(machine)-546(learning)-546(models)-546(are)]TJ -11.955 -11.955 Td [(increasingly)-414(deplo)10(yed)-414(to)-414(high-stak)10(e)-414(decision)-414(applications)]TJ 0 -11.955 Td [(such)-488(as)-489(healt)1(hcare)-489([)]TJ
0 1 0 rg 0 1 0 RG
[(4)]TJ
0 g 0 G
[(,)]TJ
0 1 0 rg 0 1 0 RG
[-289(20)]TJ
0 g 0 G
[(,)]TJ
0 1 0 rg 0 1 0 RG
[-288(52)]TJ
0 g 0 G
[(,)]TJ
0 1 0 rg 0 1 0 RG
[-289(70)]TJ
0 g 0 G
[(],)-548(self-dri)25(ving)-488(cars)-489([)]TJ
0 1 0 rg 0 1 0 RG
[(51)]TJ
0 g 0 G
[(])]TJ 0 -11.956 Td [(and)-338(\002nance)-338([)]TJ
0 1 0 rg 0 1 0 RG
[(53)]TJ
0 g 0 G
[(].)-575(F)15(or)-338(g)5(aining)-338(trust)-338(from)-339(stak)10(eholders)-338(and)]TJ 0 -11.955 Td [(model)-204(practitioners,)-214(it)-204(is)-204(important)-205(for)-204(deep)-204(neural)-205(netw)10(orks)]TJ 0 -11.955 Td [(\(DNNs\))-266(to)-266(mak)10(e)-267(decisions)-266(that)-266(are)-266(interpretable)-266(to)-266(both)-267(re-)]TJ 0 -11.955 Td [(searchers)-246(and)-246(end-users.)-309(T)80(o)-246(this)-246(end,)-247(for)-247(sensiti)26(v)14(e)-246(domains,)]TJ 0 -11.955 Td [(there)-240(is)-241(an)-240(ur)18(gent)-240(need)-241(for)-240(auditing)-241(tools)-240(which)-240(are)-241(scalable)]TJ 0 -11.955 Td [(and)-250(help)-250(domain)-250(e)15(xperts)-250(audit)-250(models.)]TJ 11.955 -12.303 Td [(Reasoning)-410(about)-410(model)-410(beha)20(vior)-410(is)-411(oft)1(en)-411(easier)-410(when)]TJ -11.955 -11.956 Td [(presented)-418(wi)1(th)-418(a)-418(subset)-417(of)-418(data)-417(points)-418(that)-417(are)-418(relati)25(v)15(ely)]TJ 0 -11.955 Td [(more)-369(dif)25(\002cult)-370(for)-369(a)-369(model)-370(to)-369(learn.)-668(Besides)-369(aiding)-370(inter)20(-)]TJ 0 -11.955 Td [(pretability)-270(through)-270(case-based)-270(reasoning)-270([)]TJ
0 1 0 rg 0 1 0 RG
[(11)]TJ
0 g 0 G
[(,)]TJ
0 1 0 rg 0 1 0 RG
[-144(30)]TJ
0 g 0 G
[(,)]TJ
0 1 0 rg 0 1 0 RG
[-143(39)]TJ
0 g 0 G
[(],)-276(it)-270(can)]TJ 0 -11.955 Td [(also)-280(be)-280(used)-280(to)-280(surf)10(ace)-280(a)-280(tractable)-280(subset)-281(of)-280(atypical)-280(e)15(xam-)]TJ 0 -11.955 Td [(ples)-264(for)-265(further)-264(human)-265(auditing)-264([)]TJ
0 1 0 rg 0 1 0 RG
[(46)]TJ
0 g 0 G
[(,)]TJ
0 1 0 rg 0 1 0 RG
[-140(73)]TJ
0 g 0 G
[(],)-268(for)-264(acti)25(v)15(e)-265(learning)]TJ 0 -11.955 Td [(to)-313(inform)-312(model)-313(impro)15(v)15(ements,)-328(and)-312(to)-313(choose)-313(not)-312(to)-313(clas-)]TJ 0 -11.956 Td [(sify)-249(some)-248(instances)-249(when)-249(the)-248(model)-249(is)-248(u)-1(nc)1(ertain)-249([)]TJ
0 1 0 rg 0 1 0 RG
[(7)]TJ
0 g 0 G
[(,)]TJ
0 1 0 rg 0 1 0 RG
[-128(14)]TJ
0 g 0 G
[(,)]TJ
0 1 0 rg 0 1 0 RG
[-128(21)]TJ
0 g 0 G
[(].)]TJ 0 -11.955 Td [(One)-388(of)-388(the)-389(biggest)-388(bottlenecks)-388(for)-388(human)-388(auditing)-389(is)-388(the)]TJ 0 -11.955 Td [(lar)18(ge)-224(scale)-223(of)-224(modern)-224(datasets)-223(and)-224(the)-224(cost)-223(of)-224(annotating)-224(in-)]TJ 0 -11.955 Td [(di)25(vidual)-301(features)-300([)]TJ
0 1 0 rg 0 1 0 RG
[(3)]TJ
0 g 0 G
[(,)]TJ
0 1 0 rg 0 1 0 RG
[-164(38)]TJ
0 g 0 G
[(,)]TJ
0 1 0 rg 0 1 0 RG
[-164(68)]TJ
0 g 0 G
[(].)-462(Methods)-300(which)-301(automatically)]TJ
0 g 0 G
0 g 0 G
258.75 487.268 Td [(surf)10(ace)-209(a)-209(subset)-210(of)-209(relati)25(v)15(ely)-209(more)-209(challenging)-209(e)15(xamples)-210(for)]TJ 0 -11.956 Td [(human)-286(inspection)-286(help)-285(prioritize)-286(limited)-286(human)-286(annotation)]TJ 0 -11.955 Td [(and)-402(auditing)-403(time.)-767(Despite)-402(the)-403(ur)18(genc)15(y)-402(of)-402(this)-403(use-case,)]TJ 0 -11.955 Td [(ranking)-261(e)15(xamples)-261(by)-261(dif)25(\002culty)-261(has)-261(had)-262(limited)-261(treatment)-261(in)]TJ 0 -11.955 Td [(the)-385(conte)15(xt)-385(of)-385(deep)-385(neural)-385(netw)10(orks)-385(due)-386(to)-385(the)-385(computa-)]TJ 0 -11.955 Td [(tional)-250(cost)-250(of)-250(ranking)-250(a)-250(high)-250(dimensional)-250(feature)-250(space.)]TJ/F57 9.9626 Tf 0 -17.909 Td [(Pr)18(esent)-365(w)10(ork.)]TJ/F58 9.9626 Tf 66.591 0 Td [(A)-365(popular)-364(interpretability)-365(tool)-365(is)-365(salienc)15(y)]TJ -66.591 -11.956 Td [(maps,)-213(where)-204(each)-203(of)-204(the)-204(features)-203(of)-204(the)-204(input)-203(data)-204(are)-204(scored)]TJ 0 -11.955 Td [(based)-334(on)-334(their)-334(contrib)20(ution)-334(t)1(o)-334(the)-334<026e616c>-334(output)-334([)]TJ
0 1 0 rg 0 1 0 RG
[(64)]TJ
0 g 0 G
[(].)-562(Ho)25(w-)]TJ 0 -11.955 Td [(e)25(v)15(er)40(,)-428(these)-393(e)15(xplanations)-392(are)-393(typically)-392(for)-393(a)-392(single)-393(predic-)]TJ 0 -11.955 Td [(tion)-340(and)-339(generated)-340(after)-339(the)-340(model)-339(is)-340(trained.)-579(Our)-339(goal)-340(is)]TJ 0 -11.955 Td [(to)-422(le)25(v)15(erage)-421(these)-422(e)15(xplanations)-421(to)-422(automatically)-421(surf)10(ace)-422(a)]TJ 0 -11.955 Td [(subset)-325(of)-324(relati)25(v)15(ely)-325(more)-325(challenging)-324(e)15(xamples)-325(for)-325(human)]TJ 0 -11.956 Td [(inspection)-316(to)-317(help)-316(prioritize)-317(limit)1(ed)-317(human)-316(annotation)-317(and)]TJ 0 -11.955 Td [(auditing)-347(time.)-603(T)80(o)-347(this)-348(end,)-372(we)-347(propose)-348(a)-347(ranking)-348(method)]TJ 0 -11.955 Td [(across)-316(all)-316(e)15(xamples)-317(that)-316(instead)-316(measures)-316(the)-317(per)20(-e)15(xample)]TJ 0 -11.955 Td [(change)-221(in)-221(e)15(xplanations)-221(o)15(v)15(er)-221(training.)-300(Examples)-221(that)-221(are)-221(dif-)]TJ 0 -11.955 Td [(\002cult)-376(for)-375(a)-376(model)-376(to)-375(learn)-376(will)-376(e)15(xhibit)-375(higher)-376(v)25(ariance)-376(in)]TJ 0 -11.955 Td [(gradient)-202(updates)-202(throughout)-202(training.)-294(On)-201(the)-202(other)-202(hand,)-212(the)]TJ 0 -11.956 Td [(backpropag)5(ated)-283(gradients)-284(of)-283(the)-284(samples)-283(that)-283(are)]TJ/F61 9.9626 Tf 198.99 0 Td [(r)37(elatively)]TJ -198.99 -11.955 Td [(easier)]TJ/F58 9.9626 Tf 28.845 0 Td [(will)-428(e)15(xhibit)-429(lo)25(wer)-428(v)25(ariance)-428(because)-429(the)-428(loss)-428(from)]TJ -28.845 -11.955 Td [(these)-366(e)15(xamples)-366(does)-367(not)-366(consistently)-366(dominate)-366(the)-367(model)]TJ 0 -11.955 Td [(training.)]TJ 11.955 -12.24 Td [(W)80(e)-268(term)-268(this)-267(class)-268(normalized)-268(ranking)-268(mechanism)]TJ/F61 9.9626 Tf 204.37 0 Td [(V)111(ari-)]TJ -216.325 -11.955 Td [(ance)-425(of)-425(Gr)15(adients)]TJ/F58 9.9626 Tf 78.96 0 Td [(\(V)129(oG\))-425(and)-425(demonstrate)-425(that)-424(V)129(oG)-425(is)-425(a)]TJ -78.96 -11.955 Td [(meaningful)-345(w)10(ay)-346(for)-345(ranking)-346(data)-345(by)-346(dif)25(\002culty)-345(and)-346(surf)10(ac-)]TJ 0 -11.955 Td [(ing)-273(a)-273(trac)1(table)-273(subset)-273(of)-273(the)-272(most)-273(challenging)-273(e)15(xamples)-273(for)]TJ 0 -11.956 Td [(human-in-the-loop)-349(auditing)-350(across)-349(a)-349(v)25(ariety)-349(of)-350(lar)18(ge-scale)]TJ 0 -11.955 Td [(datasets.)-298(V)129(oG)-212(assigns)-213(higher)-213(scores)-213(to)-212(test)-213(set)-213(e)15(xamples)-213(that)]TJ 0 -11.955 Td [(are)-219(more)-220(challenging)-219(for)-219(the)-219(model)-220(to)-219(classify)-219(and)-219(pro)15(v)15(es)-220(to)]TJ 0 -11.955 Td [(be)-315(an)-316(ef)25<026369>1(ent)-316(tool)-315(for)-315(detecting)-315(out-of-distrib)20(ution)-316(\(OoD\))]TJ 0 -11.955 Td [(samples.)-539(V)129(oG)-326(is)-327(model)-326(and)-326(domain-agnostic)-327(as)-326(all)-326(that)-327(is)]TJ 0 -11.955 Td [(required)-250(is)-250(the)-250(backpropag)5(ated)-250(gradients)-250(from)-250(the)-250(model.)]TJ/F57 9.9626 Tf 0 -17.909 Td [(Contrib)20(utions.)]TJ/F58 9.9626 Tf 68.866 0 Td [(W)80(e)-374(demonstrate)-374(consistent)-373(results)-374(across)]TJ -68.866 -11.956 Td [(tw)10(o)-359(architectures)-359(and)-359(three)-359(datasets)-359<96>-359(Cif)10(ar)20(-10,)-386(Cif)10(ar)20(-100)]TJ 0 -11.955 Td [([)]TJ
0 1 0 rg 0 1 0 RG
[(43)]TJ
0 g 0 G
[(])-273(and)-273(ImageNet)-272([)]TJ
0 1 0 rg 0 1 0 RG
[(61)]TJ
0 g 0 G
[(].)-379(Our)-272(contrib)20(utions)-273(can)-273(be)-273(enumer)20(-)]TJ 0 -11.955 Td [(ated)-250(as)-250(follo)25(ws:)]TJ
0 g 0 G
7.472 -20.779 Td [(1.)]TJ
0 g 0 G
[-500(W)80(e)-450(present)-449(V)111(ariance)-450(of)-449(Gradients)-450(\(V)129(oG\))-449<96>-450(a)-450(class-)]TJ 12.453 -11.955 Td [(normalized)-207(gradient)-208(v)25(ariance)-207(score)-208(for)-207(determining)-208(the)]TJ 0 -11.955 Td [(relati)25(v)15(e)-357(ease)-356(of)-357(learning)-357(data)-357(samples)-356(within)-357(a)-357(gi)25(v)15(en)]TJ
0 g 0 G
0 g 0 G
ET
Q
Q
Q
q
1 0 0 1 0 0 cm
/WmBvMfXQ6bj-ZWgx2Ct-3A Do
Q
Q
q
1 0 0 1 0 0 cm
/45H-hOK99s_9Q-8TCiqcHg Do
Q
endstream
endobj
17 0 obj
<< /Font << /F57 42 0 R /F58 43 0 R /F59 44 0 R /F61 45 0 R >> /ProcSet [ /PDF /Text ] /XObject << /45H-hOK99s_9Q-8TCiqcHg 46 0 R /WmBvMfXQ6bj-ZWgx2Ct-3A 47 0 R >> >>
endobj
18 0 obj
<< /Length 17845 >>
stream
q
q
q
0 g 0 G
0 g 0 G
BT
/F58 9.9626 Tf 70.037 710.037 Td [(class)-312(\(Sec.)]TJ
1 0 0 rg 1 0 0 RG
[-312(2)]TJ
0 g 0 G
[(\).)-495(V)129(oG)-312(identi\002es)-312(clusters)-312(of)-312(images)-312(with)]TJ 0 -11.955 Td [(clearly)-197(disti)1(nct)-197(semantic)-197(propert)1(ies,)-208(where)-196(images)-197(with)]TJ 0 -11.955 Td [(lo)25(w)-277(V)129(oG)-277(scores)-277(feature)-277(f)10(ar)-278(less)-277(cluttered)-277(backgrounds)]TJ 0 -11.955 Td [(and)-479(more)-479(prototypical)-479(v)25(antage)-479(points)-479(of)-479(the)-479(object)]TJ 0 -11.955 Td [(\(Fig.)]TJ
1 0 0 rg 1 0 0 RG
[-416(4)]TJ
0 g 0 G
[(\).)-808(In)-416(contrast,)-458(images)-416(with)-416(high)-416(V)129(oG)-416(scores)]TJ 0 -11.956 Td [(o)15(v)15(er)20(-inde)15(x)-297(on)-297(images)-297(with)-297(cluttered)-298(backgrounds)-297(and)]TJ 0 -11.955 Td [(atypical)-250(v)25(antage)-250(points)-250(of)-250(the)-250(object)-250(of)-250(interest.)]TJ
0 g 0 G
-12.453 -16.376 Td [(2.)]TJ
0 g 0 G
[-500(V)129(oG)-351(ef)25(fecti)25(v)15(ely)-351(surf)10(aces)-352(memorized)-351(e)15(xamples,)]TJ/F61 9.9626 Tf 207.565 0 Td [(i.e)]TJ/F58 9.9626 Tf 9.684 0 Td [(.)-351(it)]TJ -204.796 -11.955 Td [(allocates)-357(higher)-356(scores)-357(to)-357(images)-356(that)-357(require)]TJ/F61 9.9626 Tf 189.217 0 Td [(memo-)]TJ -189.217 -11.956 Td [(rization)]TJ/F58 9.9626 Tf 34.203 0 Td [(\(Sec.)]TJ
1 0 0 rg 1 0 0 RG
[-321(4)]TJ
0 g 0 G
[(\).)-523(Further)40(,)-339(V)129(oG)-321(aids)-321(in)-322(understanding)]TJ -34.203 -11.955 Td [(the)-213(model)-214(beha)20(vior)-213(at)-214(dif)25(ferent)-213(training)-214(stages)-213(and)-214(pro-)]TJ 0 -11.955 Td [(vides)-250(insight)-250(into)-250(the)-250(learning)-250(c)15(ycle)-250(of)-250(the)-250(model.)]TJ
0 g 0 G
-12.453 -16.376 Td [(3.)]TJ
0 g 0 G
[-500(W)80(e)-353(sho)25(w)-352(the)-353(reliability)-353(of)-352(V)129(oG)-353(as)-352(an)-353(OoD)-353(detection)]TJ 12.453 -11.956 Td [(technique)-359(and)-358(compare)-359(its)-359(performance)-358(to)-359(9)-359(e)15(xisting)]TJ 0 -11.955 Td [(OoD)-300(methods,)-313(where)-300(it)-300(outperforms)-300(se)25(v)15(eral)-300(methods,)]TJ 0 -11.955 Td [(such)-275(as)-276(PCA)-275([)]TJ
0 1 0 rg 0 1 0 RG
[(24)]TJ
0 g 0 G
[(])-275(and)-276(KDE)-275([)]TJ
0 1 0 rg 0 1 0 RG
[(15)]TJ
0 g 0 G
[(,)]TJ
0 1 0 rg 0 1 0 RG
[-147(54)]TJ
0 g 0 G
[(].)-386(V)129(oG)-275(presents)-276(an)]TJ 0 -11.955 Td [(o)15(v)15(erall)-305(impro)15(v)15(ement)-306(of)]TJ/F19 9.9626 Tf 97.627 0 Td [(9)]TJ/F22 9.9626 Tf 4.981 0 Td [(:)]TJ/F19 9.9626 Tf 2.767 0 Td [(26%)]TJ/F58 9.9626 Tf 21.308 0 Td [(in)-305(precision)-306(compared)]TJ -126.683 -11.955 Td [(to)-250(all)-250(other)-250(methods.)]TJ/F57 11.9552 Tf -19.925 -32.971 Td [(2.)-250(V)100(oG)-250(Framew)10(ork)]TJ/F58 9.9626 Tf 11.955 -19.148 Td [(W)80(e)-350(consider)-350(a)-349(supervised)-350(classi\002cation)-350(problem)-350(where)]TJ -11.955 -11.955 Td [(a)-290(DNN)-290(is)-290(trained)-289(to)-290(approximate)-290(the)-290(function)]TJ/F25 9.9626 Tf 186.349 0 Td [(F)]TJ/F58 9.9626 Tf 11.038 0 Td [(that)-290(maps)]TJ -197.387 -11.955 Td [(an)-309(input)-308(v)25(ariable)]TJ/F76 9.9626 Tf 70.95 0 Td [(X)]TJ/F58 9.9626 Tf 11.736 0 Td [(to)-308(an)-309(output)-309(v)25(ariable)]TJ/F76 9.9626 Tf 86.755 0 Td [(Y)]TJ/F58 9.9626 Tf 8.949 0 Td [(,)-323(formally)]TJ/F25 9.9626 Tf 43.095 0 Td [(F)]TJ/F19 9.9626 Tf 11.998 0 Td [(:)]TJ/F76 9.9626 Tf -233.483 -11.955 Td [(X)]TJ/F25 9.9626 Tf 11.447 0 Td [(7!)]TJ/F76 9.9626 Tf 12.748 0 Td [(Y)]TJ/F58 9.9626 Tf 8.948 0 Td [(,)-251(where)]TJ/F76 9.9626 Tf 31.832 0 Td [(Y)]TJ/F58 9.9626 Tf 11.449 0 Td [(is)-251(a)-251(discrete)-251(label)-251(v)15(ector)-251(associated)-251(with)]TJ -76.424 -11.955 Td [(each)-215(input)]TJ/F76 9.9626 Tf 43.025 0 Td [(X)]TJ/F58 9.9626 Tf 10.808 0 Td [(and)]TJ/F22 9.9626 Tf 16.531 0 Td [(y)]TJ/F25 9.9626 Tf 8.009 0 Td [(2)]TJ/F76 9.9626 Tf 9.409 0 Td [(Y)]TJ/F58 9.9626 Tf 11.094 0 Td [(corresponds)-215(to)-216(one)-215(of)]TJ/F22 9.9626 Tf 87.157 0 Td [(C)]TJ/F58 9.9626 Tf 9.978 0 Td [(cate)15(gories)]TJ -196.011 -11.955 Td [(or)-250(classes)-250(in)-250(the)-250(dataset.)]TJ 11.955 -12.174 Td [(A)-270(gi)25(v)15(en)-269(input)-270(image)]TJ/F76 9.9626 Tf 84.505 0 Td [(X)]TJ/F58 9.9626 Tf 11.348 0 Td [(can)-270(be)-269(decomposed)-270(into)-269(a)-270(set)-269(of)]TJ -107.808 -11.955 Td [(pix)15(els)]TJ/F22 9.9626 Tf 26.946 0 Td [(x)]TJ/F21 6.9738 Tf 5.693 -1.494 Td [(i)]TJ/F58 9.9626 Tf 3.317 1.494 Td [(,)-351(where)]TJ/F22 9.9626 Tf 33.62 0 Td [(i)]TJ/F19 9.9626 Tf 7.688 0 Td [(=)]TJ/F25 9.9626 Tf 12.005 0 Td [(f)]TJ/F19 9.9626 Tf 4.981 0 Td [(1)]TJ/F22 9.9626 Tf 4.981 0 Td [(;)-167(:)-166(:)-167(:)-167(;)-166(N)]TJ/F25 9.9626 Tf 31.23 0 Td [(g)]TJ/F58 9.9626 Tf 8.276 0 Td [(and)]TJ/F22 9.9626 Tf 17.681 0 Td [(N)]TJ/F58 9.9626 Tf 12.385 0 Td [(is)-331(the)-330(total)-331(num-)]TJ -168.803 -11.955 Td [(ber)-282(of)-282(pix)15(els)-281(in)-282(the)-282(image.)-406(F)15(or)-281(a)-282(gi)25(v)15(en)-282(image,)-290(we)-282(compute)]TJ 0 -11.955 Td [(the)-285(gradient)-285(of)-285(the)-284(acti)25(v)25(ation)]TJ/F22 9.9626 Tf 118.278 0 Td [(A)]TJ/F21 6.9738 Tf 7.472 3.615 Td [(l)]TJ 0 -6.078 Td [(p)]TJ/F58 9.9626 Tf 7.443 2.463 Td [(with)-285(respect)-285(to)-285(each)-284(pix)15(el)]TJ/F22 9.9626 Tf -133.193 -11.955 Td [(x)]TJ/F21 6.9738 Tf 5.694 -1.495 Td [(i)]TJ/F58 9.9626 Tf 3.317 1.495 Td [(,)-251(where)]TJ/F22 9.9626 Tf 31.828 0 Td [(l)]TJ/F58 9.9626 Tf 5.667 0 Td [(designates)-251(the)-251(pre-softmax)-250(layer)-251(of)-251(the)-251(netw)10(ork)]TJ -46.506 -11.956 Td [(and)]TJ/F22 9.9626 Tf 16.976 0 Td [(p)]TJ/F58 9.9626 Tf 7.602 0 Td [(is)-260(the)-260(inde)15(x)-260(of)-260(either)-260(the)-260(true)-260(or)-260(predicted)-260(class)-260(prob-)]TJ -24.578 -11.955 Td [(ability)65(.)-411(W)80(e)-284(w)10(ould)-284(lik)10(e)-283(to)-284(note)-284(that)-284(the)-283(pre-softmax)-284(layer)-284(is)]TJ 0 -11.955 Td [(responsible)-262(for)-262(connecting)-261(acti)25(v)25(ations)-262(from)-262(pre)25(vious)-262(layers)]TJ 0 -11.955 Td [(in)-323(the)-323(netw)10(ork)-323(to)-323(indi)25(vidual)-323(class)-323(scores.)-529(Hence,)-341(comput-)]TJ 0 -11.955 Td [(ing)-214(the)-213(gradients)-214(w)65(.r)55(.t.)-298(this)-213(class)-214(inde)15(x)15(ed)-214(score)-213(measures)-214(the)]TJ 0 -11.956 Td [(contrib)20(ution)-250(of)-250(features)-250(to)-250(the)-250<026e616c>-250(class)-250(prediction)-250([)]TJ
0 1 0 rg 0 1 0 RG
[(64)]TJ
0 g 0 G
[(].)]TJ 11.955 -12.173 Td [(Note)-276(our)-277(goal)-276(is)-277(to)-276(rank)-276(e)15(xamples,)-283(so)-277(for)-276(each)-277(e)15(xample,)]TJ -11.955 -11.955 Td [(we)-289(compute)-288(the)-289(pre-softmax)-289(acti)25(v)25(ation)-288(gradient)-289(inde)15(x)15(ed)-289(at)]TJ 0 -11.955 Td [(predicted/true)-381(label)-382(with)-381(respect)-381(to)-382(the)-381(input.)-704(This)-381(is)-382(f)10(ar)]TJ 0 -11.955 Td [(more)-304(computationally)-304(ef)25(\002cient)-305(than)-304(computing)-304(the)-304(full)-305(Ja-)]TJ 0 -11.956 Td [(cobian)-250(matrix)-250(with)-250(indi)25(vidual)-250(layers.)]TJ 11.955 -12.173 Td [(Let)]TJ/F76 9.9626 Tf 15.695 0 Td [(S)]TJ/F58 9.9626 Tf 8.781 0 Td [(be)-242(a)-243(matrix)-242(that)-243(represents)-242(the)-242(gradient)-243(of)]TJ/F22 9.9626 Tf 167.613 0 Td [(A)]TJ/F21 6.9738 Tf 7.472 3.616 Td [(l)]TJ 0 -6.079 Td [(p)]TJ/F58 9.9626 Tf 7.021 2.463 Td [(with)]TJ -218.537 -11.955 Td [(respect)-261(to)-261(indi)25(vidual)-261(pix)15(els)]TJ/F22 9.9626 Tf 110.17 0 Td [(x)]TJ/F21 6.9738 Tf 5.694 -1.494 Td [(i)]TJ/F58 9.9626 Tf 3.317 1.494 Td [(,)]TJ/F61 9.9626 Tf 5.117 0 Td [(i.e)]TJ/F58 9.9626 Tf 9.683 0 Td [(.)-261(for)-261(an)-261(image)-261(of)-260(size)]TJ/F19 9.9626 Tf 87.245 0 Td [(3)]TJ/F25 9.9626 Tf 7.275 0 Td [<02>]TJ/F19 9.9626 Tf -228.501 -11.955 Td [(32)]TJ/F25 9.9626 Tf 12.726 0 Td [<02>]TJ/F19 9.9626 Tf 10.511 0 Td [(32)]TJ/F58 9.9626 Tf 9.963 0 Td [(,)-343(the)-324(gradient)-325(matrix)]TJ/F76 9.9626 Tf 86.438 0 Td [(S)]TJ/F58 9.9626 Tf 9.597 0 Td [(will)-324(be)-325(of)-324(dimensions)]TJ/F19 9.9626 Tf 91.522 0 Td [(3)]TJ/F25 9.9626 Tf 7.744 0 Td [<02>]TJ/F19 9.9626 Tf -228.501 -11.955 Td [(32)]TJ/F25 9.9626 Tf 12.176 0 Td [<02>]TJ/F19 9.9626 Tf 9.963 0 Td [(32)]TJ/F58 9.9626 Tf 9.963 0 Td [(.)]TJ/F76 9.9626 Tf 66.043 -18.644 Td [(S)]TJ/F19 9.9626 Tf 9.133 0 Td [(=)]TJ/F22 9.9626 Tf 11.711 7.704 Td [(@)-56(A)]TJ/F21 6.9738 Tf 13.315 3.616 Td [(l)]TJ 0 -6.078 Td [(p)]TJ
ET
q
1 0 0 1 169.101 130.152 cm
[]0 d 0 J 0.398 w 0 0 m 17.92 0 l S
Q
BT
/F22 9.9626 Tf 170.634 120.827 Td [(@)-56(x)]TJ/F21 6.9738 Tf 11.537 -1.494 Td [(i)]TJ/F58 9.9626 Tf 92.575 8.328 Td [(\(1\))]TJ -224.634 -22.751 Td [(This)-309(formulation)-309(may)-309(feel)-309(f)10(amiliar)-309(as)-309(it)-310(is)-309(often)-309(computed)]TJ 0 -11.955 Td [(based)-222(upon)-222(the)-221(weights)-222(of)-222(a)-222(trained)-221(model)-222(and)-222(visualized)-222(as)]TJ 0 -11.955 Td [(a)-256(image)-256(heatmap)-257(for)-256(interpretability)-256(purposes)-256([)]TJ
0 1 0 rg 0 1 0 RG
[(5)]TJ
0 g 0 G
[(,)]TJ
0 1 0 rg 0 1 0 RG
[-134(31)]TJ
0 g 0 G
[(,)]TJ
0 1 0 rg 0 1 0 RG
[-135(63)]TJ
0 g 0 G
[(,)]TJ
0 1 0 rg 0 1 0 RG
[-134(64)]TJ
0 g 0 G
[(,)]TJ
0 g 0 G
0 g 0 G
0 1 0 rg 0 1 0 RG
258.75 629.037 Td [(64)]TJ
0 g 0 G
[<96>]TJ
0 1 0 rg 0 1 0 RG
[(66)]TJ
0 g 0 G
[(].)-520(In)-320(contrast)-320(to)-320(salienc)15(y)-321(m)1(aps)-321(which)-320(are)-320(inherently)]TJ 0 -11.955 Td [(local)-309(e)15(xplanation)-308(tools,)-324(we)-309(are)-308(le)25(v)15(eraging)-309(relati)25(v)15(e)-309(changes)]TJ 0 -11.955 Td [(in)-250(gradients)-250(across)-250(training)-250(to)-250(rank)-250(all)-250(e)15(xamples)-250(globally)65(.)]TJ 11.955 -11.955 Td [(F)15(ollo)25(wing)-416(se)25(v)15(eral)-417(seminal)-416(papers)-417(in)-416(e)15(xplainability)-417(lit-)]TJ -11.955 -11.955 Td [(erature)-474([)]TJ
0 1 0 rg 0 1 0 RG
[(31)]TJ
0 g 0 G
[(,)]TJ
0 1 0 rg 0 1 0 RG
[-280(63)]TJ
0 g 0 G
[<96>]TJ
0 1 0 rg 0 1 0 RG
[(66)]TJ
0 g 0 G
[(],)-530(we)-474(tak)10(e)-475(the)-474(a)20(v)15(erage)-474(o)15(v)15(er)-474(the)-475(color)]TJ 0 -11.956 Td [(channels)-310(to)-309(arri)25(v)15(e)-310(at)-310(a)-310(gradient)-309(matrix)-310([)]TJ
0 1 0 rg 0 1 0 RG
[(63)]TJ
0 g 0 G
[<96>]TJ
0 1 0 rg 0 1 0 RG
[(66)]TJ
0 g 0 G
[(])-310(where)]TJ/F76 9.9626 Tf 219.373 0 Td [(S)]TJ/F25 9.9626 Tf 10.235 0 Td [(2)]TJ/F69 9.9626 Tf -229.608 -11.955 Td [(R)]TJ/F18 6.9738 Tf 7.195 3.616 Td [(32)]TJ/F24 6.9738 Tf 7.943 0 Td [<02>]TJ/F18 6.9738 Tf 6.226 0 Td [(32)]TJ/F58 9.9626 Tf 8.441 -3.616 Td [(.)-370(F)15(or)-270(a)-270(gi)25(v)15(en)-270(set)-270(of)]TJ/F22 9.9626 Tf 78.845 0 Td [(K)]TJ/F58 9.9626 Tf 11.864 0 Td [(checkpoints,)-275(we)-270(generate)-270(the)]TJ -120.514 -11.955 Td [(abo)15(v)15(e)-266(gradient)-266(matrix)]TJ/F76 9.9626 Tf 90.105 0 Td [(S)]TJ/F58 9.9626 Tf 9.016 0 Td [(for)-266(all)-266(indi)25(vidual)-266(checkpoints,)]TJ/F61 9.9626 Tf 122.464 0 Td [(i.e)]TJ/F58 9.9626 Tf 9.684 0 Td [(.,)]TJ/F25 9.9626 Tf -231.269 -11.955 Td [(f)]TJ/F76 9.9626 Tf 4.981 0 Td [(S)]TJ/F18 6.9738 Tf 6.365 -1.494 Td [(1)]TJ/F22 9.9626 Tf 4.47 1.494 Td [(;)-167(:)-166(:)-167(:)-167(;)]TJ/F76 9.9626 Tf 22.139 0 Td [(S)]TJ/F21 6.9738 Tf 6.365 -1.494 Td [(K)]TJ/F25 9.9626 Tf 7.681 1.494 Td [(g)]TJ/F58 9.9626 Tf 4.981 0 Td [(.)-560(W)80(e)-333(then)-333(calculate)-334(the)-333(mean)-333(gradient)]TJ/F22 9.9626 Tf 159.982 0 Td [<16>]TJ/F58 9.9626 Tf 9.323 0 Td [(by)]TJ -226.287 -11.955 Td [(taking)-332(the)-332(a)20(v)15(erage)-332(of)-332(the)]TJ/F22 9.9626 Tf 104.712 0 Td [(K)]TJ/F58 9.9626 Tf 12.481 0 Td [(gradient)-332(matrices.)-556(Note,)]TJ/F22 9.9626 Tf 103.102 0 Td [<16>]TJ/F58 9.9626 Tf 9.31 0 Td [(is)]TJ -229.605 -11.955 Td [(the)-379(mean)-379(across)-379(dif)25(ferent)-379(checkpoints)-379(and)-379(is)-379(of)-379(the)-379(same)]TJ 0 -11.956 Td [(size)-341(as)-342(the)-341(gradient)-342(matrix)]TJ/F76 9.9626 Tf 111.635 0 Td [(S)]TJ/F58 9.9626 Tf 6.365 0 Td [(.)-584(W)80(e)-342(then)-341(calculate)-342(the)-341(v)25(ari-)]TJ -118 -11.955 Td [(ance)-250(of)-250(gradients)-250(across)-250(each)-250(pix)15(el)-250(as:)]TJ/F22 9.9626 Tf 87.524 -31.237 Td [<16>]TJ/F19 9.9626 Tf 8.77 0 Td [(=)]TJ 13.808 6.74 Td [(1)]TJ
ET
q
1 0 0 1 416.867 537.829 cm
[]0 d 0 J 0.398 w 0 0 m 9.174 0 l S
Q
BT
/F22 9.9626 Tf 416.867 528.504 Td [(K)]TJ/F21 6.9738 Tf 15.634 19.288 Td [(K)]TJ/F12 9.9626 Tf -3.604 -2.989 Td [(X)]TJ/F21 6.9738 Tf 0.647 -21.099 Td [(t)]TJ/F18 6.9738 Tf 3.01 0 Td [(=1)]TJ/F76 9.9626 Tf 12.394 11.634 Td [(S)]TJ/F21 6.9738 Tf 6.365 -1.494 Td [(t)]TJ/F22 9.9626 Tf 3.508 1.494 Td [(:)]TJ/F58 9.9626 Tf 78.675 0 Td [(\(2\))]TJ -164.803 -41.71 Td [(V)129(oG)]TJ/F21 6.9738 Tf 18.083 -1.495 Td [(p)]TJ/F19 9.9626 Tf 7.372 1.495 Td [(=)]TJ/F12 9.9626 Tf 10.516 15.654 Td [(r)]TJ
ET
q
1 0 0 1 414.627 509.481 cm
[]0 d 0 J 0.398 w 0 0 m 11.565 0 l S
Q
BT
/F19 9.9626 Tf 417.919 500.367 Td [(1)]TJ
ET
q
1 0 0 1 415.822 496.118 cm
[]0 d 0 J 0.398 w 0 0 m 9.174 0 l S
Q
BT
/F22 9.9626 Tf 415.822 486.794 Td [(K)]TJ/F21 6.9738 Tf 15.634 19.287 Td [(K)]TJ/F12 9.9626 Tf -3.604 -2.989 Td [(X)]TJ/F21 6.9738 Tf 0.647 -21.099 Td [(t)]TJ/F18 6.9738 Tf 3.01 0 Td [(=1)]TJ/F19 9.9626 Tf 10.734 11.635 Td [(\()]TJ/F76 9.9626 Tf 3.874 0 Td [(S)]TJ/F21 6.9738 Tf 6.365 -1.495 Td [(t)]TJ/F25 9.9626 Tf 5.722 1.495 Td [<00>]TJ/F22 9.9626 Tf 9.962 0 Td [<16>]TJ/F19 9.9626 Tf 6.003 0 Td [(\))]TJ/F18 6.9738 Tf 3.875 4.113 Td [(2)]TJ/F22 9.9626 Tf 4.469 -4.113 Td [(:)]TJ/F58 9.9626 Tf 50.983 0 Td [(\(3\))]TJ -224.634 -26.343 Td [(W)80(e)-234(a)20(v)15(erage)-235(the)-234(pix)15(el-wise)-235(v)25(ariance)-234(of)-235(gradients)-234(to)-235(compute)]TJ 0 -11.955 Td [(a)-250(scalar)-250(V)129(oG)-250(score)-250(for)-250(the)-250(gi)25(v)15(en)-250(input)-250(image:)]TJ 72.074 -31.237 Td [(V)129(oG)]TJ/F19 9.9626 Tf 20.85 0 Td [(=)]TJ 13.766 6.74 Td [(1)]TJ
ET
q
1 0 0 1 413.497 426.584 cm
[]0 d 0 J 0.398 w 0 0 m 9.091 0 l S
Q
BT
/F22 9.9626 Tf 413.497 417.259 Td [(N)]TJ/F21 6.9738 Tf 15.613 19.287 Td [(N)]TJ/F12 9.9626 Tf -3.666 -2.989 Td [(X)]TJ/F21 6.9738 Tf 0.647 -21.098 Td [(t)]TJ/F18 6.9738 Tf 3.009 0 Td [(=1)]TJ/F19 9.9626 Tf 10.734 11.634 Td [(\()]TJ/F58 9.9626 Tf 3.875 0 Td [(V)129(oG)]TJ/F21 6.9738 Tf 18.082 -1.495 Td [(p)]TJ/F19 9.9626 Tf 4.605 1.495 Td [(\))]TJ/F22 9.9626 Tf 3.874 0 Td [(;)]TJ/F58 9.9626 Tf 63.226 0 Td [(\(4\))]TJ -224.634 -30.304 Td [(where)]TJ/F22 9.9626 Tf 26.335 0 Td [(N)]TJ/F58 9.9626 Tf 11.086 0 Td [(is)-200(the)-201(total)-200(number)-200(of)-201(pix)15(els)-200(in)-200(a)-201(gi)25(v)15(en)-200(image.)-294(First)]TJ -37.421 -11.955 Td [(calculating)-300(the)-300(pix)15(el-wise)-299(v)25(ariance)-300(\(Eqn.)]TJ
1 0 0 rg 1 0 0 RG
[-300(3)]TJ
0 g 0 G
[(\))-300(and)-300(then)-300(a)20(v)15(er)20(-)]TJ 0 -11.955 Td [(age)-231(o)15(v)15(er)-232(the)-231(pix)15(els)-232(\(Eqn.)]TJ
1 0 0 rg 1 0 0 RG
[-231(4)]TJ
0 g 0 G
[(\))-232(is)-231(consistent)-232(with)-231(pre)25(vious)-232(XAI)]TJ 0 -11.955 Td [(w)10(orks)-279(where)-278(the)-279(gradients)-278(of)-279(an)-278(input)-279(image)-278(are)-279(computed)]TJ 0 -11.955 Td [(independently)-250(for)-250(each)-250(pix)15(el)-250(in)-250(an)-250(image)-250([)]TJ
0 1 0 rg 0 1 0 RG
[(64)]TJ
0 g 0 G
[<96>]TJ
0 1 0 rg 0 1 0 RG
[(66)]TJ
0 g 0 G
[(].)]TJ 11.955 -11.955 Td [(In)-335(order)-336(to)-335(account)-335(for)-336(inherent)-335(dif)25(ferences)-335(in)-336(v)25(ariance)]TJ -11.955 -11.956 Td [(between)-520(classes,)-588(we)-521(normalize)-520(the)-520(absolute)-521(V)129(oG)-520(score)]TJ 0 -11.955 Td [(by)-493(class-le)25(v)15(el)-493(V)129(oG)-492(mean)-493(and)-493(standard)-492(de)25(viation.)-1039(This)]TJ 0 -11.955 Td [(amounts)-364(to)-364(asking:)]TJ/F61 9.9626 Tf 82.911 0 Td [(What)-364(is)-364(the)-364(variance)-364(of)-364(gr)15(adients)-364(for)]TJ -82.911 -11.955 Td [(a)-380(given)-380(ima)10(g)10(e)-381(with)-380(r)37(espect)-380(to)-380(all)-380(other)-380(e)20(xemplar)10(s)-381(of)-380(this)]TJ 0 -11.955 Td [(class)-250(cate)40(gory?)]TJ/F57 10.9589 Tf 0 -31.843 Td [(2.1.)-420(V)92(alidating)-420(the)-420(beha)25(vior)-421(of)-420(V)100(oG)-420(on)-420(synthetic)]TJ 24.657 -11.955 Td [(data)]TJ/F58 9.9626 Tf -12.702 -17.933 Td [(In)-354(Fig.)]TJ
1 0 0 rg 1 0 0 RG
[-355(1a)]TJ
0 g 0 G
[(,)-380(we)-354(illustrate)-355(the)-354(principle)-354(and)-355(ef)25(fecti)25(v)15(eness)]TJ -11.955 -11.955 Td [(of)-325(V)129(oG)-326(in)-325(a)-326(controlled)-325(to)10(y)-325(e)15(xample)-326(setting.)-536(The)-325(data)-326(w)10(as)]TJ 0 -11.956 Td [(generated)-404(using)-404(tw)10(o)-403(separate)-404(isotropic)-404(Gaussian)-404(clusters.)]TJ 0 -11.955 Td [(In)-312(such)-312(a)-312(simple)-312(lo)25(w)-312(dimensional)-312(problem,)-328(the)-312(most)-312(chal-)]TJ 0 -11.955 Td [(lenging)-341(e)15(xamples)-340(for)-341(the)-341(model)-340(to)-341(classify)-340(can)-341(be)-341(quanti-)]TJ 0 -11.955 Td [<026564>-356(by)-357(distance)-356(to)-357(the)-356(decision)-357(boundary)65(.)-630(In)-356(Fig.)]TJ
1 0 0 rg 1 0 0 RG
[-357(1a)]TJ
0 g 0 G
[(,)-383(we)]TJ 0 -11.955 Td [(visualize)-305(the)-305(trained)-305(decision)-305(boundary)-305(of)-305(a)-305(multiple)-305(layer)]TJ 0 -11.955 Td [(perceptron)-271(\(MLP\))-271(with)-272(a)-271(single)-271(hidden)-271(layer)-271(trained)-271(for)]TJ/F19 9.9626 Tf 226.287 0 Td [(15)]TJ/F58 9.9626 Tf -226.287 -11.956 Td [(epochs.)-456(W)80(e)-298(compute)-299(V)129(oG)-299(for)-298(each)-299(training)-298(data)-299(point)-299(and)]TJ 0 -11.955 Td [(plot)-330<026e616c>-329(V)129(oG)-330(score)-329(for)-330(each)-329(point)-330(ag)5(ainst)-330(the)-329(distance)-330(to)]TJ 0 -11.955 Td [(the)-285(trained)-285(boundary)65(.)-415(In)-285(Fig.)]TJ
1 0 0 rg 1 0 0 RG
[-285(1b)]TJ
0 g 0 G
[(,)-294(we)-285(can)-285(see)-285(that)-285(V)129(oG)-285(suc-)]TJ 0 -11.955 Td [(cessfully)-258(ranks)-258(highest)-258(the)-258(e)15(xamples)-258(closest)-259(t)1(o)-259(the)-258(decision)]TJ
0 g 0 G
0 g 0 G
ET
Q
Q
Q
q
1 0 0 1 0 0 cm
/G_iL0Frmrc4cwMl6pw_TCg Do
Q
endstream
endobj
19 0 obj
<< /Font << /F12 48 0 R /F18 49 0 R /F19 50 0 R /F21 51 0 R /F22 52 0 R /F24 53 0 R /F25 54 0 R /F57 42 0 R /F58 43 0 R /F61 45 0 R /F69 55 0 R /F76 56 0 R >> /ProcSet [ /PDF /Text ] /XObject << /G_iL0Frmrc4cwMl6pw_TCg 57 0 R >> >>
endobj
20 0 obj
<< /Length 5970 >>
stream
q
q
q
0 g 0 G
0 g 0 G
0 g 0 G
1 0 0 1 79.187 565.148 cm
q
.48456 0 0 .48456 0 0 cm
q
1 0 0 1 0 0 cm
/Im1 Do
Q
Q
0 g 0 G
1 0 0 1 -79.187 -565.148 cm
BT
/F58 9.9626 Tf 102.577 553.791 Td [(\(a\))-250(T)80(o)10(y)-250(dataset)-250(trained)-250(decision)-250(boundary)]TJ
0 g 0 G
ET
1 0 0 1 309.995 565.428 cm
q
.48593 0 0 .48593 0 0 cm
q
412.56 0 0 317.52 0 0 cm
/Im2 Do
Q
Q
0 g 0 G
1 0 0 1 -309.995 -565.428 cm
BT
/F58 9.9626 Tf 356.489 554.07 Td [(\(b\))-250(Distance)-250(vs.)-310(V)129(oG)-250(score)]TJ
0 g 0 G
0 g 0 G
-306.377 -22.197 Td [(Figure)-304(1.)]TJ/F57 9.9626 Tf 41.202 0 Td [(Left:)]TJ/F58 9.9626 Tf 25.179 0 Td [(V)111(ariance)-304(of)-303(Gradients)-304(\(V)129(oG\))-304(for)-303(each)-304(testing)-304(data)-303(point)-304(in)-304(the)-303(tw)10(o-dimensional)-304(to)10(y)-303(problem.)]TJ/F57 9.9626 Tf 379.261 0 Td [(Right:)]TJ/F58 9.9626 Tf 31.276 0 Td [(V)129(oG)]TJ -476.918 -11.955 Td [(accords)-274(higher)-274(scores)-273(to)-274(the)-274(most)-274(challenging)-274(e)15(xamples)-274(cl)1(osest)-274(to)-274(the)-274(decision)-274(boundary)-274(\(as)-273(measured)-274(by)-274(the)-274(perpendicular)]TJ 0 -11.956 Td [(distance\).)]TJ
0 g 0 G
0 g 0 G
0 g 0 G
0 g 0 G
0 g 0 G
ET
1 0 0 1 55.613 485.656 cm
q
0 1 -1 0 0 0 cm
1 0 0 1 -55.613 -485.656 cm
BT
/F58 9.9626 Tf -162.406 485.656 Td [(C)]TJ/F58 7.9701 Tf 7.144 0 Td [(I)-62(F)12(A)-61(R)]TJ/F58 9.9626 Tf 19.536 0 Td [(-)-50(1)-50(0)-50(;)]TJ/F73 9.9626 Tf 20.284 0 Td [(plane)]TJ/F58 9.9626 Tf 67.494 0 Td [(C)]TJ/F58 7.9701 Tf 7.143 0 Td [(I)-62(F)12(A)-61(R)]TJ/F58 9.9626 Tf 19.536 0 Td [(-)-50(1)-50(0)-50(0)-50(;)]TJ/F73 9.9626 Tf 25.763 0 Td [(apple)]TJ
ET
1 0 0 1 55.613 485.656 cm
Q
1 0 0 1 -55.613 -485.656 cm
BT
/F58 9.9626 Tf 83.25 485.656 Td [(L)]TJ/F58 7.9701 Tf 6.586 0 Td [(O)-28(W)-62(E)-62(S)-61(T)]TJ/F58 9.9626 Tf 32.129 0 Td [(V)]TJ/F58 7.9701 Tf 7.332 0 Td [(O)]TJ/F58 9.9626 Tf 6.247 0 Td [(G)-5422(H)]TJ/F58 7.9701 Tf 68.896 0 Td [(I)-62(G)-62(H)-62(E)-61(S)-62(T)]TJ/F58 9.9626 Tf 33.773 0 Td [(V)]TJ/F58 7.9701 Tf 7.333 0 Td [(O)]TJ/F58 9.9626 Tf 6.246 0 Td [(G)-7129(L)]TJ/F58 7.9701 Tf 84.798 0 Td [(O)-28(W)-62(E)-62(S)-61(T)]TJ/F58 9.9626 Tf 32.13 0 Td [(V)]TJ/F58 7.9701 Tf 7.332 0 Td [(O)]TJ/F58 9.9626 Tf 6.247 0 Td [(G)-5421(H)]TJ/F58 7.9701 Tf 68.896 0 Td [(I)-62(G)-62(H)-61(E)-62(S)-61(T)]TJ/F58 9.9626 Tf 33.773 0 Td [(V)]TJ/F58 7.9701 Tf 7.332 0 Td [(O)]TJ/F58 9.9626 Tf 6.247 0 Td [(G)]TJ
ET
1 0 0 1 59.878 365.027 cm
q
.67773 0 0 .67773 0 0 cm
q
340 0 0 170 0 0 cm
/Im3 Do
Q
Q
1 0 0 1 0 -116.21 cm
q
.67773 0 0 .67773 0 0 cm
q
340 0 0 170 0 0 cm
/Im4 Do
Q
Q
0 g 0 G
1 0 0 1 -59.878 -248.817 cm
BT
/F58 9.9626 Tf 128.889 237.459 Td [(\(a\))-250(Early-stage)-250(training)]TJ
0 g 0 G
ET
1 0 0 1 304.921 365.027 cm
q
.67773 0 0 .67773 0 0 cm
q
340 0 0 170 0 0 cm
/Im5 Do
Q
Q
1 0 0 1 0 -116.21 cm
q
.67773 0 0 .67773 0 0 cm
q
340 0 0 170 0 0 cm
/Im6 Do
Q
Q
0 g 0 G
1 0 0 1 -304.921 -248.817 cm
BT
/F58 9.9626 Tf 375.591 237.459 Td [(\(b\))-250(Late-stage)-250(training)]TJ
0 g 0 G
0 g 0 G
-325.479 -21.918 Td [(Figure)-387(2.)-719(The)-387(5)]TJ/F25 9.9626 Tf 68.83 0 Td [<02>]TJ/F58 9.9626 Tf 7.748 0 Td [(5)-387(grid)-386(sho)25(ws)-387(the)-386(top-25)-387(Cif)10(ar)20(-10)-386(and)-387(Cif)10(ar)20(-100)-386(training-set)-387(images)-387(with)-386(the)-387(lo)25(west)-386(and)-387(highest)-386(V)129(oG)]TJ -76.578 -11.955 Td [(scores)-249(in)-248(the)]TJ/F61 9.9626 Tf 52.256 0 Td [(Early)]TJ/F58 9.9626 Tf 24.615 0 Td [(\(a\))-249(and)]TJ/F61 9.9626 Tf 30.401 0 Td [(Late)]TJ/F58 9.9626 Tf 20.191 0 Td [(\(b\))-249(training)-249(s)1(tage)-249(respecti)25(v)15(ely)-249(of)-249(tw)10(o)-248(randomly)-249(chosen)-249(classes.)-309(Lo)25(wer)-249(V)129(oG)-249(images)-249(e)25(vidence)]TJ -127.463 -11.955 Td [(uncluttered)-207(backgrounds)-208(\(for)-207(both)]TJ/F73 9.9626 Tf 136.641 0 Td [(apple)]TJ/F58 9.9626 Tf 23.956 0 Td [(and)]TJ/F73 9.9626 Tf 16.452 0 Td [(plane)]TJ/F58 9.9626 Tf 21.89 0 Td [(\))-207(in)-208(the)]TJ/F61 9.9626 Tf 29.44 0 Td [(Late)]TJ/F58 9.9626 Tf 19.779 0 Td [(training)-207(stage.)-296(V)129(oG)-207(also)-208(appears)-207(to)-207(capture)-208(a)-207(color)-208(bias)-207(present)]TJ -248.158 -11.955 Td [(during)-279(the)]TJ/F61 9.9626 Tf 43.753 0 Td [(Early)]TJ/F58 9.9626 Tf 24.92 0 Td [(training)-279(stage)-280(for)-279(both)]TJ/F73 9.9626 Tf 91.929 0 Td [(apple)]TJ/F58 9.9626 Tf 24.674 0 Td [(\(red\).)-398(The)-279(V)129(oG)-280(images)-279(in)]TJ/F61 9.9626 Tf 106.496 0 Td [(Late)]TJ/F58 9.9626 Tf 20.496 0 Td [(training)-279(stage)-280(present)-279(unusual)-279(v)25(antage)-280(points,)]TJ -312.268 -11.955 Td [(with)-250(images)-250(where)-250(the)-250(frame)-250(is)-250(zoomed)-250(in)-250(on)-250(the)-250(object)-250(of)-250(interest.)]TJ
0 g 0 G
0 g 0 G
0 -33.475 Td [(boundary)65(.)-294(The)-202(most)-202(challenging)-202(e)15(xamples)-202(e)15(xhibit)-203(the)-202(great-)]TJ 0 -11.955 Td [(est)-233(v)25(ariance)-233(in)-233(gradient)-233(updates)-233(o)15(v)15(er)-233(the)-233(course)-234(of)-233(the)-233(train-)]TJ 0 -11.955 Td [(ing)-228(process.)-303(In)-228(the)-229(follo)25(wing)-228(sections,)-233(we)-228(will)-228(scale)-228(this)-229(to)10(y)]TJ 0 -11.955 Td [(problem)-296(and)-297(sho)25(w)-296(consistent)-297(results)-296(across)-296(multiple)-297(archi-)]TJ 0 -11.956 Td [(tectures)-250(and)-250(datasets.)]TJ
0 g 0 G
0 g 0 G
/F57 10.9589 Tf 258.75 47.821 Td [(2.2.)-250(Experimental)-250(Setup)]TJ/F57 9.9626 Tf 0 -29.336 Td [(Datasets.)]TJ/F58 9.9626 Tf 45 0 Td [(W)80(e)-366(e)25(v)25(aluate)-365(our)-366(methodology)-366(on)-365(Cif)10(ar)20(-10)-366(and)]TJ -45 -11.955 Td [(Cif)10(ar)20(-100)-197([)]TJ
0 1 0 rg 0 1 0 RG
[(43)]TJ
0 g 0 G
[(],)-207(and)-197(ImageNet)-197([)]TJ
0 1 0 rg 0 1 0 RG
[(61)]TJ
0 g 0 G
[(])-197(datasets.)-292(F)15(or)-197(all)-197(datasets,)]TJ 0 -11.955 Td [(we)-250(compute)-250(V)129(oG)-250(for)-250(both)-250(training)-250(and)-250(test)-250(sets.)]TJ
0 g 0 G
0 g 0 G
ET
Q
Q
Q
q
1 0 0 1 0 0 cm
/1OgBxXrPKrfMemRHBqIV7Q Do
Q
endstream
endobj
21 0 obj
<< /CS /DeviceRGB /S /Transparency /Type /Group >>
endobj
22 0 obj
<< /Font << /F25 54 0 R /F57 42 0 R /F58 43 0 R /F61 45 0 R /F73 58 0 R >> /ProcSet [ /PDF /Text /ImageC ] /XObject << /1OgBxXrPKrfMemRHBqIV7Q 59 0 R /Im1 60 0 R /Im2 61 0 R /Im3 62 0 R /Im4 63 0 R /Im5 64 0 R /Im6 65 0 R >> >>
endobj
23 0 obj
<< /Length 17719 >>
stream
q
q
q
0 g 0 G
0 g 0 G
BT
/F57 9.9626 Tf 50.112 710.037 Td [(Cifar)-448(T)74(raining)15(.)]TJ/F58 9.9626 Tf 75.39 0 Td [(W)80(e)-448(use)-447(a)-448(ResNet-18)-447(netw)10(ork)-448([)]TJ
0 1 0 rg 0 1 0 RG
[(25)]TJ
0 g 0 G
[(])-447(for)]TJ -75.39 -11.955 Td [(both)-419(Cif)10(ar)20(-10)-419(and)-419(Cif)10(ar)20(-100.)-818(F)15(or)-419(each)-419(dataset,)-462(we)-419(train)]TJ 0 -11.955 Td [(the)-270(model)-270(for)]TJ/F19 9.9626 Tf 56.773 0 Td [(350)]TJ/F58 9.9626 Tf 17.637 0 Td [(epochs)-270(using)-270(stochastic)-271(gradient)-270(descent)]TJ -74.41 -11.955 Td [(\(SGD\))-237(and)-237(compute)-237(the)-237(input)-237(gradients)-237(for)-237(each)-237(sample)-237(e)25(v-)]TJ 0 -11.955 Td [(ery)]TJ/F19 9.9626 Tf 16.093 0 Td [(10)]TJ/F58 9.9626 Tf 13.333 0 Td [(epochs.)-575(W)80(e)-338(implemented)-338(standard)-339(data)-338(augmenta-)]TJ -29.426 -11.956 Td [(tion)-289(by)-289(applying)-290(cropping)-289(and)-289(horizontal)-289<03697073>-289(of)-290(input)-289(im-)]TJ 0 -11.955 Td [(ages.)-309(W)80(e)-246(use)-247(a)-246(base)-247(learning)-246(rate)-247(schedule)-246(of)]TJ/F19 9.9626 Tf 182.095 0 Td [(0)]TJ/F22 9.9626 Tf 4.981 0 Td [(:)]TJ/F19 9.9626 Tf 2.768 0 Td [(1)]TJ/F58 9.9626 Tf 7.437 0 Td [(and)-247(adap-)]TJ -197.281 -11.955 Td [(ti)25(v)15(ely)-362(change)-362(to)]TJ/F19 9.9626 Tf 69.086 0 Td [(0)]TJ/F22 9.9626 Tf 4.982 0 Td [(:)]TJ/F19 9.9626 Tf 2.767 0 Td [(01)]TJ/F58 9.9626 Tf 13.571 0 Td [(at)]TJ/F19 9.9626 Tf 10.801 0 Td [(150)]TJ/F58 6.9738 Tf 14.944 3.615 Td [(th)]TJ/F58 9.9626 Tf 9.533 -3.615 Td [(and)]TJ/F19 9.9626 Tf 17.994 0 Td [(0)]TJ/F22 9.9626 Tf 4.981 0 Td [(:)]TJ/F19 9.9626 Tf 2.768 0 Td [(001)]TJ/F58 9.9626 Tf 18.552 0 Td [(at)]TJ/F19 9.9626 Tf 10.801 0 Td [(250)]TJ/F58 6.9738 Tf 14.944 3.615 Td [(th)]TJ/F58 9.9626 Tf 9.533 -3.615 Td [(training)]TJ -205.257 -11.955 Td [(epochs.)-405(The)-281(top-1)-282(test)-281(set)-282(accurac)15(y)-281(for)-282(Cif)10(ar)20(-10)-281(and)-282(Cif)10(ar)20(-)]TJ 0 -11.955 Td [(100)-250(were)]TJ/F19 9.9626 Tf 39.282 0 Td [(89)]TJ/F22 9.9626 Tf 9.963 0 Td [(:)]TJ/F19 9.9626 Tf 2.768 0 Td [(57%)]TJ/F58 9.9626 Tf 20.755 0 Td [(and)]TJ/F19 9.9626 Tf 16.877 0 Td [(66)]TJ/F22 9.9626 Tf 9.962 0 Td [(:)]TJ/F19 9.9626 Tf 2.768 0 Td [(86%)]TJ/F58 9.9626 Tf 20.755 0 Td [(respecti)25(v)15(ely)65(.)]TJ/F57 9.9626 Tf -123.13 -18.667 Td [(ImageNet)-502(T)74(raining)15(.)]TJ/F58 9.9626 Tf 96.38 0 Td [(W)80(e)-502(use)-503(a)-502(ResNet-50)-502([)]TJ
0 1 0 rg 0 1 0 RG
[(25)]TJ
0 g 0 G
[(])-503(model)]TJ -96.38 -11.955 Td [(for)-376(training)-376(on)-377(Image)1(Net.)-689(The)-376(netw)10(ork)-376(w)10(as)-377(trained)-376(with)]TJ 0 -11.955 Td [(batch)-362(normalization)-362([)]TJ
0 1 0 rg 0 1 0 RG
[(35)]TJ
0 g 0 G
[(],)-391(weight)-362(decay)65(,)-390(decreasing)-363(learn-)]TJ 0 -11.956 Td [(ing)-322(rate)-322(schedules,)-340(and)-322(augmented)-322(training)-322(data.)-527(W)80(e)-322(train)]TJ 0 -11.955 Td [(for)]TJ/F19 9.9626 Tf 15.09 0 Td [(32)]TJ/F22 9.9626 Tf 9.962 0 Td [(;)]TJ/F19 9.9626 Tf 4.428 0 Td [(000)]TJ/F58 9.9626 Tf 18.417 0 Td [(steps)-349(\(approximately)]TJ/F19 9.9626 Tf 87.742 0 Td [(90)]TJ/F58 9.9626 Tf 13.436 0 Td [(epochs\))-349(on)-348(ImageNet)]TJ -149.075 -11.955 Td [(with)-238(a)-239(batch)-238(size)-238(of)]TJ/F19 9.9626 Tf 79.38 0 Td [(1024)]TJ/F58 9.9626 Tf 19.925 0 Td [(.)-306(W)80(e)-238(store)]TJ/F19 9.9626 Tf 42.688 0 Td [(32)]TJ/F58 9.9626 Tf 12.337 0 Td [(checkpoints)-238(o)15(v)15(er)-239(the)]TJ -154.33 -11.955 Td [(course)-351(of)-352(training,)-377(b)20(ut)-351(in)-352(practice)-351(observ)15(e)-352(that)-351(V)129(oG)-352(rank-)]TJ 0 -11.955 Td [(ing)-328(is)-329(v)15(ery)-328(stable)-329(computed)-328(with)-329(as)-328(fe)25(w)-329(as)]TJ/F19 9.9626 Tf 177.914 0 Td [(3)]TJ/F58 9.9626 Tf 8.254 0 Td [(checkpoints.)]TJ -186.168 -11.956 Td [(Our)-296(model)-296(achie)25(v)15(es)-296(a)-296(top-1)-296(accurac)15(y)-296(of)]TJ/F19 9.9626 Tf 163.941 0 Td [(76)]TJ/F22 9.9626 Tf 9.963 0 Td [(:)]TJ/F19 9.9626 Tf 2.767 0 Td [(68%)]TJ/F58 9.9626 Tf 21.213 0 Td [(and)-296(top-5)]TJ -197.884 -11.955 Td [(accurac)15(y)-250(of)]TJ/F19 9.9626 Tf 48.528 0 Td [(93)]TJ/F22 9.9626 Tf 9.962 0 Td [(:)]TJ/F19 9.9626 Tf 2.768 0 Td [(29%)]TJ/F58 9.9626 Tf 18.265 0 Td [(.)]TJ/F57 9.9626 Tf -79.523 -18.667 Td [(Number)-267(of)-268(checkpoints.)]TJ/F58 9.9626 Tf 105.514 0 Td [(The)-267(number)-268(of)-267(checkpoints)-268(used)]TJ -105.514 -11.955 Td [(to)-257(compute)-257(V)129(oG)-257(balances)-256(ef)25(\002cienc)15(y)-257(for)-257(practitioners)-257(to)-257(use)]TJ 0 -11.955 Td [(with)-255(the)-255(rob)20(ustness)-255(of)-255(ranking.)-326(This)-255(can)-255(be)-255(set)-255(by)-256(the)-255(prac-)]TJ 0 -11.955 Td [(titioner)40(,)-305(and)-294(we)-294(note)-294(that)-294(in)-294(practice)-295(t)1(h)-1(e)-294(last)-294(3)-294(checkpoints)]TJ 0 -11.955 Td [(are)-268(suf)25(\002cient)-267(for)-268(a)-267(rob)20(ust)-268(V)129(oG)-268(ranking)-267(\(minimal)-268(dif)25(ference)]TJ 0 -11.955 Td [(when)-350(restricting)-351(to)-350(the)-351(last)-350(3)-350(in)-351(Figs.)]TJ
1 0 0 rg 1 0 0 RG
[-350(5)]TJ
0 g 0 G
[(b,)]TJ
1 0 0 rg 1 0 0 RG
[(8)]TJ
0 g 0 G
[(b,)]TJ
1 0 0 rg 1 0 0 RG
[(11)]TJ
0 g 0 G
[(b)-351(vs)1(.)-612(e)25(v)25(al-)]TJ 0 -11.956 Td [(uating)-358(on)-357(all)-358(checkpoints)-358(in)-357(Fig.)]TJ
1 0 0 rg 1 0 0 RG
[-358(4)]TJ
0 g 0 G
[(\).)-633(F)15(or)-358(all)-358(e)15(xperiments,)]TJ 0 -11.955 Td [(V)129(oG\()]TJ/F61 9.9626 Tf 21.4 0 Td [(early)]TJ/F58 9.9626 Tf 20.473 0 Td [(-stage\))-345(is)-345(computed)-344(using)-345(checkpoints)-345(from)-345(the)]TJ -41.873 -11.955 Td [<02727374>-203(3)-203(epochs)-203(and)-203(V)129(oG\()]TJ/F61 9.9626 Tf 92.028 0 Td [(late)]TJ/F58 9.9626 Tf 14.944 0 Td [(-stage\))-203(is)-203(computed)-203(using)-203(check-)]TJ -106.972 -11.955 Td [(points)-315(from)-314(the)-315(last)-314(3)-315(epochs.)-504(The)-314(test)-315(set)-315(accurac)15(y)-314(at)-315(the)]TJ/F61 9.9626 Tf 0 -11.955 Td [(early)]TJ/F58 9.9626 Tf 20.473 0 Td [(-stage)-357(is)]TJ/F19 9.9626 Tf 37.548 0 Td [(44)]TJ/F22 9.9626 Tf 9.963 0 Td [(:)]TJ/F19 9.9626 Tf 2.767 0 Td [(65%)]TJ/F58 9.9626 Tf 18.265 0 Td [(,)]TJ/F19 9.9626 Tf 6.314 0 Td [(14)]TJ/F22 9.9626 Tf 9.962 0 Td [(:)]TJ/F19 9.9626 Tf 2.768 0 Td [(16%)]TJ/F58 9.9626 Tf 18.265 0 Td [(,)-384(and)]TJ/F19 9.9626 Tf 24.256 0 Td [(51)]TJ/F22 9.9626 Tf 9.963 0 Td [(:)]TJ/F19 9.9626 Tf 2.767 0 Td [(87%)]TJ/F58 9.9626 Tf 21.821 0 Td [(for)-357(Cif)10(ar)20(-10,)]TJ -185.132 -11.956 Td [(Cif)10(ar)20(-100,)-223(and)-216(ImageNet,)-223(respecti)25(v)15(ely)65(.)-299(In)-216(the)]TJ/F61 9.9626 Tf 181.024 0 Td [(late)]TJ/F58 9.9626 Tf 14.943 0 Td [(-stage)-216(it)-216(is)]TJ/F19 9.9626 Tf -195.967 -11.955 Td [(89)]TJ/F22 9.9626 Tf 9.963 0 Td [(:)]TJ/F19 9.9626 Tf 2.767 0 Td [(57%)]TJ/F58 9.9626 Tf 18.265 0 Td [(,)]TJ/F19 9.9626 Tf 5.542 0 Td [(66)]TJ/F22 9.9626 Tf 9.963 0 Td [(:)]TJ/F19 9.9626 Tf 2.767 0 Td [(86%)]TJ/F58 9.9626 Tf 18.265 0 Td [(,)-306(and)]TJ/F19 9.9626 Tf 22.868 0 Td [(76)]TJ/F22 9.9626 Tf 9.962 0 Td [(:)]TJ/F19 9.9626 Tf 2.768 0 Td [(68%)]TJ/F58 9.9626 Tf 21.204 0 Td [(for)-295(Cif)10(ar)20(-10,)-306(Cif)10(ar)20(-100,)-307(and)]TJ -124.334 -11.955 Td [(ImageNet,)-250(respecti)25(v)15(ely)65(.)]TJ/F57 11.9552 Tf 0 -27.038 Td [(3.)-250(Utility)-250(of)-250(V)100(oG)-250(as)-250(an)-250(A)50(uditing)-250(T)92(ool)]TJ/F58 9.9626 Tf 11.955 -19.971 Td [(In)-297(this)-297(section,)-309(we)-298(e)25(v)25(alua)1(te)-298(the)-297(merits)-297(of)-297(V)129(oG)-297(as)-298(an)-297(au-)]TJ -11.955 -11.955 Td [(diting)-224(tool.)-302(Speci\002cally)65(,)-229(we)-224(\(1\))-225(present)-224(the)-224(qualitati)25(v)15(e)-225(prop-)]TJ 0 -11.955 Td [(erties)-236(of)-236(images)-236(at)-236(both)-235(ends)-236(of)-236(the)-236(V)129(oG)-236(spectrum,)-239(\(2\))-236(mea-)]TJ 0 -11.956 Td [(sure)-227(ho)25(w)-227(discriminati)25(v)15(e)-227(V)129(oG)-227(is)-227(at)-227(separating)-227(easy)-227(e)15(xamples)]TJ 0 -11.955 Td [(from)-251(dif)25(\002cult,)-251(\(3\))-251(quantify)-251(the)-251(stability)-251(of)-251(the)-251(V)129(oG)-251(ranking,)]TJ 0 -11.955 Td [(\(4\))-193(use)-193(V)129(oG)-192(as)-193(an)-193(auditing)-193(tool)-192(for)-193(test)-193(dataset,)-204(and)-193(\(5\))-193(le)25(v)15(er)20(-)]TJ 0 -11.955 Td [(age)-250(V)129(oG)-250(to)-250(understand)-250(the)-250(training)-250(dynamics)-250(of)-250(a)-250(DNN.)]TJ/F57 9.9626 Tf 0 -18.667 Td [(1\))-207(Qualitati)10(v)10(e)-207(inspection)-207(of)-208(ranking)15(.)]TJ/F58 9.9626 Tf 154.494 0 Td [(A)-207(qualitati)25(v)15(e)-207(inspec-)]TJ -154.494 -11.955 Td [(tion)-280(of)-280(e)15(xamples)-280(with)-279(high)-280(and)-280(lo)25(w)-280(V)129(oG)-280(scores)-280(sho)25(ws)-280(that)]TJ 0 -11.955 Td [(there)-240(are)-241(distinct)-240(semantic)-240(properties)-240(to)-241(the)-240(images)-240(at)-241(either)]TJ 0 -11.956 Td [(end)-305(of)-305(the)-305(ranking.)-476(W)80(e)-305(visualize)]TJ/F19 9.9626 Tf 136.171 0 Td [(25)]TJ/F58 9.9626 Tf 13.003 0 Td [(images)-305(rank)10(ed)-305(lo)25(west)]TJ -149.174 -11.955 Td [(and)-218(highest)-218(according)-218(to)-218(V)129(oG)-218(for)-218(both)-219(the)-218(entire)-218(dataset)-218(\(vi-)]TJ 0 -11.955 Td [(sualized)-225(for)-224(ImageNet)-225(in)-225(Fig.)]TJ
1 0 0 rg 1 0 0 RG
[-224(7)]TJ
0 g 0 G
[(\))-225(and)-225(for)-224(speci\002c)-225(classes)-225(\(vi-)]TJ 0 -11.955 Td [(sualized)-270(for)-269(ImageNet)-270(in)-269(Fig.)]TJ
1 0 0 rg 1 0 0 RG
[-270(3)]TJ
0 g 0 G
[-269(and)-270(for)-270(Cif)10(ar)20(-10)-269(and)-270(Cif)10(ar)20(-)]TJ 0 -11.955 Td [(100)-371(in)-371(Fig.)]TJ
1 0 0 rg 1 0 0 RG
[-370(2)]TJ
0 g 0 G
[(\).)-673(Images)-370(with)]TJ/F61 9.9626 Tf 120.916 0 Td [(low)]TJ/F58 9.9626 Tf 18.09 0 Td [(V)129(oG)-371(score)-370(tend)-371(to)-371(ha)20(v)15(e)]TJ
0 g 0 G
0 g 0 G
119.744 629.037 Td [(uncluttered)-263(and)-264(often)-263(white)-263(backgrounds)-264(with)-263(the)-263(object)-264(of)]TJ 0 -11.955 Td [(interest)-259(centered)-260(clearly)-259(in)-260(the)-259(frame.)-338(Images)-260(with)-259(the)]TJ/F61 9.9626 Tf 218.537 0 Td [(high)]TJ/F58 9.9626 Tf -218.537 -11.955 Td [(V)129(oG)-209(scores)-210(ha)20(v)15(e)-209(cluttered)-209(backgrounds)-209(and)-210(the)-209(object)-209(of)-210(in-)]TJ 0 -11.955 Td [(terest)-210(is)-211(not)-210(easily)-210(distinguishable)-210(from)-211(the)-210(background.)-297(W)80(e)]TJ 0 -11.956 Td [(also)-290(note)-290(that)-291(ima)1(ges)-291(with)-290(high)-290(V)129(oG)-290(scores)-290(tend)-291(to)-290(feature)]TJ 0 -11.955 Td [(atypical)-194(v)25(ant)1(age)-194(points)-194(of)-193(the)-194(objects)-193(such)-194(as)-193(highly)-194(zoomed)]TJ 0 -11.955 Td [(frames,)-212(side)-203(pro\002les)-202(of)-203(the)-203(object)-202(or)-203(shots)-202(tak)10(en)-203(from)-203(abo)15(v)15(e.)]TJ 0 -11.955 Td [(Often,)-216(the)-207(object)-207(of)-207(interest)-208(is)-207(partially)-207(occluded)-207(or)-207(there)-208(are)]TJ 0 -11.955 Td [(image)-250(corruptions)-250(present)-250(such)-250(as)-250(hea)20(vy)-250(blur)55(.)]TJ/F57 9.9626 Tf 0 -18.397 Td [(2\))-312(T)92(est)-311(set)-312(err)18(or)-311(and)-312(V)100(oG.)]TJ/F58 9.9626 Tf 115.856 0 Td [(A)-312(v)25(aluabl)1(e)-312(property)-312(of)-311(an)-312(au-)]TJ -115.856 -11.955 Td [(diting)-334(tool)-334(is)-334(to)-333(ef)25(fecti)25(v)15(ely)-334(discriminate)-334(between)-334(easy)-334(and)]TJ 0 -11.955 Td [(challenging)-325(e)15(xamples.)-536(In)-325(Fig.)]TJ
1 0 0 rg 1 0 0 RG
[-325(4)]TJ
0 g 0 G
[(,)-345(we)-325(plot)-325(the)-325(test)-325(set)-326(error)]TJ 0 -11.955 Td [(of)-396(e)15(xamples)-397(b)20(uck)10(eted)-396(by)-396(V)129(oG)-397(decile.)-749(Note)-396(that)-396(we)-397(plot)]TJ 0 -11.955 Td [(error)40(,)-256(so)-255(lo)25(wer)-254(is)-255(better)55(.)-324(W)80(e)-255(sho)25(w)-255(that)-255(e)15(xamples)-254(at)-255(the)-255(lo)25(w-)]TJ 0 -11.956 Td [(est)-306(percentiles)-306(of)-307(V)129(oG)-306(ha)20(v)15(e)-306(lo)25(w)-306(error)-306(rates,)-321(and)-306(misclassi-)]TJ 0 -11.955 Td [(\002cation)-317(increases)-316(with)-317(an)-317(increase)-316(in)-317(V)129(oG)-317(scores.)-510(Our)-317(re-)]TJ 0 -11.955 Td [(sults)-252(are)-252(consistent)-253(across)-252(all)-252(datasets,)-253(yet)-252(the)-252(trend)-252(is)-253(more)]TJ 0 -11.955 Td [(pronounced)-379(for)-379(more)-379(comple)15(x)-379(datasets)-379(such)-379(as)-379(Cif)10(ar)20(-100)]TJ 0 -11.955 Td [(and)-263(ImageNet.)-347(W)80(e)-263(ascribe)-262(this)-263(to)-263(dif)25(ferences)-262(in)-263(underlying)]TJ 0 -11.955 Td [(model)-230(comple)15(xity)65(.)-303(Furthermore,)-234(in)-230(Fig.)]TJ
1 0 0 rg 1 0 0 RG
[-231(10)]TJ
0 g 0 G
[(,)-234(we)-230(observ)15(e)-230(that)]TJ 0 -11.956 Td [(test)-373(set)-373(error)-374(on)-373(the)-373(lo)25(west)-373(V)129(oG)-373(scored)-373(images)-374(are)-373(lo)25(wer)]TJ 0 -11.955 Td [(than)-250(the)-250(baseline)-250(test)-250(set)-250(performance.)]TJ/F57 9.9626 Tf 0 -18.396 Td [(3\))-334(Stability)-334(of)-334(V)100(oG)-333(ranking)15(.)]TJ/F58 9.9626 Tf 126.527 0 Td [(T)80(o)-334(b)20(uild)-334(trust)-334(with)-334(an)-333(end-)]TJ -126.527 -11.955 Td [(user)40(,)-240(a)-238(k)10(e)15(y)-238(desirable)-238(property)-238(of)-237(an)15(y)-238(auditing)-238(tool)-238(is)-238(consis-)]TJ 0 -11.956 Td [(tenc)15(y)-193(in)-194(performance.)-291(W)80(e)-193(w)10(ould)-194(e)15(xpect)-193(a)-193(consistent)-194(method)]TJ 0 -11.955 Td [(to)-255(produce)-254(a)-255(ranking)-255(with)-254(a)-255(closely)-254(bounded)-255(distrib)20(ution)-255(of)]TJ 0 -11.955 Td [(scores)-282(across)-283(independently)-282(trained)-282(runs)-283(for)-282(a)-282(gi)25(v)15(en)-283(model)]TJ 0 -11.955 Td [(and)-193(dataset.)-290(T)80(o)-193(measure)-193(the)-192(consistenc)15(y)-193(of)-193(the)-192(V)129(oG)-193(ranking,)]TJ 0 -11.955 Td [(we)-312(train)-313<02>26(v)14(e)-312(Cif)10(ar)20(-10)-312(netw)10(orks)-312(from)-313(random)-312(initialization)]TJ 0 -11.955 Td [(follo)25(wing)-348(the)-348(training)-347(methodology)-348(described)-348(in)-348(Sec.)]TJ
1 0 0 rg 1 0 0 RG
[-348(2.2)]TJ
0 g 0 G
[(.)]TJ 0 -11.956 Td [(Empirically)65(,)-203(Fig.)]TJ
1 0 0 rg 1 0 0 RG
[-192(6)]TJ
0 g 0 G
[-192(sho)25(ws)-192(that)-192(V)129(oG)-192(rankings)-192(e)25(vidence)-192(a)-192(con-)]TJ 0 -11.955 Td [(sistent)-271(distrib)20(ution)-271(of)-270(test-error)-271(at)-271(each)-271(percentile)-271(gi)25(v)15(en)-271(the)]TJ 0 -11.955 Td [(same)-332(model)-332(and)-332(dataset.)-556(F)15(or)-331(completeness,)-353(we)-332(also)-332(mea-)]TJ 0 -11.955 Td [(sure)-240(instance-wise)-240(V)129(oG)-239(stability)-240(by)-240(computing)-240(the)-240(standard)]TJ 0 -11.955 Td [(de)25(viation)-236(of)-236(V)129(oG)-236(scores)-236(for)-236(50k)-236(Cif)10(ar)20(-10)-237(samples)-236(across)-236(10)]TJ 0 -11.955 Td [(independent)-359(initializations.)-639(The)-359(standard)-360(de)25(viation)-359(of)-360(the)]TJ 0 -11.956 Td [(V)129(oG)-279(scores)-279(is)-280(ne)15(gligible)-279(with)-279(a)-279(mean)-279(de)25(viation)-279(of)]TJ/F19 9.9626 Tf 203.204 0 Td [(3)]TJ/F22 9.9626 Tf 4.981 0 Td [(:)]TJ/F19 9.9626 Tf 2.767 0 Td [(81)]TJ/F22 9.9626 Tf 9.963 0 Td [(e)]TJ/F24 6.9738 Tf 4.639 3.616 Td [<00>]TJ/F18 6.9738 Tf 6.227 0 Td [(9)]TJ/F58 9.9626 Tf -231.781 -15.571 Td [(across)-321(all)-320(samples.)-522(In)-321(addition,)-338(we)-321<026e64>-320(similar)-321(results)-321(for)]TJ 0 -11.955 Td [(Cif)10(ar)20(-100)-374(dataset)-373(where)-374(the)-374(output)-374(V)129(oG)-373(scores)-374(are)-374(stable)]TJ 0 -11.955 Td [(\(mean)-250(std)-250(of)]TJ/F19 9.9626 Tf 52.288 0 Td [(9)]TJ/F22 9.9626 Tf 4.981 0 Td [(:)]TJ/F19 9.9626 Tf 2.768 0 Td [(6)]TJ/F22 9.9626 Tf 4.981 0 Td [(e)]TJ/F25 9.9626 Tf 4.639 0 Td [<00>]TJ/F19 9.9626 Tf 7.749 0 Td [(6)]TJ/F58 9.9626 Tf 4.981 0 Td [(\))-250(across)-250(dif)25(ferent)-249(model)-250(initializations.)]TJ -82.387 -11.955 Td [(Finally)65(,)-342(we)-323(e)15(xtend)-324(our)-323(stability)-324(e)15(xperiments)-323(to)-324(understand)]TJ 0 -11.956 Td [(the)-218(ef)25(fect)-218(of)-218(dif)25(ferent)-218(training)-218(h)5(yperparameter)-218(settings)-218(\(e.g.,)]TJ 0 -11.955 Td [(batch)-353(size\))-353(on)-353(the)-354(V)129(oG)-353(scores.)-619(Here,)-379(we)-353(train)-354(5)-353(Cif)10(ar)20(-10)]TJ 0 -11.955 Td [(models)-207(using)-207(dif)25(ferent)-208(batch)-207(sizes,)-216(i.e.,)]TJ/F25 9.9626 Tf 155.081 0 Td [(f)]TJ/F58 9.9626 Tf 4.981 0 Td [(128,)-216(256,)-216(384,)-215(512,)]TJ -160.062 -11.955 Td [(640)]TJ/F25 9.9626 Tf 14.944 0 Td [(g)]TJ/F58 9.9626 Tf 4.981 0 Td [(,)-243(and)-242<026e64>-242(that)-241(the)-242(mean)-242(V)129(oG)-241(standard)-242(de)25(viation)-242(across)]TJ -19.925 -11.955 Td [(50k)-250(Cif)10(ar)20(-10)-250(samples)-250(w)10(as)]TJ/F19 9.9626 Tf 105.852 0 Td [(1)]TJ/F22 9.9626 Tf 4.981 0 Td [(:)]TJ/F19 9.9626 Tf 2.768 0 Td [(9)]TJ/F22 9.9626 Tf 4.981 0 Td [(e)]TJ/F25 9.9626 Tf 4.639 0 Td [<00>]TJ/F19 9.9626 Tf 7.749 0 Td [(5)]TJ/F58 9.9626 Tf 4.981 0 Td [(.)]TJ/F57 9.9626 Tf -135.951 -18.397 Td [(4\))-280(V)100(oG)-281(as)-280(an)-280(unsuper)10(vised)-281(auditing)-280(tool.)]TJ/F58 9.9626 Tf 177.706 0 Td [(Man)15(y)-280(auditing)]TJ -177.706 -11.955 Td [(tools)-332(used)-332(to)-332(e)25(v)25(aluate)-331(and)-332(understand)-332(possible)-332(model)-332(bias)]TJ 0 -11.955 Td [(require)-370(the)-371(presence)-370(of)-370(labels)-371(for)-370(protected)-370(attrib)20(utes)-371(and)]TJ 0 -11.955 Td [(underlying)-314(v)25(ariables.)-502(Ho)25(we)25(v)15(er)40(,)-330(this)-314(is)-314(highly)-314(infeasible)-314(in)]TJ 0 -11.955 Td [(real-w)10(orld)-304(settings)-305([)]TJ
0 1 0 rg 0 1 0 RG
[(68)]TJ
0 g 0 G
[(].)-473(F)15(or)-304(image)-304(and)-304(language)-305(datasets,)]TJ 0 -11.956 Td [(the)-414(high)-414(dimensionality)-414(of)-413(the)-414(problem)-414(mak)10(es)-414(it)-414(hard)-414(to)]TJ
0 g 0 G
0 g 0 G
ET
Q
Q
Q
q
1 0 0 1 0 0 cm
/nRnPo911iDPInB2Z63T0HQ Do
Q
endstream
endobj
24 0 obj
<< /Font << /F18 49 0 R /F19 50 0 R /F22 52 0 R /F24 53 0 R /F25 54 0 R /F57 42 0 R /F58 43 0 R /F61 45 0 R >> /ProcSet [ /PDF /Text ] /XObject << /nRnPo911iDPInB2Z63T0HQ 66 0 R >> >>
endobj
25 0 obj
<< /Length 8909 >>
stream
q
q
q
0 g 0 G
0 g 0 G
0 g 0 G
0 g 0 G
0 g 0 G
BT
/F58 9.9626 Tf 84.377 713.235 Td [(L)]TJ/F58 7.9701 Tf 6.585 0 Td [(O)-28(W)-62(E)-62(S)-61(T)]TJ/F58 9.9626 Tf 32.129 0 Td [(V)]TJ/F58 7.9701 Tf 7.333 0 Td [(O)]TJ/F58 9.9626 Tf 6.246 0 Td [(G)-5422(H)]TJ/F58 7.9701 Tf 68.897 0 Td [(I)-62(G)-62(H)-61(E)-62(S)-61(T)]TJ/F58 9.9626 Tf 33.772 0 Td [(V)]TJ/F58 7.9701 Tf 7.333 0 Td [(O)]TJ/F58 9.9626 Tf 6.246 0 Td [(G)-6844(L)]TJ/F58 7.9701 Tf 81.964 0 Td [(O)-28(W)-62(E)-62(S)-61(T)]TJ/F58 9.9626 Tf 32.129 0 Td [(V)]TJ/F58 7.9701 Tf 7.333 0 Td [(O)]TJ/F58 9.9626 Tf 6.246 0 Td [(G)-5422(H)]TJ/F58 7.9701 Tf 68.897 0 Td [(I)-62(G)-62(H)-61(E)-62(S)-61(T)]TJ/F58 9.9626 Tf 33.773 0 Td [(V)]TJ/F58 7.9701 Tf 7.332 0 Td [(O)]TJ/F58 9.9626 Tf 6.247 0 Td [(G)]TJ
ET
1 0 0 1 53.781 570.873 cm
q
.25012 0 0 .25012 0 0 cm
q
1 0 0 1 0 0 cm
/Im7 Do
Q
Q
1 0 0 1 245.043 -1.883 cm
q
.2571 0 0 .2571 0 0 cm
q
1 0 0 1 0 0 cm
/Im8 Do
Q
Q
0 g 0 G
0 g 0 G
1 0 0 1 -298.824 -568.99 cm
BT
/F59 7.9701 Tf 152.757 561.145 Td [(M)-150(A)-150(G)-150(P)-150(I)-150(E)-23936(P)-150(O)-150(P)-900(B)-150(O)-150(T)-150(T)-150(L)-150(E)]TJ
0 g 0 G
/F58 9.9626 Tf -102.645 -26.431 Td [(Figure)-233(3.)-304(Each)-234(5)]TJ/F25 9.9626 Tf 66.058 0 Td [<02>]TJ/F58 9.9626 Tf 7.748 0 Td [(5)-233(grid)-233(sho)25(ws)-233(the)-233(top-25)-234(ImageNet)-233(training-set)-233(images)-233(with)-233(the)-233(lo)25(west)-233(and)-233(highest)-233(V)129(oG)-233(scores)-233(for)-233(the)-234(class)]TJ/F59 9.9626 Tf -73.806 -11.955 Td [(magpie)]TJ/F58 9.9626 Tf 38.911 0 Td [(and)]TJ/F59 9.9626 Tf 17.431 0 Td [(pop)-600(bottle)]TJ/F58 9.9626 Tf 59.776 0 Td [(.)-477(T)35(raining)-306(set)-305(images)-306(with)-306(higher)-305(V)129(oG)-306(scores)-306(tend)-305(to)-306(feature)-306(zoomed-i)1(n)-306(images)-306(with)-305(atypical)]TJ -116.118 -11.955 Td [(color)-250(schemes)-250(and)-250(v)25(antage)-250(points.)]TJ
0 g 0 G
0 g 0 G
0 g 0 G
ET
1 0 0 1 57.757 380.768 cm
q
.3392 0 0 .3392 0 0 cm
q
1 0 0 1 0 0 cm
/Im9 Do
Q
Q
0 g 0 G
1 0 0 1 -57.757 -380.768 cm
BT
/F58 9.9626 Tf 110.742 369.411 Td [(\(a\))-250(Cif)10(ar)20(-10)]TJ
0 g 0 G
ET
1 0 0 1 218.651 379.014 cm
q
.35272 0 0 .35272 0 0 cm
q
1 0 0 1 0 0 cm
/Im10 Do
Q
Q
0 g 0 G
1 0 0 1 -218.651 -379.014 cm
BT
/F58 9.9626 Tf 268.867 367.656 Td [(\(b\))-250(Cif)10(ar)20(-100)]TJ
0 g 0 G
ET
1 0 0 1 379.669 378.346 cm
q
.35007 0 0 .35007 0 0 cm
q
1 0 0 1 0 0 cm
/Im11 Do
Q
Q
0 g 0 G
1 0 0 1 -379.669 -378.346 cm
BT
/F58 9.9626 Tf 432.091 366.989 Td [(\(c\))-250(ImageNet)]TJ
0 g 0 G
0 g 0 G
-381.979 -21.918 Td [(Figure)-244(4.)-308(The)-244(mean)-243(top-1)-244(test)-244(set)-244(error)-244(\(y-axis\))-244(for)-243(the)-244(e)15(xamples)-244(thresholded)-244(by)-244(V)129(oG)-243(score)-244(percentile)-244(\(x-axis\).)-308(Across)-244(Cif)10(ar)20(-)]TJ 0 -11.955 Td [(10,)-333(Cif)10(ar)20(-100)-316(and)-317(ImageNet,)-333(mis-classi\002cation)-316(increases)-317(with)-316(an)-316(increase)-317(in)-316(V)129(oG)-317(scores.)-509(Across)-316(all)-317(datasets)-316(the)-316(group)-317(of)]TJ 0 -11.955 Td [(samples)-250(in)-250(the)-250(top-10)-250(percentile)-250(V)129(oG)-250(scores)-250(ha)20(v)15(e)-250(the)-250(highest)-250(error)-250(rate,)]TJ/F61 9.9626 Tf 286.383 0 Td [(i.e)]TJ/F58 9.9626 Tf 9.684 0 Td [(.)-250(contains)-250(most)-250(number)-250(of)-250(misclassi\002ed)-250(samples.)]TJ
0 g 0 G
0 g 0 G
-296.067 -33.475 Td [(identify)-292(a)-293(priori)-292(what)-293(underlying)-292(v)25(ariables)-292(one)-293(needs)-292(to)-293(be)]TJ 0 -11.955 Td [(a)15(w)10(are)-349(of.)-607(Ev)15(en)-348(acquiring)-349(the)-349(labels)-349(for)-349(a)-349(limited)-349(number)]TJ 0 -11.955 Td [(of)-390(attrib)20(utes)-390(protected)-390(by)-390(la)15(w)-390(\(gender)40(,)-426(race\))-390(is)-390(e)15(xpensi)25(v)15(e)]TJ 0 -11.955 Td [(and/or)-226(may)-227(be)-227(percei)26(v)14(ed)-226(as)-227(int)1(rusi)25(v)15(e,)-232(leading)-226(to)-227(noisy)-226(or)-227(in-)]TJ 0 -11.955 Td [(complete)-243(labels)-243([)]TJ
0 1 0 rg 0 1 0 RG
[(2)]TJ
0 g 0 G
[(,)]TJ
0 1 0 rg 0 1 0 RG
[-118(29)]TJ
0 g 0 G
[(].)-308(This)-243(means)-243(that)-243(ranking)-243(techniques)]TJ 0 -11.956 Td [(which)-250(do)-250(not)-250(require)-250(labels)-250(at)-250(test)-250(time)-250(are)-250(v)15(ery)-250(v)25(aluable.)]TJ 11.955 -16.822 Td [(One)-197(k)10(e)15(y)-197(adv)25(antage)-197(of)-197(V)129(oG)-196(is)-197(that)-197(we)-197(sho)25(w)-197(it)-197(continues)-197(to)]TJ -11.955 -11.955 Td [(produce)-202(a)-202(reliable)-202(ranking)-201(e)25(v)15(en)-202(when)-202(the)-202(gradients)-202(are)-202(com-)]TJ 0 -11.955 Td [(puted)]TJ/F61 9.9626 Tf 24.426 0 Td [(w)74(.r)111(.t.)]TJ/F58 9.9626 Tf 21.94 0 Td [(the)-230(predicted)-230(label.)-303(In)-230(Fig.)]TJ
1 0 0 rg 1 0 0 RG
[-229(7)]TJ
0 g 0 G
[(,)-234(we)-230(include)-230(the)-230(top)]TJ -46.366 -11.955 Td [(and)-321(bottom)-321(25)-321(V)129(oG)-321(ImageNet)-321(test)-321(images)-321(using)-321(predicted)]TJ 0 -11.956 Td [(labels)-252(from)-253(the)-252(model.)-317(Finally)65(,)-253(we)-253(also)-252(computed)-252(the)-253(mean)]TJ 0 -11.955 Td [(test-error)-247(for)-248(the)-247(predicted)-247(V)129(oG)-248(distrib)20(ution,)-248(and)-247<026e64>-247(that)-248(it)]TJ 0 -11.955 Td [(also)-334(ef)25(fecti)25(v)15(ely)-334(discriminates)-334(between)-334(top-10)-335(and)-334(bottom-)]TJ 0 -11.955 Td [(10)-250(e)15(xamples,)-250(respecti)25(v)15(ely)-250(\(Fig.)]TJ
1 0 0 rg 1 0 0 RG
[-250(12a)]TJ
0 g 0 G
[(\).)]TJ/F57 9.9626 Tf 0 -22.492 Td [(5\))-374(V)100(oG)-374(understands)-374(early)-374(and)-374(late)-374(training)-374(dynamics.)]TJ/F58 9.9626 Tf 7.558 -11.955 Td [(Recent)-399(w)10(orks)-400(ha)20(v)15(e)-400(sho)25(wn)-399(that)-400(there)-399(are)-400(distinct)-399(stages)]TJ -7.558 -11.955 Td [(to)-326(training)-326(in)-326(deep)-326(neural)-326(netw)10(orks)-326([)]TJ
0 1 0 rg 0 1 0 RG
[(1)]TJ
0 g 0 G
[(,)]TJ
0 1 0 rg 0 1 0 RG
[-180(17)]TJ
0 g 0 G
[(,)]TJ
0 1 0 rg 0 1 0 RG
[-181(36)]TJ
0 g 0 G
[(,)]TJ
0 1 0 rg 0 1 0 RG
[-181(49)]TJ
0 g 0 G
[(].)-538(T)80(o)-326(this)]TJ
0 g 0 G
0 g 0 G
258.75 206.686 Td [(end,)-347(we)-328(in)40(v)15(estig)5(ate)-328(whether)-328(V)129(oG)-327(rankings)-328(are)-328(sensiti)25(v)15(e)-328(to)]TJ 0 -11.955 Td [(the)-344(stage)-344(of)-344(the)-344(training)-344(process.)-593(W)80(e)-344(compute)-344(V)129(oG)-344(sepa-)]TJ 0 -11.955 Td [(rately)-372(for)-372(tw)10(o)-373(dif)25(ferent)-372(stages)-372(of)-372(the)-372(training)-372(process:)-555(\(i\))]TJ 0 -11.955 Td [(the)]TJ/F61 9.9626 Tf 15.698 0 Td [(Early)]TJ/F58 9.9626 Tf 22.137 0 Td [(-stage)-354(\(\002rst)-353(three)-354(epochs\))-354(and)-354(\(ii\))-353(the)]TJ/F61 9.9626 Tf 156.911 0 Td [(Late)]TJ/F58 9.9626 Tf 17.714 0 Td [(-stage)]TJ -212.46 -11.955 Td [(\(last)-405(three)-405(epochs\).)-775(W)80(e)-405(plot)-406(V)129(oG)-405(sc)1(ores)-406(ag)5(ainst)-405(the)-405(test)]TJ 0 -11.956 Td [(set)-353(error)-354(at)-353(each)-354(decile)-353(in)-354(early-)-353(and)-353(late-stage)-354(and)-353<026e64>-354(a)]TJ 0 -11.955 Td [(\003ipping)-368(beha)20(vior)-367(across)-368(all)-367(datasets)-368(and)-368(netw)10(orks)-367(\(Fig.)]TJ
1 0 0 rg 1 0 0 RG
[-368(5)]TJ
0 g 0 G
0 -11.955 Td [(for)-340(ImageNet,)-362(Fig.)]TJ
1 0 0 rg 1 0 0 RG
[-340(8)]TJ
0 g 0 G
[-340(for)-340(Cif)10(ar)20(-100,)-363(and)-340(Fig.)]TJ
1 0 0 rg 1 0 0 RG
[-340(11)]TJ
0 g 0 G
[-340(for)-340(Cif)10(ar)20(-)]TJ 0 -11.955 Td [(10\).)-931(In)-457(the)-457(early)-457(training)-457(stage,)-509(samples)-457(ha)20(ving)-457(higher)]TJ 0 -11.955 Td [(V)129(oG)-389(scores)-389(ha)20(v)15(e)-389(a)-390(lo)25(wer)-389(a)20(v)15(erage)-389(error)-389(rate)-389(as)-390(the)-389(gradi-)]TJ 0 -11.956 Td [(ent)-281(updates)-281(hinge)-281(on)-282(easy)-281(e)15(xamples.)-403(This)-282(phenomenon)-281(re-)]TJ 0 -11.955 Td [(v)15(erses)-216(during)-216(the)-216(late-stage)-216(of)-215(the)-216(training,)-223(where,)-223(across)-216(all)]TJ 0 -11.955 Td [(datasets,)-272(high)-267(V)129(oG)-267(scores)-267(in)-268(the)-267(late-stage)-267(ha)20(v)15(e)-267(the)-268(highest)]TJ 0 -11.955 Td [(error)-231(rates)-230(as)-231(updates)-230(to)-231(the)-231(challenging)-230(e)15(xamples)-231(dominate)]TJ 0 -11.955 Td [(the)-288(computation)-288(of)-288(v)25(ariance.)-425(Further)40(,)-297(we)-288(note)-289(a)-288(noticeable)]TJ 0 -11.955 Td [(visual)-305(dif)25(ference)-305(between)-304(the)-305(image)-305(ranking)-305(computed)-305(for)]TJ/F61 9.9626 Tf 0 -11.956 Td [(early-)]TJ/F58 9.9626 Tf 27.651 0 Td [(and)]TJ/F61 9.9626 Tf 18.247 0 Td [(late)]TJ/F58 9.9626 Tf 14.944 0 Td [(-stages)-387(of)-388(training.)-723(As)-387(seen)-388(in)-387(Fig.)]TJ
1 0 0 rg 1 0 0 RG
[-388(2)]TJ
0 g 0 G
[(,)-422(for)]TJ -60.842 -11.955 Td [(some)-265(classes)-265(such)-265(as)]TJ/F61 9.9626 Tf 85.814 0 Td [(apple)]TJ/F58 9.9626 Tf 22.136 0 Td [(,)-269(it)-265(appears)-264(that)-265(V)129(oG)-265(scores)-265(also)]TJ
0 g 0 G
0 g 0 G
ET
Q
Q
Q
q
1 0 0 1 0 0 cm
/zZMrvvvki60IU7c-9CPGKQ Do
Q
endstream
endobj
26 0 obj
<< /CS /DeviceRGB /S /Transparency /Type /Group >>
endobj
27 0 obj
<< /Font << /F25 54 0 R /F57 42 0 R /F58 43 0 R /F59 44 0 R /F61 45 0 R >> /ProcSet [ /PDF /Text ] /XObject << /Im10 67 0 R /Im11 68 0 R /Im7 69 0 R /Im8 70 0 R /Im9 71 0 R /zZMrvvvki60IU7c-9CPGKQ 72 0 R >> >>
endobj
28 0 obj
<< /Length 9140 >>
stream
q
q
q
0 g 0 G
0 g 0 G
0 g 0 G
1 0 0 1 76.857 573.377 cm
q
.4432 0 0 .4432 0 0 cm
q
1 0 0 1 0 0 cm
/Im12 Do
Q
Q
0 g 0 G
1 0 0 1 -76.857 -573.377 cm
BT
/F58 9.9626 Tf 128.889 562.019 Td [(\(a\))-250(Early-stage)-250(training)]TJ
0 g 0 G
ET
1 0 0 1 321.9 576.279 cm
q
.42566 0 0 .42566 0 0 cm
q
1 0 0 1 0 0 cm
/Im13 Do
Q
Q
0 g 0 G
1 0 0 1 -321.9 -576.279 cm
BT
/F58 9.9626 Tf 375.591 564.922 Td [(\(b\))-250(Late-stage)-250(training)]TJ
0 g 0 G
0 g 0 G
-325.479 -24.82 Td [(Figure)-279(5.)-398(The)-279(mean)-280(top-1)-279(test)-279(set)-280(error)-279(\(y-axis\))-279(for)-279(the)-280(e)15(xamples)-279(thresholded)-279(by)-280(V)129(oG)-279(score)-279(percentile)-279(\(x-axis\))-280(in)-279(ImageNet)]TJ 0 -11.956 Td [(v)25(alidation)-350(set.)-609(The)-350(Early)-350(\(a\))-350(and)-350(Late)-350(\(b\))-350(stage)-350(V)129(oG)-349(analysis)-350(sho)25(ws)-350(in)40(v)15(erse)-350(beha)20(vior)-350(where)-350(the)-350(role)-349(of)-350(V)129(oG)-350<03697073>-350(as)-350(the)]TJ 0 -11.955 Td [(training)-250(progresses.)]TJ
0 g 0 G
0 g 0 G
0 g 0 G
ET
1 0 0 1 72.556 355.199 cm
q
.2076 0 0 .2076 0 0 cm
q
921.75 0 0 662.25 0 0 cm
/Im14 Do
Q
Q
0 g 0 G
1 0 0 1 -72.556 -355.199 cm
BT
/F58 9.9626 Tf 50.112 336.868 Td [(Figure)-330(6.)-552(The)-330(V)129(oG)-331(top-1)-330(test)-331(set)-330(error)-331(for)-330<02>25(v)15(e)-331(ResNet-18)]TJ 0 -11.955 Td [(netw)10(orks)-376(independently)-377(trained)-376(on)-376(Cif)10(ar)20(-10)-376(from)-377(random)]TJ 0 -11.955 Td [(initialization.)-628(The)-356(plot)-356(sho)25(ws)-356(that)-356(V)129(oG)-356(produces)-356(a)-356(stable)]TJ 0 -11.955 Td [(ranking)-193(with)-193(a)-193(similar)-193(distrib)20(ution)-193(of)-193(error)-194(in)-193(each)-193(percentile)]TJ 0 -11.956 Td [(across)-250(all)-250(images)]TJ
0 g 0 G
0 g 0 G
0 -32.297 Td [(capture)-369(the)-368(netw)10(ork')55(s)-369(color)-369(bias)-369(during)-368(the)]TJ/F61 9.9626 Tf 181.11 0 Td [(early)]TJ/F58 9.9626 Tf 24.147 0 Td [(training)]TJ -205.257 -11.955 Td [(stage,)-246(where)-246(images)-246(with)-245(the)-246(lo)25(west)-245(V)129(oG)-246(scores)-246(o)15(v)15(er)20(-inde)15(x)]TJ 0 -11.955 Td [(on)-250(red-colored)-250(apples.)]TJ/F57 11.9552 Tf 0 -23.322 Td [(4.)-216(Relationship)-216(between)-217(V)100(oG)-216(Scor)18(es)-216(and)-216(Mem-)]TJ 17.933 -13.948 Td [(orized/OoD)-250(Examples)]TJ/F58 9.9626 Tf -5.978 -18.929 Td [(Recent)-326(w)10(orks)-327(ha)20(v)15(e)-326(highlighted)-327(that)-326(DNNs)-326(produce)-327(un-)]TJ -11.955 -11.955 Td [(calibrated)-277(output)-277(probabilities)-277(that)-277(cannot)-277(be)-278(interpreted)-277(as)]TJ 0 -11.955 Td [(a)-311(measure)-312(of)-311(certainty)-312([)]TJ
0 1 0 rg 0 1 0 RG
[(22)]TJ
0 g 0 G
[(,)]TJ
0 1 0 rg 0 1 0 RG
[-171(26)]TJ
0 g 0 G
[(,)]TJ
0 1 0 rg 0 1 0 RG
[-170(37)]TJ
0 g 0 G
[(,)]TJ
0 1 0 rg 0 1 0 RG
[-171(44)]TJ
0 g 0 G
[(].)-495(T)80(o)-311(this)-312(e)1(n)-1(d,)-326(we)-312(ar)20(-)]TJ 0 -11.955 Td [(gue)-379(that)-380(if)-379(V)129(oG)-380(is)-379(a)-379(reliable)-380(auditing)-379(tool,)-412(it)-379(should)-380(cap-)]TJ 0 -11.955 Td [(ture)-356(model)-355(uncertainty)-356(e)25(v)15(en)-356(when)-356(it')55(s)-355(not)-356(re\003ected)-356(in)-356(the)]TJ 0 -11.956 Td [(output)-334(probabilities.)-561(W)80(e)-334(consider)-334(V)129(oG)-333(rankings)-334(on)-334(a)-334(task)]TJ 0 -11.955 Td [(where)-408(the)-409(netw)10(ork)-408(produces)-408(highly)-408(con\002dent)-409(predictions)]TJ 0 -11.955 Td [(for)-227(incorrect/out-of-distrib)20(ution)-227(input)1(s)-227(and)-227(e)25(v)25(aluate)-227(V)129(oG)-227(on)]TJ 0 -11.955 Td [(tw)10(o)-255(separate)-256(tasks:)-321(\(1\))-255(identifying)-256(e)15(xamples)-255(memorized)-256(by)]TJ
0 g 0 G
0 g 0 G
258.75 401.717 Td [(the)-250(model)-250(and)-250(\(2\))-250(detecting)-250(out-of-distrib)20(ution)-250(e)15(xamples.)]TJ/F57 10.9589 Tf 0 -30.221 Td [(4.1.)-230(Surfacing)-231(examples)-230(that)-230(r)18(equir)18(e)-231(memorization)]TJ/F58 9.9626 Tf 11.955 -21.364 Td [(Ov)15(erparameterized)-821(netw)10(orks)-821(ha)20(v)15(e)-821(been)-821(sho)25(wn)-821(to)]TJ -11.955 -11.956 Td [(achie)25(v)15(e)-206(zero)-206(training)-205(error)-206(by)-206(memorizing)-206(e)15(xamples)-205([)]TJ
0 1 0 rg 0 1 0 RG
[(19)]TJ
0 g 0 G
[(,)]TJ
0 1 0 rg 0 1 0 RG
[-57(32)]TJ
0 g 0 G
[(,)]TJ
0 1 0 rg 0 1 0 RG
0 -11.955 Td [(72)]TJ
0 g 0 G
[(].)-421(W)80(e)-287(e)15(xplore)-287(whether)-287(V)129(oG)-288(ca)1(n)-288(distinguish)-287(between)-287(e)15(x-)]TJ 0 -11.955 Td [(amples)-200(that)-200(require)-201(memorization)-200(and)-200(the)-200(rest)-200(of)-201(the)-200(dataset.)]TJ 0 -11.955 Td [(T)80(o)-416(do)-416(this,)-458(we)-416(replicate)-416(the)-416(general)-416(e)15(xperiment)-416(setup)-416(of)]TJ 0 -11.955 Td [(Zhang)-275(et)-275(al.)-276([)]TJ
0 1 0 rg 0 1 0 RG
[(72)]TJ
0 g 0 G
[(])-275(and)-275(replace)]TJ/F19 9.9626 Tf 118.527 0 Td [(20%)]TJ/F58 9.9626 Tf 21.007 0 Td [(of)-275(all)-275(labels)-276(in)-275(the)-275(train-)]TJ -139.534 -11.956 Td [(ing)-208(set)-208(with)-208(randomly)-208(shuf)25<036564>-208(labels.)-296(W)80(e)-209(re-train)-208(the)-208(model)]TJ 0 -11.955 Td [(from)-270(random)-270(initialization)-270(and)-270(compute)-271(V)129(oG)-270(scores)]TJ/F61 9.9626 Tf 210.686 0 Td [(acr)45(oss)]TJ -210.686 -11.955 Td [(tr)15(aining)]TJ/F58 9.9626 Tf 35.332 0 Td [(for)-339(all)-338(e)15(xamples)-339(in)-338(the)-339(training)-338(set.)-576(Our)-338(netw)10(ork)]TJ -35.332 -11.955 Td [(achie)25(v)15(es)]TJ/F19 9.9626 Tf 37.625 0 Td [(0%)]TJ/F58 9.9626 Tf 17.007 0 Td [(training)-374(error)-373(which)-374(w)10(ould)-374(only)-373(be)-374(possible)]TJ -54.632 -11.955 Td [(gi)25(v)15(en)-301(successful)-300(memorization)-301(of)-301(the)-300(noisy)-301(e)15(xamples)-301(with)]TJ 0 -11.955 Td [(shuf)25<036564>-306(labels.)-478(W)80(e)-306(no)25(w)-306(answer)-306(the)-306(question:)]TJ/F61 9.9626 Tf 188.65 0 Td [(Is)-306(V)111(oG)-306(able)]TJ -188.65 -11.956 Td [(to)-256(discriminate)-256(between)-256(these)-256(memorized)-256(e)20(xamples)-257(and)-256(the)]TJ 0 -11.955 Td [(r)37(est)-250(of)-250(the)-250(dataset?)]TJ/F58 9.9626 Tf 11.955 -15.387 Td [(W)80(e)-301(perform)-301(a)-301(tw)10(o-sample)]TJ/F22 9.9626 Tf 107.923 0 Td [(t)]TJ/F58 9.9626 Tf 3.597 0 Td [(-test)-301(with)-301(unequal)-301(v)25(ariances)]TJ -123.475 -11.955 Td [([)]TJ
0 1 0 rg 0 1 0 RG
[(69)]TJ
0 g 0 G
[(])-269(and)-270(sho)25(w)-269(that)-269(this)-270(dif)25(ference)-269(is)-269(statistically)-270(signi\002cant)]TJ 0 -11.955 Td [(at)-371(a)]TJ/F22 9.9626 Tf 19.014 0 Td [(p)]TJ/F58 9.9626 Tf 5.012 0 Td [(-v)25(alue)-371(of)]TJ/F19 9.9626 Tf 40.344 0 Td [(0)]TJ/F22 9.9626 Tf 4.981 0 Td [(:)]TJ/F19 9.9626 Tf 2.767 0 Td [(001)]TJ/F58 9.9626 Tf 14.944 0 Td [(,)]TJ/F61 9.9626 Tf 6.492 0 Td [(i.e)]TJ/F58 9.9626 Tf 9.683 0 Td [(.)-371(shuf)25<036564>-372(label)1(s)-372(ha)20(v)15(e)-371(a)-371(dif)25(ferent)]TJ -103.237 -11.955 Td [(V)129(oG)-304(distrib)20(ution)-304(than)-305(the)-304(non-shuf)25<036564>-304(dataset.)-473(Intuiti)25(v)15(ely)65(,)]TJ 0 -11.955 Td [(the)-340(tw)10(o-sample)]TJ/F22 9.9626 Tf 65.327 0 Td [(t)]TJ/F58 9.9626 Tf 3.597 0 Td [(-test)-340(produces)-339(a)]TJ/F22 9.9626 Tf 67.694 0 Td [(p)]TJ/F58 9.9626 Tf 5.013 0 Td [(-v)25(alue)-340(that)-339(can)-340(be)-339(used)]TJ -141.631 -11.956 Td [(to)-329(decide)-328(whether)-329(there)-328(is)-329(e)25(vidence)-328(of)-329(a)-328(signi\002cant)-329(dif)25(fer)20(-)]TJ 0 -11.955 Td [(ence)-319(between)-319(the)-319(tw)10(o)-319(distrib)20(utions)-319(of)-319(V)129(oG)-319(scores.)-517(The)]TJ/F22 9.9626 Tf 227.92 0 Td [(p)]TJ/F58 9.9626 Tf 5.012 0 Td [(-)]TJ -232.932 -11.955 Td [(v)25(alue)-261(represents)-261(the)-260(probability)-261(that)-261(the)-261(dif)25(ference)-261(between)]TJ 0 -11.955 Td [(the)-312(sample)-311(means)-312(is)-311(lar)18(ge,)]TJ/F61 9.9626 Tf 110.398 0 Td [(i.e)]TJ/F58 9.9626 Tf 9.684 0 Td [(.)-312(the)-311(smaller)-312(the)]TJ/F22 9.9626 Tf 68.586 0 Td [(p)]TJ/F58 9.9626 Tf 5.012 0 Td [(-v)25(alue,)-327(the)]TJ -193.68 -11.955 Td [(stronger)-316(is)-316(the)-316(e)25(vidence)-315(that)-316(the)-316(tw)10(o)-316(populations)-316(ha)20(v)15(e)-316(dif-)]TJ 0 -11.955 Td [(ferent)-342(means.)-587(F)15(or)-343(both)-342(Cif)10(ar)20(-10)-342(and)-343(Cif)10(ar)20(-100,)-365(we)-342<026e64>-343(a)]TJ 0 -11.956 Td [(statistically)-410(signi\002cant)-411(dif)25(ference)-410(in)-411(V)129(oG)-410(scores)-410(for)-411(each)]TJ 0 -11.955 Td [(population)-421(\()]TJ/F22 9.9626 Tf 50.134 0 Td [(p)]TJ/F58 9.9626 Tf 5.012 0 Td [(-v)25(alue)-421(is)]TJ/F22 9.9626 Tf 39.685 0 Td [(<)]TJ/F19 9.9626 Tf 13.675 0 Td [(0)]TJ/F22 9.9626 Tf 4.981 0 Td [(:)]TJ/F19 9.9626 Tf 2.767 0 Td [(001)]TJ/F58 9.9626 Tf 14.944 0 Td [(\),)-464(which)-421(sho)25(ws)-421(that)-422(V)129(oG)]TJ -131.198 -11.955 Td [(is)-283(discriminati)25(v)15(e)-284(at)-283(distinguishing)-283(between)-283(memorized)-284(and)]TJ 0 -11.955 Td [(non-memorized)-334(e)15(xamples.)-561(W)80(e)-334(include)-333(more)-334(details)-334(about)]TJ 0 -11.955 Td [(the)-250(statistical)-250(testing)-250(in)-250(Sec.)]TJ
1 0 0 rg 1 0 0 RG
[-250(C)]TJ
0 g 0 G
[(.)]TJ
0 g 0 G
0 g 0 G
ET
Q
Q
Q
q
1 0 0 1 0 0 cm
/uhmQOpx-hnD8W1eqAVtW4g Do
Q
endstream
endobj
29 0 obj
<< /CS /DeviceRGB /S /Transparency /Type /Group >>
endobj
30 0 obj
<< /Font << /F19 50 0 R /F22 52 0 R /F57 42 0 R /F58 43 0 R /F61 45 0 R >> /ProcSet [ /PDF /Text /ImageC ] /XObject << /Im12 73 0 R /Im13 74 0 R /Im14 75 0 R /uhmQOpx-hnD8W1eqAVtW4g 76 0 R >> >>
endobj
31 0 obj
<< /Length 3748 >>
stream
q
q
q
0 g 0 G
0 g 0 G
0 g 0 G
1 0 0 1 59.813 415.468 cm
q
.56395 0 0 .56395 0 0 cm
q
1 0 0 1 0 0 cm
/Im15 Do
Q
Q
0 g 0 G
1 0 0 1 -59.813 -415.468 cm
BT
/F58 9.9626 Tf 142.244 404.111 Td [(\(a\))-250(Lo)25(west)-250(V)129(oG)]TJ
0 g 0 G
ET
1 0 0 1 304.857 415.468 cm
q
.56395 0 0 .56395 0 0 cm
q
1 0 0 1 0 0 cm
/Im16 Do
Q
Q
0 g 0 G
1 0 0 1 -304.857 -415.468 cm
BT
/F58 9.9626 Tf 386.052 404.111 Td [(\(b\))-250(Highest)-250(V)129(oG)]TJ
0 g 0 G
0 g 0 G
-335.94 -21.918 Td [(Figure)-307(7.)-481(Each)-307(5)]TJ/F25 9.9626 Tf 69.29 0 Td [<02>]TJ/F58 9.9626 Tf 7.749 0 Td [(5)-307(grid)-307(sho)25(ws)-307(the)-307(top-25)-307(ImageNet)-307(test)-307(set)-307(images)-307(with)-307(the)-307(lo)25(west)-307(and)-307(highest)-307(V)129(oG)-307(scores)-307(for)-307(the)-307(top-1)]TJ -77.039 -11.955 Td [(predicted)-343(class.)-589(T)70(est)-343(set)-344(images)-343(with)-343(higher)-343(V)129(oG)-343(scores)-343(tend)-343(to)-343(feature)-343(zoomed-in)-343(images)-343(and)-344(are)-343(misclassi\002ed)-343(more)-343(as)]TJ 0 -11.956 Td [(compared)-250(to)-250(the)-250(lo)25(wer)-250(V)129(oG)-250(images)-250(which)-250(tend)-250(to)-250(feature)-250(more)-250(prototypical)-250(v)25(antage)-250(points)-250(of)-250(objects.)]TJ
0 g 0 G
0 g 0 G
0 g 0 G
ET
1 0 0 1 94.78 223.323 cm
q
.45245 0 0 .45245 0 0 cm
q
1 0 0 1 0 0 cm
/Im17 Do
Q
Q
0 g 0 G
1 0 0 1 -94.78 -223.323 cm
BT
/F58 9.9626 Tf 138.791 211.966 Td [(\(a\))-250(Early-stage)-250(training)]TJ
0 g 0 G
ET
1 0 0 1 320.019 223.323 cm
q
.45245 0 0 .45245 0 0 cm
q
1 0 0 1 0 0 cm
/Im18 Do
Q
Q
0 g 0 G
1 0 0 1 -320.019 -223.323 cm
BT
/F58 9.9626 Tf 365.689 211.966 Td [(\(b\))-250(Late-stage)-250(training)]TJ
0 g 0 G
0 g 0 G
-315.577 -21.918 Td [(Figure)-268(8.)-365(The)-268(mean)-268(top-1)-268(test)-269(s)1(et)-269(error)-268(\(y-axis\))-268(for)-268(the)-268(e)15(x)15(emplars)-269(thresholded)-268(by)-268(V)129(oG)-268(score)-268(percentile)-268(\(x-axis\))-269(in)-268(Cif)10(ar)20(-100)]TJ 0 -11.955 Td [(testing)-281(set.)-403(The)-281(early)-281(\(a\))-281(and)-281(late)-281(\(b\))-281(stage)-281(V)129(oG)-281(analysis)-281(sho)25(ws)-281(in)40(v)15(erse)-281(beha)20(vior)-281(where)-281(the)-281(role)-281(of)-281(V)129(oG)-281<03697073>-281(as)-281(the)-281(training)]TJ 0 -11.956 Td [(progresses.)-310(Results)-250(for)-250(Cif)10(ar)20(-10)-250(are)-250(sho)25(wn)-250(in)-250(Appendix)-250(Fig.)]TJ
1 0 0 rg 1 0 0 RG
[-250(11)]TJ
0 g 0 G
[(.)]TJ
0 g 0 G
0 g 0 G
/F57 10.9589 Tf 0 -33.474 Td [(4.2.)-250(OutofDistrib)20(ution)-250(detection)]TJ/F58 9.9626 Tf 11.955 -27.753 Td [(W)80(e)-349(ha)20(v)15(e)-348(already)-349(established)-348(that)-349(V)129(oG)-349(is)-348(v)15(ery)-349(ef)25(fecti)25(v)15(e)]TJ -11.955 -11.955 Td [(at)-390(distinguishing)-391(between)-390(easy)-391(and)-390(challenging)-391(e)15(xamples)]TJ 0 -11.955 Td [(\(Fig.)]TJ
1 0 0 rg 1 0 0 RG
[-202(10)]TJ
0 g 0 G
[(\).)-294(Here,)-211(we)-202(ask)-201(whether)-202(this)-202(mak)10(es)-201(V)129(oG)-202(an)-202(ef)25(fecti)25(v)15(e)]TJ
0 g 0 G
0 g 0 G
258.75 51.663 Td [(out)-312(of)-312(distrib)20(ution)-312(\(OoD\))-312(detec)1(tion)-312(tool.)-496(It)-312(also)-312(gi)25(v)15(es)-312(us)-312(a)]TJ 0 -11.955 Td [(setting)-322(in)-323(which)-322(to)-323(compare)-322(V)129(oG)-322(as)-323(a)-322(ranking)-323(mechanism)]TJ 0 -11.955 Td [(to)-250(other)-250(methods)]TJ 11.955 -15.798 Td [(Ruf)25(f)-363(et)-362(al.)-363([)]TJ
0 1 0 rg 0 1 0 RG
[(59)]TJ
0 g 0 G
[(])-363(benchmark)-362(a)-363(v)25(ariety)-363(of)-362(OoD)-363(detection)]TJ -11.955 -11.955 Td [(techniques)-288(on)-287(MNIST)92(-C)-288([)]TJ
0 1 0 rg 0 1 0 RG
[(50)]TJ
0 g 0 G
[(].)-423(F)15(or)-287(completeness,)-297(we)-288(repli-)]TJ
0 g 0 G
0 g 0 G
ET
Q
Q
Q
q
1 0 0 1 0 0 cm
/eJfhg1Q-a1BEZMs3jkOpbA Do
Q
endstream
endobj
32 0 obj
<< /CS /DeviceRGB /S /Transparency /Type /Group >>
endobj
33 0 obj
<< /Font << /F25 54 0 R /F57 42 0 R /F58 43 0 R >> /ProcSet [ /PDF /Text ] /XObject << /Im15 77 0 R /Im16 78 0 R /Im17 79 0 R /Im18 80 0 R /eJfhg1Q-a1BEZMs3jkOpbA 81 0 R >> >>
endobj
34 0 obj
<< /Length 15887 >>
stream
q
q
q
0 g 0 G
0 g 0 G
BT
/F58 9.9626 Tf 50.112 710.037 Td [(cate)-481(this)-481(precise)-481(setup)-481(by)-481(using)-481(a)-481(trained)-481(LeNet)-481(model)]TJ 0 -11.955 Td [(and)-423(e)25(v)25(aluate)-423(V)129(oG)-422(on)-423(MNIST)92(-C)-422(ag)5(ainst)-423(9)-423(other)-423(methods)]TJ 0 -11.955 Td [([)]TJ
0 1 0 rg 0 1 0 RG
[(12)]TJ
0 g 0 G
[(,)]TJ
0 1 0 rg 0 1 0 RG
[-130(41)]TJ
0 g 0 G
[(,)]TJ
0 1 0 rg 0 1 0 RG
[-130(45)]TJ
0 g 0 G
[(,)]TJ
0 1 0 rg 0 1 0 RG
[-130(56)]TJ
0 g 0 G
[<96>]TJ
0 1 0 rg 0 1 0 RG
[(58)]TJ
0 g 0 G
[(,)]TJ
0 1 0 rg 0 1 0 RG
[-130(60)]TJ
0 g 0 G
[(,)]TJ
0 1 0 rg 0 1 0 RG
[-130(62)]TJ
0 g 0 G
[(,)]TJ
0 1 0 rg 0 1 0 RG
[-130(67)]TJ
0 g 0 G
[(].)]TJ/F57 9.9626 Tf 0 -19.717 Td [(Ev)10(aluation)-391(metrics.)]TJ/F58 9.9626 Tf 91.61 0 Td [(W)80(e)-391(e)25(v)25(aluate)-390(OoD)-391(detection)-390(perfor)20(-)]TJ -91.61 -11.955 Td [(mance)-250(using)-250(the)-250(follo)25(wing)-250(metrics:)]TJ/F57 9.9626 Tf 0 -19.717 Td [(i\))-227(A)50(UR)30(OC.)]TJ/F58 9.9626 Tf 48.823 0 Td [(The)-227(Area)-227(Under)-227(the)-226(Recei)25(v)15(er)-227(Operator)-227(Charac-)]TJ -48.823 -11.955 Td [(teristic)-201(\(A)55(UR)40(OC\))-201(curv)15(e)-200(can)-201(be)-201(interpreted)-201(as)-201(the)-201(probability)]TJ 0 -11.955 Td [(that)-265(a)-265(positi)25(v)15(e)-265(e)15(xample)-264(is)-265(assigned)-265(a)-265(higher)-265(detection)-265(score)]TJ 0 -11.955 Td [(than)-250(a)-250(ne)15(g)5(ati)25(v)15(e)-250(e)15(xample)-250([)]TJ
0 1 0 rg 0 1 0 RG
[(18)]TJ
0 g 0 G
[(].)]TJ/F57 9.9626 Tf 0 -19.717 Td [(ii\))-518(A)50(UPR)-518(\(In\).)]TJ/F58 9.9626 Tf 75.987 0 Td [(The)-518(Area)-518(Under)-518(the)-518(Precision)-518(Recall)]TJ -75.987 -11.955 Td [(\(A)55(UPR\))-191(curv)15(e)-191(computes)-190(the)-191(precision-recall)-191(pairs)-191(for)-191(dif)25(fer)20(-)]TJ 0 -11.956 Td [(ent)-233(probability)-234(thresholds)-233(by)-234(considering)-233(the)-234(in-distrib)20(ution)]TJ 0 -11.955 Td [(e)15(xamples)-250(as)-250(the)-250(positi)25(v)15(e)-250(class.)]TJ/F57 9.9626 Tf 0 -19.717 Td [(iii\))-202(A)50(UPR)-201(\(Out\).)]TJ/F58 9.9626 Tf 71.472 0 Td [(A)55(UPR)-202(\(Out\))-201(is)-202(A)55(UPR)-201(as)-202(described)-202(abo)15(v)15(e,)]TJ -71.472 -11.955 Td [(b)20(ut)-203(calculated)-203(considering)-204(the)-203(OoD)-203(e)15(xamples)-203(as)-203(the)-204(positi)25(v)15(e)]TJ 0 -11.955 Td [(class.)-445(W)80(e)-296(treat)-295(this)-295(outlier)-295(class)-295(as)-295(positi)25(v)15(e)-296(by)-295(multiplying)]TJ 0 -11.955 Td [(the)-216(V)129(oG)-217(scores)-216(by)]TJ/F25 9.9626 Tf 73.739 0 Td [<00>]TJ/F19 9.9626 Tf 7.749 0 Td [(1)]TJ/F58 9.9626 Tf 7.137 0 Td [(and)-216(labelling)-217(them)-216(positi)25(v)15(e)-217(when)-216(cal-)]TJ -88.625 -11.955 Td [(culating)-250(A)55(UPR)-250(\(Out\).)]TJ
0 g 0 G
0 g 0 G
0 -26.666 Td [(T)80(able)-330(1.)-550(Comparison)-330(of)-330(V)129(oG)-330(to)-330(9)-330(e)15(xisting)-330(OoD)-330(detection)]TJ 0 -11.955 Td [(methods.)-302(Sho)25(wn)-227(are)-226(a)20(v)15(erage)-227(v)25(alues)-226(of)-227(metrics)-226(and)-227(standard)]TJ 0 -11.955 Td [(de)25(viations)-306(across)-305(15)-306(corruptions)-305(in)-306(the)-305(MNIST)92(-C)-306(datasets.)]TJ 0 -11.955 Td [(Arro)25(ws)-401(\()]TJ/F25 9.9626 Tf 36.943 0 Td [(")]TJ/F58 9.9626 Tf 4.982 0 Td [(\))-401(indicate)-401(the)-402(direction)-401(of)-401(better)-401(metric)-401(perfor)20(-)]TJ -41.925 -11.955 Td [(mance.)-310(V)129(oG)-248(outperforms)-249(most)-248(baselines)-249(by)-248(a)-249(lar)18(ge)-249(mar)18(gin.)]TJ
0 g 0 G
ET
q
1 0 0 1 73.782 387.319 cm
[]0 d 0 J 0.797 w 0 0 m 186.668 0 l S
Q
BT
/F58 8.9664 Tf 79.76 376.461 Td [(OoD)-250(methods)-1360(A)55(UR)40(OC)-250(\()]TJ/F91 8.9664 Tf 98.014 0 Td [(")]TJ/F58 8.9664 Tf 4.607 0 Td [(\))-1360(A)55(UPR)-250(OUT)-250(\()]TJ/F91 8.9664 Tf 64.498 0 Td [(")]TJ/F58 8.9664 Tf 4.608 0 Td [(\))]TJ
ET
q
1 0 0 1 73.782 371.209 cm
[]0 d 0 J 0.498 w 0 0 m 186.668 0 l S
Q
BT
/F58 8.9664 Tf 79.76 360.5 Td [(KDE)-250([)]TJ
0 1 0 rg 0 1 0 RG
[(57)]TJ
0 g 0 G
[(])-3042(57.46)]TJ/F24 6.9738 Tf 83.06 0 Td [<06>]TJ/F58 6.9738 Tf 6.226 0 Td [(32.09)]TJ/F58 8.9664 Tf 35.927 0 Td [(62.56)]TJ/F24 6.9738 Tf 20.174 0 Td [<06>]TJ/F58 6.9738 Tf 6.227 0 Td [(24.16)]TJ/F58 8.9664 Tf -151.614 -10.959 Td [(MVE)-250([)]TJ
0 1 0 rg 0 1 0 RG
[(58)]TJ
0 g 0 G
[(])-2875(62.84)]TJ/F24 6.9738 Tf 83.06 0 Td [<06>]TJ/F58 6.9738 Tf 6.226 0 Td [(21.92)]TJ/F58 8.9664 Tf 37.67 0 Td [(61.42)]TJ/F24 6.9738 Tf 20.174 0 Td [<06>]TJ/F58 6.9738 Tf 6.227 0 Td [(19.1)]TJ/F58 8.9664 Tf -153.357 -10.959 Td [(DOCC)-250([)]TJ
0 1 0 rg 0 1 0 RG
[(60)]TJ
0 g 0 G
[(])-2319(69.16)]TJ/F24 6.9738 Tf 83.06 0 Td [<06>]TJ/F58 6.9738 Tf 6.226 0 Td [(28.35)]TJ/F58 8.9664 Tf 35.927 0 Td [(70.37)]TJ/F24 6.9738 Tf 20.174 0 Td [<06>]TJ/F58 6.9738 Tf 6.227 0 Td [(23.25)]TJ/F58 8.9664 Tf -151.614 -10.958 Td [(kPCA)-250([)]TJ
0 1 0 rg 0 1 0 RG
[(12)]TJ
0 g 0 G
[(])-2555(72.12)]TJ/F24 6.9738 Tf 82.188 0 Td [<06>]TJ/F58 6.9738 Tf 6.227 0 Td [(31.00.)]TJ/F58 8.9664 Tf 36.798 0 Td [(75.39)]TJ/F24 6.9738 Tf 20.174 0 Td [<06>]TJ/F58 6.9738 Tf 6.227 0 Td [(26.37)]TJ/F58 8.9664 Tf -151.614 -10.959 Td [(SVDD)-250([)]TJ
0 1 0 rg 0 1 0 RG
[(67)]TJ
0 g 0 G
[(])-2375(74.01)]TJ/F24 6.9738 Tf 83.06 0 Td [<06>]TJ/F58 6.9738 Tf 6.226 0 Td [(21.39)]TJ/F58 8.9664 Tf 35.927 0 Td [(73.33)]TJ/F24 6.9738 Tf 20.174 0 Td [<06>]TJ/F58 6.9738 Tf 6.227 0 Td [(21.98)]TJ/F58 8.9664 Tf -151.614 -10.959 Td [(PCA)-250([)]TJ
0 1 0 rg 0 1 0 RG
[(56)]TJ
0 g 0 G
[(])-3152(77.71)]TJ/F24 6.9738 Tf 83.06 0 Td [<06>]TJ/F58 6.9738 Tf 6.226 0 Td [(30.90)]TJ/F58 8.9664 Tf 37.67 0 Td [(80.86)]TJ/F24 6.9738 Tf 20.174 0 Td [<06>]TJ/F58 6.9738 Tf 6.227 0 Td [(25.2)]TJ/F58 8.9664 Tf -153.357 -10.959 Td [(Gaussian)-250([)]TJ
0 1 0 rg 0 1 0 RG
[(45)]TJ
0 g 0 G
[(])-1431(80.57)]TJ/F24 6.9738 Tf 83.06 0 Td [<06>]TJ/F58 6.9738 Tf 6.226 0 Td [(29.71)]TJ/F58 8.9664 Tf 35.927 0 Td [(84.51)]TJ/F24 6.9738 Tf 20.174 0 Td [<06>]TJ/F58 6.9738 Tf 6.227 0 Td [(22.62)]TJ/F57 8.9664 Tf -151.614 -10.959 Td [(V)100(oG)]TJ/F58 8.9664 Tf 62.014 0 Td [(85.42)]TJ/F24 6.9738 Tf 20.174 0 Td [<06>]TJ/F58 6.9738 Tf 6.227 0 Td [(10.28.)]TJ/F58 8.9664 Tf 38.541 0 Td [(84.96)]TJ/F24 6.9738 Tf 20.174 0 Td [<06>]TJ/F58 6.9738 Tf 6.227 0 Td [(9.61)]TJ/F58 8.9664 Tf -153.357 -10.959 Td [(AE)-250([)]TJ
0 1 0 rg 0 1 0 RG
[(41)]TJ
0 g 0 G
[(])-3764(89.89)]TJ/F24 6.9738 Tf 83.06 0 Td [<06>]TJ/F58 6.9738 Tf 6.226 0 Td [(18.52)]TJ/F58 8.9664 Tf 35.927 0 Td [(89.99)]TJ/F24 6.9738 Tf 20.174 0 Td [<06>]TJ/F58 6.9738 Tf 6.227 0 Td [(18.19)]TJ/F58 8.9664 Tf -151.614 -10.959 Td [(AEGAN)-250([)]TJ
0 1 0 rg 0 1 0 RG
[(62)]TJ
0 g 0 G
[(])-1696(95.93)]TJ/F24 6.9738 Tf 83.931 0 Td [<06>]TJ/F58 6.9738 Tf 6.227 0 Td [(7.90.)]TJ/F58 8.9664 Tf 36.798 0 Td [(95.40)]TJ/F24 6.9738 Tf 20.174 0 Td [<06>]TJ/F58 6.9738 Tf 6.227 0 Td [(9.46)]TJ
ET
q
1 0 0 1 73.782 256.468 cm
[]0 d 0 J 0.797 w 0 0 m 186.668 0 l S
Q
0 g 0 G
BT
/F57 9.9626 Tf 50.112 224.462 Td [(Findings.)]TJ/F58 9.9626 Tf 46.866 0 Td [(In)-390(T)80(able)]TJ
1 0 0 rg 1 0 0 RG
[-390(1)]TJ
0 g 0 G
[(,)-425(we)-390(observ)15(e)-390(that)-391(V)129(oG)-390(outperform)1(s)]TJ -46.866 -11.955 Td [(all)-372(methods)-373(e)15(xcept)-372(AutoEncoders)-372(\(AE\))-372(and)-373(AutoEncoder)]TJ 0 -11.955 Td [(GAN)-230(\(AEGAN\).)-230(In)-230(stark)-230(contrast)-230(to)-230(V)129(oG,)-231(AE)-230(and)-230(AEGAN)]TJ 0 -11.955 Td [(require)-199(comple)15(x)-198(training)-199(of)-199(auxiliary)-199(models)-198(and)-199(do)-199(not)-199(fea-)]TJ 0 -11.956 Td [(sibly)-285(scale)-285(be)15(yond)-285(small-scale)-285(datasets)-285(lik)10(e)-285(MNIST)74(.)-285(Gi)25(v)15(en)]TJ 0 -11.955 Td [(these)-216(limitations,)-222(V)129(oG)-216(remains)-216(a)-215(v)25(aluable)-216(and)-216(scalable)-216(OoD)]TJ 0 -11.955 Td [(detection)-317(m)1(ethod)-317(as)-317(it)-316(can)-317(be)-316(used)-317(for)-316(lar)18(ge-scale)-317(datasets)]TJ 0 -11.955 Td [(\()]TJ/F61 9.9626 Tf 3.317 0 Td [(e)15(.g)]TJ/F58 9.9626 Tf 11.746 0 Td [(.)-196(ImageNet\))-195(and)-195(netw)10(orks)-196(\()]TJ/F61 9.9626 Tf 107.008 0 Td [(e)15(.g)]TJ/F58 9.9626 Tf 11.746 0 Td [(.)-195(ResNet-50\).)-292(Unlik)10(e)-196(gen-)]TJ -133.817 -11.955 Td [(erati)25(v)15(e)-273(models,)-280(V)129(oG)-273(does)-274(not)-273(require)-274(an)-273(uncorrupted)-274(train-)]TJ 0 -11.956 Td [(ing)-332(dataset)-333(for)-332(learning)-332(image)-333(distrib)20(utions.)-557(Further)40(,)-353(V)129(oG)]TJ 0 -11.955 Td [(only)-332(le)25(v)15(erages)-331(data)-332(from)-332(training)-331(itself,)-352(is)-332(computed)-332(from)]TJ 0 -11.955 Td [(checkpoints)-290(already)-291(stored)-290(o)15(v)15(er)-290(the)-291(course)-290(of)-290(training,)-301(and)]TJ 0 -11.955 Td [(does)-250(not)-250(require)-250(the)-250(true)-250(label)-250(to)-250(rank.)]TJ
0 g 0 G
0 g 0 G
/F57 11.9552 Tf 258.75 629.037 Td [(5.)-250(Related)-250(W)75(ork)]TJ/F58 9.9626 Tf 11.955 -18.929 Td [(Our)-343(w)10(ork)-342(proposes)-343(a)-342(method)-343(to)-342(rank)-343(training)-342(and)-343(test-)]TJ -11.955 -11.955 Td [(ing)-257(data)-257(by)-257(estimating)-258(e)15(xample)-257(dif)25(\002culty)65(.)-331(Gi)25(v)15(en)-257(the)-258(size)-257(of)]TJ 0 -11.955 Td [(current)-269(datasets,)-275(this)-269(can)-269(be)-270(a)-269(po)25(werful)-269(interpretability)-270(tool)]TJ 0 -11.955 Td [(to)-307(isolate)-307(a)-307(tractable)-307(subset)-307(of)-307(e)15(xamples)-307(for)-307(human-in-the-)]TJ 0 -11.955 Td [(loop)-333(auditing)-332(and)-333(aid)-333(in)-332(curriculum)-333(learning)-332([)]TJ
0 1 0 rg 0 1 0 RG
[(8)]TJ
0 g 0 G
[(])-333(or)-333(distin-)]TJ 0 -11.956 Td [(guishing)-359(between)-359(sources)-360(of)-359(uncertainty)-718([)]TJ
0 1 0 rg 0 1 0 RG
[(16)]TJ
0 g 0 G
[(,)]TJ
0 1 0 rg 0 1 0 RG
[-203(33)]TJ
0 g 0 G
[(].)-638(While)]TJ 0 -11.955 Td [(prior)-272(w)10(orks)-272(ha)20(v)15(e)-272(proposed)-272(dif)25(ferent)-272(notions)-273(of)-272(what)-272(subset)]TJ 0 -11.955 Td [(merits)-304(surf)10(acing,)-318(introduced)-305(the)-304(concept)-305(of)-304(prototypes)-305(and)]TJ 0 -11.955 Td [(quintessential)-271(e)15(xamples)-271(in)-271(the)-271(dataset,)-276(b)20(ut)-272(did)-271(not)-271(focus)-271(on)]TJ 0 -11.955 Td [(lar)18(ge-scale)-250(deep)-250(neural)-250(netw)10(orks)-250(models)-250([)]TJ
0 1 0 rg 0 1 0 RG
[(9)]TJ
0 g 0 G
[(,)]TJ
0 1 0 rg 0 1 0 RG
[-130(13)]TJ
0 g 0 G
[(,)]TJ
0 1 0 rg 0 1 0 RG
[-130(39)]TJ
0 g 0 G
[(,)]TJ
0 1 0 rg 0 1 0 RG
[-130(40)]TJ
0 g 0 G
[(,)]TJ
0 1 0 rg 0 1 0 RG
[-130(73)]TJ
0 g 0 G
[(].)]TJ 11.955 -11.955 Td [(Unlik)10(e)-335(pre)25(vious)-335(w)10(orks,)-357(we)-335(propose)-335(a)-335(measure)-336(that)-335(can)]TJ -11.955 -11.956 Td [(be)-299(e)15(xtended)-299(to)-300(rank)-299(the)-299(entire)-299(dataset)-299(by)-300(estimat)1(ing)-300(e)15(xam-)]TJ 0 -11.955 Td [(ple)-342(dif)25(\002culty)-341(\(rather)-342(than)-342(surf)10(acing)-341(a)-342(prototypical)-342(subset\).)]TJ 0 -11.955 Td [(In)-207(addition,)-215(V)129(oG)-207(is)-207(f)10(ar)-207(more)-206(ef)25(\002cient)-207(than)-207(other)-207(global)-207(rank-)]TJ 0 -11.955 Td [(ings)-250(lik)10(e)-250([)]TJ
0 1 0 rg 0 1 0 RG
[(42)]TJ
0 g 0 G
[(])-250(and)-250([)]TJ
0 1 0 rg 0 1 0 RG
[(23)]TJ
0 g 0 G
[(].)]TJ 11.955 -11.955 Td [(V)129(oG)-275(also)-276(does)-275(not)-275(require)-276(modifying)-275(the)-275(architecture)-276(or)]TJ -11.955 -11.956 Td [(making)-204(an)15(y)-205(assumptions)-204(about)-205(the)-204(statistics)-204(of)-205(the)-204(input)-205(dis-)]TJ 0 -11.955 Td [(trib)20(ution.)-361(In)-266(particular)40(,)-271(w)10(orks)-267(such)-267(as)-267([)]TJ
0 1 0 rg 0 1 0 RG
[(39)]TJ
0 g 0 G
[(])-267(require)-267(assump-)]TJ 0 -11.955 Td [(tions)-341(about)-340(the)-341(statistics)-341(of)-341(the)-340(input)-341(distrib)20(ution)-341(and)-341([)]TJ
0 1 0 rg 0 1 0 RG
[(47)]TJ
0 g 0 G
[(])]TJ 0 -11.955 Td [(requires)-206(modifying)-206(the)-206(architecture)-206(to)-206(pre\002x)-206(an)-206(autoencoder)]TJ 0 -11.955 Td [(to)-263(surf)10(ace)-263(a)-263(set)-263(of)-263(prototypes,)-263([)]TJ
0 1 0 rg 0 1 0 RG
[(55)]TJ
0 g 0 G
[(])-263(le)25(v)15(erages)-264(pruning)-263(of)-263(the)]TJ 0 -11.955 Td [(model)-232(to)-233(identify)-232(dif)25(\002cult)-232(e)15(xamples)-232(and)-233([)]TJ
0 1 0 rg 0 1 0 RG
[(6)]TJ
0 g 0 G
[(])-232(requires)-232(the)-233(ad-)]TJ 0 -11.956 Td [(dition)-250(of)-250(an)-250(auxiliary)-250(k-nn)-250(model)-250(after)-250(each)-250(layer)55(.)]TJ 11.955 -11.955 Td [(Our)-245(w)10(ork)-245(is)-245(complementary)-245(to)-245(recent)-245(w)10(orks)-245(by)-245([)]TJ
0 1 0 rg 0 1 0 RG
[(36)]TJ
0 g 0 G
[(])-245(that)]TJ -11.955 -11.955 Td [(proposes)-273(a)-272(c-score)-273(to)-273(rank)-273(e)15(xamples)-272(by)-273(aligning)-273(them)-273(with)]TJ 0 -11.955 Td [(training)-331(instances,)-330([)]TJ
0 1 0 rg 0 1 0 RG
[(30)]TJ
0 g 0 G
[(])-331(that)-331(classi\002es)-330(e)15(xamples)-331(as)-331(outliers)]TJ 0 -11.955 Td [(according)-246(to)-246(sensiti)25(vity)-247(to)-246(v)25(arying)-246(model)-246(capacity)65(,)-247(and)-247([)]TJ
0 1 0 rg 0 1 0 RG
[(10)]TJ
0 g 0 G
[(])]TJ 0 -11.955 Td [(that)-360(considers)-360(dif)25(ferent)-360(measures)-360(to)-361(i)1(solate)-361(prototypes)-360(for)]TJ 0 -11.956 Td [(ranking)-267(the)-266(entire)-267(dataset.)-360(W)80(e)-267(note)-267(that)-266(the)-267(c-score)-267(method)]TJ 0 -11.955 Td [(proposed)-277(by)-277([)]TJ
0 1 0 rg 0 1 0 RG
[(36)]TJ
0 g 0 G
[(])-277(is)-277(considerably)-277(more)-277(computationally)-277(in-)]TJ 0 -11.955 Td [(tensi)25(v)15(e)-366(to)-367(compute)-366(than)-366(V)129(oG)-366(as)-367(it)-366(requires)-366(training)-366(up)-367(to)]TJ 0 -11.955 Td [(20,000)-248(netw)10(ork)-248(replications)-249(per)-248(dataset.)-309(Se)25(v)15(eral)-248(of)-248(the)-249(pro-)]TJ 0 -11.955 Td [(totype)-285(methods)-285(considered)-285(by)-285([)]TJ
0 1 0 rg 0 1 0 RG
[(10)]TJ
0 g 0 G
[(])-286(require)-285(training)-285(ensem-)]TJ 0 -11.955 Td [(bles)-384(of)-383(models,)-417(as)-384(does)-383(the)-384(compression)-383(sensiti)25(vity)-384(mea-)]TJ 0 -11.956 Td [(sure)-328(proposed)-328(by)-328([)]TJ
0 1 0 rg 0 1 0 RG
[(30)]TJ
0 g 0 G
[(].)-544(Finally)65(,)-348(our)-328(proposed)-328(V)129(oG)-328(is)-328(both)]TJ 0 -11.955 Td [(dif)25(ferent)-360(in)-361(the)-360(formulation)-360(and)-360(can)-361(be)-360(computed)-360(using)-361(a)]TJ 0 -11.955 Td [(small)-239(number)-238(of)-239(e)15(xisting)-238(checkpoints)-239(sa)20(v)15(ed)-239(o)15(v)15(er)-238(the)-239(course)]TJ 0 -11.955 Td [(of)-250(training.)]TJ/F57 11.9552 Tf 0 -23.911 Td [(6.)-250(Conclusion)-250(and)-250(Futur)18(e)-250(W)75(ork)]TJ/F58 9.9626 Tf 11.955 -18.929 Td [(In)-354(this)-355(w)10(ork,)-380(we)-354(proposed)-354(V)129(oG)-355(as)-354(a)-354(v)25(aluable)-354(and)-355(ef)25<022d>]TJ -11.955 -11.955 Td [(cient)-304(w)10(ay)-305(to)-304(rank)-304(data)-304(by)-305(dif)25(\002culty)-304(and)-304(surf)10(ace)-304(a)-305(tractable)]TJ 0 -11.955 Td [(subset)-271(of)-271(the)-272(most)-271(challenging)-271(e)15(xamples)-271(for)-272(human-in-the-)]TJ 0 -11.955 Td [(loop)-194(auditing.)-292(High)-194(V)129(oG)-194(samples)-194(are)-194(challenging)-195(to)-194(classify)]TJ 0 -11.955 Td [(for)-313(algorithm)-313(and)-313(surf)10(aces)-313(clusters)-313(of)-314(images)-313(with)-313(distinct)]TJ 0 -11.955 Td [(visual)-317(properties.)-512(Moreo)15(v)15(er)40(,)-334(V)129(oG)-317(is)-317(domain)-317(agnostic)-318(as)-317(it)]TJ 0 -11.956 Td [(uses)-306(only)-307(the)-306(v)25(anilla)-306(gradient)-307(e)15(xplanation)-306(from)-306(the)-307(model,)]TJ 0 -11.955 Td [(and)-221(can)-221(be)-220(used)-221(to)-221(rank)-221(bot)1(h)-221(training)-221(and)-221(test)-220(e)15(xamples.)-301(W)80(e)]TJ 0 -11.955 Td [(sho)25(w)-252(that)-251(it)-252(is)-252(also)-251(a)-252(useful)-252(unsupervised)-251(protocol,)-252(as)-252(it)-252(can)]TJ 0 -11.955 Td [(ef)25(fecti)25(v)15(ely)-250(rank)-250(e)15(xamples)-250(using)-250(the)-250(predicted)-250(label.)]TJ
0 g 0 G
0 g 0 G
ET
Q
Q
Q
q
1 0 0 1 0 0 cm
/ABp51YMVDA5rqOBxDy-XTQ Do
Q
endstream
endobj
35 0 obj
<< /Font << /F19 50 0 R /F24 53 0 R /F25 54 0 R /F57 42 0 R /F58 43 0 R /F61 45 0 R /F91 82 0 R >> /ProcSet [ /PDF /Text ] /XObject << /ABp51YMVDA5rqOBxDy-XTQ 83 0 R >> >>
endobj
36 0 obj
<< /Length 14090 >>
stream
q
q
q
0 g 0 G
0 g 0 G
BT
/F57 11.9552 Tf 50.112 710.037 Td [(Refer)18(ences)]TJ
0 g 0 G
/F58 9.9626 Tf 4.981 -18.929 Td [([1])]TJ
0 g 0 G
[-500(Alessandro)-645(Achille,)-745(Matteo)-645(Ro)15(v)15(ere,)-744(and)-646(Stef)10(ano)]TJ 16.598 -11.955 Td [(Soatto.)-401(Critical)-263(learning)-262(periods)-263(in)-263(deep)-263(netw)10(orks.)-401(In)]TJ/F61 9.9626 Tf 0 -11.955 Td [(ICLR)]TJ/F58 9.9626 Tf 21.589 0 Td [(,)-250(2019.)]TJ
1 0 0 rg 1 0 0 RG
[-360(5)]TJ
0 g 0 G
0 g 0 G
-38.187 -13.981 Td [([2])]TJ
0 g 0 G
[-500(McKane)-393(Andrus,)-428(Elena)-393(Spitzer)40(,)-428(Jef)25(fre)15(y)-393(Bro)25(wn,)-429(and)]TJ 16.598 -11.955 Td [(Alice)-219(Xiang.)-293(\224what)-220(we)-219(can')18(t)-219(measure,)-225(we)-220(can')18(t)-219(under)20(-)]TJ 0 -11.955 Td [(stand\224:)-391(Challenges)-290(to)-290(demographic)-290(data)-291(procurement)]TJ 0 -11.955 Td [(in)-461(the)-460(pursuit)-461(of)-460(f)10(airness.)]TJ/F61 9.9626 Tf 117.928 0 Td [(CoRR)]TJ/F58 9.9626 Tf 23.801 0 Td [(,)-461(abs/2011.02282,)]TJ -141.729 -11.955 Td [(2020.)]TJ
1 0 0 rg 1 0 0 RG
[-360(5)]TJ
0 g 0 G
0 g 0 G
-16.598 -13.981 Td [([3])]TJ
0 g 0 G
[-500(McKane)-393(Andrus,)-428(Elena)-393(Spitzer)40(,)-428(Jef)25(fre)15(y)-393(Bro)25(wn,)-429(and)]TJ 16.598 -11.955 Td [(Alice)-244(Xiang.)-347(What)-244(we)-244(can')18(t)-244(measure,)-245(we)-244(can')18(t)-244(under)20(-)]TJ 0 -11.955 Td [(stand:)-539(Challenges)-364(to)-364(demographic)-364(data)-365(procurement)]TJ 0 -11.955 Td [(in)-250(the)-250(pursuit)-250(of)-250(f)10(airness.)-360(In)]TJ/F61 9.9626 Tf 113.613 0 Td [(F)115(AccT)]TJ/F58 9.9626 Tf 25.414 0 Td [(,)-250(2021.)]TJ
1 0 0 rg 1 0 0 RG
[-360(1)]TJ
0 g 0 G
0 g 0 G
-155.625 -13.981 Td [([4])]TJ
0 g 0 G
[-500(Marcus)-495(A)-495(Badgele)15(y)65(,)-556(John)-495(R)-495(Zech,)-556(Luk)10(e)-495(Oakden-)]TJ 16.598 -11.955 Td [(Rayner)40(,)-933(Benjamin)-797(S)-796(Glicksber)18(g,)-934(Manw)10(ay)-797(Liu,)]TJ 0 -11.955 Td [(W)40(illiam)-397(Gale,)-435(Michael)-397(V)-398(McConnell,)-434(Bethan)15(y)-398(Per)20(-)]TJ 0 -11.956 Td [(cha,)-457(Thomas)-415(M)-416(Sn)15(yder)40(,)-457(and)-415(Joel)-416(T)-415(Dudle)15(y)65(.)-890(Deep)]TJ 0 -11.955 Td [(learning)-370(predicts)-370(hip)-369(fracture)-370(using)-370(confounding)-370(pa-)]TJ 0 -11.955 Td [(tient)-663(and)-662(healthcare)-663(v)25(ariables.)-1680(In)]TJ/F61 9.9626 Tf 162.685 0 Td [(NPJ)-662(Digital)]TJ -162.685 -11.955 Td [(Medicine)]TJ/F58 9.9626 Tf 37.071 0 Td [(,)-250(2019.)]TJ
1 0 0 rg 1 0 0 RG
[-360(1)]TJ
0 g 0 G
0 g 0 G
-53.669 -13.981 Td [([5])]TJ
0 g 0 G
[-500(Da)20(vid)-279(Baehrens,)-285(T)35(imon)-279(Schroeter)40(,)-285(Stef)10(an)-279(Harmeling,)]TJ 16.598 -11.955 Td [(Motoaki)-289(Ka)15(w)10(anabe,)-300(Katja)-289(Hansen,)-299(and)-290(Klaus-Robert)]TJ 0 -11.955 Td [(M)]TJ 10.794 2.172 Td [<98>]TJ -1.937 -2.172 Td [(A)]TJ 7.746 0.05 Td [<10>]TJ -0.553 -0.05 Td [(zller)55(.)-438(Ho)25(w)-274(to)-274(e)15(xplain)-274(indi)25(vidual)-274(classi\002cation)-275(de-)]TJ -16.05 -11.955 Td [(cisions.)-360(In)]TJ/F61 9.9626 Tf 44.542 0 Td [(JMLR)]TJ/F58 9.9626 Tf 24.349 0 Td [(,)-250(2010.)]TJ
1 0 0 rg 1 0 0 RG
[-360(2)]TJ
0 g 0 G
0 g 0 G
-85.489 -13.981 Td [([6])]TJ
0 g 0 G
[-500(Robert)-253(J.)-253(N.)-253(Baldock,)-254(Hartmut)-253(Maennel,)-254(and)-253(Behnam)]TJ 16.598 -11.955 Td [(Ne)15(yshab)20(ur)55(.)-269(Deep)-209(learning)-208(through)-208(the)-208(lens)-208(of)-209(e)15(xample)]TJ 0 -11.955 Td [(dif)25(\002culty)65(.)]TJ/F61 9.9626 Tf 41.713 0 Td [(CoRR)]TJ/F58 9.9626 Tf 23.801 0 Td [(,)-250(abs/2106.09647,)-250(2021.)]TJ
1 0 0 rg 1 0 0 RG
[-360(8)]TJ
0 g 0 G
0 g 0 G
-82.112 -13.981 Td [([7])]TJ
0 g 0 G
[-500(Peter)-248(L.)-247(Bartlett)-248(and)-248(Marten)-247(H.)-248(W)80(e)15(gkamp.)-355(Classi\002ca-)]TJ 16.598 -11.955 Td [(tion)-268(with)-268(a)-267(reject)-268(option)-268(using)-268(a)-267(hinge)-268(loss.)-417(In)]TJ/F61 9.9626 Tf 187.832 0 Td [(JMLR)]TJ/F58 9.9626 Tf 24.348 0 Td [(,)]TJ -212.18 -11.955 Td [(2008.)]TJ
1 0 0 rg 1 0 0 RG
[-360(1)]TJ
0 g 0 G
0 g 0 G
-16.598 -13.981 Td [([8])]TJ
0 g 0 G
[-500(Y)110(oshua)-297(Bengio,)-308(J)]TJ 87.662 0.05 Td []TJ -0.553 -0.05 Td [(er)]TJ 8.573 0.05 Td [<88>]TJ -0.832 -0.05 Td [(ome)-297(Louradour)40(,)-308(Ronan)-296(Collobert,)]TJ -78.252 -11.955 Td [(and)-378(Jason)-377(W)80(eston.)-768(Curriculum)-378(learning.)-768(In)]TJ/F61 9.9626 Tf 188.38 0 Td [(ICML)]TJ/F58 9.9626 Tf 23.8 0 Td [(,)]TJ -212.18 -11.955 Td [(2009.)]TJ
1 0 0 rg 1 0 0 RG
[-360(8)]TJ
0 g 0 G
0 g 0 G
-16.598 -13.981 Td [([9])]TJ
0 g 0 G
[-500(Jacob)-379(Bien)-379(and)-379(Robert)-379(T)35(ibshirani.)-774(Prototype)-379(selec-)]TJ 16.598 -11.955 Td [(tion)-310(for)-310(interpretable)-310(classi\002cat)1(ion.)-552(In)]TJ/F61 9.9626 Tf 158.126 0 Td [(The)-310(Annals)-310(of)]TJ -158.126 -11.955 Td [(Applied)-250(Statistics)]TJ/F58 9.9626 Tf 69.468 0 Td [(,)-250(2011.)]TJ
1 0 0 rg 1 0 0 RG
[-360(8)]TJ
0 g 0 G
0 g 0 G
-91.047 -13.981 Td [([10])]TJ
0 g 0 G
[-500(Nicholas)-208(Carlini,)-216(Ulf)10(ar)-207(Erlingsson,)-216(and)-208(Nicolas)-208(P)15(aper)20(-)]TJ 21.579 -11.955 Td [(not.)-289(Distrib)20(ution)-217(density)65(,)-224(tails,)-223(and)-217(outliers)-218(in)-217(machine)]TJ 0 -11.955 Td [(learning:)-310(Metrics)-250(and)-250(applications.)]TJ/F61 9.9626 Tf 142.534 0 Td [(arXiv)]TJ/F58 9.9626 Tf 22.136 0 Td [(,)-250(2019.)]TJ
1 0 0 rg 1 0 0 RG
[-360(8)]TJ
0 g 0 G
0 g 0 G
-186.249 -13.981 Td [([11])]TJ
0 g 0 G
[-500(Rich)-391(Caruana.)-810(Case-based)-390(e)15(xplanation)-391(for)-391(arti\002cial)]TJ 21.579 -11.955 Td [(neural)-238(nets.)-333(In)]TJ/F61 9.9626 Tf 59.789 0 Td [(Arti\002cial)-238(Neur)15(al)-237(Networks)-238(in)-237(Medicine)]TJ -59.789 -11.955 Td [(and)-250(Biolo)10(gy)]TJ/F58 9.9626 Tf 48.328 0 Td [(,)-250(2000.)]TJ
1 0 0 rg 1 0 0 RG
[-360(1)]TJ
0 g 0 G
0 g 0 G
-69.907 -13.981 Td [([12])]TJ
0 g 0 G
[-500(Ragha)20(v)15(endra)-230(Chalapath)5(y)65(,)-234(Adi)1(tya)-230(Krishna)-230(Menon,)-234(and)]TJ 21.579 -11.955 Td [(Sanjay)-298(Cha)15(wla.)-513(Rob)20(ust,)-309(deep)-298(and)-298(inducti)25(v)15(e)-298(anomaly)]TJ 0 -11.955 Td [(detection.)-461(In)]TJ/F61 9.9626 Tf 54.705 0 Td [(J)25(oint)-281(Eur)45(opean)-282(Confer)37(ence)-281(on)-282(Mac)15(hine)]TJ -54.705 -11.955 Td [(Learning)-519(and)-520(Knowledg)10(e)-519(Disco)10(very)-519(in)-520(Databases)]TJ/F58 9.9626 Tf 212.18 0 Td [(,)]TJ -212.18 -11.955 Td [(pages)-250(36\22651.)-250(Springer)40(,)-250(2017.)]TJ
1 0 0 rg 1 0 0 RG
[-360(8)]TJ
0 g 0 G
0 g 0 G
-21.579 -13.981 Td [([13])]TJ
0 g 0 G
[-500(Ha)15(w-Shiuan)-200(Chang,)-211(Erik)-200(Learned-Miller)40(,)-210(and)-201(Andre)25(w)]TJ 21.579 -11.955 Td [(McCallum.)-451(Acti)25(v)15(e)-279(bias:)-367(T)35(raining)-279(more)-278(accurate)-279(neu-)]TJ 0 -11.955 Td [(ral)-362(netw)10(orks)-361(by)-362(emphasizing)-361(high)-362(v)25(ariance)-362(samples.)]TJ 0 -11.955 Td [(In)]TJ/F61 9.9626 Tf 10.789 0 Td [(NeurIPS)]TJ/F58 9.9626 Tf 34.312 0 Td [(,)-250(2017.)]TJ
1 0 0 rg 1 0 0 RG
[-360(8)]TJ
0 g 0 G
0 g 0 G
0 g 0 G
0 g 0 G
192.07 629.037 Td [([14])]TJ
0 g 0 G
[-500(Corinna)-274(Cortes,)-279(Giulia)-274(DeSalv)20(o,)-279(and)-274(Mehryar)-274(Mohri.)]TJ 21.579 -11.955 Td [(Boosting)-250(with)-250(abstention.)-360(In)]TJ/F61 9.9626 Tf 116.502 0 Td [(NeurIPS)]TJ/F58 9.9626 Tf 34.311 0 Td [(,)-250(2016.)]TJ
1 0 0 rg 1 0 0 RG
[-360(1)]TJ
0 g 0 G
0 g 0 G
-172.392 -15.281 Td [([15])]TJ
0 g 0 G
[-500(Richard)-300(A)-300(Da)20(vis,)-313(K)25(eh-Shin)-300(Lii,)-313(and)-300(Dimitris)-301(N)-300(Poli-)]TJ 21.579 -11.955 Td [(tis.)-726(Remarks)-364(on)-364(some)-365(nonparametric)-364(estimates)-364(of)-365(a)]TJ 0 -11.955 Td [(density)-249(function.)-358(In)]TJ/F61 9.9626 Tf 81.308 0 Td [(Selected)-249(W)92(orks)-249(of)-249(Murr)15(ay)-249(Rosen-)]TJ -81.308 -11.956 Td [(blatt)]TJ/F58 9.9626 Tf 18.271 0 Td [(.)-250(Springer)40(,)-250(2011.)]TJ
1 0 0 rg 1 0 0 RG
[-360(2)]TJ
0 g 0 G
0 g 0 G
-39.85 -15.28 Td [([16])]TJ
0 g 0 G
[-500(Daniel)-474(D')55(souza,)-531(Zach)-474(Nussbaum,)-530(Chirag)-475(Ag)5(arw)10(al,)]TJ 21.579 -11.956 Td [(and)-250(Sara)-250(Hook)10(er)55(.)-360(A)-250(tale)-250(of)-250(tw)10(o)-250(long)-250(tails,)-250(2021.)]TJ
1 0 0 rg 1 0 0 RG
[-360(8)]TJ
0 g 0 G
0 g 0 G
-21.579 -15.28 Td [([17])]TJ
0 g 0 G
[-500(F)15(artash)-346(F)15(aghri,)-371(Da)20(vid)-346(Duv)15(enaud,)-371(Da)20(vid)-346(J)-346(Fleet,)-371(and)]TJ 21.579 -11.956 Td [(Jimmy)-225(Ba.)-305(A)-225(study)-225(of)-225(gradient)-225(v)25(ariance)-225(in)-225(deep)-225(learn-)]TJ 0 -11.955 Td [(ing.)]TJ/F61 9.9626 Tf 18.809 0 Td [(arXiv)]TJ/F58 9.9626 Tf 22.137 0 Td [(,)-250(2020.)]TJ
1 0 0 rg 1 0 0 RG
[-360(5)]TJ
0 g 0 G
0 g 0 G
-62.525 -15.281 Td [([18])]TJ
0 g 0 G
[-500(T)80(om)-252(F)15(a)15(wcett.)-365(An)-251(introduction)-252(to)-251(roc)-252(analysis.)-365(In)]TJ/F61 9.9626 Tf 219.891 0 Td [(P)80(at-)]TJ -198.312 -11.955 Td [(tern)-250(r)37(eco)10(gnition)-250(letter)10(s)]TJ/F58 9.9626 Tf 91.306 0 Td [(,)-250(2006.)]TJ
1 0 0 rg 1 0 0 RG
[-360(8)]TJ
0 g 0 G
0 g 0 G
-112.885 -15.281 Td [([19])]TJ
0 g 0 G
[-500(V)60(italy)-223(Feldman.)-302(Does)-223(learning)-224(require)-223(memorization?)]TJ 21.579 -11.955 Td [(a)-233(short)-232(tale)-233(about)-232(a)-233(long)-233(tail.)-322(In)]TJ/F61 9.9626 Tf 125.962 0 Td [(A)30(CM)-233(SIGA)30(CT)-232(Sympo-)]TJ -125.962 -11.955 Td [(sium)-250(on)-250(Theory)-250(of)-250(Computing)]TJ/F58 9.9626 Tf 119.003 0 Td [(,)-250(2020.)]TJ
1 0 0 rg 1 0 0 RG
[-360(6)]TJ
0 g 0 G
0 g 0 G
-140.582 -15.281 Td [([20])]TJ
0 g 0 G
[-500(Ross)-485(Gruetzemacher)40(,)-544(Ashish)-485(Gupta,)-544(and)-485(Da)20(vid)-485(B.)]TJ 21.579 -11.955 Td [(P)15(aradice.)-723(3d)-363(deep)-364(learning)-363(for)-363(detecting)-364(pulmonary)]TJ 0 -11.955 Td [(nodules)-250(in)-250(ct)-250(scans.)-360(In)]TJ/F61 9.9626 Tf 91.855 0 Td [(J)40(AMIA)]TJ/F58 9.9626 Tf 27.815 0 Td [(,)-250(2018.)]TJ
1 0 0 rg 1 0 0 RG
[-360(1)]TJ
0 g 0 G
0 g 0 G
-141.249 -15.281 Td [([21])]TJ
0 g 0 G
[-500(Abhijit)-786(Guha)-787(Ro)10(y,)-920(Jie)-786(Ren,)-920(Shek)10(oofeh)-787(Azizi,)]TJ 21.579 -11.955 Td [(Aaron)-500(Loh,)-563(V)60(i)25(v)15(ek)-501(Natarajan,)-563(B)1(asil)-501(Mustaf)10(a,)-563(Nick)]TJ 0 -11.955 Td [(P)15(a)15(wlo)25(wski,)-547(Jan)-488(Fre)15(yber)18(g,)-547(Y)111(uan)-487(Liu,)-547(Zach)-488(Bea)20(v)15(er)40(,)]TJ 0 -11.956 Td [(Nam)-643(V)129(o,)-740(Pe)15(ggy)-643(Bui,)-741(Sa)1(mantha)-643(W)40(inter)40(,)-741(P)15(atricia)]TJ 0 -11.955 Td [(MacW)40(illiams,)-375(Gre)15(g)-350(S.)-350(Corrado,)-375(Umesh)-351(T)70(elang,)-375(Y)111(un)]TJ 0 -11.955 Td [(Liu,)-487(T)80(aylan)-439(Cemgil,)-487(Alan)-439(Karthik)10(esaling)5(am,)-487(Balaji)]TJ 0 -11.955 Td [(Lakshminarayanan,)-245(and)-244(Jim)-244(W)40(ink)10(ens.)-346(Does)-244(your)-244(der)20(-)]TJ 0 -11.955 Td [(matology)-346(classi\002er)-346(kno)25(w)-346(what)-346(it)-346(doesn')18(t)-346(kno)25(w?)-598(de-)]TJ 0 -11.956 Td [(tecting)-409(the)-408(long-tail)-409(of)-408(unseen)-409(conditions.)]TJ/F61 9.9626 Tf 182.024 0 Td [(Medical)]TJ -182.024 -11.955 Td [(Ima)10(g)10(e)-250(Analysis)]TJ/F58 9.9626 Tf 60.951 0 Td [(,)-250(75:102274,)-250(2022.)]TJ
1 0 0 rg 1 0 0 RG
[-360(1)]TJ
0 g 0 G
0 g 0 G
-82.53 -15.281 Td [([22])]TJ
0 g 0 G
[-500(Chuan)-229(Guo,)-233(Geof)25(f)-229(Pleiss,)-233(Y)111(u)-228(Sun,)-233(and)-229(Kilian)-229(Q)-229(W)80(ein-)]TJ 21.579 -11.955 Td [(ber)18(ger)55(.)-422(On)-270(calibration)-269(of)-270(modern)-269(neural)-269(netw)10(orks.)-423(In)]TJ/F61 9.9626 Tf 0 -11.955 Td [(ICML)]TJ/F58 9.9626 Tf 23.801 0 Td [(,)-250(2017.)]TJ
1 0 0 rg 1 0 0 RG
[-360(6)]TJ
0 g 0 G
0 g 0 G
-45.38 -15.281 Td [([23])]TJ
0 g 0 G
[-500(Hrayr)-558(Harutyun)15(yan,)-634(Alessandro)-558(Achille,)-635(Gio)15(v)25(anni)]TJ 21.579 -11.955 Td [(P)15(aolini,)-606(Orchid)-535(Majumder)40(,)-607(A)74(vinash)-535(Ra)20(vichandran,)]TJ 0 -11.955 Td [(Rahul)-279(Bhotika,)-286(and)-280(Ste)1(f)10(ano)-280(Soatto.)-453(Estimating)-279(infor)20(-)]TJ 0 -11.955 Td [(mati)25(v)15(eness)-357(of)-358(samples)-357(with)-358(smooth)-357(unique)-358(informa-)]TJ 0 -11.955 Td [(tion.)-360(In)]TJ/F61 9.9626 Tf 32.368 0 Td [(ICLR)]TJ/F58 9.9626 Tf 21.589 0 Td [(,)-250(2021.)]TJ
1 0 0 rg 1 0 0 RG
[-360(8)]TJ
0 g 0 G
0 g 0 G
-75.536 -15.281 Td [([24])]TJ
0 g 0 G
[-500(Douglas)-262(M.)-262(Ha)15(wkins.)-399(The)-262(detection)-262(of)-262(errors)-262(in)-262(mul-)]TJ 21.579 -11.956 Td [(ti)25(v)25(ariate)-289(data)-289(using)-289(principal)-289(components.)-486(In)]TJ/F61 9.9626 Tf 183.927 0 Td [(J)25(ournal)]TJ -183.927 -11.955 Td [(of)-250(the)-250(American)-250(Statistical)-250(Association)]TJ/F58 9.9626 Tf 154.977 0 Td [(,)-250(1974.)]TJ
1 0 0 rg 1 0 0 RG
[-360(2)]TJ
0 g 0 G
0 g 0 G
-176.556 -15.281 Td [([25])]TJ
0 g 0 G
[-500(Kaiming)-240(He,)-243(Xiangyu)-240(Zhang,)-243(Shaoqing)-240(Ren,)-242(and)-241(Jian)]TJ 21.579 -11.955 Td [(Sun.)-353(Deep)-247(residual)-247(learning)-247(for)-247(image)-246(recognition.)-354(In)]TJ/F61 9.9626 Tf 0 -11.955 Td [(CVPR)]TJ/F58 9.9626 Tf 24.906 0 Td [(,)-250(2016.)]TJ
1 0 0 rg 1 0 0 RG
[-360(4)]TJ
0 g 0 G
[(,)]TJ
1 0 0 rg 1 0 0 RG
[-250(14)]TJ
0 g 0 G
0 g 0 G
-46.485 -15.281 Td [([26])]TJ
0 g 0 G
[-500(Dan)-240(Hendrycks)-240(and)-240(K)25(e)25(vin)-241(Gimpel.)-338(A)-240(baseline)-241(for)-240(de-)]TJ 21.579 -11.955 Td [(tecting)-271(misclassi\002ed)-271(and)-271(out-of-distrib)20(ution)-271(e)15(xamples)]TJ 0 -11.955 Td [(in)-250(neural)-250(netw)10(orks.)]TJ/F61 9.9626 Tf 80.129 0 Td [(ICLR)]TJ/F58 9.9626 Tf 21.589 0 Td [(,)-250(2017.)]TJ
1 0 0 rg 1 0 0 RG
[-360(6)]TJ
0 g 0 G
[(,)]TJ
1 0 0 rg 1 0 0 RG
[-250(14)]TJ
0 g 0 G
0 g 0 G
-123.297 -15.281 Td [([27])]TJ
0 g 0 G
[-500(Dan)-358(Hendrycks,)-384(Xiao)10(yuan)-358(Liu,)-384(Eric)-358(W)80(allace,)-385(Adam)]TJ 21.579 -11.955 Td [(Dziedzic,)-415(Rishabh)-383(Krishnan,)-415(and)-382(Da)15(wn)-382(Song.)-784(Pre-)]TJ 0 -11.955 Td [(trained)-749(T)35(ransformers)-749(Impro)15(v)15(e)-750(Out-of-Distrib)20(ution)]TJ 0 -11.956 Td [(Rob)20(ustness.)-360(page)-250(arXi)25(v)65(,)-250(Apr)55(.)-250(2020.)]TJ
1 0 0 rg 1 0 0 RG
[-360(14)]TJ
0 g 0 G
0 g 0 G
0 g 0 G
ET
Q
Q
Q
q
1 0 0 1 0 0 cm
/S2aaOBObXr0Hc_03kE5ZFg Do
Q
endstream
endobj
37 0 obj
<< /Font << /F57 42 0 R /F58 43 0 R /F61 45 0 R >> /ProcSet [ /PDF /Text ] /XObject << /S2aaOBObXr0Hc_03kE5ZFg 84 0 R >> >>
endobj
38 0 obj
<< /Length 14035 >>
stream
q
q
q
0 g 0 G
0 g 0 G
0 g 0 G
BT
/F58 9.9626 Tf 50.112 710.037 Td [([28])]TJ
0 g 0 G
[-500(Dan)-448(Hendrycks,)-498(K)25(e)25(vin)-448(Zhao,)-497(Ste)25(v)15(en)-448(Basart,)-498(Jacob)]TJ 21.579 -11.955 Td [(Steinhardt,)-371(and)-348(Da)15(wn)-347(Song.)-670(Natural)-348(adv)15(ersarial)-347(e)15(x-)]TJ 0 -11.955 Td [(amples.)-360(In)]TJ/F61 9.9626 Tf 45.09 0 Td [(CVPR)]TJ/F58 9.9626 Tf 24.907 0 Td [(,)-250(2021.)]TJ
1 0 0 rg 1 0 0 RG
[-360(13)]TJ
0 g 0 G
0 g 0 G
-91.576 -14.837 Td [([29])]TJ
0 g 0 G
[-500(Sara)-344(Hook)10(er)55(.)-663(Mo)15(ving)-344(be)15(yond)-344(\223algorithmic)-345(bias)-344(is)-345(a)]TJ 21.579 -11.956 Td [(data)-250(problem\224.)]TJ/F61 9.9626 Tf 62.794 0 Td [(P)80(atterns)]TJ/F58 9.9626 Tf 32.966 0 Td [(,)-250(2\(4\):100241,)-250(2021.)]TJ
1 0 0 rg 1 0 0 RG
[-360(5)]TJ
0 g 0 G
0 g 0 G
-117.339 -14.837 Td [([30])]TJ
0 g 0 G
[-500(Sara)-352(Hook)10(er)40(,)-378(Aaron)-352(Courville,)-378(Gre)15(gory)-352(Clark,)-378(Y)100(ann)]TJ 21.579 -11.955 Td [(Dauphin,)-414(and)-381(Andrea)-381(Frome.)-780(What)-381(do)-382(compressed)]TJ 0 -11.955 Td [(deep)-250(neural)-250(netw)10(orks)-250(for)18(get?)]TJ/F61 9.9626 Tf 119.82 0 Td [(arXiv)]TJ/F58 9.9626 Tf 22.136 0 Td [(,)-250(2019.)]TJ
1 0 0 rg 1 0 0 RG
[-360(1)]TJ
0 g 0 G
[(,)]TJ
1 0 0 rg 1 0 0 RG
[-250(8)]TJ
0 g 0 G
0 g 0 G
-163.535 -14.838 Td [([31])]TJ
0 g 0 G
[-500(Sara)-282(Hook)10(er)40(,)-291(Dumitru)-282(Erhan,)-290(Pieter)20(-Jan)-283(Kindermans,)]TJ 21.579 -11.955 Td [(and)-195(Been)-195(Kim.)-242(A)-195(benchmark)-195(for)-196(interpreta)1(bility)-196(meth-)]TJ 0 -11.955 Td [(ods)-250(in)-250(deep)-250(neural)-250(netw)10(orks.)-360(In)]TJ/F61 9.9626 Tf 128.547 0 Td [(NeurIPS)]TJ/F58 9.9626 Tf 34.311 0 Td [(,)-250(2019.)]TJ
1 0 0 rg 1 0 0 RG
[-360(2)]TJ
0 g 0 G
0 g 0 G
-184.437 -14.838 Td [([32])]TJ
0 g 0 G
[-500(Sara)-194(Hook)10(er)40(,)-206(Nyalleng)-194(Moorosi,)-206(Gre)15(gory)-194(Clark,)-206(Samy)]TJ 21.579 -11.955 Td [(Bengio,)-471(and)-427(Emily)-427(Denton.)-926(Characterising)-427(bias)-427(in)]TJ 0 -11.955 Td [(compressed)-250(models.)]TJ/F61 9.9626 Tf 84.383 0 Td [(arXiv)]TJ/F58 9.9626 Tf 22.136 0 Td [(,)-250(2020.)]TJ
1 0 0 rg 1 0 0 RG
[-360(6)]TJ
0 g 0 G
0 g 0 G
-128.098 -14.837 Td [([33])]TJ
0 g 0 G
[-500(Niel)-290(T)70(eng)-291(Hu,)-300(Xin)15(yu)-290(Hu,)-301(Rosanne)-290(Liu,)-300(Sara)-291(Hook)10(er)40(,)]TJ 21.579 -11.956 Td [(and)-273(Jason)-273(Y)110(osinski.)-434(When)-273(does)-274(loss-based)-273(prioritiza-)]TJ 0 -11.955 Td [(tion)-250(f)10(ail?,)-250(2021.)]TJ
1 0 0 rg 1 0 0 RG
[-360(8)]TJ
0 g 0 G
0 g 0 G
-21.579 -14.837 Td [([34])]TJ
0 g 0 G
[-500(Gao)-383(Huang,)-416(Zhuang)-383(Liu,)-417(Laurens)-383(V)111(an)-383(Der)-383(Maaten,)]TJ 21.579 -11.955 Td [(and)-300(Kilian)-299(Q)-300(W)80(einber)18(ger)55(.)-519(De)1(n)-1(s)1(ely)-300(connected)-300(con)40(v)20(o-)]TJ 0 -11.956 Td [(lutional)-250(netw)10(orks.)-360(In)]TJ/F61 9.9626 Tf 86.226 0 Td [(CVPR)]TJ/F58 9.9626 Tf 24.906 0 Td [(,)-250(2017.)]TJ
1 0 0 rg 1 0 0 RG
[-360(14)]TJ
0 g 0 G
0 g 0 G
-132.711 -14.837 Td [([35])]TJ
0 g 0 G
[-500(Ser)18(ge)15(y)-242(Iof)25(fe)-242(and)-242(Christian)-241(Sze)15(gedy)65(.)-343(Batch)-242(normaliza-)]TJ 21.579 -11.955 Td [(tion:)-390(Accelerating)-290(deep)-290(netw)10(ork)-290(training)-290(by)-290(reducing)]TJ 0 -11.955 Td [(internal)-250(co)15(v)25(ariate)-250(shift.)-360(In)]TJ/F61 9.9626 Tf 106.111 0 Td [(ICML)]TJ/F58 9.9626 Tf 23.8 0 Td [(,)-250(2015.)]TJ
1 0 0 rg 1 0 0 RG
[-360(4)]TJ
0 g 0 G
0 g 0 G
-151.49 -14.838 Td [([36])]TJ
0 g 0 G
[-500(Ziheng)-523(Jiang,)-592(Chiyuan)-523(Zhang,)-592(K)15(unal)-523(T)80(al)10(w)10(ar)40(,)-592(and)]TJ 21.579 -11.955 Td [(Michael)-340(C)-339(Mozer)55(.)-647(Characterizing)-339(structural)-340(re)15(gular)20(-)]TJ 0 -11.955 Td [(ities)-301(of)-302(labeled)-301(data)-302(i)1(n)-302(o)15(v)15(erparameterized)-301(models.)-525(In)]TJ/F61 9.9626 Tf 0 -11.955 Td [(ICML)]TJ/F58 9.9626 Tf 23.801 0 Td [(,)-250(2021.)]TJ
1 0 0 rg 1 0 0 RG
[-360(5)]TJ
0 g 0 G
[(,)]TJ
1 0 0 rg 1 0 0 RG
[-250(8)]TJ
0 g 0 G
0 g 0 G
-45.38 -14.838 Td [([37])]TJ
0 g 0 G
[-500(Ale)15(x)-208(K)25(endall)-207(and)-208(Y)100(arin)-207(Gal.)-268(What)-208(uncertainties)-207(do)-208(we)]TJ 21.579 -11.955 Td [(need)-341(in)-341(bayesian)-340(deep)-341(learning)-341(for)-341(computer)-341(vision?)]TJ 0 -11.955 Td [(In)]TJ/F61 9.9626 Tf 10.789 0 Td [(NeurIPS)]TJ/F58 9.9626 Tf 34.312 0 Td [(,)-250(2017.)]TJ
1 0 0 rg 1 0 0 RG
[-360(6)]TJ
0 g 0 G
0 g 0 G
-66.68 -14.838 Td [([38])]TJ
0 g 0 G
[-500(Zaid)-304(Khan)-304(and)-304(Y)111(un)-304(Fu.)-532(One)-304(label,)-318(one)-304(billion)-304(f)10(aces.)]TJ 21.579 -11.955 Td [(In)]TJ/F61 9.9626 Tf 10.789 0 Td [(F)115(AccT)]TJ/F58 9.9626 Tf 25.415 0 Td [(,)-250(2021.)]TJ
1 0 0 rg 1 0 0 RG
[-360(1)]TJ
0 g 0 G
0 g 0 G
-57.783 -14.837 Td [([39])]TJ
0 g 0 G
[-500(Been)-228(Kim,)-232(Raji)25(v)-228(Khanna,)-232(and)-227(Oluw)10(asanmi)-228(O)-228(K)35(o)10(yejo.)]TJ 21.579 -11.955 Td [(Examples)-279(are)-279(not)-279(enough,)-287(learn)-279(to)-279(criticize!)-398(criticism)]TJ 0 -11.956 Td [(for)-250(interpretability)65(.)-360(In)]TJ/F61 9.9626 Tf 89.543 0 Td [(NeurIPS)]TJ/F58 9.9626 Tf 34.311 0 Td [(,)-250(2016.)]TJ
1 0 0 rg 1 0 0 RG
[-360(1)]TJ
0 g 0 G
[(,)]TJ
1 0 0 rg 1 0 0 RG
[-250(8)]TJ
0 g 0 G
0 g 0 G
-145.433 -14.837 Td [([40])]TJ
0 g 0 G
[-500(Been)-403(Kim,)-440(Cynthia)-403(Rudin,)-441(and)-402(Julie)-403(A)-402(Shah.)-849(The)]TJ 21.579 -11.955 Td [(bayesian)-580(case)-581(model:)-970(A)-580(generati)25(v)15(e)-580(approach)-581(for)]TJ 0 -11.955 Td [(case-based)-304(reasoning)-303(and)-304(prototype)-304(classi\002cation.)-532(In)]TJ/F61 9.9626 Tf 0 -11.956 Td [(NeurIPS)]TJ/F58 9.9626 Tf 34.311 0 Td [(,)-250(2014.)]TJ
1 0 0 rg 1 0 0 RG
[-360(8)]TJ
0 g 0 G
0 g 0 G
-55.89 -14.837 Td [([41])]TJ
0 g 0 G
[-500(Ki)-759(Hyun)-758(Kim,)-886(Sangw)10(oo)-758(Shim,)-886(Y)110(ongsub)-759(Lim,)]TJ 21.579 -11.955 Td [(Jongseob)-245(Jeon,)-247(Jeongw)10(oo)-245(Choi,)-247(Byungchan)-245(Kim,)-247(and)]TJ 0 -11.955 Td [(Andre)-312(S)-312(Y)110(oon.)-558(Rapp:)-434(No)15(v)15(elty)-312(detection)-312(with)-312(recon-)]TJ 0 -11.955 Td [(struction)-352(along)-353(projection)-352(pathw)10(ay)65(.)-687(In)]TJ/F61 9.9626 Tf 162.089 0 Td [(International)]TJ -162.089 -11.956 Td [(Confer)37(ence)-250(on)-250(Learning)-250(Repr)37(esentations)]TJ/F58 9.9626 Tf 163.356 0 Td [(,)-250(2019.)]TJ
1 0 0 rg 1 0 0 RG
[-360(8)]TJ
0 g 0 G
0 g 0 G
-184.935 -14.837 Td [([42])]TJ
0 g 0 G
[-500(P)15(ang)-220(W)80(ei)-220(K)35(oh)-220(and)-219(Perc)15(y)-220(Liang.)-295(Understanding)-220(black-)]TJ 21.579 -11.955 Td [(box)-453(predictions)-452(via)-453(in\003uence)-453(functions.)-1008(In)]TJ/F61 9.9626 Tf 188.38 0 Td [(ICML)]TJ/F58 9.9626 Tf 23.8 0 Td [(,)]TJ -212.18 -11.955 Td [(2017.)]TJ
1 0 0 rg 1 0 0 RG
[-360(8)]TJ
0 g 0 G
0 g 0 G
-21.579 -14.838 Td [([43])]TJ
0 g 0 G
[-500(Ale)15(x)-405(Krizhe)25(vsk)15(y)65(,)-445(Geof)25(fre)15(y)-405(Hinton,)-445(et)-405(al.)-858(Learning)]TJ 21.579 -11.955 Td [(multiple)-375(layers)-375(of)-375(features)-375(from)-375(tin)15(y)-375(images.)-760(2009.)]TJ
1 0 0 rg 1 0 0 RG
0 -11.955 Td [(1)]TJ
0 g 0 G
[(,)]TJ
1 0 0 rg 1 0 0 RG
[-250(3)]TJ
0 g 0 G
0 g 0 G
0 g 0 G
0 g 0 G
237.171 629.037 Td [([44])]TJ
0 g 0 G
[-500(Balaji)-660(Lakshminarayanan,)-762(Ale)15(xander)-660(Pritzel,)-763(and)]TJ 21.579 -11.955 Td [(Charles)-525(Blundell.)-1240(Simple)-525(and)-525(scalable)-525(predicti)25(v)15(e)]TJ 0 -11.955 Td [(uncertainty)-558(estimation)-558(using)-558(deep)-558(ensembles.)-1346(In)]TJ/F61 9.9626 Tf 0 -11.955 Td [(NeurIPS)]TJ/F58 9.9626 Tf 34.311 0 Td [(,)-250(2017.)]TJ
1 0 0 rg 1 0 0 RG
[-360(6)]TJ
0 g 0 G
0 g 0 G
-55.89 -14.562 Td [([45])]TJ
0 g 0 G
[-500(Kimin)-473(Lee,)-529(Kibok)-473(Lee)1(,)-529(Honglak)-473(Lee,)-529(and)-473(Jinw)10(oo)]TJ 21.579 -11.955 Td [(Shin.)-583(A)-320(simple)-320(uni\002ed)-319(frame)25(w)10(ork)-320(for)-320(detecting)-320(out-)]TJ 0 -11.955 Td [(of-distrib)20(ution)-378(samples)-378(and)-378(adv)15(ersarial)-377(attacks.)]TJ/F61 9.9626 Tf 200.285 0 Td [(Ad-)]TJ -200.285 -11.956 Td [(vances)-312(in)-312(neur)15(al)-312(information)-312(pr)45(oces)1(sing)-312(systems)]TJ/F58 9.9626 Tf 196.466 0 Td [(,)-327(31,)]TJ -196.466 -11.955 Td [(2018.)]TJ
1 0 0 rg 1 0 0 RG
[-360(8)]TJ
0 g 0 G
0 g 0 G
-21.579 -14.562 Td [([46])]TJ
0 g 0 G
[-500(Christian)-333(Leibig,)-354(V)111(aneeda)-334(Allk)10(en,)-354(Murat)-333(Sec)388-55(kin)-334(A)92(y-)]TJ 21.579 -11.955 Td [(han,)-305(Philipp)-294(Berens,)-305(and)-293(Sie)15(gfried)-294(W)80(ahl.)-501(Le)25(v)15(eraging)]TJ 0 -11.955 Td [(uncertainty)-196(information)-195(from)-196(deep)-195(neural)-196(netw)10(orks)-196(for)]TJ 0 -11.955 Td [(disease)-250(detection.)-360(In)]TJ/F61 9.9626 Tf 84.652 0 Td [(Scienti\002c)-250(r)37(eports)]TJ/F58 9.9626 Tf 67.426 0 Td [(,)-250(2017.)]TJ
1 0 0 rg 1 0 0 RG
[-360(1)]TJ
0 g 0 G
0 g 0 G
-173.657 -14.562 Td [([47])]TJ
0 g 0 G
[-500(Oscar)-213(Li,)-221(Hao)-213(Liu,)-220(Chaof)10(an)-213(Chen,)-221(and)-213(Cynthia)-213(Rudin.)]TJ 21.579 -11.955 Td [(Deep)-203(learning)-204(for)-203(case-based)-204(reasoning)-203(through)-204(proto-)]TJ 0 -11.955 Td [(types:)-411(A)-300(neural)-301(netw)10(ork)-300(that)-301(e)15(xplains)-300(its)-301(predictions.)]TJ 0 -11.955 Td [(In)]TJ/F61 9.9626 Tf 10.79 0 Td [(AAAI)]TJ/F58 9.9626 Tf 21.579 0 Td [(,)-250(2018.)]TJ
1 0 0 rg 1 0 0 RG
[-360(8)]TJ
0 g 0 G
0 g 0 G
-53.948 -14.562 Td [([48])]TJ
0 g 0 G
[-500(Shiyu)-350(Liang,)-374(Y)55(ixuan)-350(Li,)-374(and)-350(R.)-349(Srikant.)-679(Enhancing)]TJ 21.579 -11.955 Td [(the)-224(reliability)-225(of)-224(out-of-distrib)20(ution)-225(image)-224(detection)-225(in)]TJ 0 -11.955 Td [(neural)-250(netw)10(orks.)-360(In)]TJ/F61 9.9626 Tf 80.677 0 Td [(ICLR)]TJ/F58 9.9626 Tf 21.589 0 Td [(,)-250(2018.)]TJ
1 0 0 rg 1 0 0 RG
[-360(14)]TJ
0 g 0 G
0 g 0 G
-123.845 -14.562 Td [([49])]TJ
0 g 0 G
[-500(Karttik)10(e)15(ya)-393(Mang)5(alam)-393(and)-392(V)60(inay)-393(Uday)-393(Prabhu.)-817(Do)]TJ 21.579 -11.955 Td [(deep)-378(neural)-379(netw)10(orks)-378(learn)-379(shallo)25(w)-378(learnable)-379(e)15(xam-)]TJ 0 -11.956 Td [(ples)-317(\002rst?)-769(In)]TJ/F61 9.9626 Tf 58.248 0 Td [(ICML)-317(W)92(orkshop)-317(on)-317(Deep)-317(Phenomena)]TJ/F58 9.9626 Tf 153.932 0 Td [(,)]TJ -212.18 -11.955 Td [(2019.)]TJ
1 0 0 rg 1 0 0 RG
[-360(5)]TJ
0 g 0 G
0 g 0 G
-21.579 -14.561 Td [([50])]TJ
0 g 0 G
[-500(Norman)-371(Mu)-371(and)-370(Justin)-371(Gilmer)55(.)-747(Mnist-c)1(:)-552(A)-371(rob)20(ust-)]TJ 21.579 -11.956 Td [(ness)-278(benchmark)-278(for)-277(computer)-278(vision.)-449(In)]TJ/F61 9.9626 Tf 164.123 0 Td [(ICML)-278(W)92(ork-)]TJ -164.123 -11.955 Td [(shop)-327(on)-328(Uncertainty)-327(and)-328(Rob)20(ustness)-327(in)-327(Deep)-328(Learn-)]TJ 0 -11.955 Td [(ing)]TJ/F58 9.9626 Tf 12.732 0 Td [(,)-250(2019.)]TJ
1 0 0 rg 1 0 0 RG
[-360(7)]TJ
0 g 0 G
0 g 0 G
-34.311 -14.562 Td [([51])]TJ
0 g 0 G
[-500(NHTSA.)-340(T)70(echnical)-241(report,)-243(U.S.)-241(Department)-241(of)-241(T)35(rans-)]TJ 21.579 -11.955 Td [(portation,)-266(Nati)1(onal)-263(Highw)10(ay)-262(T)35(raf)25<02632c>-266(T)70(esla)-262(Crash)-263(Pre-)]TJ 0 -11.955 Td [(liminary)-199(Ev)25(aluation)-199(Report)-199(Safety)-199(Administration.)]TJ/F61 9.9626 Tf 202.497 0 Td [(PE)]TJ -202.497 -11.955 Td [(16-007)]TJ/F58 9.9626 Tf 28.224 0 Td [(,)-250(Jan)-250(2017.)]TJ
1 0 0 rg 1 0 0 RG
[-360(1)]TJ
0 g 0 G
0 g 0 G
-49.803 -14.562 Td [([52])]TJ
0 g 0 G
[-500(Luk)10(e)-763(Oakden-Rayner)40(,)-891(Jared)-763(Dunnmon,)-892(Gusta)20(v)20(o)]TJ 21.579 -11.955 Td [(Carneiro,)-412(and)-380(Christopher)-380(R)]TJ 117.655 0.05 Td []TJ -0.553 -0.05 Td [(e.)-776(Hidden)-380(strati\002cation)]TJ -117.102 -11.955 Td [(causes)-196(clinically)-195(meaningful)-196(f)10(ailures)-195(in)-196(machine)-196(learn-)]TJ 0 -11.956 Td [(ing)-498(for)-499(medical)-498(imaging.)-1156(In)]TJ/F61 9.9626 Tf 130.709 0 Td [(A)30(CM)-498(confer)37(ence)-499(on)]TJ -130.709 -11.955 Td [(Health,)-250(Infer)37(ence)10(,)-250(and)-250(Learning)]TJ/F58 9.9626 Tf 128.198 0 Td [(,)-250(2020.)]TJ
1 0 0 rg 1 0 0 RG
[-360(1)]TJ
0 g 0 G
0 g 0 G
-149.777 -14.561 Td [([53])]TJ
0 g 0 G
[-500(Ahmet)-210(Murat)-209(Ozbayoglu,)-218(Mehmet)-209(Ugur)-210(Gudelek,)-218(and)]TJ 21.579 -11.956 Td [(Omer)-304(Berat)-304(Sezer)55(.)-534(Deep)-304(learning)-304(for)-305(\002nancial)-304(appli-)]TJ 0 -11.955 Td [(cations:)-411(A)-301(surv)15(e)15(y)65(.)-522(In)]TJ/F61 9.9626 Tf 89.875 0 Td [(Applied)-301(Soft)-300(Computing)]TJ/F58 9.9626 Tf 96.769 0 Td [(,)-313(2020.)]TJ
1 0 0 rg 1 0 0 RG
-186.644 -11.955 Td [(1)]TJ
0 g 0 G
0 g 0 G
-21.579 -14.562 Td [([54])]TJ
0 g 0 G
[-500(Emanuel)-287(P)15(arzen.)-480(On)-288(estimation)-287(of)-288(a)-287(probability)-288(den-)]TJ 21.579 -11.955 Td [(sity)-304(function)-303(and)-304(mode.)]TJ/F61 9.9626 Tf 100.988 0 Td [(The)-304(Annals)-303(of)-304(mathematical)]TJ -100.988 -11.955 Td [(statistics)]TJ/F58 9.9626 Tf 34.878 0 Td [(,)-250(1962.)]TJ
1 0 0 rg 1 0 0 RG
[-360(2)]TJ
0 g 0 G
0 g 0 G
-56.457 -14.562 Td [([55])]TJ
0 g 0 G
[-500(Mansheej)-311(P)15(aul,)-327(Surya)-312(Ganguli,)-327(and)-311(Gintare)-312(Karolina)]TJ 21.579 -11.955 Td [(Dziug)5(aite.)-494(Deep)-292(learning)-292(on)-291(a)-292(data)-292(diet:)-394(Finding)-292(im-)]TJ 0 -11.955 Td [(portant)-250(e)15(xamples)-250(early)-250(in)-250(training,)-250(2021.)]TJ
1 0 0 rg 1 0 0 RG
[-360(8)]TJ
0 g 0 G
0 g 0 G
-21.579 -14.562 Td [([56])]TJ
0 g 0 G
[-500(Karl)-250(Pearson.)-359(Li)1(ii.)-250(on)-250(lines)-249(and)-250(planes)-249(of)-250(closest)-249<0274>-250(to)]TJ 21.579 -11.955 Td [(systems)-326(of)-326(points)-325(in)-326(space.)]TJ/F61 9.9626 Tf 115.566 0 Td [(The)-326(London,)-345(Edinb)20(ur)37(gh,)]TJ -115.566 -11.955 Td [(and)-191(Dublin)-190(philosophical)-191(ma)10(gazine)-191(and)-190(journal)-191(of)-191(sci-)]TJ 0 -11.955 Td [(ence)]TJ/F58 9.9626 Tf 18.251 0 Td [(,)-250(2\(11\):559\226572,)-250(1901.)]TJ
1 0 0 rg 1 0 0 RG
[-360(8)]TJ
0 g 0 G
0 g 0 G
0 g 0 G
ET
Q
Q
Q
q
1 0 0 1 0 0 cm
/xAK7ubL752GOzViFwXZd8w Do
Q
endstream
endobj
39 0 obj
<< /Font << /F58 43 0 R /F61 45 0 R >> /ProcSet [ /PDF /Text ] /XObject << /xAK7ubL752GOzViFwXZd8w 85 0 R >> >>
endobj
40 0 obj
<< /Length 9344 >>
stream
q
q
q
0 g 0 G
0 g 0 G
0 g 0 G
BT
/F58 9.9626 Tf 50.112 710.037 Td [([57])]TJ
0 g 0 G
[-500(M)-361(Rosenblatt.)-716(Remarks)-362(on)-361(some)-361(nonparametric)-362(es-)]TJ 21.579 -11.955 Td [(timates)-305(of)-305(a)-304(density)-305(function.)-305(annals)-305(of)-305(mathematical)]TJ 0 -11.955 Td [(statistics.)-360(1956.)]TJ
1 0 0 rg 1 0 0 RG
[-360(8)]TJ
0 g 0 G
0 g 0 G
-21.579 -16.972 Td [([58])]TJ
0 g 0 G
[-500(Peter)-320(J)-319(Rousseeuw)-320(and)-320(Annick)-320(M)-320(Lero)10(y)65(.)]TJ/F61 9.9626 Tf 194.34 0 Td [(Rob)20(ust)-320(r)37(e-)]TJ -172.761 -11.956 Td [(gr)37(ession)-282(and)-282(outlier)-281(detection)]TJ/F58 9.9626 Tf 119.854 0 Td [(,)-290(v)20(olume)-282(589.)-461(John)-282(wi-)]TJ -119.854 -11.955 Td [(le)15(y)-250(&)-250(sons,)-250(2005.)]TJ
1 0 0 rg 1 0 0 RG
[-360(8)]TJ
0 g 0 G
0 g 0 G
-21.579 -16.972 Td [([59])]TJ
0 g 0 G
[-500(Lukas)-381(Ruf)25(f,)-414(Jacob)-381(R)-381(Kauf)25(fmann,)-414(Robert)-382(A)-381(V)111(ander)20(-)]TJ 21.579 -11.955 Td [(meulen,)-364(Gr)]TJ 46.513 0.05 Td []TJ -0.553 -0.05 Td [(egoire)-341(Monta)20(v)20(on,)-365(W)80(ojciech)-341(Samek,)-364(Mar)20(-)]TJ -45.96 -11.955 Td [(ius)-480(Kloft,)-537(Thomas)-480(G)-480(Dietterich,)-538(and)-480(Klaus-Robert)]TJ 0 -11.956 Td [(M)]TJ 9.689 0.05 Td []TJ -0.832 -0.05 Td [(uller)55(.)-1312(A)-548(unifying)-547(re)25(vie)25(w)-548(of)-547(deep)-548(and)-548(shallo)25(w)]TJ -8.857 -11.955 Td [(anomaly)-235(detection.)-326(In)]TJ/F61 9.9626 Tf 89.554 0 Td [(Pr)45(oceedings)-235(of)-234(the)-235(IEEE)]TJ/F58 9.9626 Tf 97.873 0 Td [(,)-235(2021.)]TJ
1 0 0 rg 1 0 0 RG
-187.427 -11.955 Td [(7)]TJ
0 g 0 G
0 g 0 G
-21.579 -16.972 Td [([60])]TJ
0 g 0 G
[-500(Lukas)-377(Ruf)25(f,)-409(Robert)-377(A)-378(V)111(anderm)1(eulen,)-410(Ni)1(co)-378(G)]TJ 211.349 0.049 Td []TJ -0.832 -0.049 Td [(ornitz,)]TJ -188.938 -11.955 Td [(Ale)15(xander)-431(Binder)40(,)-475(Emmanuel)-431(M)]TJ 136.731 0.049 Td []TJ -0.831 -0.049 Td [(uller)40(,)-476(Klaus-Robert)]TJ -135.9 -11.956 Td [(M)]TJ 9.689 0.05 Td []TJ -0.832 -0.05 Td [(uller)40(,)-587(and)-520(Marius)-519(Kloft.)-1223(Deep)-520(semi-supervised)]TJ -8.857 -11.955 Td [(anomaly)-260(detection.)]TJ/F61 9.9626 Tf 79.805 0 Td [(arXiv)-260(pr)37(eprint)-259(arXiv:1906.02694)]TJ/F58 9.9626 Tf 132.375 0 Td [(,)]TJ -212.18 -11.955 Td [(2019.)]TJ
1 0 0 rg 1 0 0 RG
[-360(8)]TJ
0 g 0 G
0 g 0 G
-21.579 -16.972 Td [([61])]TJ
0 g 0 G
[-500(Olg)5(a)-640(Russak)10(o)15(vsk)15(y)65(,)-737(Jia)-640(Deng,)-737(Hao)-640(Su,)-738(Jonathan)]TJ 21.579 -11.956 Td [(Krause,)-295(Sanjee)25(v)-286(Satheesh,)-295(Sean)-286(Ma,)-296(Zhiheng)-286(Huang,)]TJ 0 -11.955 Td [(Andrej)-333(Karpath)5(y)65(,)-353(Aditya)-333(Khosla,)-353(Michael)-333(Bernstein,)]TJ 0 -11.955 Td [(et)-397(al.)-829(Imagenet)-397(lar)18(ge)-397(scale)-396(visual)-397(recognition)-397(chal-)]TJ 0 -11.955 Td [(lenge.)-360(In)]TJ/F61 9.9626 Tf 38.446 0 Td [(IJCV)]TJ/F58 9.9626 Tf 20.473 0 Td [(,)-250(2015.)]TJ
1 0 0 rg 1 0 0 RG
[-360(1)]TJ
0 g 0 G
[(,)]TJ
1 0 0 rg 1 0 0 RG
[-250(3)]TJ
0 g 0 G
0 g 0 G
-80.498 -16.972 Td [([62])]TJ
0 g 0 G
[-500(Thomas)-236(Schle)15(gl,)-239(Philipp)-236(Seeb)]TJ 141.981 0.049 Td []TJ -0.832 -0.049 Td [(ock,)-239(Sebastian)-236(M)-236(W)80(ald-)]TJ -119.57 -11.956 Td [(stein,)-272(Ursula)-268(Schmidt-Erfurth,)-272(and)-267(Geor)18(g)-268(Langs.)-417(Un-)]TJ 0 -11.955 Td [(supervised)-348(anomaly)-347(detection)-348(with)-348(generati)25(v)15(e)-348(adv)15(er)20(-)]TJ 0 -11.955 Td [(sarial)-237(netw)10(orks)-236(to)-237(guide)-236(mark)10(er)-237(disco)15(v)15(ery)65(.)-331(In)]TJ/F61 9.9626 Tf 182.024 0 Td [(Interna-)]TJ -182.024 -11.955 Td [(tional)-300(confer)37(ence)-299(on)-300(information)-299(pr)45(ocessing)-300(in)-300(medi-)]TJ 0 -11.955 Td [(cal)-250(ima)10(ging)]TJ/F58 9.9626 Tf 47.223 0 Td [(,)-250(pages)-250(146\226157.)-250(Springer)40(,)-250(2017.)]TJ
1 0 0 rg 1 0 0 RG
[-360(8)]TJ
0 g 0 G
0 g 0 G
-68.802 -16.973 Td [([63])]TJ
0 g 0 G
[-500(A)74(v)25(anti)-585(Shrikumar)40(,)-669(Pe)15(yton)-585(Greenside,)-669(and)-585(Anshul)]TJ 21.579 -11.955 Td [(K)15(undaje.)-637(Learning)-337(important)-336(features)-337(through)-337(prop-)]TJ 0 -11.955 Td [(ag)5(ating)-250(acti)25(v)25(ation)-250(dif)25(ferences.)-360(In)]TJ/F61 9.9626 Tf 133.926 0 Td [(ICML)]TJ/F58 9.9626 Tf 23.801 0 Td [(,)-250(2017.)]TJ
1 0 0 rg 1 0 0 RG
[-360(2)]TJ
0 g 0 G
0 g 0 G
-179.306 -16.972 Td [([64])]TJ
0 g 0 G
[-500(Karen)-375(Simon)15(yan,)-407(Andrea)-375(V)111(edaldi,)-407(and)-375(Andre)25(w)-376(Zis-)]TJ 21.579 -11.956 Td [(serman.)-451(Deep)-278(inside)-279(con)40(v)20(olutional)-278(netw)10(orks:)-367(V)60(isual-)]TJ 0 -11.955 Td [(ising)-330(image)-330(classi\002cation)-330(models)-330(and)-331(salienc)15(y)-330(maps.)]TJ 0 -11.955 Td [(In)]TJ/F61 9.9626 Tf 10.79 0 Td [(W)92(orkshop)-250(at)-250(ICLR)]TJ/F58 9.9626 Tf 73.802 0 Td [(,)-250(2014.)]TJ
1 0 0 rg 1 0 0 RG
[-360(1)]TJ
0 g 0 G
[(,)]TJ
1 0 0 rg 1 0 0 RG
[-250(2)]TJ
0 g 0 G
0 g 0 G
-106.171 -16.972 Td [([65])]TJ
0 g 0 G
[-500(Daniel)-332(Smilk)10(o)15(v)65(,)-353(Nikhil)-333(Thorat,)-352(Been)-333(Kim,)-353(Fernanda)]TJ 21.579 -11.956 Td [(V)60(i)]TJ 9.918 0.05 Td []TJ -0.553 -0.05 Td [(eg)5(as,)-271(and)-267(Martin)-267(W)80(attenber)18(g.)-414(Smoothgrad:)-344(remo)15(v-)]TJ -9.365 -11.955 Td [(ing)-284(noise)-284(by)-285(adding)-284(noise.)-469(In)]TJ/F61 9.9626 Tf 121.496 0 Td [(ICML)-284(W)92(orkshop)-284(on)-284(V)74(i-)]TJ -121.496 -11.955 Td [(sualization)-250(for)-250(Deep)-250(Learning)]TJ/F58 9.9626 Tf 120.387 0 Td [(,)-250(2017.)]TJ
1 0 0 rg 1 0 0 RG
[-360(2)]TJ
0 g 0 G
0 g 0 G
-141.966 -16.972 Td [([66])]TJ
0 g 0 G
[-500(Mukund)-210(Sundararajan,)-217(Ankur)-210(T)80(aly)65(,)-218(and)-209(Qiqi)-210(Y)100(an.)-273(Ax-)]TJ 21.579 -11.955 Td [(iomatic)-233(attrib)20(ution)-232(for)-233(deep)-233(netw)10(orks.)-322(In)]TJ/F61 9.9626 Tf 163.645 0 Td [(ICML)]TJ/F58 9.9626 Tf 23.801 0 Td [(,)-233(2017.)]TJ
1 0 0 rg 1 0 0 RG
-187.446 -11.956 Td [(2)]TJ
0 g 0 G
0 g 0 G
-21.579 -16.972 Td [([67])]TJ
0 g 0 G
[-500(Da)20(vid)-394(MJ)-394(T)80(ax)-395(and)-394(Robert)-394(PW)-395(Duin.)-821(Support)-395(v)15(ec-)]TJ 21.579 -11.955 Td [(tor)-304(data)-303(description.)]TJ/F61 9.9626 Tf 85.778 0 Td [(Mac)15(hine)-304(learning)]TJ/F58 9.9626 Tf 71.498 0 Td [(,)-317(54\(1\):45\22666,)]TJ -157.276 -11.955 Td [(2004.)]TJ
1 0 0 rg 1 0 0 RG
[-360(8)]TJ
0 g 0 G
0 g 0 G
-21.579 -16.973 Td [([68])]TJ
0 g 0 G
[-500(Michael)-445(V)111(eale)-444(and)-445(Reuben)-444(Binns.)-983(F)15(airer)-445(machine)]TJ 21.579 -11.955 Td [(learning)-338(in)-338(the)-337(real)-338(w)10(orld:)-486(Mitig)5(ating)-338(discrimination)]TJ 0 -11.955 Td [(without)-251(collecting)-250(sensiti)25(v)15(e)-251(data.)-362(In)]TJ/F61 9.9626 Tf 145.192 0 Td [(Big)-251(Data)-250(&)-251(Soci-)]TJ -145.192 -11.955 Td [(ety)]TJ/F58 9.9626 Tf 11.616 0 Td [(,)-250(2017.)]TJ
1 0 0 rg 1 0 0 RG
[-360(1)]TJ
0 g 0 G
[(,)]TJ
1 0 0 rg 1 0 0 RG
[-250(4)]TJ
0 g 0 G
0 g 0 G
0 g 0 G
0 g 0 G
225.555 629.037 Td [([69])]TJ
0 g 0 G
[-500(Bernard)-670(L)-669(W)80(elch.)-1702(The)-670(generalization)-669(of)-670(`stu-)]TJ 21.579 -11.955 Td [(dent')55(s'problem)-255(when)-254(se)25(v)15(eral)-255(dif)25(ferent)-255(population)-255(v)25(ar)20(-)]TJ 0 -11.955 Td [(lances)-250(are)-250(in)40(v)20(olv)15(ed.)-360(In)]TJ/F61 9.9626 Tf 93.03 0 Td [(Biometrika)]TJ/F58 9.9626 Tf 44.274 0 Td [(,)-250(1947.)]TJ
1 0 0 rg 1 0 0 RG
[-360(6)]TJ
0 g 0 G
0 g 0 G
-158.883 -14.944 Td [([70])]TJ
0 g 0 G
[-500(Hongtao)-351(Xie,)-375(Dongbao)-351(Y)100(ang,)-375(Nannan)-351(Sun,)-376(Zhineng)]TJ 21.579 -11.955 Td [(Chen,)-366(and)-342(Y)110(ongdong)-342(Zhang.)-656(Automated)-343(pulmonary)]TJ 0 -11.955 Td [(nodule)-190(detection)-190(in)-191(ct)-190(images)-190(using)-190(deep)-191(con)40(v)20(olutional)]TJ 0 -11.956 Td [(neural)-250(netw)10(orks.)-360(In)]TJ/F61 9.9626 Tf 80.677 0 Td [(P)80(attern)-250(Reco)10(gnition)]TJ/F58 9.9626 Tf 79.63 0 Td [(,)-250(2019.)]TJ
1 0 0 rg 1 0 0 RG
[-360(1)]TJ
0 g 0 G
0 g 0 G
-181.886 -14.944 Td [([71])]TJ
0 g 0 G
[-500(Ser)18(ge)15(y)-217(Zagoruyk)10(o)-217(and)-217(Nik)10(os)-217(K)35(omodakis.)-289(W)40(ide)-217(resid-)]TJ 21.579 -11.955 Td [(ual)-250(netw)10(orks.)-360(In)]TJ/F61 9.9626 Tf 67.955 0 Td [(BMVC)]TJ/F58 9.9626 Tf 27.118 0 Td [(,)-250(2016.)]TJ
1 0 0 rg 1 0 0 RG
[-360(14)]TJ
0 g 0 G
0 g 0 G
-116.652 -14.944 Td [([72])]TJ
0 g 0 G
[-500(Chiyuan)-434(Zhang,)-481(Samy)-434(Bengio,)-480(Moritz)-434(Hardt,)-481(Ben-)]TJ 21.579 -11.955 Td [(jamin)-304(Recht,)-318(and)-305(Oriol)-304(V)60(in)15(yals.)-534(Understanding)-305(deep)]TJ 0 -11.955 Td [(learning)-307(requires)-308(rethinking)-307(generalization.)-544(In)]TJ/F61 9.9626 Tf 190.592 0 Td [(ICLR)]TJ/F58 9.9626 Tf 21.588 0 Td [(,)]TJ -212.18 -11.955 Td [(2017.)]TJ
1 0 0 rg 1 0 0 RG
[-360(6)]TJ
0 g 0 G
0 g 0 G
-21.579 -14.944 Td [([73])]TJ
0 g 0 G
[-500(Jianping)-681(Zhang.)-1741(Selecting)-681(typical)-681(instances)-682(in)]TJ 21.579 -11.955 Td [(instance-based)-389(learning.)-805(In)]TJ/F61 9.9626 Tf 117.862 0 Td [(Mac)15(hine)-389(Learning)-389(Pr)45(o-)]TJ -117.862 -11.956 Td [(ceedings)]TJ/F58 9.9626 Tf 34.859 0 Td [(.)-250(Else)25(vier)40(,)-250(1992.)]TJ
1 0 0 rg 1 0 0 RG
[-360(1)]TJ
0 g 0 G
[(,)]TJ
1 0 0 rg 1 0 0 RG
[-250(8)]TJ
0 g 0 G
0 g 0 G
0 g 0 G
0 g 0 G
0 g 0 G
ET
Q
Q
Q
q
1 0 0 1 0 0 cm
/JUC0Mj_A09vfdRnNE74yOQ Do
Q
endstream
endobj
41 0 obj
<< /Font << /F58 43 0 R /F61 45 0 R >> /ProcSet [ /PDF /Text ] /XObject << /JUC0Mj_A09vfdRnNE74yOQ 86 0 R >> >>
endobj
42 0 obj
<< /BaseFont /VWNWHD+NimbusRomNo9L-Medi /Encoding 87 0 R /FirstChar 2 /FontDescriptor 88 0 R /LastChar 173 /Subtype /Type1 /ToUnicode 89 0 R /Type /Font /Widths 90 0 R >>
endobj
43 0 obj
<< /BaseFont /KIXDLN+NimbusRomNo9L-Regu /Encoding 87 0 R /FirstChar 2 /FontDescriptor 91 0 R /LastChar 184 /Subtype /Type1 /ToUnicode 92 0 R /Type /Font /Widths 93 0 R >>
endobj
44 0 obj
<< /BaseFont /AAUIUS+NimbusMonL-Regu /Encoding 87 0 R /FirstChar 46 /FontDescriptor 94 0 R /LastChar 122 /Subtype /Type1 /ToUnicode 95 0 R /Type /Font /Widths 96 0 R >>
endobj
45 0 obj
<< /BaseFont /UZEBGQ+NimbusRomNo9L-ReguItal /Encoding 87 0 R /FirstChar 2 /FontDescriptor 97 0 R /LastChar 122 /Subtype /Type1 /ToUnicode 98 0 R /Type /Font /Widths 99 0 R >>
endobj
46 0 obj
<< /BBox [ 0 0 612 792 ] /Matrix [ 1 0 0 1 0 0 ] /Resources << /Font 100 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ] >> /Subtype /Form /Type /XObject /Length 115 /Filter /FlateDecode >>
stream
x-0D|i; iW$:1EB*CN7OBrIW؊X;7^W-~g!VA'(s8 s+q/Ӏ7$|3
endstream
endobj
47 0 obj
<< /BBox [ 0 0 612 792 ] /Group << /CS /DeviceRGB /I true /S /Transparency /Type /Group >> /Resources << /ExtGState << /a0 << /CA 1 /ca 1 >> /a1 << /CA 0.5 /ca 0.5 >> >> /XObject << /x6 101 0 R >> >> /Subtype /Form /Type /XObject /Length 382 /Filter /FlateDecode >>
stream
xe[n!Y7P|=Q'y(Ji;#$v0U-V\oZWɽ:5db,6IR6@6Pc"3K+GQ؎3T<9j`UT@
v5ڳtHN;9+7[^S؇:[E"8+sW\qEeКu6Bhsj;)y;Tp0nؚgʦFǃS-="z7ۺ9*&Q~ZPuí}C9嬂8l?Jܪmx%e8w](Rlq鞋t×e:=p=y0`XeJCWxau~uof{`r
endstream
endobj
48 0 obj
<< /BaseFont /LSOBOM+CMEX10 /FirstChar 88 /FontDescriptor 102 0 R /LastChar 114 /Subtype /Type1 /ToUnicode 103 0 R /Type /Font /Widths 104 0 R >>
endobj
49 0 obj
<< /BaseFont /PWAWJI+CMR7 /FirstChar 49 /FontDescriptor 105 0 R /LastChar 61 /Subtype /Type1 /ToUnicode 106 0 R /Type /Font /Widths 107 0 R >>
endobj
50 0 obj
<< /BaseFont /XWMBIK+CMR10 /FirstChar 37 /FontDescriptor 108 0 R /LastChar 61 /Subtype /Type1 /ToUnicode 109 0 R /Type /Font /Widths 110 0 R >>
endobj
51 0 obj
<< /BaseFont /HHPBWQ+CMMI7 /FirstChar 75 /FontDescriptor 111 0 R /LastChar 116 /Subtype /Type1 /ToUnicode 112 0 R /Type /Font /Widths 113 0 R >>
endobj
52 0 obj
<< /BaseFont /NUNTVA+CMMI10 /FirstChar 11 /FontDescriptor 114 0 R /LastChar 121 /Subtype /Type1 /ToUnicode 115 0 R /Type /Font /Widths 116 0 R >>
endobj
53 0 obj
<< /BaseFont /QDWLYD+CMSY7 /FirstChar 0 /FontDescriptor 117 0 R /LastChar 6 /Subtype /Type1 /ToUnicode 118 0 R /Type /Font /Widths 119 0 R >>
endobj
54 0 obj
<< /BaseFont /HPOBOQ+CMSY10 /FirstChar 0 /FontDescriptor 120 0 R /LastChar 103 /Subtype /Type1 /ToUnicode 121 0 R /Type /Font /Widths 122 0 R >>
endobj
55 0 obj
<< /BaseFont /QDTWCG+MSBM10 /FirstChar 82 /FontDescriptor 123 0 R /LastChar 82 /Subtype /Type1 /ToUnicode 124 0 R /Type /Font /Widths 125 0 R >>
endobj
56 0 obj
<< /BaseFont /ZLWGLI+CMBX10 /FirstChar 83 /FontDescriptor 126 0 R /LastChar 89 /Subtype /Type1 /ToUnicode 127 0 R /Type /Font /Widths 128 0 R >>
endobj
57 0 obj
<< /BBox [ 0 0 612 792 ] /Matrix [ 1 0 0 1 0 0 ] /Resources << /Font 129 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ] >> /Subtype /Form /Type /XObject /Length 114 /Filter /FlateDecode >>
stream
x-0D|C4+LD0
ӳ\28QlAJ,؝\+|ЖJS=Y+`> );ki/UrV76/"4
endstream
endobj
58 0 obj
<< /BaseFont /MBNBOP+CMSS10 /FirstChar 97 /FontDescriptor 130 0 R /LastChar 112 /Subtype /Type1 /ToUnicode 131 0 R /Type /Font /Widths 132 0 R >>
endobj
59 0 obj
<< /BBox [ 0 0 612 792 ] /Matrix [ 1 0 0 1 0 0 ] /Resources << /Font 133 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ] >> /Subtype /Form /Type /XObject /Length 114 /Filter /FlateDecode >>
stream
x-0D|CD+LD0
ӳ"!Xn5WST>X-lO)1c l7JW*oHʳ,
endstream
endobj
60 0 obj
<< /BBox [ 0 0 436.718 319.575 ] /Filter /FlateDecode /FormType 1 /Group 21 0 R /PTEX.FileName (/compile/figures/toy_dataset_decision_boundary.pdf) /PTEX.InfoDict 134 0 R /PTEX.PageNumber 1 /Resources << /ExtGState << /A1 << /CA 0 /Type /ExtGState /ca 1 >> /A2 << /CA 1 /Type /ExtGState /ca 1 >> /A3 << /CA 1 /Type /ExtGState /ca 0.4 >> /A4 << /CA 0.8 /Type /ExtGState /ca 0.8 >> >> /Font << /F1 135 0 R /F2 136 0 R >> /Pattern << >> /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ] /Shading << >> /XObject << /F1-DejaVuSans-Bold-minus 137 0 R /M0 138 0 R /M1 139 0 R /P0 140 0 R >> >> /Subtype /Form /Type /XObject /Length 159599 >>
stream
x̽M-[s7E3Y=&WIȕս#Kss2*2ʯ5~/#=[ok5m|eW?{Wg?2ɟm3~UgxZ
ԯv'o~KJU*~ǿcswGVu}hϿ_վ~ßg߯ׯuodo~c:џj5~_GcU?r_=}b/~^">װ?__?w~;_8~W=?~ԡ/~gԃ;U~}wKg~>SN_>}V_uǿêIPg?(|=F1/nz{~}v/v|ԯ~>Nox_=~jl^`Nu8?t^_v5qLEZSn7=־@5Qaߣ^qTRF2OccN8*?{T>+
?xkNn|RW.kXcux?_^ǷCaE~쭾3i6/o;[ؿt;!Oo=>_:)eKwd0;<Ǿ[Yuf+]Ue0Vs23Ub֜ϷgU~3֦Ͼ}njcfQc0V]cf6/٪U^m6cЩ03M`fgފ3SӘmd6ז9_k5cϑ9vT?h|~>Ǜ}3>EfHT?`m:تgˏg00VQ(NT?`YT?`iT?Φ˫ɎlAf_;R>MV4f0jS9NT?`ƨ~V[R?k*SR?Mt~ڗ}dR?GǮc
)\?Βiv%P~}B[}mFO?O`bOhIaoF[ǮM4v~L!3V~)3/3YR?K5.cc#c1v|~׳ĽfqLϴK.3xJ/;#ILәEdB_R?&s$h{0)"~LXY"cJ2~L*cYR?&-KF8]D>R?&1ԏpHNc"'kB\?&Ƒ1`2w#c2p"1E`)}~L&ct!Iҥ~LcbbϧRYc1V,OS7 ǘQc&F>%?iQc{~3-S
(1\?Lr3c(1{c&F~p[~ۅ3Qbafjǘi%(Y?WOR?&fiR?
bTAJ4QJ4hQI4hQ1* F~:ĨO]t1*!F~:ĨO1Y?25*c*a~6fjTCJJPp~6VgcmW31*3 F~ĨgbTg@JLQl`lτ1A~6fZTgBJLQ 1*c"a~6fbTD\ς10Y?ll,F~L4LJ$hL*!F~6Ĩԏ G1;HlQ
1*!F}m~ԏ%٘iQ1*s F~Ĩԏ!٘u\1*s F~p~6fU7D?[llGsuNE?&F~;p&F~\?~Eb
iQtϵBr31c̪ǘι~Y+1fbǘI1c&F~*ĨOAJ;iPR?犇Jij&F~Ĩno~&gc&F~:ĨOEb4LJ R?bTg@J;ĒR?bTg@JLX\w~x!٘Q 5*c*a~6fPdBl䨼Q 1*3!F~LTLJ,QtH,Qg#X?YR?bTgAJHXR?bTgCJL謟
1E?sxF~LT@J$(LLJQ1*c~6fbT@JQ1JL#,Lr3]c̴(OóKLr31cluLr31
(1\?ƬS25c.f~1fj~6fj>W\?
&\Lr/c&F~1fb ;llAށgc[D?
1sI|x(L*E?~66u~ @Dy<;lA>-<$~>;|LAށgcUD?:?Ɔ"M[D?:?hL<t~ @D<D?[G4vt~ @Du<<`lh [:?DDv"'QIA$ ru~9:?DDN"&QIA$ ru~9:?DDN""%QIA0Dv";QIAbDDN"'QIA$`CN2?d~ A)`Kt&
ى2?Vd~ HA.`C-2?vd~!'Uk2?e~l ؔA%`[6$ XAg2?Vy~ XA!`Sm;2?ؐH E52?6d~l ؒA@Gbe~ XA&d~ ؐA)`KlIA"`Uu2?6e~Z.O#vd~!%QUk2?e~Й
2?d~l ؑD+2?Ve~ XA!`Sm:2?(`E52?6d~l[2?e~`CN2?x~ ؐA)`K6_w!'QUk2?e~l A%`[ ؑAlyd~ Xm;B#&^hO[jK)j/oJKyLqCobR:cR)(-vNF'.҉ޕ3t,7Sɷ_CĈ[QjkY)&tJ.ml{Uj)]HJ47RZT#
> 75Ҿ|'fɏP="YvQ1ꬰ<|Tj:*SHzS
|ʵ/ͣ cTMh>
%ScW*t%MfJPW>?wG
cJe֢ts(>Z
aKiRk7:N[m=]i>cR
`(4Et gp]JfNJ 3Qu]g5ueLbqrk TZ}=S3ŭtFi7жR^DO4Nt9"u:Oy5ܡrfE;]E@'N|ov:zӁ" 5]Ėt:P<3 M;X@1ɝ~.t
v:qk
hE:hy5K@
JmK;(6
QA;]g@R