======================================================================
KEEP-vs-REFIT DECISION  (sentiment)
======================================================================

 budget B=25
   keep    NLL_impr=+0.0000[+0.0000,+0.0000]  harm_rate=0.00  refit_rate=0.00
   always  NLL_impr=-0.0358[-0.0615,-0.0156]  harm_rate=0.45  refit_rate=1.00
   shrink  NLL_impr=-0.0200[-0.0465,-0.0029]  harm_rate=0.27  refit_rate=0.98
   cv      NLL_impr=-0.0208[-0.0462,-0.0013]  harm_rate=0.20  refit_rate=0.52
   crit    NLL_impr=-0.0303[-0.0547,-0.0069]  harm_rate=0.30  refit_rate=0.40
   [reference] fraction of pairs where full-250 oracle beats keep by >0.005: 0.57

 budget B=50
   keep    NLL_impr=+0.0000[+0.0000,+0.0000]  harm_rate=0.00  refit_rate=0.00
   always  NLL_impr=+0.0028[+0.0002,+0.0047]  harm_rate=0.30  refit_rate=1.00
   shrink  NLL_impr=+0.0044[+0.0026,+0.0065]  harm_rate=0.10  refit_rate=1.00
   cv      NLL_impr=-0.0002[-0.0021,+0.0022]  harm_rate=0.10  refit_rate=0.28
   crit    NLL_impr=-0.0012[-0.0053,+0.0022]  harm_rate=0.18  refit_rate=0.32
   [reference] fraction of pairs where full-250 oracle beats keep by >0.005: 0.57

 budget B=100
   keep    NLL_impr=+0.0000[+0.0000,+0.0000]  harm_rate=0.00  refit_rate=0.00
   always  NLL_impr=+0.0085[+0.0051,+0.0119]  harm_rate=0.15  refit_rate=1.00
   shrink  NLL_impr=+0.0085[+0.0055,+0.0119]  harm_rate=0.05  refit_rate=1.00
   cv      NLL_impr=+0.0061[+0.0030,+0.0091]  harm_rate=0.05  refit_rate=0.55
   crit    NLL_impr=+0.0068[+0.0032,+0.0104]  harm_rate=0.05  refit_rate=0.38
   [reference] fraction of pairs where full-250 oracle beats keep by >0.005: 0.57

 budget B=250
   keep    NLL_impr=+0.0000[+0.0000,+0.0000]  harm_rate=0.00  refit_rate=0.00
   always  NLL_impr=+0.0124[+0.0085,+0.0169]  harm_rate=0.03  refit_rate=1.00
   shrink  NLL_impr=+0.0119[+0.0085,+0.0158]  harm_rate=0.00  refit_rate=0.98
   cv      NLL_impr=+0.0095[+0.0059,+0.0134]  harm_rate=0.00  refit_rate=0.53
   crit    NLL_impr=+0.0118[+0.0080,+0.0161]  harm_rate=0.02  refit_rate=0.58
   [reference] fraction of pairs where full-250 oracle beats keep by >0.005: 0.57

 secondary metrics at B=50 (mean over pairs):
   keep    ECE=0.0532  Brier=0.1781  NLL=0.5352
   always  ECE=0.0495  Brier=0.1775  NLL=0.5324
   crit    ECE=0.0547  Brier=0.1783  NLL=0.5363

 secondary metrics at B=100 (mean over pairs):
   keep    ECE=0.0532  Brier=0.1781  NLL=0.5352
   always  ECE=0.0420  Brier=0.1762  NLL=0.5267
   crit    ECE=0.0435  Brier=0.1765  NLL=0.5283
======================================================================
KEEP-vs-REFIT DECISION  (acs)
======================================================================

 budget B=25
   keep    NLL_impr=+0.0000[+0.0000,+0.0000]  harm_rate=0.00  refit_rate=0.00
   always  NLL_impr=-0.0082[-0.0654,+0.0244]  harm_rate=0.57  refit_rate=1.00
   shrink  NLL_impr=+0.0057[-0.0501,+0.0387]  harm_rate=0.24  refit_rate=0.99
   cv      NLL_impr=-0.0003[-0.0757,+0.0433]  harm_rate=0.14  refit_rate=0.43
   crit    NLL_impr=-0.0021[-0.0613,+0.0311]  harm_rate=0.22  refit_rate=0.43
   [reference] fraction of pairs where full-250 oracle beats keep by >0.005: 0.35

 budget B=50
   keep    NLL_impr=+0.0000[+0.0000,+0.0000]  harm_rate=0.00  refit_rate=0.00
   always  NLL_impr=+0.0312[+0.0210,+0.0422]  harm_rate=0.31  refit_rate=1.00
   shrink  NLL_impr=+0.0364[+0.0278,+0.0450]  harm_rate=0.17  refit_rate=0.99
   cv      NLL_impr=+0.0359[+0.0272,+0.0444]  harm_rate=0.16  refit_rate=0.47
   crit    NLL_impr=+0.0338[+0.0242,+0.0435]  harm_rate=0.15  refit_rate=0.40
   [reference] fraction of pairs where full-250 oracle beats keep by >0.005: 0.35

 budget B=100
   keep    NLL_impr=+0.0000[+0.0000,+0.0000]  harm_rate=0.00  refit_rate=0.00
   always  NLL_impr=+0.0407[+0.0301,+0.0494]  harm_rate=0.16  refit_rate=1.00
   shrink  NLL_impr=+0.0424[+0.0331,+0.0502]  harm_rate=0.01  refit_rate=1.00
   cv      NLL_impr=+0.0412[+0.0319,+0.0490]  harm_rate=0.05  refit_rate=0.54
   crit    NLL_impr=+0.0411[+0.0313,+0.0498]  harm_rate=0.06  refit_rate=0.41
   [reference] fraction of pairs where full-250 oracle beats keep by >0.005: 0.35

 budget B=250
   keep    NLL_impr=+0.0000[+0.0000,+0.0000]  harm_rate=0.00  refit_rate=0.00
   always  NLL_impr=+0.0427[+0.0345,+0.0503]  harm_rate=0.05  refit_rate=1.00
   shrink  NLL_impr=+0.0431[+0.0349,+0.0505]  harm_rate=0.02  refit_rate=0.99
   cv      NLL_impr=+0.0422[+0.0335,+0.0500]  harm_rate=0.01  refit_rate=0.62
   crit    NLL_impr=+0.0423[+0.0341,+0.0498]  harm_rate=0.04  refit_rate=0.50
   [reference] fraction of pairs where full-250 oracle beats keep by >0.005: 0.35

 secondary metrics at B=50 (mean over pairs):
   keep    ECE=0.0713  Brier=0.1745  NLL=0.5362
   always  ECE=0.0613  Brier=0.1662  NLL=0.5051
   crit    ECE=0.0580  Brier=0.1657  NLL=0.5024

 secondary metrics at B=100 (mean over pairs):
   keep    ECE=0.0713  Brier=0.1745  NLL=0.5362
   always  ECE=0.0505  Brier=0.1640  NLL=0.4955
   crit    ECE=0.0492  Brier=0.1639  NLL=0.4951