Coverage for src/bob/measure/plot.py: 58%

1#!/usr/bin/env python

2# vim: set fileencoding=utf-8 :

3# Mon 23 May 2011 14:36:14 CEST

4import warnings

6import numpy

9def log_values(min_step=-4, counts_per_step=4):

10 """Computes log-scaled values between :math:`10^{M}` and 1

12 This function computes log-scaled values between :math:`10^{M}` and 1

13 (including), where :math:`M` is the ``min_ste`` argument, which needs to be a

14 negative integer. The integral ``counts_per_step`` value defines how many

15 values between two adjacent powers of 10 will be created. The total number

16 of values will be ``-min_step * counts_per_step + 1``.

19 Parameters:

21 min_step (:py:class:`int`, optional): The power of 10 that will be the

22 minimum value. E.g., the default ``-4`` will result in the first number

23 to be :math:`10^{-4}` = ``0.00001`` or ``0.01%``

25 counts_per_step (:py:class:`int`, optional): The number of values that will

26 be put between two adjacent powers of 10. With the default value ``4``

27 (and default values of ``min_step``), we will get ``log_list[0] ==

28 1e-4``, ``log_list[4] == 1e-3``, ..., ``log_list[16] == 1``.

31 Returns:

33 :py:class:`list`: A list of logarithmically scaled values between

34 :math:`10^{M}` and 1.

36 """

38 import math

40 return [

41 math.pow(10.0, i * 1.0 / counts_per_step)

42 for i in range(min_step * counts_per_step, 0)

43 ] + [1.0]

46def _semilogx(x, y, **kwargs):

47 # remove points were x is 0

48 x, y = numpy.asarray(x), numpy.asarray(y)

49 zero_index = x == 0

50 x = x[~zero_index]

51 y = y[~zero_index]

52 from matplotlib import pyplot

54 return pyplot.semilogx(x, y, **kwargs)

57def roc(

58 negatives,

59 positives,

60 npoints=2000,

61 CAR=None,

62 min_far=-8,

63 tpr=False,

64 semilogx=False,

65 **kwargs,

66):

67 """Plots Receiver Operating Characteristic (ROC) curve.

69 This method will call ``matplotlib`` to plot the ROC curve for a system which

70 contains a particular set of negatives (impostors) and positives (clients)

71 scores. We use the standard :py:func:`matplotlib.pyplot.plot` command. All

72 parameters passed with exception of the three first parameters of this method

73 will be directly passed to the plot command.

75 The plot will represent the false-alarm on the horizontal axis and the

76 false-rejection on the vertical axis. The values for the axis will be

77 computed using :py:func:`bob.measure.roc`.

79 .. note::

81 This function does not initiate and save the figure instance, it only

82 issues the plotting command. You are the responsible for setting up and

83 saving the figure as you see fit.

86 Parameters

87 ----------

88 negatives : array

89 1D float array that contains the scores of the

90 "negative" (noise, non-class) samples of your classifier. See

91 (:py:func:`bob.measure.roc`)

93 positives : array

94 1D float array that contains the scores of the

95 "positive" (signal, class) samples of your classifier. See

96 (:py:func:`bob.measure.roc`)

98 npoints : :py:class:`int`, optional

99 The number of points for the plot. See

100 (:py:func:`bob.measure.roc`)

101

102 min_far : float, optional

103 The minimum value of FPR and FNR that is used for ROC computations.

104

105 tpr : bool, optional

106 If True, will plot TPR (TPR = 1 - FNR) on the y-axis instead of FNR.

107

108 semilogx : bool, optional

109 If True, will use pyplot.semilogx to plot the ROC curve.

110

111 CAR : :py:class:`bool`, optional

112 This option is deprecated. Please use ``TPR`` and ``semilogx`` options instead.

113 If set to ``True``, it will plot the CPR

114 (CAR) over FPR in using :py:func:`matplotlib.pyplot.semilogx`, otherwise the

115 FPR over FNR linearly using :py:func:`matplotlib.pyplot.plot`.

116

117 **kwargs

118 Extra plotting parameters, which are

119 passed directly to :py:func:`matplotlib.pyplot.plot`.

120

121 Returns

122 -------

123 object

124 `list` of :py:class:`matplotlib.lines.Line2D`: The lines that

125 were added as defined by the return value of

126 :py:func`matplotlib.pyplot.plot`.

127 """

128 if CAR is not None:

129 warnings.warn(

130 "CAR argument is deprecated. Please use TPR and semilogx arguments instead.",

131 DeprecationWarning,

132 )

133 tpr = semilogx = CAR

134

135 from matplotlib import pyplot

136

137 from . import roc as calc

138

139 fpr, fnr = calc(negatives, positives, npoints, min_far=min_far)

140

141 if tpr:

142 fnr = 1 - fnr # plot tpr instead of fnr

143

144 if not semilogx:

145 return pyplot.plot(fpr, fnr, **kwargs)

146 else:

147 return _semilogx(fpr, fnr, **kwargs)

148

149

150def roc_for_far(

151 negatives, positives, far_values=log_values(), CAR=True, **kwargs

152):

153 """Plots the ROC curve for the given list of False Positive Rates (FAR).

154

155 This method will call ``matplotlib`` to plot the ROC curve for a system which

156 contains a particular set of negatives (impostors) and positives (clients)

157 scores. We use the standard :py:func:`matplotlib.pyplot.semilogx` command.

158 All parameters passed with exception of the three first parameters of this

159 method will be directly passed to the plot command.

160

161 The plot will represent the False Positive Rate (FPR) on the horizontal

162 axis and the Correct Positive Rate (CPR) on the vertical axis. The values

163 for the axis will be computed using :py:func:`bob.measure.roc_for_far`.

164

165 .. note::

166

167 This function does not initiate and save the figure instance, it only

168 issues the plotting command. You are the responsible for setting up and

169 saving the figure as you see fit.

170

171

172 Parameters:

173

174 negatives (array): 1D float array that contains the scores of the

175 "negative" (noise, non-class) samples of your classifier. See

176 (:py:func:`bob.measure.roc`)

177

178 positives (array): 1D float array that contains the scores of the

179 "positive" (signal, class) samples of your classifier. See

180 (:py:func:`bob.measure.roc`)

181

182 far_values (:py:class:`list`, optional): The values for the FPR, where the

183 CPR (CAR) should be plotted; each value should be in range [0,1].

184

185 CAR (:py:class:`bool`, optional): If set to ``True``, it will plot the CPR

186 (CAR) over FPR in using :py:func:`matplotlib.pyplot.semilogx`, otherwise the

187 FPR over FNR linearly using :py:func:`matplotlib.pyplot.plot`.

188

189 kwargs (:py:class:`dict`, optional): Extra plotting parameters, which are

190 passed directly to :py:func:`matplotlib.pyplot.plot`.

191

192

193 Returns:

194

195 :py:class:`list` of :py:class:`matplotlib.lines.Line2D`: The lines that

196 were added as defined by the return value of

197 :py:func:`matplotlib.pyplot.semilogx`.

198

199 """

200 warnings.warn(

201 "roc_for_far is deprecated. Please use the roc function instead."

202 )

203

204 from matplotlib import pyplot

205

206 from . import roc_for_far as calc

207

208 out = calc(negatives, positives, far_values)

209 if not CAR:

210 return pyplot.plot(out[0, :], out[1, :], **kwargs)

211 else:

212 return _semilogx(out[0, :], (1 - out[1, :]), **kwargs)

213

214

215def precision_recall_curve(negatives, positives, npoints=2000, **kwargs):

216 """Plots a Precision-Recall curve.

217

218 This method will call ``matplotlib`` to plot the precision-recall curve for a

219 system which contains a particular set of ``negatives`` (impostors) and

220 ``positives`` (clients) scores. We use the standard

221 :py:func:`matplotlib.pyplot.plot` command. All parameters passed with

222 exception of the three first parameters of this method will be directly

223 passed to the plot command.

224

225 .. note::

226

227 This function does not initiate and save the figure instance, it only

228 issues the plotting command. You are the responsible for setting up and

229 saving the figure as you see fit.

230

231

232 Parameters:

233

234 negatives (array): 1D float array that contains the scores of the

235 "negative" (noise, non-class) samples of your classifier. See

236 (:py:func:`bob.measure.precision_recall_curve`)

237

238 positives (array): 1D float array that contains the scores of the

239 "positive" (signal, class) samples of your classifier. See

240 (:py:func:`bob.measure.precision_recall_curve`)

241

242 npoints (:py:class:`int`, optional): The number of points for the plot. See

243 (:py:func:`bob.measure.precision_recall_curve`)

244

245 kwargs (:py:class:`dict`, optional): Extra plotting parameters, which are

246 passed directly to :py:func:`matplotlib.pyplot.plot`.

247

248

249 Returns:

250

251 :py:class:`list` of :py:class:`matplotlib.lines.Line2D`: The lines that

252 were added as defined by the return value of

253 :py:func:`matplotlib.pyplot.plot`.

254

255 """

256

257 from matplotlib import pyplot

258

259 from . import precision_recall_curve as calc

260

261 out = calc(negatives, positives, npoints)

262 return pyplot.plot(100.0 * out[0, :], 100.0 * out[1, :], **kwargs)

263

264

265def epc(

266 dev_negatives,

267 dev_positives,

268 test_negatives,

269 test_positives,

270 npoints=100,

271 **kwargs,

272):

273 """Plots Expected Performance Curve (EPC) as defined in the paper:

274

275 Bengio, S., Keller, M., Mariéthoz, J. (2004). The Expected Performance Curve.

276 International Conference on Machine Learning ICML Workshop on ROC Analysis in

277 Machine Learning, 136(1), 1963–1966. IDIAP RR. Available:

278 http://eprints.pascal-network.org/archive/00000670/

279

280 This method will call ``matplotlib`` to plot the EPC curve for a system which

281 contains a particular set of negatives (impostors) and positives (clients)

282 for both the development and test sets. We use the standard

283 :py:func:`matplotlib.pyplot.plot` command. All parameters passed with

284 exception of the five first parameters of this method will be directly passed

285 to the plot command.

286

287 The plot will represent the minimum HTER on the vertical axis and the cost on

288 the horizontal axis.

289

290 .. note::

291

292 This function does not initiate and save the figure instance, it only

293 issues the plotting commands. You are the responsible for setting up and

294 saving the figure as you see fit.

295

296

297 Parameters:

298

299 dev_negatives (array): 1D float array that contains the scores of the

300 "negative" (noise, non-class) samples of your classifier, from the

301 development set. See (:py:func:`bob.measure.epc`)

302

303 dev_positives (array): 1D float array that contains the scores of the

304 "positive" (signal, class) samples of your classifier, from the

305 development set. See (:py:func:`bob.measure.epc`)

306

307 test_negatives (array): 1D float array that contains the scores of the

308 "negative" (noise, non-class) samples of your classifier, from the test

309 set. See (:py:func:`bob.measure.epc`)

310

311 test_positives (array): 1D float array that contains the scores of the

312 "positive" (signal, class) samples of your classifier, from the test set.

313 See (:py:func:`bob.measure.epc`)

314

315 npoints (:py:class:`int`, optional): The number of points for the plot. See

316 (:py:func:`bob.measure.epc`)

317

318 kwargs (:py:class:`dict`, optional): Extra plotting parameters, which are

319 passed directly to :py:func:`matplotlib.pyplot.plot`.

320

321

322 Returns:

323

324 :py:class:`list` of :py:class:`matplotlib.lines.Line2D`: The lines that

325 were added as defined by the return value of

326 :py:func:`matplotlib.pyplot.plot`.

327

328 """

329

330 from matplotlib import pyplot

331

332 from . import epc as calc

333

334 out = calc(

335 dev_negatives, dev_positives, test_negatives, test_positives, npoints

336 )

337 return pyplot.plot(out[0, :], 100.0 * out[1, :], **kwargs)

338

339

340def det(negatives, positives, npoints=2000, min_far=-8, **kwargs):

341 """Plots Detection Error Trade-off (DET) curve as defined in the paper:

342

343 Martin, A., Doddington, G., Kamm, T., Ordowski, M., & Przybocki, M. (1997).

344 The DET curve in assessment of detection task performance. Fifth European

345 Conference on Speech Communication and Technology (pp. 1895-1898). Available:

346 http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.117.4489&rep=rep1&type=pdf

347

348 This method will call ``matplotlib`` to plot the DET curve(s) for a system

349 which contains a particular set of negatives (impostors) and positives

350 (clients) scores. We use the standard :py:func:`matplotlib.pyplot.plot`

351 command. All parameters passed with exception of the three first parameters

352 of this method will be directly passed to the plot command.

353

354 The plot will represent the false-alarm on the horizontal axis and the

355 false-rejection on the vertical axis.

356

357 This method is strongly inspired by the NIST implementation for Matlab,

358 called DETware, version 2.1 and available for download at the NIST website:

359

360 http://www.itl.nist.gov/iad/mig/tools/

361

362 .. note::

363

364 This function does not initiate and save the figure instance, it only

365 issues the plotting commands. You are the responsible for setting up and

366 saving the figure as you see fit.

367

368 .. note::

369

370 If you wish to reset axis zooming, you must use the Gaussian scale rather

371 than the visual marks showed at the plot, which are just there for

372 displaying purposes. The real axis scale is based on

373 :py:func:`bob.measure.ppndf`. For example, if you wish to set the x and y

374 axis to display data between 1% and 40% here is the recipe:

375

376 .. code-block:: python

377

378 import bob.measure

379 from matplotlib import pyplot

380 bob.measure.plot.det(...) #call this as many times as you need

381 #AFTER you plot the DET curve, just set the axis in this way:

382 pyplot.axis([bob.measure.ppndf(k/100.0) for k in (1, 40, 1, 40)])

383

384 We provide a convenient way for you to do the above in this module. So,

385 optionally, you may use the :py:func:`bob.measure.plot.det_axis` method

386 like this:

387

388 .. code-block:: python

389

390 import bob.measure

391 bob.measure.plot.det(...)

392 # please note we convert percentage values in det_axis()

393 bob.measure.plot.det_axis([1, 40, 1, 40])

394

395

396 Parameters:

397

398 negatives (array): 1D float array that contains the scores of the

399 "negative" (noise, non-class) samples of your classifier. See

400 (:py:func:`bob.measure.det`)

401

402 positives (array): 1D float array that contains the scores of the

403 "positive" (signal, class) samples of your classifier. See

404 (:py:func:`bob.measure.det`)

405

406 npoints (:py:class:`int`, optional): The number of points for the plot. See

407 (:py:func:`bob.measure.det`)

408

409 axisfontsize (:py:class:`str`, optional): The size to be used by

410 x/y-tick-labels to set the font size on the axis

411

412 kwargs (:py:class:`dict`, optional): Extra plotting parameters, which are

413 passed directly to :py:func:`matplotlib.pyplot.plot`.

414

415

416 Returns:

417

418 :py:class:`list` of :py:class:`matplotlib.lines.Line2D`: The lines that

419 were added as defined by the return value of

420 :py:func:`matplotlib.pyplot.plot`.

421

422 """

423

424 # these are some constants required in this method

425 desiredTicks = [

426 "0.000001",

427 "0.000002",

428 "0.000005",

429 "0.00001",

430 "0.00002",

431 "0.00005",

432 "0.0001",

433 "0.0002",

434 "0.0005",

435 "0.001",

436 "0.002",

437 "0.005",

438 "0.01",

439 "0.02",

440 "0.05",

441 "0.1",

442 "0.2",

443 "0.4",

444 "0.6",

445 "0.8",

446 "0.9",

447 "0.95",

448 "0.98",

449 "0.99",

450 "0.995",

451 "0.998",

452 "0.999",

453 "0.9995",

454 "0.9998",

455 "0.9999",

456 "0.99995",

457 "0.99998",

458 "0.99999",

459 ]

460

461 desiredLabels = [

462 "0.0001",

463 "0.0002",

464 "0.0005",

465 "0.001",

466 "0.002",

467 "0.005",

468 "0.01",

469 "0.02",

470 "0.05",

471 "0.1",

472 "0.2",

473 "0.5",

474 "1",

475 "2",

476 "5",

477 "10",

478 "20",

479 "40",

480 "60",

481 "80",

482 "90",

483 "95",

484 "98",

485 "99",

486 "99.5",

487 "99.8",

488 "99.9",

489 "99.95",

490 "99.98",

491 "99.99",

492 "99.995",

493 "99.998",

494 "99.999",

495 ]

496

497 # this will actually do the plotting

498 from matplotlib import pyplot

499

500 from . import det as calc

501 from . import ppndf

502

503 out = calc(negatives, positives, npoints, min_far)

504 retval = pyplot.plot(out[0, :], out[1, :], **kwargs)

505

506 # now the trick: we must plot the tick marks by hand using the PPNDF method

507 pticks = ppndf(numpy.array(desiredTicks, dtype=float))

508 ax = pyplot.gca() # and finally we set our own tick marks

509 ax.set_xticks(pticks)

510 ax.set_xticklabels(desiredLabels)

511 ax.set_yticks(pticks)

512 ax.set_yticklabels(desiredLabels)

513

514 return retval

515

516

517def det_axis(v, **kwargs):

518 """Sets the axis in a DET plot.

519

520 This method wraps the :py:func:`matplotlib.pyplot.axis` by calling

521 :py:func:`bob.measure.ppndf` on the values passed by the user so they are

522 meaningful in a DET plot as performed by :py:func:`bob.measure.plot.det`.

523

524

525 Parameters:

526

527 v (``sequence``): A sequence (list, tuple, array or the like) containing

528 the X and Y limits in the order ``(xmin, xmax, ymin, ymax)``. Expected

529 values should be in percentage (between 0 and 100%). If ``v`` is not a

530 list or tuple that contains 4 numbers it is passed without further

531 inspection to :py:func:`matplotlib.pyplot.axis`.

532

533 kwargs (:py:class:`dict`, optional): Extra plotting parameters, which are

534 passed directly to :py:func:`matplotlib.pyplot.axis`.

535

536

537 Returns:

538

539 object: Whatever is returned by :py:func:`matplotlib.pyplot.axis`.

540

541 """

542

543 import logging

544

545 logger = logging.getLogger("bob.measure")

546

547 from matplotlib import pyplot

548

549 from . import ppndf

550

551 # treat input

552 try:

553 tv = list(v) # normal input

554 if len(tv) != 4:

555 raise IndexError

556 tv = [ppndf(float(k) / 100) for k in tv]

557 cur = pyplot.axis()

558

559 # limits must be within bounds

560 if tv[0] < cur[0]:

561 logger.warn("Readjusting xmin: the provided value is out of bounds")

562 tv[0] = cur[0]

563 if tv[1] > cur[1]:

564 logger.warn("Readjusting xmax: the provided value is out of bounds")

565 tv[1] = cur[1]

566 if tv[2] < cur[2]:

567 logger.warn("Readjusting ymin: the provided value is out of bounds")

568 tv[2] = cur[2]

569 if tv[3] > cur[3]:

570 logger.warn("Readjusting ymax: the provided value is out of bounds")

571 tv[3] = cur[3]

572

573 except Exception:

574 tv = v

575

576 return pyplot.axis(tv, **kwargs)

577

578

579def cmc(cmc_scores, logx=True, **kwargs):

580 """Plots the (cumulative) match characteristics and returns the maximum rank.

581

582 This function plots a CMC curve using the given CMC scores (:py:class:`list`:

583 A list of tuples, where each tuple contains the

584 ``negative`` and ``positive`` scores for one probe of the database).

585

586

587 Parameters:

588

589 cmc_scores (array): 1D float array containing the CMC values (See

590 :py:func:`bob.measure.cmc`)

591

592 logx (:py:class:`bool`, optional): If set (the default), plots the rank

593 axis in logarithmic scale using :py:func:`matplotlib.pyplot.semilogx` or

594 in linear scale using :py:func:`matplotlib.pyplot.plot`

595

596 kwargs (:py:class:`dict`, optional): Extra plotting parameters, which are

597 passed directly to :py:func:`matplotlib.pyplot.plot`.

598

599

600 Returns:

601

602 int: The number of classes (clients) in the given scores.

603

604 """

605

606 from matplotlib import pyplot

607

608 from . import cmc as calc

609

610 out = calc(cmc_scores)

611

612 if logx:

613 _semilogx(range(1, len(out) + 1), out, **kwargs)

614 else:

615 pyplot.plot(range(1, len(out) + 1), out, **kwargs)

616

617 return len(out)

618

619

620def detection_identification_curve(

621 cmc_scores, far_values=log_values(), rank=1, logx=True, **kwargs

622):

623 """Plots the Detection & Identification curve over the FPR

624

625 This curve is designed to be used in an open set identification protocol, and

626 defined in Chapter 14.1 of [LiJain2005]_. It requires to have at least one

627 open set probe item, i.e., with no corresponding gallery, such that the

628 positives for that pair are ``None``.

629

630 The detection and identification curve first computes FPR thresholds based on

631 the out-of-set probe scores (negative scores). For each probe item, the

632 **maximum** negative score is used. Then, it plots the detection and

633 identification rates for those thresholds, which are based on the in-set

634 probe scores only. See [LiJain2005]_ for more details.

635

636 .. [LiJain2005] **Stan Li and Anil K. Jain**, *Handbook of Face Recognition*, Springer, 2005

637

638

639 Parameters:

640

641 cmc_scores (array): 1D float array containing the CMC values (See

642 :py:func:`bob.measure.cmc`)

643

644 rank (:py:class:`int`, optional): The rank for which the curve should be

645 plotted

646

647 far_values (:py:class:`list`, optional): The values for the FPR (FAR), where the

648 CPR (CAR) should be plotted; each value should be in range [0,1].

649

650 logx (:py:class:`bool`, optional): If set (the default), plots the rank

651 axis in logarithmic scale using :py:func:`matplotlib.pyplot.semilogx` or

652 in linear scale using :py:func:`matplotlib.pyplot.plot`

653

654 kwargs (:py:class:`dict`, optional): Extra plotting parameters, which are

655 passed directly to :py:func:`matplotlib.pyplot.plot`.

656

657

658 Returns:

659

660 :py:class:`list` of :py:class:`matplotlib.lines.Line2D`: The lines that

661 were added as defined by the return value of

662 :py:func:`matplotlib.pyplot.plot`.

664 """

666 import math

668 import numpy

670 from matplotlib import pyplot

671

672 from . import detection_identification_rate, far_threshold

673

674 # for each probe, for which no positives exists, get the highest negative

675 # score; and sort them to compute the FAR thresholds

676 negatives = sorted(

677 max(neg)

678 for neg, pos in cmc_scores

679 if (pos is None or not numpy.array(pos).size) and neg is not None

680 )

681 if not negatives:

682 raise ValueError(

683 "There need to be at least one pair with only negative scores"

684 )

685

686 # compute thresholds based on FAR values

687 thresholds = [far_threshold(negatives, [], v, True) for v in far_values]

688

689 # compute detection and identification rate based on the thresholds for

690 # the given rank

691 rates = [

692 detection_identification_rate(cmc_scores, t, rank)

693 if not math.isnan(t)

694 else numpy.nan

695 for t in thresholds

696 ]

697

698 # plot curve

699 if logx:

700 return _semilogx(far_values, rates, **kwargs)

701 else:

702 return pyplot.plot(far_values, rates, **kwargs)