Coverage for src/bob/measure/script/common_options.py: 92%
543 statements
« prev ^ index » next coverage.py v7.0.5, created at 2023-06-16 14:10 +0200
1"""Stores click common options for plots"""
3import functools
4import logging
6import click
7import matplotlib.pyplot as plt
8import tabulate
10from clapper.click import verbosity_option
11from click.types import FLOAT, INT
12from matplotlib.backends.backend_pdf import PdfPages
14LOGGER = logging.getLogger(__name__)
17def bool_option(name, short_name, desc, dflt=False, **kwargs):
18 """Generic provider for boolean options
20 Parameters
21 ----------
22 name : str
23 name of the option
24 short_name : str
25 short name for the option
26 desc : str
27 short description for the option
28 dflt : bool or None
29 Default value
30 **kwargs
31 All kwargs are passed to click.option.
33 Returns
34 -------
35 ``callable``
36 A decorator to be used for adding this option.
37 """
39 def custom_bool_option(func):
40 def callback(ctx, param, value):
41 ctx.meta[name.replace("-", "_")] = value
42 return value
44 return click.option(
45 "-%s/-n%s" % (short_name, short_name),
46 "--%s/--no-%s" % (name, name),
47 default=dflt,
48 help=desc,
49 show_default=True,
50 callback=callback,
51 is_eager=True,
52 **kwargs,
53 )(func)
55 return custom_bool_option
58def list_float_option(name, short_name, desc, nitems=None, dflt=None, **kwargs):
59 """Get option to get a list of float f
61 Parameters
62 ----------
63 name : str
64 name of the option
65 short_name : str
66 short name for the option
67 desc : str
68 short description for the option
69 nitems : obj:`int`, optional
70 If given, the parsed list must contains this number of items.
71 dflt : :any:`list`, optional
72 List of default values for axes.
73 **kwargs
74 All kwargs are passed to click.option.
76 Returns
77 -------
78 ``callable``
79 A decorator to be used for adding this option.
80 """
82 def custom_list_float_option(func):
83 def callback(ctx, param, value):
84 if value is None or not value.replace(" ", ""):
85 value = None
86 elif value is not None:
87 tmp = value.split(",")
88 if nitems is not None and len(tmp) != nitems:
89 raise click.BadParameter(
90 "%s Must provide %d axis limits" % (name, nitems)
91 )
92 try:
93 value = [float(i) for i in tmp]
94 except Exception:
95 raise click.BadParameter("Inputs of %s be floats" % name)
96 ctx.meta[name.replace("-", "_")] = value
97 return value
99 return click.option(
100 "-" + short_name,
101 "--" + name,
102 default=dflt,
103 show_default=True,
104 help=desc + " Provide just a space (' ') to cancel default values.",
105 callback=callback,
106 **kwargs,
107 )(func)
109 return custom_list_float_option
112def open_file_mode_option(**kwargs):
113 """Get open mode file option
115 Parameters
116 ----------
117 **kwargs
118 All kwargs are passed to click.option.
120 Returns
121 -------
122 ``callable``
123 A decorator to be used for adding this option.
124 """
126 def custom_open_file_mode_option(func):
127 def callback(ctx, param, value):
128 if value not in ["w", "a", "w+", "a+"]:
129 raise click.BadParameter("Incorrect open file mode")
130 ctx.meta["open_mode"] = value
131 return value
133 return click.option(
134 "-om",
135 "--open-mode",
136 default="w",
137 help="File open mode",
138 callback=callback,
139 **kwargs,
140 )(func)
142 return custom_open_file_mode_option
145def scores_argument(min_arg=1, force_eval=False, **kwargs):
146 """Get the argument for scores, and add `dev-scores` and `eval-scores` in
147 the context when `--eval` flag is on (default)
149 Parameters
150 ----------
151 min_arg : int
152 the minimum number of file needed to evaluate a system. For example,
153 vulnerability analysis needs licit and spoof and therefore min_arg = 2
155 Returns
156 -------
157 callable
158 A decorator to be used for adding score arguments for click commands
159 """
161 def custom_scores_argument(func):
162 def callback(ctx, param, value):
163 min_a = min_arg or 1
164 mutli = 1
165 error = ""
166 if (
167 "evaluation" in ctx.meta and ctx.meta["evaluation"]
168 ) or force_eval:
169 mutli += 1
170 error += "- %d evaluation file(s) \n" % min_a
171 if "train" in ctx.meta and ctx.meta["train"]:
172 mutli += 1
173 error += "- %d training file(s) \n" % min_a
174 # add more test here if other inputs are needed
176 min_a *= mutli
177 ctx.meta["min_arg"] = min_a
178 if len(value) < 1 or len(value) % ctx.meta["min_arg"] != 0:
179 raise click.BadParameter(
180 "The number of provided scores must be > 0 and a multiple of %d "
181 "because the following files are required:\n"
182 "- %d development file(s)\n" % (min_a, min_arg or 1)
183 + error,
184 ctx=ctx,
185 )
186 ctx.meta["scores"] = value
187 return value
189 return click.argument(
190 "scores", type=click.Path(exists=True), callback=callback, **kwargs
191 )(func)
193 return custom_scores_argument
196def alpha_option(dflt=1, **kwargs):
197 """An alpha option for plots"""
199 def custom_eval_option(func):
200 def callback(ctx, param, value):
201 ctx.meta["alpha"] = value
202 return value
204 return click.option(
205 "-a",
206 "--alpha",
207 default=dflt,
208 type=click.FLOAT,
209 show_default=True,
210 help="Adjusts transparency of plots.",
211 callback=callback,
212 **kwargs,
213 )(func)
215 return custom_eval_option
218def no_legend_option(dflt=True, **kwargs):
219 """Get option flag to say if legend should be displayed or not"""
220 return bool_option(
221 "disp-legend", "dl", "If set, no legend will be printed.", dflt=dflt
222 )
225def eval_option(**kwargs):
226 """Get option flag to say if eval-scores are provided"""
228 def custom_eval_option(func):
229 def callback(ctx, param, value):
230 ctx.meta["evaluation"] = value
231 return value
233 return click.option(
234 "-e",
235 "--eval",
236 "evaluation",
237 is_flag=True,
238 default=False,
239 show_default=True,
240 help="If set, evaluation scores must be provided",
241 callback=callback,
242 **kwargs,
243 )(func)
245 return custom_eval_option
248def hide_dev_option(dflt=False, **kwargs):
249 """Get option flag to say if dev plot should be hidden"""
251 def custom_hide_dev_option(func):
252 def callback(ctx, param, value):
253 ctx.meta["hide_dev"] = value
254 return value
256 return click.option(
257 "--hide-dev",
258 is_flag=True,
259 default=dflt,
260 show_default=True,
261 help="If set, hide dev related plots",
262 callback=callback,
263 **kwargs,
264 )(func)
266 return custom_hide_dev_option
269def sep_dev_eval_option(dflt=True, **kwargs):
270 """Get option flag to say if dev and eval plots should be in different
271 plots"""
272 return bool_option(
273 "split",
274 "s",
275 "If set, evaluation and dev curve in different plots",
276 dflt,
277 )
280def linestyles_option(dflt=False, **kwargs):
281 """Get option flag to turn on/off linestyles"""
282 return bool_option(
283 "line-styles",
284 "S",
285 "If given, applies a different line style to each line.",
286 dflt,
287 **kwargs,
288 )
291def cmc_option(**kwargs):
292 """Get option flag to say if cmc scores"""
293 return bool_option(
294 "cmc", "C", "If set, CMC score files are provided", **kwargs
295 )
298def semilogx_option(dflt=False, **kwargs):
299 """Option to use semilog X-axis"""
300 return bool_option(
301 "semilogx", "G", "If set, use semilog on X axis", dflt, **kwargs
302 )
305def tpr_option(dflt=False, **kwargs):
306 """Option to use TPR (true positive rate) on y-axis"""
307 return bool_option(
308 "tpr",
309 "tpr",
310 "If set, use TPR (also called 1-FNR, 1-FNMR, or 1-BPCER) on Y axis",
311 dflt,
312 **kwargs,
313 )
316def print_filenames_option(dflt=True, **kwargs):
317 """Option to tell if filenames should be in the title"""
318 return bool_option(
319 "show-fn", "P", "If set, show filenames in title", dflt, **kwargs
320 )
323def const_layout_option(dflt=True, **kwargs):
324 """Option to set matplotlib constrained_layout"""
326 def custom_layout_option(func):
327 def callback(ctx, param, value):
328 ctx.meta["clayout"] = value
329 plt.rcParams["figure.constrained_layout.use"] = value
330 return value
332 return click.option(
333 "-Y/-nY",
334 "--clayout/--no-clayout",
335 default=dflt,
336 show_default=True,
337 help="(De)Activate constrained layout",
338 callback=callback,
339 **kwargs,
340 )(func)
342 return custom_layout_option
345def axes_val_option(dflt=None, **kwargs):
346 """Option for setting min/max values on axes"""
347 return list_float_option(
348 name="axlim",
349 short_name="L",
350 desc="min/max axes values separated by commas (e.g. ``--axlim "
351 " 0.1,100,0.1,100``)",
352 nitems=4,
353 dflt=dflt,
354 **kwargs,
355 )
358def thresholds_option(**kwargs):
359 """Option to give a list of thresholds"""
360 return list_float_option(
361 name="thres",
362 short_name="T",
363 desc="Given threshold for metrics computations, e.g. "
364 "0.005,0.001,0.056",
365 nitems=None,
366 dflt=None,
367 **kwargs,
368 )
371def lines_at_option(dflt="1e-3", **kwargs):
372 """Get option to draw const far line"""
373 return list_float_option(
374 name="lines-at",
375 short_name="la",
376 desc="If given, draw vertical lines at the given axis positions. "
377 "Your values must be separated with a comma (,) without space. "
378 "This option works in ROC and DET curves.",
379 nitems=None,
380 dflt=dflt,
381 **kwargs,
382 )
385def x_rotation_option(dflt=0, **kwargs):
386 """Get option for rotartion of the x axis lables"""
388 def custom_x_rotation_option(func):
389 def callback(ctx, param, value):
390 value = abs(value)
391 ctx.meta["x_rotation"] = value
392 return value
394 return click.option(
395 "-r",
396 "--x-rotation",
397 type=click.INT,
398 default=dflt,
399 show_default=True,
400 help="X axis labels ration",
401 callback=callback,
402 **kwargs,
403 )(func)
405 return custom_x_rotation_option
408def legend_ncols_option(dflt=3, **kwargs):
409 """Get option for number of columns for legends"""
411 def custom_legend_ncols_option(func):
412 def callback(ctx, param, value):
413 value = abs(value)
414 ctx.meta["legends_ncol"] = value
415 return value
417 return click.option(
418 "-lc",
419 "--legends-ncol",
420 type=click.INT,
421 default=dflt,
422 show_default=True,
423 help="The number of columns of the legend layout.",
424 callback=callback,
425 **kwargs,
426 )(func)
428 return custom_legend_ncols_option
431def subplot_option(dflt=111, **kwargs):
432 """Get option to set subplots"""
434 def custom_subplot_option(func):
435 def callback(ctx, param, value):
436 value = abs(value)
437 nrows = value // 10
438 nrows, ncols = divmod(nrows, 10)
439 ctx.meta["n_col"] = ncols
440 ctx.meta["n_row"] = nrows
441 return value
443 return click.option(
444 "-sp",
445 "--subplot",
446 type=click.INT,
447 default=dflt,
448 show_default=True,
449 help="The order of subplots.",
450 callback=callback,
451 **kwargs,
452 )(func)
454 return custom_subplot_option
457def cost_option(**kwargs):
458 """Get option to get cost for FPR"""
460 def custom_cost_option(func):
461 def callback(ctx, param, value):
462 if value < 0 or value > 1:
463 raise click.BadParameter("Cost for FPR must be betwen 0 and 1")
464 ctx.meta["cost"] = value
465 return value
467 return click.option(
468 "-C",
469 "--cost",
470 type=float,
471 default=0.99,
472 show_default=True,
473 help="Cost for FPR in minDCF",
474 callback=callback,
475 **kwargs,
476 )(func)
478 return custom_cost_option
481def points_curve_option(**kwargs):
482 """Get the number of points use to draw curves"""
484 def custom_points_curve_option(func):
485 def callback(ctx, param, value):
486 if value < 2:
487 raise click.BadParameter(
488 "Number of points to draw curves must be greater than 1",
489 ctx=ctx,
490 )
491 ctx.meta["points"] = value
492 return value
494 return click.option(
495 "-n",
496 "--points",
497 type=INT,
498 default=2000,
499 show_default=True,
500 help="The number of points use to draw curves in plots",
501 callback=callback,
502 **kwargs,
503 )(func)
505 return custom_points_curve_option
508def n_bins_option(**kwargs):
509 """Get the number of bins in the histograms"""
510 possible_strings = [
511 "auto",
512 "fd",
513 "doane",
514 "scott",
515 "rice",
516 "sturges",
517 "sqrt",
518 ]
520 def custom_n_bins_option(func):
521 def callback(ctx, param, value):
522 if value is None:
523 value = ["doane"]
524 else:
525 tmp = value.split(",")
526 try:
527 value = [
528 int(i) if i not in possible_strings else i for i in tmp
529 ]
530 except Exception:
531 raise click.BadParameter("Incorrect number of bins inputs")
532 ctx.meta["n_bins"] = value
533 return value
535 return click.option(
536 "-b",
537 "--nbins",
538 type=click.STRING,
539 default="doane",
540 help="The number of bins for the different quantities to plot, "
541 "seperated by commas. For example, if you plot histograms "
542 "of negative and positive scores "
543 ", input something like `100,doane`. All the "
544 "possible bin options can be found in https://docs.scipy.org/doc/"
545 "numpy/reference/generated/numpy.histogram.html. Be aware that "
546 "for some corner cases, the option `auto` and `fd` can lead to "
547 "MemoryError.",
548 callback=callback,
549 **kwargs,
550 )(func)
552 return custom_n_bins_option
555def table_option(dflt="rst", **kwargs):
556 """Get table option for tabulate package
557 More informnations: https://pypi.org/project/tabulate/
558 """
560 def custom_table_option(func):
561 def callback(ctx, param, value):
562 ctx.meta["tablefmt"] = value
563 return value
565 return click.option(
566 "--tablefmt",
567 type=click.Choice(tabulate.tabulate_formats),
568 default=dflt,
569 show_default=True,
570 help="Format of printed tables.",
571 callback=callback,
572 **kwargs,
573 )(func)
575 return custom_table_option
578def output_plot_file_option(default_out="plots.pdf", **kwargs):
579 """Get options for output file for plots"""
581 def custom_output_plot_file_option(func):
582 def callback(ctx, param, value):
583 """Save ouput file and associated pdf in context list,
584 print the path of the file in the log"""
585 ctx.meta["output"] = value
586 ctx.meta["PdfPages"] = PdfPages(value)
587 LOGGER.debug("Plots will be output in %s", value)
588 return value
590 return click.option(
591 "-o",
592 "--output",
593 default=default_out,
594 show_default=True,
595 help="The file to save the plots in.",
596 callback=callback,
597 **kwargs,
598 )(func)
600 return custom_output_plot_file_option
603def output_log_metric_option(**kwargs):
604 """Get options for output file for metrics"""
606 def custom_output_log_file_option(func):
607 def callback(ctx, param, value):
608 if value is not None:
609 LOGGER.debug("Metrics will be output in %s", value)
610 ctx.meta["log"] = value
611 return value
613 return click.option(
614 "-l",
615 "--log",
616 default=None,
617 type=click.STRING,
618 help="If provided, computed numbers are written to "
619 "this file instead of the standard output.",
620 callback=callback,
621 **kwargs,
622 )(func)
624 return custom_output_log_file_option
627def no_line_option(**kwargs):
628 """Get option flag to say if no line should be displayed"""
630 def custom_no_line_option(func):
631 def callback(ctx, param, value):
632 ctx.meta["no_line"] = value
633 return value
635 return click.option(
636 "--no-line",
637 is_flag=True,
638 default=False,
639 show_default=True,
640 help="If set does not display vertical lines",
641 callback=callback,
642 **kwargs,
643 )(func)
645 return custom_no_line_option
648def criterion_option(
649 lcriteria=["eer", "min-hter", "far"], check=True, **kwargs
650):
651 """Get option flag to tell which criteriom is used (default:eer)
653 Parameters
654 ----------
655 lcriteria : :any:`list`
656 List of possible criteria
657 """
659 def custom_criterion_option(func):
660 list_accepted_crit = (
661 lcriteria if lcriteria is not None else ["eer", "min-hter", "far"]
662 )
664 def callback(ctx, param, value):
665 if value not in list_accepted_crit and check:
666 raise click.BadParameter(
667 "Incorrect value for `--criterion`. "
668 "Must be one of [`%s`]" % "`, `".join(list_accepted_crit)
669 )
670 ctx.meta["criterion"] = value
671 return value
673 return click.option(
674 "-c",
675 "--criterion",
676 default="eer",
677 help="Criterion to compute plots and "
678 "metrics: %s)" % ", ".join(list_accepted_crit),
679 callback=callback,
680 is_eager=True,
681 show_default=True,
682 **kwargs,
683 )(func)
685 return custom_criterion_option
688def decimal_option(dflt=1, short="-d", **kwargs):
689 """Get option to get decimal value"""
691 def custom_decimal_option(func):
692 def callback(ctx, param, value):
693 ctx.meta["decimal"] = value
694 return value
696 return click.option(
697 short,
698 "--decimal",
699 type=click.INT,
700 default=dflt,
701 help="Number of decimals to be printed.",
702 callback=callback,
703 show_default=True,
704 **kwargs,
705 )(func)
707 return custom_decimal_option
710def far_option(far_name="FAR", **kwargs):
711 """Get option to get far value"""
713 def custom_far_option(func):
714 def callback(ctx, param, value):
715 if value is not None and (value > 1 or value < 0):
716 raise click.BadParameter(
717 "{} value should be between 0 and 1".format(far_name)
718 )
719 ctx.meta["far_value"] = value
720 return value
722 return click.option(
723 "-f",
724 "--{}-value".format(far_name.lower()),
725 "far_value",
726 type=click.FLOAT,
727 default=None,
728 help="The {} value for which to compute threshold. This option "
729 "must be used alongside `--criterion far`.".format(far_name),
730 callback=callback,
731 show_default=True,
732 **kwargs,
733 )(func)
735 return custom_far_option
738def min_far_option(far_name="FAR", dflt=1e-4, **kwargs):
739 """Get option to get min far value"""
741 def custom_min_far_option(func):
742 def callback(ctx, param, value):
743 if value is not None and (value > 1 or value < 0):
744 raise click.BadParameter(
745 "{} value should be between 0 and 1".format(far_name)
746 )
747 ctx.meta["min_far_value"] = value
748 return value
750 return click.option(
751 "-M",
752 "--min-{}-value".format(far_name.lower()),
753 "min_far_value",
754 type=click.FLOAT,
755 default=dflt,
756 help="Select the minimum {} value used in ROC and DET plots; "
757 "should be a power of 10.".format(far_name),
758 callback=callback,
759 show_default=True,
760 **kwargs,
761 )(func)
763 return custom_min_far_option
766def figsize_option(dflt="4,3", **kwargs):
767 """Get option for matplotlib figsize
769 Parameters
770 ----------
771 dflt : str
772 matplotlib default figsize for the command. must be a a list of int
773 separated by commas.
775 Returns
776 -------
777 callable
778 A decorator to be used for adding score arguments for click commands
779 """
781 def custom_figsize_option(func):
782 def callback(ctx, param, value):
783 ctx.meta["figsize"] = (
784 value if value is None else [float(x) for x in value.split(",")]
785 )
786 if value is not None:
787 plt.rcParams["figure.figsize"] = ctx.meta["figsize"]
788 return value
790 return click.option(
791 "--figsize",
792 default=dflt,
793 show_default=True,
794 help="If given, will run "
795 "``plt.rcParams['figure.figsize']=figsize)``. "
796 "Example: --figsize 4,6",
797 callback=callback,
798 **kwargs,
799 )(func)
801 return custom_figsize_option
804def legend_loc_option(dflt="best", **kwargs):
805 """Get the legend location of the plot"""
807 def custom_legend_loc_option(func):
808 def callback(ctx, param, value):
809 ctx.meta["legend_loc"] = value.replace("-", " ") if value else value
810 return value
812 return click.option(
813 "-ll",
814 "--legend-loc",
815 default=dflt,
816 show_default=True,
817 type=click.Choice(
818 [
819 "best",
820 "upper-right",
821 "upper-left",
822 "lower-left",
823 "lower-right",
824 "right",
825 "center-left",
826 "center-right",
827 "lower-center",
828 "upper-center",
829 "center",
830 ]
831 ),
832 help="The legend location code",
833 callback=callback,
834 **kwargs,
835 )(func)
837 return custom_legend_loc_option
840def line_width_option(**kwargs):
841 """Get line width option for the plots"""
843 def custom_line_width_option(func):
844 def callback(ctx, param, value):
845 ctx.meta["line_width"] = value
846 return value
848 return click.option(
849 "--line-width",
850 type=FLOAT,
851 help="The line width of plots",
852 callback=callback,
853 **kwargs,
854 )(func)
856 return custom_line_width_option
859def marker_style_option(**kwargs):
860 """Get marker style otpion for the plots"""
862 def custom_marker_style_option(func):
863 def callback(ctx, param, value):
864 ctx.meta["marker_style"] = value
865 return value
867 return click.option(
868 "--marker-style",
869 type=FLOAT,
870 help="The marker style of the plots",
871 callback=callback,
872 **kwargs,
873 )(func)
875 return custom_marker_style_option
878def legends_option(**kwargs):
879 """Get the legends option for the different systems"""
881 def custom_legends_option(func):
882 def callback(ctx, param, value):
883 if value is not None:
884 value = value.split(",")
885 ctx.meta["legends"] = value
886 return value
888 return click.option(
889 "-lg",
890 "--legends",
891 type=click.STRING,
892 default=None,
893 help="The legend for each system comma separated. "
894 "Example: --legends ISV,CNN",
895 callback=callback,
896 **kwargs,
897 )(func)
899 return custom_legends_option
902def title_option(**kwargs):
903 """Get the title option for the different systems"""
905 def custom_title_option(func):
906 def callback(ctx, param, value):
907 ctx.meta["title"] = value
908 return value
910 return click.option(
911 "-t",
912 "--title",
913 type=click.STRING,
914 default=None,
915 help="The title of the plots. Provide just a space (-t ' ') to "
916 "remove the titles from figures.",
917 callback=callback,
918 **kwargs,
919 )(func)
921 return custom_title_option
924def titles_option(**kwargs):
925 """Get the titles option for the different plots"""
927 def custom_title_option(func):
928 def callback(ctx, param, value):
929 if value is not None:
930 value = value.split(",")
931 ctx.meta["titles"] = value or []
932 return value or []
934 return click.option(
935 "-ts",
936 "--titles",
937 type=click.STRING,
938 default=None,
939 help="The titles of the plots seperated by commas. "
940 'For example, if the figure has two plots, "MyTitleA,MyTitleB" '
941 "is a possible input."
942 " Provide just a space (-ts ' ') to "
943 "remove the titles from figures.",
944 callback=callback,
945 **kwargs,
946 )(func)
948 return custom_title_option
951def x_label_option(dflt=None, **kwargs):
952 """Get the label option for X axis"""
954 def custom_x_label_option(func):
955 def callback(ctx, param, value):
956 ctx.meta["x_label"] = value
957 return value
959 return click.option(
960 "-xl",
961 "--x-label",
962 type=click.STRING,
963 default=dflt,
964 show_default=True,
965 help="Label for x-axis",
966 callback=callback,
967 **kwargs,
968 )(func)
970 return custom_x_label_option
973def y_label_option(dflt=None, **kwargs):
974 """Get the label option for Y axis"""
976 def custom_y_label_option(func):
977 def callback(ctx, param, value):
978 ctx.meta["y_label"] = value
979 return value
981 return click.option(
982 "-yl",
983 "--y-label",
984 type=click.STRING,
985 default=dflt,
986 help="Label for y-axis",
987 callback=callback,
988 **kwargs,
989 )(func)
991 return custom_y_label_option
994def style_option(**kwargs):
995 """Get option for matplotlib style"""
997 def custom_style_option(func):
998 def callback(ctx, param, value):
999 ctx.meta["style"] = value
1000 plt.style.use(value)
1001 return value
1003 return click.option(
1004 "--style",
1005 multiple=True,
1006 type=click.types.Choice(sorted(plt.style.available)),
1007 help="The matplotlib style to use for plotting. You can provide "
1008 "multiple styles by repeating this option",
1009 callback=callback,
1010 **kwargs,
1011 )(func)
1013 return custom_style_option
1016def metrics_command(
1017 docstring,
1018 criteria=("eer", "min-hter", "far"),
1019 far_name="FAR",
1020 check_criteria=True,
1021 **kwarg,
1022):
1023 def custom_metrics_command(func):
1024 func.__doc__ = docstring
1026 @click.command(**kwarg)
1027 @scores_argument(nargs=-1)
1028 @eval_option()
1029 @table_option()
1030 @output_log_metric_option()
1031 @criterion_option(criteria, check=check_criteria)
1032 @thresholds_option()
1033 @far_option(far_name=far_name)
1034 @legends_option()
1035 @open_file_mode_option()
1036 @verbosity_option(LOGGER)
1037 @click.pass_context
1038 @decimal_option()
1039 @functools.wraps(func)
1040 def wrapper(*args, **kwds):
1041 return func(*args, **kwds)
1043 return wrapper
1045 return custom_metrics_command
1048METRICS_HELP = """Prints a table that contains {names} for a given
1049 threshold criterion ({criteria}).
1050 {hter_note}
1052 You need to provide one or more development score file(s) for each
1053 experiment. You can also provide evaluation files along with dev files. If
1054 evaluation scores are provided, you must use flag `--eval`.
1056 {score_format}
1058 Resulting table format can be changed using the `--tablefmt`.
1060 Examples:
1062 $ {command} -v scores-dev
1064 $ {command} -e -l results.txt sys1/scores-{{dev,eval}}
1066 $ {command} -e {{sys1,sys2}}/scores-{{dev,eval}}
1068 """
1071def roc_command(docstring, far_name="FAR"):
1072 def custom_roc_command(func):
1073 func.__doc__ = docstring
1075 @click.command()
1076 @scores_argument(nargs=-1)
1077 @titles_option()
1078 @legends_option()
1079 @no_legend_option()
1080 @legend_loc_option(dflt=None)
1081 @sep_dev_eval_option()
1082 @output_plot_file_option(default_out="roc.pdf")
1083 @eval_option()
1084 @tpr_option(True)
1085 @semilogx_option(True)
1086 @lines_at_option()
1087 @axes_val_option()
1088 @min_far_option(far_name=far_name)
1089 @x_rotation_option()
1090 @x_label_option()
1091 @y_label_option()
1092 @points_curve_option()
1093 @const_layout_option()
1094 @figsize_option()
1095 @style_option()
1096 @linestyles_option()
1097 @alpha_option()
1098 @verbosity_option(LOGGER)
1099 @click.pass_context
1100 @functools.wraps(func)
1101 def wrapper(*args, **kwds):
1102 return func(*args, **kwds)
1104 return wrapper
1106 return custom_roc_command
1109ROC_HELP = """Plot ROC (receiver operating characteristic) curve.
1110 The plot will represent the false match rate on the horizontal axis and the
1111 false non match rate on the vertical axis. The values for the axis will be
1112 computed using :py:func:`bob.measure.roc`.
1114 You need to provide one or more development score file(s) for each
1115 experiment. You can also provide evaluation files along with dev files. If
1116 evaluation scores are provided, you must use flag `--eval`.
1118 {score_format}
1120 Examples:
1122 $ {command} -v scores-dev
1124 $ {command} -e -v sys1/scores-{{dev,eval}}
1126 $ {command} -e -v -o my_roc.pdf {{sys1,sys2}}/scores-{{dev,eval}}
1127 """
1130def det_command(docstring, far_name="FAR"):
1131 def custom_det_command(func):
1132 func.__doc__ = docstring
1134 @click.command()
1135 @scores_argument(nargs=-1)
1136 @output_plot_file_option(default_out="det.pdf")
1137 @titles_option()
1138 @legends_option()
1139 @no_legend_option()
1140 @legend_loc_option(dflt="upper-right")
1141 @sep_dev_eval_option()
1142 @eval_option()
1143 @axes_val_option(dflt="0.01,95,0.01,95")
1144 @min_far_option(far_name=far_name)
1145 @x_rotation_option(dflt=45)
1146 @x_label_option()
1147 @y_label_option()
1148 @points_curve_option()
1149 @lines_at_option()
1150 @const_layout_option()
1151 @figsize_option()
1152 @style_option()
1153 @linestyles_option()
1154 @alpha_option()
1155 @verbosity_option(LOGGER)
1156 @click.pass_context
1157 @functools.wraps(func)
1158 def wrapper(*args, **kwds):
1159 return func(*args, **kwds)
1161 return wrapper
1163 return custom_det_command
1166DET_HELP = """Plot DET (detection error trade-off) curve.
1167 modified ROC curve which plots error rates on both axes
1168 (false positives on the x-axis and false negatives on the y-axis).
1170 You need to provide one or more development score file(s) for each
1171 experiment. You can also provide evaluation files along with dev files. If
1172 evaluation scores are provided, you must use flag `--eval`.
1174 {score_format}
1176 Examples:
1178 $ {command} -v scores-dev
1180 $ {command} -e -v sys1/scores-{{dev,eval}}
1182 $ {command} -e -v -o my_det.pdf {{sys1,sys2}}/scores-{{dev,eval}}
1183 """
1186def epc_command(docstring):
1187 def custom_epc_command(func):
1188 func.__doc__ = docstring
1190 @click.command()
1191 @scores_argument(min_arg=1, force_eval=True, nargs=-1)
1192 @output_plot_file_option(default_out="epc.pdf")
1193 @titles_option()
1194 @legends_option()
1195 @no_legend_option()
1196 @legend_loc_option(dflt="upper-center")
1197 @points_curve_option()
1198 @const_layout_option()
1199 @x_label_option()
1200 @y_label_option()
1201 @figsize_option()
1202 @style_option()
1203 @linestyles_option()
1204 @alpha_option()
1205 @verbosity_option(LOGGER)
1206 @click.pass_context
1207 @functools.wraps(func)
1208 def wrapper(*args, **kwds):
1209 return func(*args, **kwds)
1211 return wrapper
1213 return custom_epc_command
1216EPC_HELP = """Plot EPC (expected performance curve).
1217 plots the error rate on the eval set depending on a threshold selected
1218 a-priori on the development set and accounts for varying relative cost
1219 in [0; 1] of FPR and FNR when calculating the threshold.
1221 You need to provide one or more development score and eval file(s)
1222 for each experiment.
1224 {score_format}
1226 Examples:
1228 $ {command} -v scores-{{dev,eval}}
1230 $ {command} -v -o my_epc.pdf {{sys1,sys2}}/scores-{{dev,eval}}
1231 """
1234def hist_command(docstring, far_name="FAR"):
1235 def custom_hist_command(func):
1236 func.__doc__ = docstring
1238 @click.command()
1239 @scores_argument(nargs=-1)
1240 @output_plot_file_option(default_out="hist.pdf")
1241 @eval_option()
1242 @hide_dev_option()
1243 @n_bins_option()
1244 @titles_option()
1245 @no_legend_option()
1246 @legend_ncols_option()
1247 @criterion_option()
1248 @far_option(far_name=far_name)
1249 @no_line_option()
1250 @thresholds_option()
1251 @subplot_option()
1252 @const_layout_option()
1253 @print_filenames_option()
1254 @figsize_option(dflt=None)
1255 @style_option()
1256 @x_label_option()
1257 @y_label_option()
1258 @verbosity_option(LOGGER)
1259 @click.pass_context
1260 @functools.wraps(func)
1261 def wrapper(*args, **kwds):
1262 return func(*args, **kwds)
1264 return wrapper
1266 return custom_hist_command
1269HIST_HELP = """ Plots histograms of positive and negatives along with threshold
1270 criterion.
1272 You need to provide one or more development score file(s) for each
1273 experiment. You can also provide evaluation files along with dev files. If
1274 evaluation scores are provided, you must use the `--eval` flag. The
1275 threshold is always computed from development score files.
1277 By default, when eval-scores are given, only eval-scores histograms are
1278 displayed with threshold line computed from dev-scores.
1280 {score_format}
1282 Examples:
1284 $ {command} -v scores-dev
1286 $ {command} -e -v sys1/scores-{{dev,eval}}
1288 $ {command} -e -v --criterion min-hter {{sys1,sys2}}/scores-{{dev,eval}}
1289 """
1292def evaluate_command(
1293 docstring, criteria=("eer", "min-hter", "far"), far_name="FAR"
1294):
1295 def custom_evaluate_command(func):
1296 func.__doc__ = docstring
1298 @click.command()
1299 @scores_argument(nargs=-1)
1300 @legends_option()
1301 @sep_dev_eval_option()
1302 @table_option()
1303 @eval_option()
1304 @criterion_option(criteria)
1305 @far_option(far_name=far_name)
1306 @output_log_metric_option()
1307 @output_plot_file_option(default_out="eval_plots.pdf")
1308 @lines_at_option()
1309 @points_curve_option()
1310 @const_layout_option()
1311 @figsize_option(dflt=None)
1312 @style_option()
1313 @linestyles_option()
1314 @verbosity_option(LOGGER)
1315 @click.pass_context
1316 @functools.wraps(func)
1317 def wrapper(*args, **kwds):
1318 return func(*args, **kwds)
1320 return wrapper
1322 return custom_evaluate_command
1325EVALUATE_HELP = """Runs error analysis on score sets.
1327 \b
1328 1. Computes the threshold using a criteria (EER by default) on
1329 development set scores
1330 2. Applies the above threshold on evaluation set scores to compute the
1331 HTER if a eval-score (use --eval) set is provided.
1332 3. Reports error rates on the console or in a log file.
1333 4. Plots ROC, DET, and EPC curves and score distributions to a multi-page
1334 PDF file
1336 You need to provide 1 or 2 score files for each biometric system in this
1337 order:
1339 \b
1340 * development scores
1341 * evaluation scores
1343 {score_format}
1345 Examples:
1347 $ {command} -v dev-scores
1349 $ {command} -v /path/to/sys-{{1,2,3}}/scores-dev
1351 $ {command} -e -v /path/to/sys-{{1,2,3}}/scores-{{dev,eval}}
1353 $ {command} -v -l metrics.txt -o my_plots.pdf dev-scores
1355 This command is a combination of metrics, roc, det, epc, and hist commands.
1356 If you want more flexibility in your plots, please use the individual
1357 commands.
1358 """
1361def evaluate_flow(
1362 ctx, scores, evaluation, metrics, roc, det, epc, hist, **kwargs
1363):
1364 # open_mode is always write in this command.
1365 ctx.meta["open_mode"] = "w"
1366 criterion = ctx.meta.get("criterion")
1367 if criterion is not None:
1368 click.echo("Computing metrics with %s..." % criterion)
1369 ctx.invoke(metrics, scores=scores, evaluation=evaluation)
1370 if ctx.meta.get("log") is not None:
1371 click.echo("[metrics] => %s" % ctx.meta["log"])
1373 # avoid closing pdf file before all figures are plotted
1374 ctx.meta["closef"] = False
1375 if evaluation:
1376 click.echo("Starting evaluate with dev and eval scores...")
1377 else:
1378 click.echo("Starting evaluate with dev scores only...")
1379 click.echo("Computing ROC...")
1380 # set axes limits for ROC
1381 ctx.forward(roc) # use class defaults plot settings
1382 click.echo("Computing DET...")
1383 ctx.forward(det) # use class defaults plot settings
1384 if evaluation:
1385 click.echo("Computing EPC...")
1386 ctx.forward(epc) # use class defaults plot settings
1387 # the last one closes the file
1388 ctx.meta["closef"] = True
1389 click.echo("Computing score histograms...")
1390 ctx.meta["criterion"] = "eer" # no criterion passed in evaluate
1391 ctx.forward(hist)
1392 click.echo("Evaluate successfully completed!")
1393 click.echo("[plots] => %s" % (ctx.meta["output"]))
1396def n_protocols_option(required=True, **kwargs):
1397 """Get option for number of protocols."""
1399 def custom_n_protocols_option(func):
1400 def callback(ctx, param, value):
1401 value = abs(value)
1402 ctx.meta["protocols_number"] = value
1403 return value
1405 return click.option(
1406 "-pn",
1407 "--protocols-number",
1408 type=click.INT,
1409 show_default=True,
1410 required=required,
1411 help="The number of protocols of cross validation.",
1412 callback=callback,
1413 **kwargs,
1414 )(func)
1416 return custom_n_protocols_option
1419def multi_metrics_command(
1420 docstring, criteria=("eer", "min-hter", "far"), far_name="FAR", **kwargs
1421):
1422 def custom_metrics_command(func):
1423 func.__doc__ = docstring
1425 @click.command("multi-metrics", **kwargs)
1426 @scores_argument(nargs=-1)
1427 @eval_option()
1428 @n_protocols_option()
1429 @table_option()
1430 @output_log_metric_option()
1431 @criterion_option(criteria)
1432 @thresholds_option()
1433 @far_option(far_name=far_name)
1434 @legends_option()
1435 @open_file_mode_option()
1436 @verbosity_option(LOGGER)
1437 @click.pass_context
1438 @decimal_option()
1439 @functools.wraps(func)
1440 def wrapper(*args, **kwds):
1441 return func(*args, **kwds)
1443 return wrapper
1445 return custom_metrics_command
1448MULTI_METRICS_HELP = """Multi protocol (cross-validation) metrics.
1450 Prints a table that contains mean and standard deviation of {names} for a given
1451 threshold criterion ({criteria}). The metrics are averaged over several protocols.
1452 The idea is that each protocol corresponds to one fold in your cross-validation.
1454 You need to provide as many as development score files as the number of
1455 protocols per system. You can also provide evaluation files along with dev
1456 files. If evaluation scores are provided, you must use flag `--eval`. The
1457 number of protocols must be provided using the `--protocols-number` option.
1459 {score_format}
1461 Resulting table format can be changed using the `--tablefmt`.
1463 Examples:
1465 $ {command} -vv -pn 3 {{p1,p2,p3}}/scores-dev
1467 $ {command} -vv -pn 3 -e {{p1,p2,p3}}/scores-{{dev,eval}}
1469 $ {command} -vv -pn 3 -e {{sys1,sys2}}/{{p1,p2,p3}}/scores-{{dev,eval}}
1470 """