
    UgE                       d dl Z d dlZd dlZd dlZd dlmZ d dlmZm	Z	 d dl
mZ d dlmZ d dlmZ d dlmZmZmZmZmZmZmZmZmZmZmZmZmZ d dlmZm Z  d d	l!m"Z" d d
l#m$Z$ d dl%m&Z& d dl'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z- d dl.m/Z/ d dl0m1Z1 d dl2m3Z3m4Z4m5Z5 eeegZ6ddZ7d Z8d Z9d Z:d Z;ej<        =                    dddg          d             Z>d Z?d Z@d ZAd ZBd ZCd ZDd ZEd  ZFd! ZGd" ZHd# ZIej<        =                    d$ ejJ        g d%          g d&f ejJ        g d%          dfg d'g d(fg d'dfg          d)             ZKej<        =                    d$ ejJ        g d*          g d&f ejJ        g d+          g d,fg          d-             ZLej<        =                    d$ ejJ        g d.          dfg d/dfg d.g d&fg d/g d(fg          d0             ZMej<        =                    d1g d2          d3             ZNd4 ZOej<        =                    d5d6 ejJ        g d.          g d7fd6 ejJ        g d/          g d8fd9 ejJ        g d*          dfd: ejJ        g d/          g d;fd< ejJ        g d.          d d=gfd< ejJ        g d/          d>d?gfd@ ejJ        g d.          g dAfd@ ejJ        g d/          g dBfdC ejJ        g dD          g d(fdC ejJ        g dB          g d(fdC ejJ        g dA          g d&fg          ej<        =                    dEdFdGg          dH                         ZPej<        =                    dIdJdKdFdLfdMdKdGdLfdNdFg dOfdPdFdQdRfdSi fg          dT             ZQdU ZRej<        =                    dVe6          dW             ZSej<        =                    dVe6          dX             ZT e-eUY          ej<        =                    dVeeg          ej<        =                    dZd[d\g          d]                                     ZVej<        =                    dVe6          d^             ZWej<        =                    dddg          d_             ZXd` ZYej<        =                    dddg          da             ZZdb Z[dc Z\dd Z]de Z^df Z_dg Z`ej<        =                    dhg dig djd gd gfg dig dkd gd gfg dig dld gd gfg dig dmd gd gfg dng djdQgd gfg dng dkdQgd gfg dng dldQgd gfg dng dmdQgd gfg dog djdpgd gfg dog dkd gd gfg dog dld gd gfg dog dmd gd gfg dqg djg drg dofg dqg dkg drg dofg dqg dlg drg dofg dqg dmg drg dofg dsg djg dtg dufg dsg dkg dtg dufg dsg dlg dtg dufg dsg dmg dtg dufg          dv             Zaej<        =                    dhd=d gdQdQgd=gd gfd d=gdQdQgd=gd gfg dig dwdQgd gfg dng dwdQgd gfg dog dwd gd gfg dqg dwd=gd gfg dsg dwd=gd gfg dtg dwd=gd gfg          dx             Zbdy Zcej<        =                    dzg d{          d|             Zdej<        =                    d}g d~g dg dg dg dg          d             Zeej<        =                    dd d=gg djdfg dod dQgdfg dg djdfg dg djdfg dg ddfg          d             Zfd Zgd Zhd Zid Zjd Zkd Zld Zm	 ddZnej<        =                    dehelekeif          ej<        =                    deemf          d                         Zod Zpej<        =                    dd          ej<        =                    dd          ej<        =                    d eqd=                    d                                     Zrd Zsd Ztd Zuej<        =                    dg dsg dwfg dsg dwgfg dsgg dwfg          d             Zvd Zwej<        =                    de1          d             Zxd Zyd Zzd Z{d Z|d Z}d Z~d Zd Zej<        =                    dddg          d             Zd Zd Zd Zd Zej<        =                    dg dAd=dfg dAddQfg dAddfg          d             Zej<        =                    d ejJ        g d          d=d=f ejJ        g d          d=dQf ejJ        g d          dd=f ejJ        g d          d=d=f ejJ        g d          d=dQf ejJ        g d          dd=fg          d             Zej<        =                    d ejJ        g dĢ          dg dAf ejJ        g dŢ          dQg dAf ejJ        g dƢ          dQg dAf ejJ        g dǢ          dg dȢfg          ej<        =                    dddg          dʄ                         Zd˄ Zej<        =                    dg dAd=dfg dAddQfg dAdd=fg          d̄             Zej<        =                    dg dAdfg dAdfg          dτ             Zej<        =                    dg dѢg dҢg dӢg dԢg dբgddfg dAg dҢg dӢg dԢg dբgddfg dآg dҢg dӢg dԢg dբgg d/dfg dآg dҢg dӢg dԢg dբgg d;dfg dۢg dҢg dӢg dԢg dբgg dAdfg dۢg dҢg dӢg dԢg dբgg dݢdfd d=gg dߢg dgddfg          d             Zej<        =                    de1          d             Zej<        =                    deeeeg          ej<        =                    dg d          d                         Zd Zd ZdS )    N)stats)datasetssvm)make_multilabel_classification)UndefinedMetricWarning)LogisticRegression)accuracy_scoreaucaverage_precision_scorecoverage_error	dcg_score	det_curve%label_ranking_average_precision_scorelabel_ranking_loss
ndcg_scoreprecision_recall_curveroc_auc_score	roc_curvetop_k_accuracy_score)_dcg_sample_scores_ndcg_sample_scores)train_test_split)label_binarize)_sparse_random_matrix)_convert_containerassert_allcloseassert_almost_equalassert_array_almost_equalassert_array_equalignore_warnings)softmax)CSR_CONTAINERS)check_arraycheck_consistent_lengthcheck_random_stateFc                    | t          j                    } | j        }| j        }|r||dk              ||dk              }}|j        \  }}t          j        |          }t          d          }|                    |           ||         ||         }}t          |dz            }t
          j
                            d          }t
          j        ||                    |d|z            f         }t          j        ddd          }	|	                    |d|         |d|                                       ||d                   }
|r|
ddd	f         }
|	                    ||d                   }||d         }|||
fS )
zMake some classification predictions on a toy dataset using a SVC

    If binary is True restrict to a binary classification problem instead of a
    multiclass classification problem
    N   %   r      linearT)kernelprobabilityrandom_state   )r   	load_irisdatatargetshapenparanger%   shuffleintrandomRandomStatec_randnr   SVCfitpredict_probapredict)datasetbinaryXy	n_samples
n_featuresprnghalfclfy_scorey_predy_trues                a/var/www/surfInsights/venv3-11/lib/python3.11/site-packages/sklearn/metrics/tests/test_ranking.pymake_predictionrM   9   sy    $&&AA "Qx1q51GIz
	)A
R
 
 CKKNNNQ41qAy1}D )


"
"C
a9cJ&67778A 't!
D
D
DCggah%4%))77$%%AAG   !!!Q$-[[455""FtuuXF67""    c                 N   t          j        |           d         }|| |k             }|| |k             }|                    dd          |                    dd          z
  }t          j        |dk              }|t	          t          |          t          |          z            z  S )zKAlternative implementation to check for correctness of
    `roc_auc_score`.r.   r   )r3   uniquereshapesumfloatlen)rK   rI   	pos_labelposnegdiff_matrix	n_corrects          rL   _aucr[   i   s     	&!!!$I &I%
&C
&I%
&C++a$$s{{2q'9'99K{Q''IuSXXC01111rN   c                    t          j        |           d         }t          j        | |k              }t          j        |          ddd         }||         }| |         } d}t	          t          |                    D ]D}| |         |k    r6d}t	          d|dz             D ]}| |         |k    r|dz  }||dz   z  }||z  }E||z  S )a>  Alternative implementation to check for correctness of
    `average_precision_score`.

    Note that this implementation fails on some edge cases.
    For example, for constant predictions e.g. [0.5, 0.5, 0.5],
    y_true = [1, 0, 0] returns an average precision of 0.33...
    but y_true = [0, 0, 1] returns 1.0.
    r.   NrP   r         ?)r3   rQ   rS   argsortrangerU   )	rK   rI   rV   n_posorderscoreiprecjs	            rL   _average_precisionrf   x   s     	&!!!$IF6Y&''EJw"%EenGE]FE3w<<   	 	!9	!! D1a!e__    !9	))CKDAGODTME5=rN   c                 "   t          | |          \  }}}t          t          |                    }t          t          |                    }d}t          dt	          |                    D ]"}|||         ||         ||dz
           z
  z  z  }#|S )ao  A second alternative implementation of average precision that closely
    follows the Wikipedia article's definition (see References). This should
    give identical results as `average_precision_score` for all inputs.

    References
    ----------
    .. [1] `Wikipedia entry for the Average precision
       <https://en.wikipedia.org/wiki/Average_precision>`_
    r   r.   )r   listreversedr_   rU   )rK   rI   	precisionrecall	thresholdaverage_precisionrc   s          rL   _average_precision_slowrn      s     $:&'#J#J IvyXi(())I(6""##F1c)nn%% H HYq\VAYA-FGGrN   c                     d } || ||          \  }}t          ||          }d}|}d||z
  z  ||z   z  }	||z
  }
dd||	z
  |
|	z
  z  z   z  S )zcAlternative implementation to check for correctness of `roc_auc_score`
    with `max_fpr` set.
    c                 X   t          | |          \  }}}|||k             }t          j        ||          }|||k             }t          j        ||k              }|dz
  }	||	         ||         g}
||	         ||         g}t          j        |t          j        ||
|                    }||fS )Nr.   )r   r3   appendargmaxinterp)rK   	y_predictmax_fprfprtpr_new_fprnew_tpridx_outidx_inx_interpy_interps               rL   _partial_rocz,_partial_roc_auc_score.<locals>._partial_roc   s    	22S!cWn%)GW--cWn%)C'M**1KW.KW.)GRYw(%K%KLL!!rN   r         ?r.   )r
   )rK   rt   ru   r   ry   rz   partial_aucfpr1fpr2min_areamax_areas              rL   _partial_roc_auc_scorer      s    

" 
" 
" $|FIw??GWgw''K DDdTk"dTk2Hd{H!{X-(X2EFFGGrN   dropTc                 B   t          d          \  }}}t          ||          }t          |||           \  }}}t          ||          }t	          ||d           t          |t          ||                     |j        |j        k    sJ |j        |j        k    sJ d S )NTr@   drop_intermediater'   decimal)rM   r[   r   r
   r   r   r   r2   )	r   rK   rx   rI   expected_aucrv   rw   
thresholdsroc_aucs	            rL   test_roc_curver      s     )555FAw((L$VWMMMCj#smmGg|Q????vw!?!?@@@9	!!!!9
(((((((rN   c                  \   t           j                            d          } t          j        dgdz  dgdz  z             }|                     dd          }t          ||d          \  }}}|d         dk    sJ |d	         dk    sJ |j        |j        k    sJ |j        |j        k    sJ d S )
Nr   2   r.      d   sizeTr   rP   )r3   r7   r8   arrayrandintr   r2   )rF   rK   rJ   rv   rw   thrs         rL   test_roc_curve_end_pointsr      s     )


"
"CXqcBh!r)**F[[[%%FffEEEMCcq6Q;;;;r7a<<<<9	!!!!9	!!!!!!rN   c                  d   t          d          \  } }}t          | |          \  }}}g }|D ]L}t          j        ||k    | z            }t          j        |           }	|                    d|z  |	z             Mt          ||d           |j        |j        k    sJ |j        |j        k    sJ d S )NTr   r]   r'   r   )rM   r   r3   rS   rq   r   r2   )
rK   rx   rI   rv   rw   r   tpr_correctttprE   s
             rL   test_roc_returns_consistencyr      s     )555FAw$VW55Cj K ) )VW\V+,,F6NN38a<(((( c;::::9	!!!!9
(((((((rN   c                      t          d          \  } }}t          j        t                    5  t	          | |           d d d            d S # 1 swxY w Y   d S )NFr   )rM   pytestraises
ValueErrorr   )rK   rx   rI   s      rL   test_roc_curve_multir      s    (666FAw	z	"	" # #&'"""# # # # # # # # # # # # # # # # # #s   AAAc                      t          d          \  } }}t          | |dz
            \  }}}t          ||          }t          |dd           |j        |j        k    sJ |j        |j        k    sJ d S )NTr   r   ?r'   r   )rM   r   r
   r   r2   )rK   rx   rI   rv   rw   r   r   s          rL   test_roc_curve_confidencer      s    (555FAw$VWs];;Cj#smmGgtQ77779	!!!!9
(((((((rN   c                     t          d          \  } }}t          j        | j                  }t	          | |          \  }}}t          ||          }t          |dd           |j        |j        k    sJ |j        |j        k    sJ t          j        | j                  }t	          | |          \  }}}t          ||          }t          |dd           |j        |j        k    sJ |j        |j        k    sJ t	          | |          \  }}}t          ||          }t          |dd           |j        |j        k    sJ |j        |j        k    sJ d S )NTr   r   r'   r   g(\?)rM   r3   onesr2   r   r
   r   zeros)rK   predrI   trivial_predrv   rw   r   r   s           rL   test_roc_curve_hardr     sp   +4888FD' 76<((L$V\::Cj#smmGgtQ77779	!!!!9
((((( 8FL))L$V\::Cj#smmGgtQ77779	!!!!9
((((( %VT22Cj#smmGgtQ77779	!!!!9
(((((((rN   c                     g d} g d}d}t          j        t          |          5  t          | |          \  }}}d d d            n# 1 swxY w Y   t	          |t          j        t          |          t
          j                             |j	        |j	        k    sJ |j	        |j	        k    sJ d}t          j        t          |          5  t          d | D             |          \  }}}d d d            n# 1 swxY w Y   t	          |t          j        t          |          t
          j                             |j	        |j	        k    sJ |j	        |j	        k    sJ d S )N)
r.   r.   r.   r.   r.   r.   r.   r.   r.   r.   )
r   r.   r   r.   r   r.   r   r.   r   r.   INo negative samples in y_true, false positive value should be meaninglessmatchHNo positive samples in y_true, true positive value should be meaninglessc                     g | ]}d |z
  S )r.    ).0xs     rL   
<listcomp>z,test_roc_curve_one_label.<locals>.<listcomp>6  s    )@)@)@A!a%)@)@)@rN   )
r   warnsr   r   r   r3   fullrU   nanr2   )rK   rJ   expected_messagerv   rw   r   s         rL   test_roc_curve_one_labelr   "  s   +++F+++F 	T  
,4D	E	E	E 9 9(88S*9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 sBGC
OORV<<===9	!!!!9
((((( 	S  
,4D	E	E	E J J()@)@)@)@)@&IIS*J J J J J J J J J J J J J J J sBGC
OORV<<===9	!!!!9
(((((((s#   AAAC88C<?C<c                     ddg} ddg}t          | |          \  }}}t          | |          }t          |g d           t          |g d           t          |d           ddg} ddg}t          | |          \  }}}t          | |          }t          |g d           t          |g d           t          |d           ddg} ddg}t          | |          \  }}}t          | |          }t          |ddg           t          |ddg           t          |d           ddg} ddg}t          | |          \  }}}t          | |          }t          |g d           t          |g d           t          |d           ddg} ddg}t          | |          \  }}}t          | |          }t          |ddg           t          |ddg           t          |d           ddg} dd	g}d
}t	          j        t          |          5  t          | |          \  }}}d d d            n# 1 swxY w Y   t	          j        t                    5  t          | |           d d d            n# 1 swxY w Y   t          |g d           t          |t          j
        t          j
        t          j
        g           ddg} dd	g}d}t	          j        t          |          5  t          | |          \  }}}d d d            n# 1 swxY w Y   t	          j        t                    5  t          | |           d d d            n# 1 swxY w Y   t          |t          j
        t          j
        t          j
        g           t          |g d           t          j        ddgddgg          } t          j        ddgddgg          }t	          j        t                    5  t          | |d           d d d            n# 1 swxY w Y   t	          j        t                    5  t          | |d           d d d            n# 1 swxY w Y   t          t          | |d          d           t          t          | |d          d           t          j        ddgddgg          } t          j        ddgddgg          }t	          j        t                    5  t          | |d           d d d            n# 1 swxY w Y   t	          j        t                    5  t          | |d           d d d            n# 1 swxY w Y   t          t          | |d          d           t          t          | |d          d           t          j        ddgddgg          } t          j        ddgddgg          }t          t          | |d          d           t          t          | |d          d           t          t          | |d          d           t          t          | |d          d           t          j        ddgddgg          } t          j        ddgddgg          }t          t          | |d          d           t          t          | |d          d           t          t          | |d          d           t          t          | |d          d           d S )Nr   r.   r   r   r.   r   r.   r.   r]           r         ?      ?r   r   r   r   r]   r   macroaverageweightedsamplesmicro)r   r   r   r   r   r   r   r   r   r3   r   r   )rK   rI   rw   rv   rx   r   r   s          rL   test_roc_curve_toydatar   =  s   VF!fGFG,,KCaFG,,Gc999---c999---%%%VF!fGFG,,KCaFG,,Gc999---c999---%%%VF!fGFG,,KCaFG,,GcAq6***cAq6***%%%VF!fGFG,,KCaFG,,Gc999---c999---%%%VFCjGFG,,KCaFG,,GcAq6***cAq6***%%%VFTlG 	S  
,4D	E	E	E 1 100S!1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 
z	"	" ' 'fg&&&' ' ' ' ' ' ' ' ' ' ' ' ' ' 'c???333cBFBFBF#;<<<VFTlG 	T  
,4D	E	E	E 1 100S!1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 
z	"	" ' 'fg&&&' ' ' ' ' ' ' ' ' ' ' ' ' ' 'cBFBFBF#;<<<c???333 X1v1v&''FhAA'((G	z	"	" 8 8fgw77778 8 8 8 8 8 8 8 8 8 8 8 8 8 8	z	"	" ; ;fgz::::; ; ; ; ; ; ; ; ; ; ; ; ; ; ;fgyIII3OOOfgwGGGMMMX1v1v&''FhAA'((G	z	"	" 8 8fgw77778 8 8 8 8 8 8 8 8 8 8 8 8 8 8	z	"	" ; ;fgz::::; ; ; ; ; ; ; ; ; ; ; ; ; ; ;fgyIII3OOOfgwGGGMMMX1v1v&''FhAA'((GfgwGGGKKKfgzJJJANNNfgyIII1MMMfgwGGGKKKX1v1v&''Fhc
S#J/00GfgwGGGMMMfgzJJJCPPPfgyIII3OOOfgwGGGMMMMMs   H''H+.H+I((I,/I,K<<L L  L==MMO77O;>O;P::P>P>S11S58S5T44T8;T8c            	          g d} g d}t          | |d          \  }}}t          |t          j        dddg           g d} g d	}t          | |d          \  }}}t          |t          j        dd
dddg           d S )Nr   r   r   r   r.   r.   r   皙?r   333333?ffffff?r]   Tr   r]   r   r   r   r   r   r   r   r   r   r.   r.   r.   r.   r.   r.   r   皙?r   r   r   皙?r   r   r   r   r   r   r]   r   r   )r   r   r3   inf)rK   rI   rw   rv   r   s        rL    test_roc_curve_drop_intermediater     s    F,,,G$VWMMMCjj263S*ABBB 544FOOOG$VWMMMCjj263S#s*KLLLLLrN   c                  ,   g d} g d}t          j        dd          }t          | ||          \  }}}t          j        |          dk                                     dk    sJ t          j        |          dk                                     dk    sJ d S )Nr   r   r.   r.   r.   )r   r   333333?皙?r   r      sample_weightr   )r3   repeatr   diffrS   )rK   rI   r   rv   rw   rx   s         rL   !test_roc_curve_fpr_tpr_increasingr     s     __F'''GIc1%%MFG=IIIKCaGCLL1!!##q((((GCLL1!!##q((((((rN   c                     ddg} ddg}t          t          | |          d           ddg} ddg}t          t          | |          d           g d} g d}t          t          | |          d           ddg} ddg}t          t          | |          d           g d} g d}t          t          | |          d           d S )Nr   r.   r   r.   r   r   r   r   r   r.   )r   r
   )r   rB   s     rL   test_aucr     s    	
AA	
AAc!Qii---	
AA	
AAc!Qii---		A		Ac!Qii---	
AA	
AAc!Qii+++AAc!Qii-----rN   c                      t          j        t                    5  t          g dddg           d d d            n# 1 swxY w Y   t          j        t                    5  t          dgdg           d d d            n# 1 swxY w Y   g d} g d}d                    t          j        |                     }t          j        t          t          j        |                    5  t          | |           d d d            d S # 1 swxY w Y   d S )	Nr   r   r   r   )r'   r.   r      )r            z+x is neither increasing nor decreasing : {}r   )	r   r   r   r
   formatr3   r   reescape)r   rB   error_messages      rL   test_auc_errorsr     s   	z	"	" ) )OOOc3Z((() ) ) ) ) ) ) ) ) ) ) ) ) ) ) 
z	"	"  SEC5               	AAAHHRSUUM	z=)A)A	B	B	B  Aq			                 s1   ;??A>>BB%DD
Dzy_true, labels)r   r.   r   r'   r   r.   r'   )abr   c)r   r   r   c           	         t          j        g dg dg dg dg          }t          g dg d          }t          g dg d          }||z   d	z  }t          g d
g d          }t          g dg d          }||z   d	z  }t          ddgddg          }	t          ddgddg          }
|	|
z   d	z  }||z   |z   dz  }t          t          | ||d          |           |||g}g d}t          j        ||          }t          t          | ||dd          |           d}t          j        t          |          5  t          | ||dd            d d d            d S # 1 swxY w Y   d S )Nr   r   r   r   r   r   ffffff?r   333333?r   r   r   r.   r   r.   )r   r   r   r   r.   r   )r   r   r   r'   r.   r.   r   )r   r   r   r   )r   r   r   r.   r   r   r   r   r   r   ovolabelsmulti_class)r   r   r   )weightsr   r   r  r   z6average=None is not implemented for multi_class='ovo'.r   )r3   r   r   r   r   r   r   NotImplementedError)rK   r   y_scoresscore_01score_10average_score_01score_02score_20average_score_02score_12score_21average_score_12ovo_unweighted_scorepair_scores
prevalenceovo_weighted_scorer   s                    rL   #test_multiclass_ovo_roc_auc_toydatar    sQ    x	///+<+<+<mmmL H YYY(8(8(899HYYY88H 8+q0 YYY77HYYY(8(8(899H 8+q0 aVc3Z00HaVc3Z00H 8+q0 -/??BRRVWWfhv5III   $%57GHK###JKDDDHV
	
 	
 	
 		   MM	*-	@	@	@ X Xfhv5RVWWWWX X X X X X X X X X X X X X X X X Xs   ;EE!$E!)r   r'   r   r'   )r   dr   r  )r   r   r  c           	      ,   t          j        g dg dg dg dg          }t          g dg d          }t          g dg d          }||z   d	z  }t          t          | ||d
          |           t          t          | ||d
d          |           d S )N)r   r   r   )r   r   r   )皙?r   ?)r   r   r   )r.   r   r.   r   )r   r   r  r   )r   r.   r   r.   )r   r   r  r   r'   r   r   r   r  r3   r   r   r   )rK   r   r  r  r  	ovo_scores         rL   *test_multiclass_ovo_roc_auc_toydata_binaryr    s     x	///+<+<+<oooN H \\\+@+@+@AAH\\\+@+@+@AAHH$)Ifhv5III9  
 HV
	
 	
 	
 		    rN   )r   r.   r'   r'   )r   r   r   r   c           	         t          j        g dg dg dg dg          }t          g d|d d df                   }t          g d|d d df                   }t          g d	|d d d
f                   }t          t          | |d|d           |||g           ||z   |z   dz  }t          t          | |d|          |           |dz  |dz  z   |dz  z   }t          t          | |d|d          |           d S )Nr]   r   r   )r   r   r   )r   r   r   )r   r   r   r.   r   r   r   r   )r   r.   r   r   r.   r   r   r.   r.   r'   ovr)r  r   r   g      @)r  r   r   r   r   r  )rK   r   r  out_0out_1out_2result_unweightedresult_weighteds           rL   #test_multiclass_ovr_roc_auc_toydatar%  B  s]    x	///???OOOL H
 ,,,A77E,,,A77E,,,A77EfhE&RVWWW	u   .#5fhE&III   dlUT\1ECK?OH%
	
 	
 	
 		    rN   zmulti_class, average))r  r   )r  r   )r   r   c                 b   t          j        g d          }g dg dg dg dg}t          t          ||| |          d           g dg dg dg dg}t          ||| |          dk     sJ dt          j        d	          z  }t          ||| |          t          j        d
          k    sJ d S )N)r   r.   r'   r   )r   r   r   r]   )r   r]   r   r   )r   r   r]   r   )r   皙?r'  r   r  r   r]   r   )r   r   r   )r3   r   r   r   r   r   approx)r  r   rK   	y_perfecty_imperfecty_chances         rL   0test_perfect_imperfect_chance_multiclass_roc_aucr-  m  s#    Xlll##F 	   	I fi['RRR   		K 	fk{GTTT
	 	 	 	
 bgfoo%Hk7  	s		     rN   c                   	 | 	t           j                            g dd	          }t          j        	fd|D                       }t          |g d          }t          |                                |                                          \  }}}t          ||          }t          ||dd	          }|t          j        |          k    sJ d S )
N)       @r]   r     )r   r-   c                 x    g | ]6}t           j                            d |                                          7S )r.   )nrE   r-   )r   multinomialrvsrr   )r   y_pred_iseeds     rL   r   z3test_micro_averaged_ovr_roc_auc.<locals>.<listcomp>  sM     	
 	
 	
 !!A!EELLNN	
 	
 	
rN   r   )classesr  r   r(  )r   	dirichletr4  r3   asarrayr   r   ravelr
   r   r   r)  )
global_random_seedrJ   rK   y_onehotrv   rw   rx   roc_auc_by_handroc_auc_autor6  s
            @rL   test_micro_averaged_ovr_roc_aucr?    s    D
 _  t$ OOFZ	
 	
 	
 	
"	
 	
 	
 F fiii888HHNN,,fllnn==KCa#smmO UGTTTLfmL99999999rN   zmsg, y_true, labelsz!Parameter 'labels' must be uniquer   r'   r   )r   r   r   zKNumber of classes in y_true not equal to the number of columns in 'y_score'z"Parameter 'labels' must be ordered)r   r   r   zMNumber of given labels, 2, not equal to the number of columns in 'y_score', 3r.   r   r   zMNumber of given labels, 4, not equal to the number of columns in 'y_score', 3)r   r.   r'   r   )r   r   r   r  z2'y_true' contains labels not in parameter 'labels')r   r   r   er  r   r  c                     t          j        g dg dg dg dg          }t          j        t          |           5  t          ||||           d d d            d S # 1 swxY w Y   d S )Nr   r   r   r   r   r   )r3   r   r   r   r   r   )msgrK   r   r  r  s        rL   *test_roc_auc_score_multiclass_labels_errorrD    s    R x	///+<+<+<mmmL H 
z	-	-	- P Pfhv;OOOOP P P P P P P P P P P P P P P P P Ps   AA!$A!zmsg, kwargszLaverage must be one of \('macro', 'weighted', None\) for multiclass problemsr   )r   r  zUaverage must be one of \('micro', 'macro', 'weighted', None\) for multiclass problemszksample_weight is not supported for multiclass one-vs-one ROC AUC, 'sample_weight' must be None in this case)r  r   z|Partial AUC computation not available in multiclass setting, 'max_fpr' must be set to `None`, received `max_fpr=0.5` insteadr   )r  ru   z'multi_class must be in \('ovo', 'ovr'\)c                 "   t          d          }|                    dd          }t          |          }|                    ddd          }t	          j        t          |           5  t          ||fi | d d d            d S # 1 swxY w Y   d S )N     r   r   r   r   )r%   randr!   r   r   r   r   r   )rC  kwargsrF   rI   y_probrK   s         rL   #test_roc_auc_score_multiclass_errorrK    s    N S
!
!Chhr1ooGWF[[AB[''F	z	-	-	- 0 0ff/////0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0s   (BBBc                  &   t          d          } |                     d          }t          j        dd          }d}t	          j        t          |          5  t          ||           d d d            n# 1 swxY w Y   t          j        dd          }t	          j        t          |          5  t          ||           d d d            n# 1 swxY w Y   t          j	        ddd          }t	          j        t          |          5  t          ||           d d d            n# 1 swxY w Y   t          j        d	          5  t          d          } |                     d          }t          j        dd          }t	          j        t          |          5  t          ||           d d d            n# 1 swxY w Y   t          j        dd          }t	          j        t          |          5  t          ||           d d d            n# 1 swxY w Y   t          j	        ddd          }t	          j        t          |          5  t          ||           d d d            n# 1 swxY w Y   d d d            d S # 1 swxY w Y   d S )
NrF  
   r6   dtypezROC AUC score is not definedr   rP   T)record)r%   rH  r3   r   r   r   r   r   r   r   warningscatch_warnings)rF   rJ   rK   err_msgs       rL   test_auc_score_non_binary_classrT  +  s    S
!
!CXXb\\FXb&&&F,G	z	1	1	1 & &ff%%%& & & & & & & & & & & & & & &WRu%%%F	z	1	1	1 & &ff%%%& & & & & & & & & & & & & & &WR5)))F	z	1	1	1 & &ff%%%& & & & & & & & & & & & & & & 
	 	-	-	- * * %%""E***]:W555 	* 	*&&)))	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	*5)))]:W555 	* 	*&&)))	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	*Ru---]:W555 	* 	*&&)))	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	** * * * * * * * * * * * * * * * * *s   A55A9<A91CCCD((D,/D,AJF;/J;F?	?JF?	4J7HJH	JH	5JI."J.I2	2J5I2	6JJ
J

curve_funcc                    t          d          }|                    ddd          }|                    d          }d}t          j        t
          |          5   | ||           d d d            d S # 1 swxY w Y   d S )NrF  r   r   rM  r   z"multiclass format is not supportedr   )r%   r   rH  r   r   r   )rU  rF   rK   rJ   rC  s        rL   &test_binary_clf_curve_multiclass_errorrW  K  s    
S
!
!C[[AB[''FXXb\\F
.C	z	-	-	- # #
66"""# # # # # # # # # # # # # # # # # #s   A44A8;A8c                    d}t          j        t          |          5   | t          j        ddgd          ddg           d d d            n# 1 swxY w Y   t          j        t          |          5   | t          j        ddgt
                    ddg           d d d            n# 1 swxY w Y   g d	} | g d
|          } | g d|          }t          ||          D ]%\  }}t          j                            ||           &d S )Nzy_true takes value in {'a', 'b'} and pos_label is not specified: either make y_true take value in {0, 1} or {-1, 1} or pass pos_label explicitly.r   r   r   z<U1rN  r   r]   )r   r]   r   gzG?)r   r.   r.   r   )r   r]   r]   r   )	r   r   r   r3   r   objectziptestingr   )rU  rC  rJ   	int_curvefloat_curveint_curve_partfloat_curve_parts          rL   (test_binary_clf_curve_implicit_pos_labelr`  U  s   
	  
z	-	-	- B B
28S#Je444sCjAAAB B B B B B B B B B B B B B B 
z	-	-	- C C
28S#Jf555SzBBBC C C C C C C C C C C C C C C
 #""F
<<<00I*1116::K,/	;,G,G E E((

"">3CDDDDE Es#   %AAA5*B++B/2B/)categorylabels_typerh   r   c                     t          ddg|          }d}t          j        t          |          5   | |ddg           d d d            d S # 1 swxY w Y   d S )N   a   bzy_true takes value in {b'a', b'b'} and pos_label is not specified: either make y_true take value in {0, 1} or {-1, 1} or pass pos_label explicitly.r   r   r]   )r   r   r   r   )rU  rb  r   rC  s       rL   .test_binary_clf_curve_implicit_bytes_pos_labelrf  o  s      tk::F	0 
 
z	-	-	- ' '
6C:&&&' ' ' ' ' ' ' ' ' ' ' ' ' ' ' ' ' 's   AAAc                     g d}g d}g d} | |||          } | |d d         |d d         |d d                   }t          ||          D ]\  }}t          ||           d S )Nr   )r   r   r   r   r   )r.   r.   r.   r   r   r   rP   )rZ  r   )rU  rK   rI   r   result_1result_2arr_1arr_2s           rL   (test_binary_clf_curve_zero_sample_weightrl    s    __F'''G%%%Mz&'GGGHz&"+wss|=QTRTQTCUVVVHHh// & &uu%%%%& &rN   c                    t          d          \  }}}t          |||            t          |dd          |dd          |           \  }}}|d         dk    sJ |d         |dd                                          k    sJ d|t	          j        |dk              <   |                                }t          |||            t          ||           g d}g d	}	t          ||	|           \  }}}| r7t          |g d
           t          |g d           t          |g d           n6t          |g d           t          |g d           t          |g d	           |j	        |j	        k    sJ |j	        |j	        dz   k    sJ d S )NTr   r.   r   r   r]   rP   r.   r   r   r.   )r.   r'   r   r   )r   QUU?r]   r]   )r]   r   r   r   )r.   r'   r   )r   ro  r   r]   r]   )r]   r   r   r   r   )
rM   _test_precision_recall_curver   meanr3   wherecopyr   r   r   )
r   rK   rx   rI   rE   rr   y_true_copyr   predict_probass
             rL   test_precision_recall_curverw    s   (555FAw $777 %VABBZPTUUUGAq!Q43;;;;Q46!"":??$$$$$$ %'F28FaK  !++--K $777{F+++\\F!\\N$V^tTTTGAq! )666777///000999%%%%;;;<<<444555<<<(((6QV6QVaZrN   c                    t          | ||          \  }}}t          | |          }t          |dd           t          |t          | |                     t	          t          | |          |d           |j        |j        k    sJ |j        |j        dz   k    sJ t          | t          j        |          |          \  }}}|j        |j        k    sJ |j        |j        dz   k    sJ d S )Nr   grh|?r   r'   r   r.   )	r   rn   r   r   r   rf   r   r3   
zeros_like)rK   rI   r   rE   rt  r   precision_recall_aucs          rL   rp  rp    s"   -fgQUVVVAq*267CC2E1===5fgFF   67++-A1    6QV6Z_q(((((-g&&$  Aq* 6QV6Z_q(((((((rN   c           	      (   t          j        d          5  ddg}ddg}t          |||           \  }}}t          ||          }t	          |g d           t	          |g d           t          |d           ddg}ddg}t          |||           \  }}}t          ||          }t	          |g d	           t	          |g d
           t          |d           ddg}ddg}t          |||           \  }}}t          ||          }t	          |ddg           t	          |ddg           t          |d           ddg}ddg}t          |||           \  }}}t          ||          }t	          |g d           t	          |g d           t          |d           ddg}ddg}t          |||           \  }}}t          ||          }t	          |ddg           t	          |ddg           t          |d           ddg}ddg}t          j        t          d          5  t          |||           \  }}}d d d            n# 1 swxY w Y   t          j        t          d          5  t          ||          }d d d            n# 1 swxY w Y   t          |g d           t          |g d           t          |d           ddg}ddg}t          |||           \  }}}t          t          ||          d           t	          |g d           t	          |g d           t          j
        ddgddgg          }t          j
        ddgddgg          }t          j        t          d          5  t          t          ||d          d           d d d            n# 1 swxY w Y   t          j        t          d          5  t          t          ||d          d           d d d            n# 1 swxY w Y   t          t          ||d          d           t          t          ||d          d           t          j
        ddgddgg          }t          j
        ddgddgg          }t          j        t          d          5  t          t          ||d          d           d d d            n# 1 swxY w Y   t          j        t          d          5  t          t          ||d          d           d d d            n# 1 swxY w Y   t          t          ||d          d           t          t          ||d          d           t          j
        ddgddgg          }t          j
        ddgddgg          }t          t          ||d          d           t          t          ||d          d           t          t          ||d          d           t          t          ||d          d           t          j
        ddgddgg          }t          j
        ddgddgg          }t          j        t          d          5  t          t          ||d          d           d d d            n# 1 swxY w Y   t          t          ||d          d           t          j        t          d          5  t          t          ||d          d           d d d            n# 1 swxY w Y   t          j        t          d          5  t          t          ||d          d           d d d            n# 1 swxY w Y   t          j
        ddgddgg          }t          j
        ddgddgg          }t          t          ||d          d           t          t          ||d          d           t          t          ||d          d           t          t          ||d          d           t          j
        ddgddgg          }t          j
        ddgddgg          }t          t          ||d          d           t          t          ||d          d           t          t          ||d          d           t          t          ||d          d           d d d            n# 1 swxY w Y   t          j        d          5  t          j
        ddgddgg          }t          j
        ddgddgg          }t          j        t          d          5  t          t          ||d          d           d d d            n# 1 swxY w Y   d d d            d S # 1 swxY w Y   d S )Nraise)allr   r.   r   )r   r.   r.   r   r]   )r   r   r]   r  r   r   r   r   z!No positive class found in y_truer   r   )r]   r]   r]   )r.   r   r   r   r   r   r   r   ignore)r3   errstater   r   r   r   r   r   UserWarningr   r   )r   rK   rI   rE   rt  rx   auc_prcs          rL   #test_precision_recall_curve_toydatar    s   		!	!	! V
 V
Qa&(DQQQ1a)&'::!![[[111!!YYY///GS)))Qa&(DQQQ1a)&'::!!___555!!___555 	GS)))Qa&(DQQQ1a)&'::!!c1X...!!c1X...GS)))Qa&(DQQQ1a)&'::!![[[111!!YYY///GS)))Q*(DQQQ1a)&'::!!c1X...!!aX...GS)))Q,\+-PQQQ 	V 	V,VWPTUUUGAq!	V 	V 	V 	V 	V 	V 	V 	V 	V 	V 	V 	V 	V 	V 	V\+-PQQQ 	? 	?-fg>>G	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	?999%%%999%%%###Q,(DQQQ1a3FGDDcJJJ!!___555!!]]]333 Aq6Aq6*++(QFQF+,,\+-PQQQ 	 	'III3  	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 \+-PQQQ 	 	'LLLc  	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	#FGYGGG	
 	
 	
 	/QQQSVWWWAq6Aq6*++(QFQF+,,\+-PQQQ 	 	'III3  	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 \+-PQQQ 	 	'LLLc  	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	#FGYGGG	
 	
 	
 	/QQQSVWWWAq6Aq6*++(QFQF+,,#FGWEEEs	
 	
 	
 	#FGZHHH#	
 	
 	
 	#FGYGGG	
 	
 	
 	#FGWEEEs	
 	
 	
 Aq6Aq6*++(QFQF+,,\+-PQQQ 	 	'III3  	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	#FGZHHH#	
 	
 	
 \+-PQQQ 	 	'KKKS  	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 \+-PQQQ 	 	'III3  	 	 	 	 	 	 	 	 	 	 	 	 	 	 	
 Aq6Aq6*++(QFQF+,,/QQQSVWWW#FGZHHH#	
 	
 	
 	#FGYGGG	
 	
 	
 	/QQQSVWWWAq6Aq6*++(S#Jc
344#FGWEEEs	
 	
 	
 	#FGZHHH#	
 	
 	
 	#FGYGGG	
 	
 	
 	#FGWEEEs	
 	
 	
iV
 V
 V
 V
 V
 V
 V
 V
 V
 V
 V
 V
 V
 V
 V
p 
	"	"	"  Aq6Aq6*++(QFQF+,,\+-PQQQ 	 	'LLLa  	 	 	 	 	 	 	 	 	 	 	 	 	 	 		                 s  Hc$I;cI	cI	c-J
>c
J	cJ	C&c8!N%c%N)	)c,N)	-c!O8,c8O<	<c?O<	 Bc!R?3c?S	cS	c%!TcT	cT	Ec !ZcZ	cZ	>c!\ 4c \	c\	c&!]c]	c]	E+ccc2Af!e/#f/e3	3f6e3	7fffc                     g d} g d}t          | |d          \  }}}t          |g d           g d} g d}t          | |d          \  }}}t          |g d           g d	} g d
}t          | |d          \  }}}t          |ddg           g d} g d
}t          | |d          \  }}}t          |g d
           dS )z9Check the behaviour of the `drop_intermediate` parameter.r   r   Tr   )r   r   r]   r   r   )r   r   r   r   r   r]   r   r   r   r   )r   r   r   r   r   r   r.   r.   r.   r.   N)r   r   )rK   rI   rj   rk   r   s        rL   -test_precision_recall_curve_drop_intermediater  f  sH   F,,,G$:4% % %!Ivz J000 544FOOOG$:4% % %!Ivz J > > >??? \\F"""G$:4% % %!Ivz Jc
+++ \\F"""G$:4% % %!Ivz J 4 4 455555rN   c                      t          j        dt                    } d| d d d<   t          j        d          }t	          | |          dk    sJ d S )Nr   rN  r.   r   r   )r3   r   r6   r   r   rK   rI   s     rL   &test_average_precision_constant_valuesr    sW    
 Xc%%%FF33Q3KgcllG #6733t;;;;;;rN   c                      t          j        ddg          } t          j        ddg          }d}t          j        t          |          5  t          | |d           d d d            d S # 1 swxY w Y   d S )Nr   r.   z>pos_label=2 is not a valid label. It should be one of \[0, 1\]r   r'   rV   r3   r   r   r   r   r   rK   rJ   rS  s      rL   4test_average_precision_score_binary_pos_label_errorsr    s    Xq!fFXq!fFOG	z	1	1	1 = =!<<<<= = = = = = = = = = = = = = = = = =s   
A**A.1A.c                     t          j        ddgddgddgddgg          } t          j        ddgddgddgddgg          }d}t          j        t          |          5  t          | |d	           d d d            d S # 1 swxY w Y   d S )
Nr.   r   r   r   r   r   znParameter pos_label is fixed to 1 for multilabel-indicator y_true. Do not set pos_label or set pos_label to 1.r   r  r  r  s      rL   8test_average_precision_score_multilabel_pos_label_errorsr    s     X1v1v1v1v677FXSzC:SzC:FGGF	6  
z	1	1	1 = =!<<<<= = = = = = = = = = = = = = = = = =s   A>>BBc            	         t          j        g d          } t          j        g dg dg dg dg dg dg          }d}t          j        t          |          5  t          | |d	           d d d            d S # 1 swxY w Y   d S )
N)r   r.   r'   r   r.   r'   )r   r   r   )r   r   r   )r   r   r   )r   r   r   zdParameter pos_label is fixed to 1 for multiclass y_true. Do not set pos_label or set pos_label to 1.r   r   r  r  r  s      rL   8test_average_precision_score_multiclass_pos_label_errorsr    s    X((())FXOOOOOOOOOOOO	
	 	F	6  
z	1	1	1 = =!<<<<= = = = = = = = = = = = = = = = = =s   A::A>A>c                     t          d          \  } }}t          | |          }t          | d|z            }t          | d|z            }t          | |dz
            }||k    sJ ||k    sJ ||k    sJ t          | |          }t          | d|z            }t          | d|z            }	t          | |dz
            }
||k    sJ ||	k    sJ ||
k    sJ d S )NTr   r   gư>rM  )rM   r   r   )rK   rx   rI   r   roc_auc_scaled_uproc_auc_scaled_downroc_auc_shiftedpr_aucpr_auc_scaled_uppr_auc_scaled_downpr_auc_shifteds              rL   test_score_scale_invariancer    s    )555FAwFG,,G%fcGm<<'w??#FGbL99O''''')))))o%%%%$VW55F.vsW}EE0HH,VWr\BBN%%%%%'''''^######rN   z(y_true,y_score,expected_fpr,expected_fnrr   r   )r   r   r   )r   r   r.   r   r   r   r   r   r   r   )r.   r.   r   r   r   )r   r   r   c                 n    t          | |          \  }}}t          ||           t          ||           d S Nr   r   rK   rI   expected_fprexpected_fnrrv   fnrrx   s          rL   test_det_curve_toydatar    s?    6 FG,,KCaC&&&C&&&&&rN   r   r   r   c                 n    t          | |          \  }}}t          ||           t          ||           d S r  r  r  s          rL   test_det_curve_tie_handlingr    s?     FG,,KCaC&&&C&&&&&rN   c                  n    t          t          g dg d          t          g dg d                     d S )Nr   r   r   )r   r   r   r   r.   r.   )r   r   r   rN   rL   test_det_curve_sanity_checkr    sN    )))[[[))$$$&<&<&<==    rN   rI   )r   r   r   r   r.   c                     t          g dt          j        d|                     \  }}}t          |dg           t          |dg           t          || g           d S )N)r   r.   r   r.   r   r.   r   r  r.   r   )r   r3   r   r   )rI   rv   r  rl   s       rL   test_det_curve_constant_scoresr    sr    #!!!271g+>+>  Ci C!C!Iy)))))rN   rK   )r   r   r   r   r   r.   r   )r   r   r   r.   r.   r.   )r   r   r.   r.   r.   r.   )r   r.   r.   r.   r.   r.   c                 t    t          | |           \  }}}t          |dg           t          |dg           d S )Nr  r   r  )rK   rv   r  rx   s       rL   test_det_curve_perfect_scoresr  "  sF     66:::KCaC!C!rN   zy_true, y_pred, err_msgzinconsistent numbers of samplesr   r   r   z Only one class present in y_truer.   r.   r.   )cancerr  
not cancer)r   r   r   zpos_label is not specifiedc                     t          j        t          |          5  t          | |           d d d            d S # 1 swxY w Y   d S )Nr   )r   r   r   r   r  s      rL   test_det_curve_bad_inputr  3  s      
z	1	1	1 " "&&!!!" " " " " " " " " " " " " " " " " "   :>>c                     dgdz  dgdz  z   } t          j        g d          }d|z
  }t          | |d          \  }}}t          | |d          \  }}}|d         t          j        d	          k    sJ |d         t          j        d
          k    sJ t          ||d d d                    t          ||d d d                    d S )Nr  r   r  r   )
r   r   r   r   r   r   r   r   r   r   r.   r  r   r   r   rP   )r3   r   r   r   r)  r   )	rK   y_pred_pos_not_cancery_pred_pos_cancerfpr_pos_cancerfnr_pos_cancerth_pos_cancerfpr_pos_not_cancerfnr_pos_not_cancerth_pos_not_cancers	            rL   test_det_curve_pos_labelr  G  s   Z!^|nq00FH%W%W%WXX114=5 5 51NNM
 AJA A A=*,= v}S111111Q6=#5#55555 N$6ttt$<===N$6ttt$<=====rN   c                    t           | ddggddgg          d           t           | ddggddgg          d           t           | ddggddgg          d           t           | g dgg dg          d           t           | g dgg dg          d           t           | g d	gg dg          d           t           | g d
gg dg          d           t           | g dgg dg          d           t           | g dgg dg          d           t           | g dgg dg          d           t           | g dgg dg          d           t           | g d	gg dg          d           t           | g d
gg dg          d           t           | g dgg dg          d           t           | g dgg dg          d           t           | g dgg dg          d           t           | g dgg dg          d           t           | g dgg dg          d           t           | g d	gg dg          d           t           | g d
gg dg          d           t           | g dgg dg          d           t           | g dgg dg          d           t           | g dgg dg          d           t           | ddggddgg          d           t           | ddggddgg          d           t           | ddggddgg          d           t           | g dgg dg          d           t           | g dgg dg          d           t           | g d	gg dg          d           t           | g d
gg dg          d           t           | g dgg dg          d           t           | g dgg dg          d           t           | g dgg dg          d           t           | g dgg dg          d           t           | g dgg dg          d           d S )Nr   r.   r   r   r   r   r  r   r   r   gUUUUUU?r   g?r   g?r   r   r   r  r   r   r   r  )r   r   r   UUUUUU?)r.   r.   r.   r   )r   r   r   r   )r   
lrap_scores    rL   check_lrap_toyr  a  s/   

QF8tTl^<<a@@@

QF8tTl^<<eDDD

QF8tTl^<<a@@@

III;1B1B1B0CDDaHHH

III;1B1B1B0CDDeLLL

III;1B1B1B0CDDaHHH

III;1B1B1B0CDDeLLL
III;!2!2!2 3446I   
III;!2!2!2 3446I   

III;1B1B1B0CDDeLLL

III;1B1B1B0CDDeLLL
III;!2!2!2 3446I   

III;1B1B1B0CDDaHHH

III;1B1B1B0CDDoVVV

III;1B1B1B0CDDaHHH

III;1B1B1B0CDDaHHH

III;1B1B1B0CDDeLLL

III;1B1B1B0CDDaHHH

III;1B1B1B0CDDoVVV

III;1B1B1B0CDDeLLL
III;!2!2!2 3446I   

III;1B1B1B0CDDaHHH

III;1B1B1B0CDDaHHH 

QF8sCj\::C@@@

QF8sCj\::C@@@

QF8sCj\::A>>>

III;1A1A1A0BCCSIII

III;1A1A1A0BCCSIII

III;1A1A1A0BCCQGGG

III;1A1A1A0BCCUKKK
III;!1!1!1 2335H   
III;!1!1!1 2335H   

III;1A1A1A0BCCQGGG

III;0ABBEJJJ

LLL>4H4H4H3IJJERRRRRrN   c           	         t          d          }t          dd          D ]}|                    d|f          }t          j        |          }t          j        d|f          } | ||          dk    sJ  | ||          dk    sJ t          j        d|f          } | ||          dk    sJ  | ||          dk    sJ t           | dgdgdgdggdgdgdgdgg          d           d S )Nr   r'   r   r.   r   r]   r   )r%   r_   uniformr3   ry  r   r   r   )r  r-   n_labelsrI   y_score_tiesrK   s         rL   !check_zero_or_all_relevant_labelsr    sA   %a((L!QKK 7 7&&QM&::}W-- 1h-((z&'**c1111z&,//36666 !X''z&'**c1111z&,//366666 
QC!qcA3'3%#u)EFF    rN   c                    t          j        t                    5   | g dg d           d d d            n# 1 swxY w Y   t          j        t                    5   | g dg dg dg dg           d d d            n# 1 swxY w Y   t          j        t                    5   | g dg dg dg dg           d d d            n# 1 swxY w Y   t          j        t                    5   | ddgddggddg           d d d            n# 1 swxY w Y   t          j        t                    5   | ddgddggddgg           d d d            n# 1 swxY w Y   t          j        t                    5   | ddgddggdgdgg           d d d            n# 1 swxY w Y   t          j        t                    5   | ddggddgddgg           d d d            n# 1 swxY w Y   t          j        t                    5   | dgdggddgddgg           d d d            n# 1 swxY w Y   t          j        t                    5   | ddgddggdgdgg           d d d            d S # 1 swxY w Y   d S )	Nr   )r   r   r   r   )r   r   r   )r   r   r   )r   r   r   r   r.   )r   r   r   r  s    rL   check_lrap_error_raisedr    s   	z	"	" 0 0
999...///0 0 0 0 0 0 0 0 0 0 0 0 0 0 0	z	"	" U U
999000///???STTTU U U U U U U U U U U U U U U	z	"	" 
 

OO//////R	
 	
 	

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
z	"	" - -
QFQF#aV,,,- - - - - - - - - - - - - - -	z	"	" / /
QFQF#q!fX.../ / / / / / / / / / / / / / /	z	"	" 1 1
QFQF#qcA3Z0001 1 1 1 1 1 1 1 1 1 1 1 1 1 1	z	"	" / /
QF8q!fq!f-.../ / / / / / / / / / / / / / /	z	"	" 1 1
QC!:AA/0001 1 1 1 1 1 1 1 1 1 1 1 1 1 1	z	"	" 1 1
QFQF#qcA3Z0001 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1s   7;;A??BB#CCC+DDD0EEE6FF F=GG#&G#H&&H*-H*
I..I25I2c           	      &   t          dd          D ]}t          j        d|f          }t          d|          D ]V}t          ||z
            D ]A}t          j        d|f          }d|d|||z   f<   t	           | ||          ||z             BWd S )Nr'   rM  r.   r   )r_   r3   r   r   r   )r  r  rI   
n_relevantrW   rK   s         rL   check_lrap_only_tiesr    s     !RLL 	X 	X'1h-((  8,, 	X 	XJX
233 X X1h-0045q#j 0001#JJvw$?$?hAVWWWWX	X		X 	XrN   c                 .   t          dd          D ] }|t          j        |                              d|f          dz   z
  }t          j        d|f          }d|d<   d|d<   t           | ||          d|z  dz   dz             t          d|          D ]zt          |z
            D ]et          j        d|f          }d|dz   f<   t           | ||          t          fdt                    D                                  f{d S )Nr'   rM  r.   )r   r   )r   rP   r   c              3   :   K   | ]}|d z   |z   d z   z  z  V  dS )r.   Nr   )r   rt  r  rW   s     rL   	<genexpr>z>check_lrap_without_tie_and_increasing_score.<locals>.<genexpr>  sM         QC!GaK:#=>     rN   )r_   r3   r4   rR   r   r   rS   )r  r  rI   rK   r  rW   s       @@rL   +check_lrap_without_tie_and_increasing_scorer    sv    !RLL  bi11991h-HH1LM 1h-((tuJJvw77!h,:Ja9OPPP  8,, 	 	JX
233 	 	1h-0045q#j 0001#Jvw//     !&z!2!2       		 rN   c                    t          | |           t          |           } t          |          }| j        \  }}t          j        |f          }t          |          D ]}t          j        ||         d          \  }}|j        }||z
  t          j        |dz             	                                }	|	         | |         
                                d         }
|
j        dk    s|
j        |k    rd||<   d||<   |
D ]7t          fd|
D                       }||xx         |         z  z  cc<   8||xx         |
j        z  cc<   |                                S )z8Simple implementation of label ranking average precisionT)return_inverser.   )	minlengthr   r   c              3   <   K   | ]}|                  k    V  d S r  r   )r   rt  labelranks     rL   r  z_my_lrap.<locals>.<genexpr>  s0       J JAaDK!7 J J J J J JrN   )r$   r#   r2   r3   emptyr_   rQ   r   bincountcumsumnonzerorS   rq  )rK   rI   rC   r  rb   rc   unique_rankinv_rankn_ranks	corr_rankrelevantn_ranked_abover  r  s               @@rL   _my_lrapr    s   FG,,,  F'""G ,IxHi\""E9 " " !#	'!*T J J JX"! K!<<<CCEE	!9$$&&q)=A(!:!:E!Ha 	5 	5E ! J J J J J J J JJJN !HHHe44HHHHaHM!::<<rN   r   rG  c                    t          dd|||          \  }}t          |j        d         |j        d         |          }t          |d          r|                                }t          ||          }t          ||          }t          ||           t          |          }|	                    ||f          }t          ||          }t          ||          }t          ||           d S )Nr.   F)rD   allow_unlabeledr-   	n_classesrC   r   )n_componentsrD   r-   toarrayr   )
r   r   r2   hasattrr  r   r  r   r%   r  )	r  r  rC   r-   rx   rK   rI   
score_lrapscore_my_lraps	            rL   %check_alternative_lrap_implementationr    s    /!  IAv $\!_<?!  G w	"" $//##6vwGGJVW--M
M222 &l33L""I(>"??G6vwGGJVW--M
M22222rN   checkfuncc                      | |           d S r  r   )r  r  s     rL   test_label_ranking_avpr  =  s     
E$KKKKKrN   c                  .    t          t                     d S r  )r  r   r   rN   rL   test_lrap_error_raisedr  K  s    ABBBBBrN   rC   )r.   r'   r   rG  r  )r'   r   rM  r-   c                 4    t          t          || |           d S r  )r  r   )rC   r  r-   s      rL   $test_alternative_lrap_implementationr  O  s)     *-y)\    rN   c                  t   t          j        g dg dg dgt                    } t          j        g dg dg dg          }t          j        g d          }t          j        g d	          }t          t	          | ||
          t          j        ||z            t          j        |          z             d S )Nr  rn  r  rN  )r   r   r   r   )r   r   r   r   r   r   r   r   )r   r   r]   )r]   r]   r   r   )r3   r   boolr   r   rS   )rK   rI   samplewise_lrapsr   s       rL   &test_lrap_sample_weighting_zero_labelsr  X  s     X|||\\\<<<@MMMFh			3335I5I5IJ G x 0 0 011H___--M-G=	
 	
 	
 	}//0026-3H3HH	    rN   c            	         t          t          ddggddgg          d           t          t          ddggddgg          d           t          t          ddggddgg          d           t          t          ddggddgg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g d	gg dg          d           t          t          g d
gg dg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g d	gg dg          d           t          t          g d
gg dg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g d	gg dg          d           t          t          g d
gg dg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g d	g dgg dg dg          d           t          t          g d	g dg d
gg dg dg dg          d           t          t          g d	g dg d
gg dg dg dg          d           d S )Nr   r.   r   r   r'   r  r  r   r   r   r   r   r   r   r  r  r  r   g      $@r   r.   r   r/  r   rM  r  r@  g@r   r.   r   r   r   r   rN   rL   test_coverage_errorr  m  st   Ax4,@@!DDDAx4,@@!DDDAx4,@@!DDDAx4,@@!DDD			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL			{5F5F5F4GHH!LLL 			999-/KLL  
 YY			999-yyy)))/T	
 	
 		   YY			999-yyy)))/T	
 	
 		    rN   c                  f   t          t          ddggddgg          d           t          t          ddggddgg          d           t          t          ddggddgg          d           t          t          ddggddgg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g d	gg dg          d           t          t          g d
gg dg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           d S )Nr   r   r.   r'   r  r  r   r   r   r   r   r   r   r  r  r   rN   rL   test_coverage_tie_handlingr    s   Ax3*>>BBBAx3*>>BBBAx3*>>BBBAx3*>>BBB			{5E5E5E4FGGKKK			{5E5E5E4FGGKKK			{5E5E5E4FGGKKK			{5E5E5E4FGGKKK			{5E5E5E4FGGKKK			{5E5E5E4FGGKKK			{5E5E5E4FGGKKK			{5E5E5E4FGGKKKKKrN   zy_true, y_scorec                     t          j        t          d          5  t          | |           d d d            d S # 1 swxY w Y   d S )Nz'Expected 2D array, got 1D array insteadr   )r   r   r   r   r  s     rL   test_coverage_1d_error_messager    s     
z)S	T	T	T ( (vw'''( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( (r  c            	         t          t          ddggddgg          d           t          t          ddggddgg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g d	gg dg          d           t          t          g d
gg dg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          ddggddgg          d           t          t          ddggddgg          d           t          t          ddggddgg          d           t          t          ddggddgg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g dg dgg dg dg          d           t          t          g dg dg d	gg dg dg dg          d           t          t          g dg dg d	gg dg dg dg          d           d S )Nr   r.   r   r   r   r  r   r   r   r   r]   r   r   r  r  r  r  r  r  r  r@  r   r   r   r   rN   rL   test_label_ranking_lossr	    s   *QF8tTl^DDaHHH*QF8tTl^DDaHHH*III;9J9J9J8KLLaPPP*III;9J9J9J8KLLeTTT*III;9J9J9J8KLLaPPP*III;9J9J9J8KLLeTTT*III;9J9J9J8KLLeTTT*III;9J9J9J8KLLeTTT *QF8tTl^DDaHHH*QF8tTl^DDaHHH*QF8sCj\BBAFFF*QF8sCj\BBAFFF*III;9J9J9J8KLLaPPP*III;9J9J9J8KLLaPPP*III;9I9I9I8JKKQOOO*III;9I9I9I8JKKQOOO IIIyyy1OOOYYY3OPP  
 YY			999-yyy)))/T	
 	
 	"	   YY			999-yyy)))/T	
 	
 	"	    rN   csr_containerc                     t          t           | t          j        g dg dg                    g dg dg          d           d S )Nr   r   r  r   r   )r   r   r3   r   )r
  s    rL   test_label_ranking_loss_sparser    sc    M"(IIIyyy#9::;;mmmYYY=W	
 	
 		    rN   c                     t          j        t                    5  t          ddgddggddg           d d d            n# 1 swxY w Y   t          j        t                    5  t          ddgddggddgg           d d d            n# 1 swxY w Y   t          j        t                    5  t          ddgddggdgdgg           d d d            n# 1 swxY w Y   t          j        t                    5  t          ddggddgddgg           d d d            n# 1 swxY w Y   t          j        t                    5  t          dgdggddgddgg           d d d            n# 1 swxY w Y   t          j        t                    5  t          ddgddggdgdgg           d d d            d S # 1 swxY w Y   d S )Nr   r.   )r   r   r   r   r   rN   rL   $test_ranking_appropriate_input_shaper    sE   	z	"	" 5 5QFQF+aV4445 5 5 5 5 5 5 5 5 5 5 5 5 5 5	z	"	" 7 7QFQF+q!fX6667 7 7 7 7 7 7 7 7 7 7 7 7 7 7	z	"	" 9 9QFQF+qcA3Z8889 9 9 9 9 9 9 9 9 9 9 9 9 9 9	z	"	" 7 7QF8q!fq!f%56667 7 7 7 7 7 7 7 7 7 7 7 7 7 7	z	"	" 9 9QC!:AA'78889 9 9 9 9 9 9 9 9 9 9 9 9 9 9	z	"	" 9 9QFQF+qcA3Z8889 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9si   ?AA#B		BB-CCC8DD"%D"E))E-0E-F55F9<F9c                  F   t          t          ddggddgg          d           t          t          ddggddgg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g dgg dg          d           t          t          g d	gg dg          d           t          t          g d
gg dg          d           d S )Nr.   r   r   r   r  r   r   r   r   r   r  r   rN   rL   test_ranking_loss_ties_handlingr    sX   *QF8sCj\BBAFFF*QF8sCj\BBAFFF*III;9I9I9I8JKKUSSS*III;9I9I9I8JKKUSSS*III;9I9I9I8JKKQOOO*III;9I9I9I8JKKQOOO*III;9I9I9I8JKKQOOO*III;9I9I9I8JKKQOOOOOrN   c                      t          dd          \  } }| dz   }t          ||           t          j                            d                              d          \  }}t          ||           d S Nr   rM  )r-   r  r.   )r'   r   rM  )r   _test_dcg_score_forr3   r7   r8   random_samplerx   rK   rI   s      rL   test_dcg_scorer    sp    .ALLLIAvgkG(((i++A..<<\JJOFG(((((rN   c                 :   t          j        t          j        | j        d                   dz             }t	          | |           }t	          | |          }||k                                    sJ t	          | | d          |k                                    sJ |j        | j        d         fk    sJ |j        | j        d         fk    sJ |t          j        t          j        |           d d d d df         |z  	                    d                    k    sJ d S )Nr.   r'   r   kr   rP   axis)
r3   log2r4   r2   r   r}  r   r)  sortrS   )rK   rI   discountidealrb   s        rL   r  r    s   wrya11A566Hvv..Evw//EUN!!!!!vv333u<AACCCCC;6<?,,,,,;6<?,,,,,FM276??111ddd7#;h#F"K"KQR"K"S"STTTTTTTTrN   c            	         t          j        t          j        d          g          } t          j        | j                  }t          | |          }t          | |d          }dt          j        t          j        dd                    z  }|t          j        |	                                | 
                                z  g          k    sJ |t          j        || d d d d df         z  	                                g          k    sJ d|dd	d f<   t          | |          }t          | |d          }|t          j        || d d d d df         z  	                                g          k    sJ |t          j        |d d         	                                | dd	d f         
                                z  |dd          	                                | dd d	f         
                                z  z   g          k    sJ d S )
Nr   Tignore_tiesr.   r'   r   rP   r   r   )r3   r9  r4   r   r2   r   r  r   r)  rS   rq  )rK   rI   dcgdcg_ignore_ties	discountss        rL   test_dcg_tiesr&  '  s   Z1''Fhv|$$G
VW
-
-C(dKKKOBGBIaOO,,,I&-6;;==!@ ABBBBBBfmi&DDbD/.I-N-N-P-P,QRRRRRRGAqrrEN
VW
-
-C(dKKKOfmi&DDbD/.I-N-N-P-P,QRRRRRR&-bqbM&ABB-"4"4"6"66m!!F1bqb5M$6$6$8$889	
       rN   c            	          t          j        d                              d          } t          | | dd          t	          j        t          | | dd                    k    sJ d S )N   )r'   r   r   T)r  r"  )r3   r4   rR   r   r   r)  )r   s    rL   test_ndcg_ignore_ties_with_kr)  ;  sq    
	"f%%AaaT222fm1a1$///7 7      rN   c                      t          j        g dg          } t          j        g dg          }d}t          j        t          |          5  t          | |           ddd           dS # 1 swxY w Y   dS )zDCheck `ndcg_score` exception when `y_true` contains negative values.)g{Gzg(\gGz޿g(\?gQ?)gQ?gףp=
?r   gQ?gHzG?z7ndcg_score should not be used on negative y_true valuesr   N)r3   r   r   r   r   r   )rK   rI   r   s      rL    test_ndcg_negative_ndarray_errorr+  B  s    X8889::Fh666788GP	z)9	:	:	: $ $67###$ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $s   A**A.1A.c                     t          j        d                              dd          } | t           j                            d                              dd| j                  z   }t          | |          }t          | |d	          }|t          j	        |          k    sJ |t          j	        d
          k    sJ |dz  }t          | |          t          j	        d
          k    sJ d S )NF   r   rM  r   皙ɿr   r   Tr!  r]   r0  )
r3   r4   rR   r7   r8   r  r2   r   r   r)  )rK   rI   ndcgndcg_no_tiess       rL   test_ndcg_invariantr1  K  s    Yr]]""1b))Fry,,Q//77c7UUUGfg&&Dfg4@@@L6=......6=%%%%%%tOGfg&&&-*<*<<<<<<<rN   r"  c           
      v   dt          j        d          d d         z  }t          j        t          j        ddd          d          }|t           j                            d                              dd	|j        
          z   }t          |||           t          j
        dt          j        t          j        dd                    z            k    sJ t          |||           t          j
        dt          j        t          j        dd                    z            k    sJ t          |||           t          j
        dt          j        t          j        dd                    z            k    sJ t          ||d|           t          j
        dt          j        t          j        dd                    z            k    sJ t          |||           t          j
        dt          j        t          j        dd                    z                                            k    sJ t!          |||           t          j
        dt          j        t          j        dd                    z                                            k    sJ dt          j        d          z  }dt          j        t          j        dd                    z                                  }t          |||           t          j
        |t          j        d          z            k    sJ t          |||           t          j
        t          j        d                    k    sJ t!          |||           t          j
        |          k    sJ t          |||           t          j
        d          k    sJ d S )Nr   r   r   r   rP   )r   r.   r   r.  r   r   r!  r'   r.   rM  )log_baser"  )r   r   	   r]   )r3   eyetiler4   r7   r8   r  r2   r   r   r)  r  r   log10r   rq  r   r   rS   )r"  rK   rI   y_score_noisyexpected_dcg_scores        rL   test_ndcg_toy_examplesr:  V  s   2A2Fgbi2r**F33Gbi33A66>>c ?   M [  	q2729Q??333	4	45 5 5 5 ;  	q2729Q??333	4	45 5 5 5 [  	q2729Q??333	4	45 5 5 5 "+  	q28BIaOO444	5	56 6 6 6 fg;???6=	
RWRYq!__%%	%++--D D     VW+>>>&-	
RWRYq!__%%	%++--C C      Fbgbi1oo666;;==[  	)BGAJJ6	7	78 8 8 8 [  	rwqzz	"	"# # # # VW+>>>&-C C     fg;???6=QTCUCUUUUUUUrN   c                      d} t          j        t          |           5  t          dggdgg           ddd           dS # 1 swxY w Y   dS )zhCheck that we raise an informative error message when trying to
    compute NDCG with a single document.zTComputing NDCG is only meaningful when there is more than 1 document. Got 1 instead.r   r.   N)r   r   r   r   )rS  s    rL   test_ndcg_error_single_documentr<  }  s    	  
z	1	1	1 ! !QC5A3%   ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! !s   A  AAc                      t          dd          \  } }| dz   }t          ||           t          j                            d                              d          \  }}t          ||           d S r  )r   _test_ndcg_score_forr3   r7   r8   r  r  s      rL   test_ndcg_scorer?    sp    .ALLLIAvgkG)))i++A..<<\JJOFG)))))rN   c                 J   t          | |           }t          | |          }||k                                    sJ | dk                        d          }||          t          j        t	          j        |                                                     k    sJ ||         t          j        t	          j        |                                                    k    sJ ||          t          j        t          | |          |          t          | |           |          z            k    sJ ||         t          j        t	          j        |                                                    k    sJ |j	        | j	        d         fk    sJ |j	        | j	        d         fk    sJ d S )Nr   r.   r  )
r   r}  r   r)  r3   r   rS   r   r   r2   )rK   rI   r  rb   all_zeros        rL   r>  r>    s   //E00EUN!!!!!!  a ((H(v}RWxi__5F5F-G-GHHHHHH?fmBHX\\^^,D,DEEEEEE(v}67++XI6
VV
,
,hY
7	8        ?fmBHX\\^^,D,DEEEEEE;6<?,,,,,;6<?,,,,,,,rN   c            	         t          j        g d          } t          | | d          dk    sJ t          | | d          dk    sJ t          j        t
                    5  t          | | d          sJ 	 d d d            n# 1 swxY w Y   t          j        t
                    5  t          | | d          sJ 	 d d d            n# 1 swxY w Y   t          j        t
                    5  t          | | d          sJ 	 d d d            n# 1 swxY w Y   t          j        g d          }t          | |d          }t          | |          }||k    sJ t          | |d	          d
k    sJ t          d          \  } }}t          j        ddd          D ]1}t          t          | ||          t          | ||                     2d S )Nr  r.   )ru   gMbP?gg?r   )r   r   r   g{Gz?r   r   Tr   g-C6?r   )
r3   r   r   r   r   r   rM   linspacer   r   )rK   r  roc_auc_with_max_fpr_oneunconstrained_roc_aucrJ   rx   ru   s          rL   test_partial_roc_auc_scorerF    s   Xlll##F333q88887771<<<<	z	"	" ; ;VVT:::::::; ; ; ; ; ; ; ; ; ; ; ; ; ; ;	z	"	" : :VVS9999999: : : : : : : : : : : : : : :	z	"	" 8 8VVQ77777778 8 8 8 8 8 8 8 8 8 8 8 8 8 8 x+++,,H,VXqIII)&(;;#'<<<<<37773>>>>'t444FFA;tQ** 
 
&&':::"667;;	
 	
 	
 	

 
s6    BB	B&CCC,DDDzy_true, k, true_scorer   r'   r   r   c                     t          j        g dg dg dg dg          }t          | ||          }|t          j        |          k    sJ d S )Nr  r   r   r   r   r   r   r   r   r   r   r   r   r  r3   r   r   r   r)  )rK   r  
true_scorerI   rb   s        rL   test_top_k_accuracy_scorerM    su     h            		
 G !A666EFM*--------rN   zy_score, k, true_score)rP   rP   r.   r.   )rP   r.   rP   r.   )r   r   r   r   )r   r   r   r   c                 V   g d}|                                  dk    r|                                 dk    rdnd}|dk    r#| |k                        t          j                  n|}t          || |          }t          ||          }||cxk    rt          j        |          k    sn J d S )Nr  r   r.   r   r  )	minmaxastyper3   int64r   r	   r   r)  )rI   r  rL  rK   rl   rJ   rb   	score_accs           rL    test_top_k_accuracy_score_binaryrT    s     \\F{{}}))gkkmmq.@.@aI78Avvg	!))"(3336F A666Evv..II::::z!:!:::::::::rN   zy_true, true_score, labels)r   r.   r.   r'   )r   r.   r.   r.   r  )r   rA  rA  r   )r   r   r  rA  labels_as_ndarrayc                     |rt          j        |          }t          j        g dg dg dg dg          }t          | |d|          }|t	          j        |          k    sJ dS )z,Test when labels and y_score are multiclass.r  rH  rI  rJ  r'   r  r   N)r3   r9  r   r   r   r)  )rK   rL  r   rU  rI   rb   s         rL   0test_top_k_accuracy_score_multiclass_with_labelsrX    s      $F##h            		
 G !AfEEEEFM*--------rN   c                     t          j        dddd          \  t          d          \  } }}}t          d                              | |           t          | |f||f          D ]O\  fdt          dd          D             }t          j        t          j	        |          dk              sJ Pd S )NrM  r0  r   )r  rC   n_informativer-   )r-   c                 Z    g | ]'}t                                        |           (S )r  )r   r=   )r   r  rA   rH   rB   s     rL   r   z8test_top_k_accuracy_score_increasing.<locals>.<listcomp>  sD     
 
 
CD C$5$5a$8$8A>>>
 
 
rN   r'   )
r   make_classificationr   r   r<   rZ  r_   r3   r}  r   )X_trainX_testy_trainy_testscoresrA   rH   rB   s        @@@rL   $test_top_k_accuracy_score_increasingrb    s    'BQ  DAq (811'M'M'M$GVWf
!
,
,
,CGGGWWf%'899 + +1
 
 
 
 
 
HMaQS
 
 
 vbgfoo)******+ +rN   c                     t          j        g dg dg dg dg          }t          | ||          t          j        |          k    sJ d S )N)r   r   r   r   )r.   r   r   r   )r   r   r   r   r  r  rK  )rK   r  rL  rI   s       rL   test_top_k_accuracy_score_tiesrd    sh     hLLLLLLLL		
 G  1555z9R9RRRRRRRrN   z	y_true, kr   c                     t          j        g dg dg dg dg          }d}t          j        t          |          5  t          | ||          }d d d            n# 1 swxY w Y   |dk    sJ d S )	Nr  )r   r   r   r   )r   r   r   r   )r   r   r   r   zu'k' \(\d+\) greater than or equal to 'n_classes' \(\d+\) will result in a perfect score and is therefore meaningless.r   r  r.   )r3   r   r   r   r   r   )rK   r  rI   r   rb   s        rL   !test_top_k_accuracy_score_warningrf  +  s     h            		
 G	6  
,4D	E	E	E ; ;$VW:::; ; ; ; ; ; ; ; ; ; ; ; ; ; ;A::::::s   AA!$A!zy_true, y_score, labels, msg)r   g=
ףp=?r.   r'   )r   r   r   )r   r   r   r   )r   r   r   z9y type must be 'binary' or 'multiclass', got 'continuous'zZNumber of classes in 'y_true' \(4\) not equal to the number of classes in 'y_score' \(3\).)r   r   r   r   z"Parameter 'labels' must be unique.z#Parameter 'labels' must be ordered.)r   r   r.   r'   zSNumber of given labels \(4\) not equal to the number of classes in 'y_score' \(3\).r  z3'y_true' contains labels not in parameter 'labels'.)r   r   r   )r   r   r   z}`y_true` is binary while y_score is 2d with 3 classes. If `y_true` does not contain all the labels, `labels` must be providedc                     t          j        t          |          5  t          | |d|           d d d            d S # 1 swxY w Y   d S )Nr   r'   rW  )r   r   r   r   )rK   rI   r   rC  s       rL   test_top_k_accuracy_score_errorrh  D  s    f 
z	-	-	- B BVW&AAAAB B B B B B B B B B B B B B B B B Bs   =AAc                      | g dg dg          }t          j        g dg dg          }t          ||          }|t          j        d          k    sJ d S )Nr   r   )r   r   r   r  )r3   r   r   r   r)  )r
  rK   rI   results       rL   Otest_label_ranking_avg_precision_score_should_allow_csr_matrix_for_y_true_inputrk    sl     ]IIIyyy122Fh344G267CCFV]5))))))))rN   metricr7  ))FT)r   r.   )r   r]   )zeroonec                 $   t           j                            d          }d|d         }}|                    ||d          }|                    |          } | |||          }t          |t                    rt          j        |          rJ dS |\  }}	}
t          j        |                                          rJ t          j        |	                                          rJ t          j        |
                                          rJ dS )zCheck that the metric works with different types of `pos_label`.

    We can expect `pos_label` to be a bool, an integer, a float, a string.
    No error should be raised for those types.
    *   rM  rP   T)r   replacer  N)	r3   r7   r8   choicerH  
isinstancerT   isnanany)rl  r7  rF   rC   rV   rK   y_probarj  metric_1metric_2r   s              rL   #test_ranking_metric_pos_label_typesry    s
    )


#
#Cwr{yIZZiZ>>Fhhy!!GVFGy999F&%   .8F#######)/&(J8H%%))+++++8H%%))+++++8J''++-------rN   c                     t           j                            |           }|                    ddd          }|                    d          }t          ||          \  }}}t          j        |d                   sJ dS )zCheck that thresholds do not exceed 1.0 when `y_score` is a probability
    estimate.

    Non-regression test for:
    https://github.com/scikit-learn/scikit-learn/issues/26193
    r   r'   rM  r   N)r3   r7   r8   r   rH  r   isinf)r;  rF   rK   rI   rx   r   s         rL   (test_roc_curve_with_probablity_estimatesr|    sv     )

 2
3
3C[[AB[''FhhrllG 11Aq*8JqM"""""""rN   c                  J   t          d          \  } }}d}t          j        t          |          5  t	          | |           ddd           n# 1 swxY w Y   d}t          j        t          |          5  t	          | ||           ddd           dS # 1 swxY w Y   dS )	z)Check the message for future deprecation.Tr   z)probas_pred was deprecated in version 1.5r   )probas_predNz4`probas_pred` and `y_score` cannot be both specified)r~  rI   )rM   r   r   FutureWarningr   r   r   )rK   rx   rI   warn_msg	error_msgs        rL   /test_precision_recall_curve_deprecation_warningr    sH    )555FAw:H	m8	4	4	4 
 
	
 	
 	
 	

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 GI	z	3	3	3 
 
	
 	
 	
 	

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
s#   AAA8BBB)NF)r   rG  r   )r   rQ  numpyr3   r   scipyr   sklearnr   r   sklearn.datasetsr   sklearn.exceptionsr   sklearn.linear_modelr   sklearn.metricsr	   r
   r   r   r   r   r   r   r   r   r   r   r   sklearn.metrics._rankingr   r   sklearn.model_selectionr   sklearn.preprocessingr   sklearn.random_projectionr   sklearn.utils._testingr   r   r   r   r   r    sklearn.utils.extmathr!   sklearn.utils.fixesr"   sklearn.utils.validationr#   r$   r%   CURVE_FUNCSrM   r[   rf   rn   r   markparametrizer   r   r   r   r   r   r   r   r   r   r   r   r   r  r  r%  r-  r?  rD  rK  rT  rW  r`  r  rf  rl  rw  rp  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r_   r  r  r  r  r  r	  r  r  r  r  r  r&  r)  r+  r1  r:  r<  r?  r>  rF  rM  rT  rX  rb  rd  rf  rh  rk  ry  r|  r  r   rN   rL   <module>r     s   				             ! ! ! ! ! ! ! ! ; ; ; ; ; ; 5 5 5 5 5 5 3 3 3 3 3 3                              M L L L L L L L 4 4 4 4 4 4 0 0 0 0 0 0 ; ; ; ; ; ;                * ) ) ) ) ) . . . . . .          )# )# )# )#`2 2 2  <  &H H H8 $//
) 
) 0/
)
" 
" 
") ) )&# # #) ) )) ) )8) ) )6eN eN eNPM M M	) 	) 	). . .&  " 	,,,		+	,,,		&			/			t$	 /X /X /Xd 	,,,		+	&&&	'	'9   : 	,,,		&			t$	yyy!			/	   D      D: : :( 	,hbh|||.D.DiiiP/BH)))**OO	
 BH\\\""	
 1BH)))**OO	
 BH\\\""F	
 BH)))**#J	
 BH\\\""LL	
 BH)))**   	
 ABH)))**OO	
 ABH)))**OO	
 ABH\\\""II	
}CF FN 77P P 87OF FPP ' "%88	
' "%88	
F "B77	
 "c22	
 
4R8=" "F	0 	0G" "F	0* * *@ {33# # 43# {33E E 43E2 -((((>	'JKK(9::
' 
' ;: LK )(
' {33	& 	& 43	& $//    0/ >) ) ), $//` ` 0/`F6 6 6D< < <= = =
= 
= 
== = =*$ $ $2 .	KKK!qc*	NNNQC!-	NNNQC!-	%%%sQC0	KKK#,	NNNSEA3/	NNNSEA3/	%%%uqc2	KKK#,	NNNQC!-	NNNQC!-	%%%sQC0	KKKiii8	NNNKKK;	NNNKKK;	%%%{{{III>	KKKMMM:	NNNIII}}}=	NNNIII}}}=	%%%yyy---@) 2' '3 2' .
Q#saS1#&
Q#saS1#&	$$$seaS1	$$$seaS1	$$$qcA3/	$$$qcA3/	$$$qcA3/	$$$qcA3/	 ' ' '   $E$E$EFF* * GF* 																 	 	 	 
Q?@	QH?@	KKK!CD	KKK!CD...OO(	

 " " "> > >48S 8S 8Sv  .1 1 14X X X  6# # #N 9:3 3 3 3@ 3)	  "G!RSS  TS C C C m44j11q22  32 21 54  *4 4 4nL L L  	$$$%	%%%&'
))&&&' ( ( (( ( (V .99  :99 9 9 	P 	P 	P) ) )U U U  (  $ $ $= = = u66#V #V 76#VL! ! !* * *- - - 
 
 
4 	q$	q#	q$ 
. 
. 
. 	...	!	!1a(	...	!	!1c*	...	!	!1a(	&&&	'	'A.	&&&	'	'C0	&&&	'	'A.
 
	; 	;
 
	;  	,,,		|||4	,,,		lll3	,,,		lll3	&&&	'	'/C/C/CD	  ,tUm<<. . =< .&+ + +& 	q$	q#	q! 
S 
S 
S 	q	q   $ " OO	 G
	
 LL	 +	
 !  	 !  0
	
 !  	 OO1
	
 LL	 LL 	
 LL	 IIA
	
 F__ooo.W	
KNQ QdB BeQ QdB
 .99* * :9* &	3I9U  CCC . .  .(# # #
 
 
 
 
rN   