
    Ug[                    ,   d Z ddlZddlZddlZddlZddlZddlmZ ddlm	Z	 ddl
mZ ddlZddlZddlmZ ddlmZmZ ddlmZ dd	lmZmZmZmZmZmZ dd
lmZ ddlm Z  ddl!m"Z" ddl#m$Z$m%Z%m&Z&m'Z'm(Z( ddl)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1m2Z2 ddl3m4Z4 ddl5m6Z6m7Z7m8Z8m9Z9m:Z:m;Z;m<Z<m=Z=m>Z>m?Z?m@Z@mAZAmBZBmCZCmDZD ddlEmFZFmGZGmHZH ddlImJZJ ddlKmLZL ddlMmNZN ddlOmPZP ddlQmRZR ddlSmTZT ddlUmVZVmWZW ddlXmYZYmZZZ ddl[m\Z\m]Z]m^Z^m_Z_m`Z` ddlambZb ddlcmdZdmeZe ddlfmgZgmhZhmiZimjZj ddlkmlZlmmZm ddlnmoZo  G d  d!e          Zp G d" d#ep          Zq G d$ d%e          Zr G d& d'er          Zs G d( d)          Zt eju        d*          Zv ejw        g d+          Zx ejw        g d,          Zy ejz        d-          Z{ej|        }                    d.el          d/             Z~d0 Zd1 Zd2 Zej|        }                    d3d4d5g          ej|        }                    d6em          d3efd7                        Zd8 Zd9 Zd: Zej|                            d;          d<             Zd= Zd> Zej|        }                    d.el          d?             Zd@ ZdA ZdB Zej|        }                    d.el          dC             ZdD ZdE ZdF ZdG Zej|        }                    d.el          dH             ZdI ZdJ ZdK Zej|        }                    d.el          dL             Zej|                            d;          dM             ZdN ZdO Zej|        }                    d.el          dP             ZdQ ZdR ZdS ZdT ZdU ZdV ZdW ZdX ZdY ZdZ Zd[ Zd\ Zd] Zd^ Zd_ Zd` Zda Zdb Zej|        }                    d6em          dc             Zdd Zde Zdf Zdg Zdh Zdi Zdj Zdk Zdl Z G dm dne          Zdo Zdp Zdq Zdr Zds Zdt Zej|                            d;          du             Zdv Zdw Z G dx dye          Zej|        }                    dzej        dg          d{             Zej|        }                    dzej        dg          d|             Zd} Zej|                            d~          ej|        }                    dzej        ddg          d                         Zej|                            d~          ej|        }                    dzej        ddg          ej|        }                    dd5d4g          ej|        }                    dd4d5g          d                                                 Zd Zej|        }                    dd4eddddfd5 e4eed          ddddfd4 e4eed          ddddfg          d             Zd Zd Zd Zd Zd Zd Zej|                            d          ej|        }                    deAe@e?g          d                         Zej|                            d          ej|        }                    deAe@e?g          d                         Zej|                            d          ej|        }                    deAe@e?g          d                         ZdS )zTest the validation module    N)partial)StringIO)sleep)issparse)BaseEstimatorclone)KMeans)load_diabetesload_digits	load_irismake_classificationmake_multilabel_classificationmake_regression)RandomForestClassifier)FitFailedWarning)SimpleImputer)LogisticRegressionPassiveAggressiveClassifierRidgeRidgeClassifierSGDClassifier)	accuracy_scorecheck_scoringconfusion_matrixexplained_variance_scoremake_scorermean_squared_errorprecision_recall_fscore_supportprecision_scorer2_score)_MultimetricScorer)GridSearchCV
GroupKFoldGroupShuffleSplitKFoldLeaveOneGroupOutLeaveOneOutLeavePGroupsOutShuffleSplitStratifiedKFoldcross_val_predictcross_val_scorecross_validatelearning_curvepermutation_test_scorevalidation_curve)_check_is_permutation_fit_and_score_score)OneTimeSplitter)FailingClassifier)OneVsRestClassifier)KNeighborsClassifier)MLPRegressor)Pipeline)LabelEncoderscale)SVC	LinearSVC)ConsumingClassifierConsumingScorerConsumingSplitter	_Registrycheck_recorded_metadata)shuffle)CheckingClassifierMockDataFrame)assert_allcloseassert_almost_equalassert_array_almost_equalassert_array_equal)COO_CONTAINERSCSR_CONTAINERS)_num_samplesc                   4    e Zd ZdZd ZddZd Zd	dZd ZdS )
MockImprovingEstimatorz+Dummy classifier to test the learning curvec                 0    || _         d| _        d | _        d S Nr   )n_max_train_sizestrain_sizesX_subset)selfrQ   s     l/var/www/surfInsights/venv3-11/lib/python3.11/site-packages/sklearn/model_selection/tests/test_validation.py__init__zMockImprovingEstimator.__init__d   s    !2    Nc                 8    || _         |j        d         | _        | S rP   rS   shaperR   rT   rS   y_subsets      rU   fitzMockImprovingEstimator.fiti        #>!,rW   c                     t           NNotImplementedErrorrT   Xs     rU   predictzMockImprovingEstimator.predictn       !!rW   c                     |                      |          rdt          | j                  | j        z  z
  S t          | j                  | j        z  S )Ng       @)_is_training_datafloatrR   rQ   rT   rd   Ys      rU   scorezMockImprovingEstimator.scoreq   sP    !!!$$ 	Dt/0043IIII)**T-CCCrW   c                     || j         u S r`   rS   rc   s     rU   rh   z(MockImprovingEstimator._is_training_datax       DM!!rW   r`   NN	__name__
__module____qualname____doc__rV   r]   re   rl   rh    rW   rU   rN   rN   a   ss        55  
   
" " "D D D D" " " " "rW   rN   c                   2     e Zd ZdZd fd	Zd ZddZ xZS )!MockIncrementalImprovingEstimatorz*Dummy classifier that provides partial_fitNc                 f    t                                          |           d | _        || _        d S r`   )superrV   xexpected_fit_params)rT   rQ   r|   	__class__s      rU   rV   z*MockIncrementalImprovingEstimator.__init__   s1    *+++#6   rW   c                     | j         |v S r`   r{   rc   s     rU   rh   z3MockIncrementalImprovingEstimator._is_training_data   s    v{rW   c                    | xj         |j        d         z  c_         |d         | _        | j        rt	          | j                  t	          |          z
  }|r t          dt          |           d          |                                D ]c\  }}|| j        v rSt          |          t          |          k    r3t          d| dt          |           dt          |           d          bd S d S )Nr   zExpected fit parameter(s) z
 not seen.zFit parameter z has length z; expected .)	rR   rZ   r{   r|   setAssertionErrorlistitemsrL   )rT   rd   yparamsmissingkeyvalues          rU   partial_fitz-MockIncrementalImprovingEstimator.partial_fit   s.   AGAJ&1# 	$233c&kkAG $JgJJJ   %llnn  
U$222|8 8!!__8% 8% )9 9 9,u:M:M 9 9&21oo9 9 9  	 	 rW   r`   )rr   rs   rt   ru   rV   rh   r   __classcell__r}   s   @rU   rx   rx   |   sf        447 7 7 7 7 7
         rW   rx   c                   4    e Zd ZdZd	dZd Zd Zd
dZd ZdS )MockEstimatorWithParameterz-Dummy classifier to test the validation curve      ?c                 "    d | _         || _        d S r`   )rS   param)rT   r   s     rU   rV   z#MockEstimatorWithParameter.__init__   s    


rW   c                 8    || _         |j        d         | _        | S rP   rY   r[   s      rU   r]   zMockEstimatorWithParameter.fit   r^   rW   c                     t           r`   ra   rc   s     rU   re   z"MockEstimatorWithParameter.predict   rf   rW   Nc                 N    |                      |          r| j        n	d| j        z
  S )N   )rh   r   )rT   rd   r   s      rU   rl   z MockEstimatorWithParameter.score   s'    !33A66JtzzA
NJrW   c                     || j         u S r`   rn   rc   s     rU   rh   z,MockEstimatorWithParameter._is_training_data   ro   rW   )r   rp   rq   rv   rW   rU   r   r      ss        77     
" " "K K K K" " " " "rW   r   c                   (     e Zd ZdZ fdZd Z xZS )%MockEstimatorWithSingleFitCallAllowedz<Dummy classifier that disallows repeated calls of fit methodc                     t          | d          r
J d            d| _        t                                          ||          S )Nfit_called_zfit is called the second timeT)hasattrr   rz   r]   )rT   rS   r\   r}   s      rU   r]   z)MockEstimatorWithSingleFitCallAllowed.fit   sB    4//PP1PPPPww{{8X...rW   c                     t           r`   ra   rc   s     rU   re   z-MockEstimatorWithSingleFitCallAllowed.predict   rf   rW   )rr   rs   rt   ru   r]   re   r   r   s   @rU   r   r      sM        FF/ / / / /
" " " " " " "rW   r   c                   P    e Zd ZdZddZ	 	 	 	 	 	 	 	 	 ddZd Zd Zdd	Zdd
Z	dS )MockClassifierz-Dummy classifier to test the cross-validationr   Fc                 "    || _         || _        d S r`   aallow_nd)rT   r   r   s      rU   rV   zMockClassifier.__init__   s     rW   Nc           	         || _         || _        |	| _        |
 |
|            | j        r#|                    t          |          d          }|j        dk    r| j        st          d          |P|j        d         |j        d         k    s4J d	                    |j        d         |j        d                               ||j        d         t          t          j        t                              k    sMJ d	                    |j        d         t          t          j        t                                                    |Rd}|j        d         |j        d         k    s4J |	                    |j        d         |j        d                               |md	}|j        t          j        k    sVJ |	                    |j        d         |j        d
         t          j        d         t          j        d
                               | S )zThe dummy arguments are to test that this fit function can
        accept non-array arguments through cross-validation, such as:
            - int
            - str (this is actually array-like)
            - object
            - function
        N   zX cannot be dr   zKMockClassifier extra fit_param sample_weight.shape[0] is {0}, should be {1}zIMockClassifier extra fit_param class_prior.shape[0] is {0}, should be {1}zRMockClassifier extra fit_param sparse_sample_weight.shape[0] is {0}, should be {1}zUMockClassifier extra fit_param sparse_param.shape is ({0}, {1}), should be ({2}, {3})r   )	dummy_int	dummy_str	dummy_objr   reshapelenndim
ValueErrorrZ   formatnpuniquer   P)rT   rd   rk   sample_weightclass_priorsparse_sample_weightsparse_paramr   r   r   callbackfmts               rU   r]   zMockClassifier.fit   s   ( #""HTNNN= 	&		#a&&"%%A6Q;;t};_---$ &q)QWQZ777??Ev!'*AGAJ@ @ 877 "$Q'3ry||+<+<<<<))/0A!0Dc")TU,,FWFW)X)X =<<  +2  (-a0AGAJ>>>

$*1-qwqzA A>>> #6   %000#**"1%"1%

	3 3000 rW   c                 n    | j         r#|                    t          |          d          }|d d df         S )Nr   r   )r   r   r   rT   Ts     rU   re   zMockClassifier.predict  s6    = 	&		#a&&"%%AAwrW   c                     |S r`   rv   r   s     rU   predict_probazMockClassifier.predict_proba  s    rW   c                 @    ddt          j        | j                  z   z  S )N      ?r   )r   absr   rj   s      rU   rl   zMockClassifier.score
  s    a"&..())rW   c                      | j         | j        dS )Nr   r   )rT   deeps     rU   
get_paramszMockClassifier.get_params  s    V777rW   )r   F)	NNNNNNNNNrp   )F)
rr   rs   rt   ru   rV   r]   re   r   rl   r   rv   rW   rU   r   r      s        77! ! ! ! != = = =~  
  * * * *8 8 8 8 8 8rW   r   )
      )
r   r   r   r   r   r   r   r      r   )
r   r   r   r   r   r   r   r   r   r      coo_containerc                    t                      } | t                    }t          dd          D ]%}||_        t	          |t          t
                    }t          ||                    t          t
                               t          j	        t
          t
          d d d         g          }t	          |||          }t          ||                    ||                     t	          ||t
                    }t          ||                    |t
                               t	          |||          }t          ||                    ||                     'd }t          |          }t	          |t                                          t
                                          d          }t          |          }t	          |t          t
                                          d          }t          d d d d t          j        f         }t          d	
          }t	          ||t
                    }t          d
          }t          j        t                    5  t	          ||t
          d           d d d            d S # 1 swxY w Y   d S )Nir   r   c                 ,    t          | t                    S r`   
isinstancer   r   s    rU   <lambda>z&test_cross_val_score.<locals>.<lambda>3      :a.. rW   check_Xr   cvcheck_yT)r   Fraiseerror_score)r   rd   ranger   r,   y2rI   rl   r   column_stackrD   tolistnewaxispytestraisesr   )r   clfX_sparser   scoresmultioutput_y
list_checkX_3ds           rU   test_cross_val_scorer     sc   


C}QH3^^ G G a,,6399Q#3#3444 R"X77 h>>6399X}#E#EFFF h336399Xr#:#:;;; !h>>6399X}#E#EFFFF /.J
Z
0
0
0CS!((**biikka@@@F
Z
0
0
0CS!RYY[[Q777F QQQ2:D
$
'
'
'CS$++F
%
(
(
(C	z	"	" < <T27;;;;< < < < < < < < < < < < < < < < < <s   I<<J J c                      t          d          \  } }t          d          }t          |dddgi          }t          || |d	
           d S )NT
return_X_yauto)gammaCr   r   )
param_gridr   )n_jobs)r   r<   r"   r-   )rd   r   r   grids       rU   test_cross_validate_many_jobsr   D  s_     %%%DAq
F


Cq"g777D4Aa((((((rW   c            
         t          d          \  } }t                      }d}t          j        t          |          5  t          || |t          t                    t          t                    f           d d d            n# 1 swxY w Y   t          j        t          |          5  t          || |t          t                    f           d d d            n# 1 swxY w Y   t          j        t          |dz             5  t          || |d           d d d            n# 1 swxY w Y   t          j        t          |dz             5  t          || |d	           d d d            n# 1 swxY w Y   t          j        t          |          5  t          || |t          t                    gg           d d d            n# 1 swxY w Y   t          j        t          d
          5  t          || |t                                 d d d            n# 1 swxY w Y   t          t                    }dt          j         d}t          j        t          |          5  t          || ||           d d d            n# 1 swxY w Y   t          j        t          |          5  t          || |d|i           d d d            d S # 1 swxY w Y   d S )Nr   random_statez.*must be unique strings.*matchscoringzEmpty list.*rv   zDuplicate.*)f1_micror   zAn empty dict[Scoring failed. The score on this train-test partition for these parameters will be set to z. Details: 
foo)r   r   r   r   r   r-   r   r   r   dictr   r   nanwarnsUserWarning)rd   r   	estimatorerror_message_regexpmulticlass_scorerwarning_messages         rU   )test_cross_validate_invalid_scoring_paramr  N  s,   A...DAq  I 8 
z)=	>	>	> 
 
 11;~3N3NO		
 	
 	
 	

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
z)=	>	>	> Q Qy!Q_1M1M0OPPPPQ Q Q Q Q Q Q Q Q Q Q Q Q Q Q 
z)=)N	O	O	O 4 4y!Q33334 4 4 4 4 4 4 4 4 4 4 4 4 4 4 
z)=)M	N	N	N J Jy!Q0HIIIIJ J J J J J J J J J J J J J J 
z)=	>	>	> R Ry!Q+o2N2N1O0PQQQQR R R R R R R R R R R R R R R 
z	9	9	9 : :y!Q9999: : : : : : : : : : : : : : : $$CDD
	9;	 	 	  
k	9	9	9 C Cy!Q0ABBBBC C C C C C C C C C C C C C C 
k	9	9	9 L Ly!Q8I0JKKKKL L L L L L L L L L L L L L L L L Ls   :BB	B	+'CC"%C"D''D+.D+E00E47E4(G

GG0 HH #H &JJ
J
,KKKc                      t          d          \  } }t          dt                      fdt                      fg          }t	          || |d          }|d         }t          |t                    sJ t          d |D                       sJ d S )NTr   imputer
classifier)return_estimatorr   c              3   @   K   | ]}t          |t                    V  d S r`   )r   r9   ).0r   s     rU   	<genexpr>z7test_cross_validate_nested_estimator.<locals>.<genexpr>  s,      KK9z)X..KKKKKKrW   )r   r9   r   r   r-   r   r   all)rd   r   pipelineresults
estimatorss        rU   $test_cross_validate_nested_estimatorr    s     $'''FQ(>++,	
 H Xq!dCCCG%Jj$'''''KK
KKKKKKKKKKrW   
use_sparseFTcsr_containerc           
         t                      }t          dd          \  }}t          d          }t          dd          \  }}t	          dd          }| r ||          } ||          }|||f|||ffD ]\  }	}
}t          |d          }t          |d	          }g }g }g }g }g }|                    |	|
          D ]\  }}t          |                              |	|         |
|                   }|	                     |||	|         |
|                              |	                     |||	|         |
|                              |	                     |||	|         |
|                              |	                     |||	|         |
|                              |	                    |           t          j        |          }t          j        |          }t          j        |          }t          j        |          }t          j        |          }|||||f}t          ||	|
||           t          ||	|
||           d S )
N   r   	n_samplesr   r   linearkernelr   neg_mean_squared_errorr   r2)r%   r   r   r   r<   r   splitr   r]   appendr   array"check_cross_validate_single_metric!check_cross_validate_multi_metric)r  r  r   X_regy_regregX_clfy_clfr   rd   r   est
mse_scorer	r2_scorertrain_mse_scorestest_mse_scorestrain_r2_scorestest_r2_scoresfitted_estimatorstraintestr   s                         rU   test_cross_validater-    s    
B #Ra@@@LE5
Q


C '!DDDLE5
XA
.
.
.C %e$$e$$eS)E5#+>? $A $A	1c"30HIII
!#t444	88Aq>> 	* 	*KE4**..51U844C##JJsAeHah$G$GHHH""99S!E(AeH#E#EFFF""::c1T7AdG#D#DEEE!!))C4!D'"B"BCCC$$S))))8$455(?33(?33.11H%677 
 	+31fbAAA)#q!VR@@@@I$A $ArW   c           	          |\  }}}}}	dD ]\  }
}|
r,t          | ||dd|          }t          |d         |           nt          | ||dd|          }t          |t                    sJ t	          |          |k    sJ t          |d         |           |
r.t          | ||dgd|          }t          |d	         |d           nt          | ||dgd|          }t          |t                    sJ t	          |          |k    sJ t          |d
         |           t          | ||dd|          }t          |d                   D ]\  }}|j                                        }t          |          r|	                                }|	|         j                                        }t          |          r|	                                }t          ||           t          |j        |	|         j                   d S )N))Tr   )Fr   r  Tr   return_train_scorer   train_scoreF
test_scorer  train_r2test_r2)r   r  r   r   )r-   rH   r   r   r   	enumeratecoef_copyr   toarrayrG   
intercept_)r   rd   r   r   r   r&  r'  r(  r)  r*  r0  dict_lenmse_scores_dictr2_scores_dictkr#  est_coeffitted_est_coefs                     rU   r  r    s    	 )@ &M &M$H 	,0#'  O &om&DFVWWWW,0#(  O /400000?##x////!/,"?QQQ  		+QD6dr  N &nZ&@/SWXXXX+QD6e  N .$/////>""h....!.";^LLLL %Q3dr  O OK899 
M 
M39>>##H 	*''))H+A.499;;O$$ 	8-5577OHo666CN,=a,@,KLLLL
M 
MrW   c           
      @   |\  }}}}}	d }
dt          t                    dd|
f}h d}|                    ddh          }dD ]}|D ]}|rBt          | |||d|	          }t	          |d         |           t	          |d         |           nt          | |||d
|	          }t          |t                    sJ t          |                                          |r|n|k    sJ t	          |d         |           t	          |d         |           t          |d         t          j
                  sJ t          |d         t          j
                  sJ t          |d         t          j
                  sJ t          |d         t          j
                  sJ t          j        |d         dk              sJ t          j        |d         dk               sJ t          j        |d         dk              sJ t          j        |d         dk               sJ Ԑd S )Nc                 p    |                      |          }t          ||          t          ||           dS )Nr  r  )re   r    r   )r   rd   r   y_preds       rU   custom_scorerz8check_cross_validate_multi_metric.<locals>.custom_scorer  s=    Q1f%%'9!V'D'D&D
 
 	
rW   rB  r  >   r4  fit_time
score_timetest_neg_mean_squared_errorr3  train_neg_mean_squared_error)TFTr/  Fr4  rG  rE  rF  r   r   )r   r    unionr-   rH   r   r   r   keysr   ndarrayr	  )r   rd   r   r   r   r&  r'  r(  r)  r*  rD  all_scoringkeys_sans_trainkeys_with_trainr0  r   
cv_resultss                    rU   r  r    s    	
 
 
 	)h''&>	
 	
 	K  O &++	34 O , "9 "9" !	9 !	9G! +Aw4B  
 **Z*@/RRR)=>@P    ,Aw5R  
 j$/////z(())#5J?    &j&;^LLL%89?  
 j3RZ@@@@@j)FGTTTTTj4bjAAAAAj6
CCCCC 6*Z0A5666666*Z025666666*\2a7888886*\2R7888888C!	9"9 "9rW   c                     t          ddd          \  } }t          d          }t                      t          d          t	                      t                      g}d}|D ]}t          j        t          |          5  t          || ||	           d d d            n# 1 swxY w Y   t          j        t          |          5  t          || ||	           d d d            n# 1 swxY w Y   d S )
N   r   r   )r  	n_classesr   r  r  z*The 'groups' parameter should not be None.r   )r   rd   r   r   )r   r<   r&   r(   r#   r$   r   r   r   r,   r+   )rd   r   r   	group_cvserror_messager   s         rU   #test_cross_val_score_predict_groupsrV  Y  s}    qqIIIDAq
X


C 		I AM > >]:];;; 	< 	<cQ!;;;;	< 	< 	< 	< 	< 	< 	< 	< 	< 	< 	< 	< 	< 	< 	<]:];;; 	> 	>qA"====	> 	> 	> 	> 	> 	> 	> 	> 	> 	> 	> 	> 	> 	> 	>> >s$   >BB"	%B"	C$$C(	+C(	z(ignore: Using or importing the ABCs fromc                  6  	 t           t           fg} 	 ddlm}m} |                     ||f           n# t
          $ r Y nw xY w| D ]S\  	 t                     	t                    }}fd}	fd}t          ||          }t          |||d           Td S )Nr   	DataFrameSeriesc                 $    t          |           S r`   r   r{   InputFeatureTypes    rU   r   z-test_cross_val_score_pandas.<locals>.<lambda>}      Z+;<< rW   c                 $    t          |           S r`   r\  r{   
TargetTypes    rU   r   z-test_cross_val_score_pandas.<locals>.<lambda>~      Az!:!: rW   r   r   r   r   )
rE   pandasrY  rZ  r  ImportErrorrd   r   rD   r,   
typesrY  rZ  X_dfy_sercheck_dfcheck_seriesr   r^  rb  s
           @@rU   test_cross_val_score_pandasrm  o  s     ]+,E,,,,,,,,fi())))   (- 0 0$
$ '&q))::b>>e<<<<:::: <HHHT5Q/////0 0   3 
A A c                  $   t          d          } t                      }|j        |j        }}t	          d          }t          | |||          }t	          d          }g }|                    ||          D ]v\  }}t          j        t          |          t                    }	t          j        t          |          t                    }
d|	|<   d|
|<   |                    ||f           wt          | |||          }t          ||           d S )Nr  rS  r   r   dtyper   )r<   r   datatargetr%   r,   r  r   zerosr   boolr  rI   )svmirisrd   r   kfoldscores_indicescv_masksr+  r,  
mask_train	mask_testscores_maskss               rU   test_cross_val_score_maskr~    s   
X


C;;D9dkqA!HHE$S!Q5999N!HHEH{{1a(( ' 'tXc!ffD111
HSVV4000	
5	$&&&&"31:::L~|44444rW   c                     t          d          } t                      }|j        |j        }}t	          j        ||j                  }t          | ||          }t          d          } t          | ||          }t          ||           t          d           } t          | ||          }t          ||           t          d          } t          j
        t                    5  t          | ||           d d d            n# 1 swxY w Y   t          j
        t                    5  t          | |                                |           d d d            d S # 1 swxY w Y   d S )NprecomputedrS  r  c                 6    t          j        | |j                  S r`   )r   dotr   )r{   r   s     rU   r   z2test_cross_val_score_precomputed.<locals>.<lambda>  s    "&AC.. rW   )r<   r   rr  rs  r   r  r   r,   rH   r   r   r   r   )rv  rw  rd   r   linear_kernelscore_precomputedscore_linearscore_callables           rU    test_cross_val_score_precomputedr    s   
]
#
#
#C;;D9dkqAF1acNNM']A>>
X


C"31--L/>>> 00
1
1
1C$S!Q//N/@@@ ]
#
#
#C	z	"	" # #Q"""# # # # # # # # # # # # # # #
 
z	"	" 8 8]1133Q7778 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8s$   $DD	D&$EEEc           	      8  	
 t                      }t          j        d         }t          t	          j        t                              } | t	          j        dg          t	          j        dg          t	          j        dg          ffd          } | t	          j        d                    }dd
t                      		
fd}t	          j
        |          t	          j        |d	|z            ||
	|d
}t          |t          t          |           d S )Nr   r   )r   r   rZ   r   *   42c                 V    | j         k    sJ | j        k    sJ | j        k    sJ d S r`   )r   r   r   )r   	DUMMY_INT	DUMMY_OBJ	DUMMY_STRs    rU   assert_fit_paramsz:test_cross_val_score_fit_params.<locals>.assert_fit_params  sD     }	))))}	))))}	))))))rW   r   )r   r   r   r   r   r   r   r   )r   )r   rd   rZ   r   r   r   r   r  eyeobjectonesfullr,   )r   r   r  rR  W_sparseP_sparser  
fit_paramsr  r  r  s           @@@rU   test_cross_val_score_fit_paramsr    s!   


C
IBIaLL!!I}	1#1#!67w  H }RVAYY''HIII* * * * * * * ++wy#	/:: ( %	 	J CAj111111rW   c                  4   t                      } g fd}t          j        d          5  t          |          }t	          | t
          t          |d          }d d d            n# 1 swxY w Y   t          |g d           t                    dk    sJ d S )Nc                 6                         | |f           dS )Nr   )r  )y_test	y_predict_score_func_argss     rU   
score_funcz3test_cross_val_score_score_func.<locals>.score_func  s!     3444srW   Trecordr   )r   r   )r   r   r   )	r   warningscatch_warningsr   r,   rd   r   rI   r   )r   r  r   rl   r  s       @rU   test_cross_val_score_score_funcr    s   


C     
	 	-	-	- B Bj))Q7qAAAB B B B B B B B B B B B B B B uooo...  A%%%%%%s   .A&&A*-A*c                  ^   t                      } t          d          }t          || j        | j                  }t          |g dd           t          || j        | j        d          }t          |g dd           t          || j        | j        d          }t          |g dd           d S )Nr  rS  )
ףp=
?r   r  r  r   r   accuracyr   f1_weighted)r   r<   r,   rr  rs  rH   )rw  r   r   	zo_scores	f1_scoress        rU   3test_cross_val_score_with_score_func_classificationr    s    ;;D
X


C S$)T[99Ff&B&B&BAFFF  TYZPPPIi)E)E)EqIII  TY]SSSIi)E)E)EqIIIIIrW   c                     t          dddd          \  } }t                      }t          || |          }t          |g dd           t          || |d	          }t          |g dd           t          || |d
	          }t	          j        g d          }t          ||d           t          t                    }t          || ||	          }t          |g dd           d S )Nr  rQ  r   r   )r  
n_featuresn_informativer   )gGz?r  r  gGz?gq=
ףp?r   r  r   r  )g(\؇gzGIgGz&qg\(qg)\G)r   r   r,   rH   r   r  r   r   )	rd   r   r   r   	r2_scoresneg_mse_scoresexpected_neg_mser   	ev_scoress	            rU   /test_cross_val_score_with_score_func_regressionr    s   RBaVWXXXDAq
''C S!Q''Ff&D&D&DaHHH  Q4888Ii)G)G)GKKK %S!Q8PQQQNx N N NOOn.>BBB 233GQ7;;;Ii)G)G)GKKKKKrW   c                    t                      }|j        } | |          }|j        }t          d          }t	          d          }t          |||d|d          \  }}}	|dk    sJ t          |	dd	           t          |||d|dt          j        |j	                  d
          \  }
}}|
|k    sJ ||	k    sJ t          d          }t	          d          }t          |||d|dt          j        |j	                  d
          \  }
}}|
|k    sJ ||	k    sJ d }t          |          }t          |||d||d
          \  }}}	t          |dd           t          |	dd           t          j        t          j        t          |                    d          }t          |||d|d          \  }}}	|dk     sJ |	dk    sJ d S )Nr  rS  r   r  r  )n_permutationsr   r   g?        r   r   )r  r   r   groupsr   c                     | |k                                     | |k                                     z
  | j        d         z  S rP   )sumrZ   )y_truerC  s     rU   custom_scorez,test_permutation_score.<locals>.custom_score>  s;    6!&&((Ff,<+A+A+C+CCv|TUVVrW   d   )r  r   r   r   g(\?g{Gz?r   r   皙?)r   rr  rs  r<   r*   r/   rG   r   r  sizer   modaranger   )r   rw  rd   r   r   rv  r   rl   r   pvaluescore_group_pvalue_group
svm_sparse	cv_sparser  scorers                    rU   test_permutation_scorer    sK   ;;D	A}QHA
X


C			B2Q"Z  E66 3;;;;Q'''#9		wqv	$ 	$ 	$ KL %6!!!! H%%%J""I#9	wqv	$ 	$ 	$ KL %6!!!!W W W &&F-Q#v"1  E1f tQ'''a((( 	ryQ  !$$A2Q"Z  E66 3;;;;C<<<<<<rW   c                  t   t          j        dt           j                                      dd          } t           j        | dd d f<   t          j        ddg| j        d         dz            }t          dt          d	t           j        
          fdt                      fg          }t          || |           d S N   rp  r   r   r   r   r   r  mean)strategymissing_valuesr  )r   r  float64r   r   repeatrZ   r9   r   r   r/   rd   r   ps      rU   &test_permutation_test_score_allow_nansr  S  s    
	#RZ(((00R88AfAadG
	1a&!'!*q.))AvbfMMMN>++,	
	 	A 1a#####rW   c            
      F   t          j        d                              dd          } t          j        dgdz  dgdz  z             }t	          d          }d}t          j        t          |	          5  t          || |           d d d            n# 1 swxY w Y   d
}t          j        t          |	          5  t          || |dt          j
        d          i           d d d            n# 1 swxY w Y   t          || |dt          j
        d          i           d S )Nr  r   r   r   r   Texpected_sample_weight#Expected sample_weight to be passedr   /sample_weight.shape == \(1,\), expected \(8,\)!r   )r  )r   r  r   r  rD   r   r   r   r/   r   r  rd   r   r   err_msgs       rU   &test_permutation_test_score_fit_paramsr  a  s   
	#r2&&A
!qA37"##A
D
9
9
9C4G	~W	5	5	5 * *sAq)))* * * * * * * * * * * * * * * AG	z	1	1	1 T TsAqorwqzz5RSSSST T T T T T T T T T T T T T T31/272;;1OPPPPPPs$   5BBB;(C//C36C3c                  t   t          j        dt           j                                      dd          } t           j        | dd d f<   t          j        ddg| j        d         dz            }t          dt          d	t           j        
          fdt                      fg          }t          || |           d S r  )r   r  r  r   r   r  rZ   r9   r   r   r,   r  s      rU   test_cross_val_score_allow_nansr  p  s    
	#RZ(((00R88AfAadG
	1a&!'!*q.))AvbfMMMN>++,	
	 	A Aq!rW   c                  P   t          j        ddgddgddgddgddgddgddgddgddgddgg
          } t          j        ddgddgddgddgddgddgddgddgddgddgg
          }t          d	          }t          t          d
          }t          t          d          }t          t          d          }t          || ||          }t          || ||          }t          || ||          }t          |g d           t          |g d           t          |g d           d S )Nr   r   r   r   r   r   )n_neighborsmicro)averagemacrosamplesr   )r   r         ?r   gUUUUUU?)r   r   r  r         ?)r   r  r7   r   r   r,   rG   )	rd   r   r   scoring_microscoring_macroscoring_samplesscore_microscore_macroscore_sampless	            rU   test_cross_val_score_multilabelr  ~  s   
GFFFGGFHHG	
	 	A 	
Q!Q!Q!Q!Q!Q!Q!Q!QRSUVQWX	 	A 1
-
-
-CAAAMAAAM!/9EEEO!#q!]CCCK!#q!]CCCK#CAGGGM%D%D%DEEE%D%D%DEEE'F'F'FGGGGGrW   c           
         t          d          \  }}t                      }t                      }t          j        |          }|                    ||          D ]E\  }}|                    ||         ||                    |                    ||                   ||<   Ft          ||||          }t          ||           t          |||          }t          |          t          |          k    sJ t                      }t          ||||          }t          |          t          |          k    sJ |                                }	|	|	t          j        |	          k    z  }	 | |	          }	t          ||	|          }t          t          |          t          |                     t          t          d          |          }t          |          t          |          k    sJ  G d d          }
t          j        t"                    5  t          ||| |
                       d d d            n# 1 swxY w Y   t%          d          \  }}d}t          j        t(          |	          5  t          t+          d
          ||dt          d                     d d d            d S # 1 swxY w Y   d S )NTr   r   r   )n_initc                       e Zd ZddZdS )%test_cross_val_predict.<locals>.BadCVNc              3      K   t          d          D ]0}t          j        g d          t          j        g d          fV  1d S )Nr   )r   r   r   r   )r   r            )r   r   r  )rT   rd   r   r  is        rU   r  z+test_cross_val_predict.<locals>.BadCV.split  sY      1XX H Hh|||,,bh.G.GGGGGGH HrW   rp   )rr   rs   rt   r  rv   rW   rU   BadCVr    s.        	H 	H 	H 	H 	H 	HrW   r  zNumber of classes in training fold \(2\) does not match total number of classes \(3\). Results may not be appropriate for your use case.r   	liblinearsolverr   r   methodr   )r
   r%   r   r   
zeros_liker  r]   re   r+   rH   r   r'   r7  medianr	   r   r   r   r   r   RuntimeWarningr   )r   rd   r   r   r#  preds2r+  r,  predsXspr  r   s               rU   test_cross_val_predictr    s<   D)))DAq	B
''C ]1Fxx1~~ , ,t%!E(###{{1T7++tc1aB///EeV,,,c1a((Eu::Q	Bc1aB///Eu::Q
&&((C33C
-

Cc3**Ec%jj#a&&111fF333Q77Eu::QH H H H H H H H
 
z	"	" 1 1#q!00001 1 1 1 1 1 1 1 1 1 1 1 1 1 1 %%%DAq	< 
 
nO	<	<	< 
 
k222"Qxx	
 	
 	
 	

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
s$   H<<I I 70J44J8;J8c            
      z   t          ddd          \  } }t          t          d          | |d          }|j        d	k    sJ t	          d
          \  } }t          t          d          | |d          }|j        dk    sJ | d d         } |d d         }d}t          j        t          |          5  t          t                      | |dt          d                     d d d            n# 1 swxY w Y   t          d
          \  } }t          dd          }t          || |d          }|j        dk    sJ t          j        |          }| |         ||         }} d}t          j        t          |          5  t          || |t          d          d           d d d            d S # 1 swxY w Y   d S )Nr   2   r   rR  r  r   r  r  decision_functionr  )r  Tr      r   r  zOnly 1 class/es in training fold, but 2 in overall dataset. This is not supported for decision_function with imbalanced folds. To fix this, use a cross-validation technique resulting in properly stratified foldsr   r  r  ovo)r  decision_function_shape)i  -   zOutput shape \(599L?, 21L?\) of decision_function does not match number of classes \(7\) in fold. Irregular decision_function .*r   n_splitsr   r  )r   r+   r   rZ   r   r   r   r   r   r%   r   r<   r   argsort)rd   r   r  rU  r#  indr   s          rU   .test_cross_val_predict_decision_function_shaper    s   bqIIIDAq+...1=P  E ;%%%%DAq+...1=P  E ;("""" 	
$3$A	$3$A	1  
z	7	7	7 
 
q!,?E!HH	
 	
 	
 	

 
 
 
 
 
 
 
 
 
 
 
 
 
 

 $'''DAq
Xu
=
=
=Cc1a0CDDDE;*$$$$
*Q--CS61S6qA	  
z)=	>	>	> W W#q!q(9(9(9BUVVVVW W W W W W W W W W W W W W W W W Ws$   6.C00C47C4 #F00F47F4c                     t          ddd          \  } }t          t          d          | |d          }|j        d	k    sJ t	          d
          \  } }t          t          d          | |d          }|j        dk    sJ d S )Nr   r  r   r  r  r  r   r  r  r   Tr   r	  r   r+   r   rZ   r   rd   r   r  s      rU   *test_cross_val_predict_predict_proba_shaper  	  s    bqIIIDAq+...1_  E ;'!!!!%%%DAq+...1_  E ;(""""""rW   c                     t          ddd          \  } }t          t          d          | |d          }|j        d	k    sJ t	          d
          \  } }t          t          d          | |d          }|j        dk    sJ d S )Nr   r  r   r  r  r  predict_log_probar  r  Tr   r	  r  r  s      rU   .test_cross_val_predict_predict_log_proba_shaper    s    bqIIIDAq+...1=P  E ;'!!!!%%%DAq+...1=P  E ;(""""""rW   c                 N   t                      }|j        |j        }} | |          }t          j        ||d d d         g          }t          dd          }t          |||          }|j        dk    sJ t          |||          }|j        dk    sJ t          |||          }t          |j        d           t          |||          }t          |j        d           d }t          |          }t          ||
                                |
                                          }t          |	          }t          |||
                                          }t          t          d
          |
                                |
                                d          }t          t          d
          ||
                                d          }|d d d d t          j        f         }	d }
t          |
          }t          ||	|          }t          |j        d           d S )Nr   Fr   )fit_interceptr   )r
  )r
  r   c                 ,    t          | t                    S r`   r   r   s    rU   r   z4test_cross_val_predict_input_types.<locals>.<lambda>B  r   rW   r   r   r  r  r  r  c                     | j         dk    S )Nr   )r   r   s    rU   r   z4test_cross_val_predict_input_types.<locals>.<lambda>Y  s    1 rW   )r   rr  rs  r   r   r   r+   rZ   rI   rD   r   r   r   )r   rw  rd   r   r   r   r   predictionsr   r   check_3ds              rU   "test_cross_val_predict_input_typesr"  )  s#   ;;D9dkqA}QHOQ$$B$L11M
e!
4
4
4C $CA..K&&&& $C=AAK((((#C155K{(&111 $C=AAK{((333 /.J
Z
0
0
0C#CQXXZZ@@K
Z
0
0
0C#CAHHJJ77K $+...	

	

"	  K $+...		

"	  K QQQ2:D$$H
X
.
.
.C#Cq11K{(&11111rW   c                  6  	 t           t           fg} 	 ddlm}m} |                     ||f           n# t
          $ r Y nw xY w| D ]S\  	 t                     	t                    }}fd}	fd}t          ||          }t          |||d           Td S )Nr   rX  c                 $    t          |           S r`   r\  r]  s    rU   r   z/test_cross_val_predict_pandas.<locals>.<lambda>m  r_  rW   c                 $    t          |           S r`   r\  ra  s    rU   r   z/test_cross_val_predict_pandas.<locals>.<lambda>n  rc  rW   rd  r   r   )
rE   re  rY  rZ  r  rf  rd   r   rD   r+   rg  s
           @@rU   test_cross_val_predict_pandasr&  _  s     ]+,E,,,,,,,,fi())))   (- 2 2$
$&&q))::b>>e<<<<:::: <HHH#tUq111112 2rn  c                     t          dddddd          \  } }d|d<   t          dd          }t          d          }t          |                    | |                    \  }}t          || ||d	
          }||d                  d         dk    sJ t          j        ||d                  d d df         dk              sJ t          j        ||d                  d d ddf         dk              sJ t          j        ||d                  dk              sJ t          |	                    d          t          j
        |j                  d           d S )Nr  r   r   r   )r  r  n_redundantr  n_clusters_per_classr   r  r   r  r  r   r  axis   )decimal)r   r   r*   r   r  r+   r   r	  rH   r  r  rZ   )rd   r   r   r   r+  r,  
yhat_probas          rU   !test_cross_val_predict_unbalancedr0  s  sk     DAq AaD
!K
@
@
@C	!	$	$	$Brxx1~~&&KE4"31OLLLJT!W:a=A6*T!W%aaad+q0111116*T!W%aaa1f-1222226*T!W%)*****jnn!n44bgag6F6FPRSSSSSSrW   c                  0   t                      } t          j                            d          }|                    dd          }t          | |d dd          }t          |d d df         |           t          | |d dd          }t          ||           d S )	Nr  r  r   r   re   )r   r   r  r   r   )r   r   randomRandomStaterandr+   rF   )mock_classifierrngrd   y_haty_hat_probas        rU   test_cross_val_predict_y_noner9    s    $&&O
)


#
#CbAoqDQyQQQEAaaadGU####dq  K A{#####rW   c                    t                      }|j        |j        }}t                      }d | t	          j        |j        d                             i}t          ||||d          }t          |t	          j	        d                     d S )Nr   r   r   )r   r   )
r   rr  rs  r   r   r  rZ   r,   rI   r  )r   rw  rd   r   r   r  r   s          rU   &test_cross_val_score_sparse_fit_paramsr;    s}    ;;D9dkqA


C(--qwqz8J8J*K*KLJQ*;;;Aq"'!**%%%%%rW   c                     d} d}t          | dddddd          \  }}t          | |dz
  |z  z            }dD ]9}t          j        d	          5 }t	          |||t          |
          t          j        ddd          |d          \  }}}	}
}d d d            n# 1 swxY w Y   t          |          dk    rt          d|d         j
        z            |j        dk    sJ |	j        dk    sJ |
j        dk    sJ |j        dk    sJ t          |t          j        ddd                     t          |                    d          t          j        ddd                     t          |	                    d          t          j        ddd                     |
j        dk    sJ |j        dk    sJ t          j        d	          5 }t	          |||t!          ||           t          j        ddd          |          \  }}}d d d            n# 1 swxY w Y   t          |          dk    rt          d|d         j
        z            t          ||           t          ||	           ;d S )Nr  r   r   r   r   r  r  r  r(  rR  r)  r   FTTr  r  皙?r   r   )r   rR   rC   return_timesUnexpected warning: %r)r   r   rQ  r+  ffffff?r  r  r  )r   rR   rC   )r   rN   r  r  r.   r%   r   linspacer   RuntimeErrormessagerZ   rI   rH   r  rq  r4   )r  r  rd   r   r   shuffle_trainwrR   train_scorestest_scores	fit_timesscore_timestrain_sizes2train_scores2test_scores2s                  rU   test_learning_curverP    sK   IH  DAq 'yX\X4M'NOOI& -= -=$D111 	Q (+++KS"55%!  	 	 	 	 	 	 	 	 	 	 	 	 	 	 	  q66A::7!A$,FGGG!W,,,, G++++')))) G++++;Ar2(>(>???!,"3"3"3";";R[cSU=V=VWWW!+"2"2"2":":BKSRT<U<UVVV )++++ I---- $D111 	Q8F"H	JJJKS"55%9 9 95L-	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 q66A::7!A$,FGGG!->>>!,<<<<[-= -=s$   ?BB	"B	=HH	H	c                     t          ddddddd          \  } }t          d          }t          || d dt          j        dd	d
                    \  }}}t          |t          j        ddd
                     t          |                    d          t          j        dd	d
                     t          |                    d          t          j        dd	d
                     d S )Nr  r   r   r   r=  rQ  r   r?  r   r   )r   r   rR   r+  rB  )r   rN   r.   r   rD  rI   rH   r  rd   r  r   rR   rI  rJ  s         rU    test_learning_curve_unsupervisedrS    s      DAq 'r**I-;1Cb0I0I. . .*K{ {BK2r$:$:;;;l//Q/77S#r9R9RSSSk..A.66Cb8Q8QRRRRRrW   c            	         t          ddddddd          \  } }t          d          }t          j        }t	                      t          _        	 t          || |dd          \  }}}t          j                                        }t          j                                         |t          _        nM# t          j                                        }t          j                                         |t          _        w xY wd	|v sJ d S )
Nr  r   r   r   r=  rQ  r   )r   verbosez[learning_curve])r   rN   sysstdoutr   r.   getvalueclose)rd   r   r   
old_stdoutrR   rI  rJ  outs           rU   test_learning_curve_verboser\    s     DAq 'r**IJCJ 1?q!12
 2
 2
.\; j!!##


 j!!##

$$$$$$s   B/ /A
C9c            	          t          ddddddd          \  } }t          d          }t          j        t                    5  t          || |d           d d d            d S # 1 swxY w Y   d S )Nr   r   r   r=  T)exploit_incremental_learningr   rN   r   r   r   r.   rd   r   r   s      rU   5test_learning_curve_incremental_learning_not_possiblera  
  s      DAq 'q))I	z	"	" K Ky!QTJJJJK K K K K K K K K K K K K K K K K Ks   A##A'*A'c                     t          ddddddd          \  } }t          d          }dD ]}t          || |dd	t          j        d
dd          |          \  }}}t          |t          j        ddd                     t          |                    d          t          j        ddd                     t          |                    d          t          j        d
dd                     d S )Nr  r   r   r   r=  rQ  r>  r   Tr?  r   r   )r   r^  rR   rC   r+  rB  r   rx   r.   r   rD  rI   rH   r  )rd   r   r   rG  rR   rI  rJ  s          rU   (test_learning_curve_incremental_learningrd    s     DAq 2"55I& W W1?)-Cb11!2
 2
 2
.\; 	;Ar2(>(>???!,"3"3"3";";R[cSU=V=VWWW!+"2"2"2":":BKSRT<U<UVVVVW WrW   c                     t          ddddddd          \  } }t          d          }t          || d ddt          j        d	d
d                    \  }}}t          |t          j        ddd                     t          |                    d          t          j        dd
d                     t          |                    d          t          j        d	d
d                     d S )Nr  r   r   r   r=  rQ  r   Tr?  r   r   )r   r   r^  rR   r+  rB  rc  rR  s         rU   5test_learning_curve_incremental_learning_unsupervisedrf  4  s      DAq 2"55I-;	
%)KS"--. . .*K{ {BK2r$:$:;;;l//Q/77S#r9R9RSSSk..A.66Cb8Q8QRRRRRrW   c            	         t          ddddddd          \  } }t          j        ddd          }t          dd d	
          }t	          || ||dd          \  }}}t	          || |d|d	          \  }}}	t          ||           t          |                    d          |                    d                     t          |                    d          |	                    d                     d S )Nr  r   r   r   r=  r  r   r   Fmax_itertolrC   r   T)rR   r   r^  )r   rR   r^  r+  )r   r   rD  r   r.   rI   rH   r  )
rd   r   rR   r   train_sizes_inctrain_scores_inctest_scores_inctrain_sizes_batchtrain_scores_batchtest_scores_batchs
             rU   <test_learning_curve_batch_and_incremental_learning_are_equalrq  L  sP     DAq +c3**K+QD%PPPI9G		%): : :6O% @N		%*@ @ @<)+< (9:::1%%'9'>'>A'>'F'F   !$$&7&<&<!&<&D&D    rW   c            	         t          ddddddd          \  } }t          d          }t          j        t                    5  t          || |dddg           d d d            n# 1 swxY w Y   t          j        t                    5  t          || |dd	d
g           d d d            n# 1 swxY w Y   t          j        t                    5  t          || |dddg           d d d            n# 1 swxY w Y   t          j        t                    5  t          || |dddg           d d d            n# 1 swxY w Y   t          j        t                    5  t          || |dddg           d d d            d S # 1 swxY w Y   d S )Nr  r   r   r   r=  rQ  r   r   rR   r  r   r?  g?   r_  r`  s      rU   0test_learning_curve_n_sample_range_out_of_boundsru  s  s/     DAq 'r**I	z	"	" B By!Q11a&AAAAB B B B B B B B B B B B B B B	z	"	" F Fy!Q13*EEEEF F F F F F F F F F F F F F F	z	"	" F Fy!Q13*EEEEF F F F F F F F F F F F F F F	z	"	" C Cy!Q11b'BBBBC C C C C C C C C C C C C C C	z	"	" C Cy!Q11b'BBBBC C C C C C C C C C C C C C C C C CsZ   A%%A),A)	B,,B03B0C33C7:C7D::D>D>FF	Fc                  <   t          ddddddd          \  } }t          d          }d}t          j        t          |          5  t          || |dt          j        dd	d          
          \  }}}d d d            n# 1 swxY w Y   t          |ddg           d S )Nr   r   r   r   r=  zzRemoved duplicate entries from 'train_sizes'. Number of ticks will be less than the size of 'train_sizes': 2 instead of 3.r   gQ?r   rs  )	r   rN   r   r   r  r.   r   rD  rI   )rd   r   r   r   rR   r  s         rU   1test_learning_curve_remove_duplicate_sample_sizesrw    s     DAq 'q))I	G  
nO	<	<	< 
 
*q!r{4a/H/H
 
 
Q
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 {QF+++++s   -A??BBc                     t          ddddddd          \  } }t          d          }t          d          }t          || ||t	          j        d	d
d                    \  }}}t          |t	          j        ddd                     t          |                    d          t	          j        dd
d                     t          |                    d          t	          j        d	d
d                     d S )Nr  r   r   r   r=  rQ  r   r  r?  r   r   rs  r+  rB  )	r   rN   r%   r.   r   rD  rI   rH   r  )rd   r   r   r   rR   rI  rJ  s          rU   (test_learning_curve_with_boolean_indicesry    s     DAq 'r**I				B-;1aBBKS",E,E. . .*K{ {BK2r$:$:;;;l//Q/77S#r9R9RSSSk..A.66Cb8Q8QRRRRRrW   c                  ^   t          j        ddgddgddgddgd	d
gddgddgddgddgddgddgd	d
gddgddgddgg          } t          j        g d          }t          j        g d          }t          dd d          }t          d          }t	          || ||dt          j        ddd          |dd	  	        \  }}}t          |                    d          t          j        g d                     t          |                    d          t          j        g d                      t          j	        t                    5  t	          || ||dt          j        ddd          |d!"           d d d            n# 1 swxY w Y   t	          || ||dt          j        ddd          |ddd#
  
        \  }}	}
t          |	                    d          |                    d                     t          |
                    d          |                    d                     d S )$Nr   r   r   r   r   r  r  r     r-                       rQ  	   r   )r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   )r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   Frh  r  333333?r   T)r   r   rR   r  rC   r   r+  )r  r  Iq?)r  r  r  r   )r   r   rR   r  r   )r   r   rR   r  rC   r   r^  )r   r  r   r#   r.   rD  rH   r  r   r   r   )rd   r   r  r   r   rn  ro  rp  rk  rl  rm  s              rU    test_learning_curve_with_shuffler    s    	FFFFHHHHHFGHHHH	
	 	A& 	>>>??AXCCCDDF ,QD%PPPI	Q			B?M		KS!,,
@ 
@ 
@<)+< Q''2I2I2I)J)J   A&&1I1I1I(J(J   
z	"	" 

 

Ca00		
 		
 		
 		


 

 

 

 

 

 

 

 

 

 

 

 

 

 

 :H		KS!,,%): : :6O% 1%%'9'>'>A'>'F'F   !$$&7&<&<!&<&D&D    s   	,FFFc                  N   t          j        d                              dd          } t          j        dgdz  dgdz  z             }t	          d          }d}t          j        t          |	          5  t          || |d
           d d d            n# 1 swxY w Y   d}t          j        t          |	          5  t          || |d
dt          j
        d          i           d d d            n# 1 swxY w Y   t          || |d
dt          j
        d          i           d S )Nr  r   r   r   r   Tr  r  r   r   r   z/sample_weight.shape == \(1,\), expected \(2,\)!r   )r   r  )r   r  r   r  rD   r   r   r   r.   r   r  r  s       rU   test_learning_curve_fit_paramsr    s   
	#r2&&A
!qA37"##A
D
9
9
9C4G	~W	5	5	5 7 7sAqg66667 7 7 7 7 7 7 7 7 7 7 7 7 7 7 AG	z	1	1	1 
 
A7QR7T	
 	
 	
 	

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 QwORWR[[3Q     s$   5BBB=)C22C69C6c                  ~   t          ddddddd          \  } }t          ddg          }d}t          j        t          |	          5  t          || |d
dt          j        ddd          d           d d d            n# 1 swxY w Y   d}t          j        t          |	          5  t          || |d
dt          j        ddd          ddt          j        d
          i           d d d            n# 1 swxY w Y   t          || |d
dt          j        ddd          ddt          j        d          i           d S )Nr  r   r   r   r=  rQ  r   z9Expected fit parameter\(s\) \['sample_weight'\] not seen.r   r   Tr?  r   r   r   )r   r^  rR   r   z2Fit parameter sample_weight has length 3; expected)r   r^  rR   r   r  )	r   rx   r   r   r   r.   r   rD  r  )rd   r   r   r  s       rU   3test_learning_curve_incremental_learning_fit_paramsr    s     DAq 2"6GHHIJG	~W	5	5	5 	
 	
)-Cb11	
 	
 	
 	
	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 CG	~W	5	5	5 

 

)-Cb11'4		
 		
 		
 		


 

 

 

 

 

 

 

 

 

 

 

 

 

 

 		%)KS"--#RWQZZ0	 	 	 	 	 	s%   +A??BB'A C33C7:C7c            	         t          ddddddd          \  } }t          j        ddd          }t          j        d          5 }t          t                      | |d|d	          \  }}d d d            n# 1 swxY w Y   t          |          dk    rt          d
|d         j	        z            t          |                    d          |           t          |                    d          d|z
             d S )Nr   r   r   r=  r   Tr  r   
param_nameparam_ranger   rA  r+  )r   r   rD  r  r  r0   r   r   rE  rF  rH   r  )rd   r   r  rH  rI  rJ  s         rU   test_validation_curver  E  sR     DAq +aB''K		 	-	-	- 
$4&((#%
 %
 %
!k
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 1vvzz3adlBCCCl//Q/77EEEk..A.66KHHHHHs   %A66A:=A:c            	          t          ddddddd          \  } }t          j        ddd          }t          t	                      | |d|d          \  }}d S )Nr   r   r   r=  r   r   r  )r   r   rD  r0   r   )rd   r   r  r  s       rU   %test_validation_curve_clone_estimatorr  `  sx      DAq +aB''K-//		  DAqqqrW   c                     d} d}t          dd          \  }}t          t          dd          ||dg dt          || 	          
          }t	          t          j        t          j        |          dd d f         d            t          t          dd          ||dg dt          |d          
          }t	          t          j        t          j        |          dd d f         d            t          t          dd          ||dg dt          |          
          }t	          t          j	        |          t          j	        |                     d S )Nr  r   r   r  r  r  r   )r?  r?  r  r  rC  r  )r   r   r   r   r   Tr  rC   r  )
r   r0   r<   r4   rH   r   vsplithstackr%   r  )r  r  rd   r   scores1scores2scores3s          rU   +test_validation_curve_cv_splits_consistencyr  v  s~   IH1===DAq8!,,,		(((H	BBB  G ry7););L!!!O)LaPPQQ8!,,,		((((D111  G ry7););L!!!O)LaPPQQ8!,,,		((((###  G bhw//'1B1BCCCCCrW   c                  f   t          j        d                              dd          } t          j        dgdz  dgdz  z             }t	          d          }d}t          j        t          |	          5  t          || |d
g dd           d d d            n# 1 swxY w Y   d}t          j        t          |	          5  t          || |d
g dddt          j
        d          i           d d d            n# 1 swxY w Y   t          || |d
g dddt          j
        d          i           d S )Nr  r   r   r   r   Tr  r  r   	foo_param)r   r   r   r   )r  r  r   r  r   )r  r  r   r  )r   r  r   r  rD   r   r   r   r0   r   r  r  s       rU    test_validation_curve_fit_paramsr    s   
	#r2&&A
!qA37"##A
D
9
9
9C4G	~W	5	5	5 
 
"!			
 	
 	
 	

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 AG	z	1	1	1 	
 	
"!		'4	
 	
 	
 	
	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 		II#RWR[[1     s$   5BB B-C::C>C>c                  ~   t           j                            d          } t          j        d          }|                     |           t          |d          sJ t          t          j        |d          d          rJ d|d<   t          |d          rJ t          t          j        |df          d          rJ d S )Nr   r     )r   r2  r3  r  rC   r1   deleter  )r6  r  s     rU   test_check_is_permutationr    s    
)


"
"C
	#AKKNNN C((((($RYq"%5%5s;;;;;AaD$Q,,,,, %RY1v%6%6<<<<<<<rW   c                 .   t          ddddd          \  }} | |          } | |          }t          t          d                    }t          |||d	          }t          |||d	          }|                                }t          ||           d S )
Nr   r   FT)rR  n_labelsallow_unlabeledreturn_indicatorr   r  rS  r   r   )r   r6   r<   r+   r8  rH   )r  rd   r   r   y_sparseclassifr  preds_sparses           rU   (test_cross_val_predict_sparse_predictionr    s     *  DAq }QH}QH!#X"6"6"677Ggq!333E$WhRHHHL''))LlE22222rW   c           
      &   t          dd          }|j        dk    r(|dk    rt          |          fnt          |          df}n|j        }t	          j        |          }|                    ||          D ]V\  }}t          |                               ||         ||                   }  t          | |          ||                   ||<   W||dz   |dz
  |
                    d          fD ]$}	t          t          | ||	||          |           %d	S )
z@Helper for tests of cross_val_predict with binary classificationr   Fr  r   r  r   strr  N)r%   r   r   rZ   r   rt  r  r   r]   getattrastyperF   r+   )
r#  rd   r   r  r   	exp_shapeexpected_predictionsr+  r,  tgs
             rU   check_cross_val_predict_binaryr    s.   	5	)	)	)B 	v{{!'+>!>!>SVVIISVVQK		G	8I..xx1~~ C CtCjjnnQuXqx00%9WS&%9%9!D'%B%BT"" !a%Q0 
 
c1bB???AU	
 	
 	
 	

 
rW   c           
         t          dd          }t          j        t          j                  j        }||dd}t          j        t          |          t          t          |                    f||         t          j                  }t          j        |d          \  }}	|	                    ||	          D ]\  }
}t          |                               ||
         |	|
                   }  t          | |          ||                   }t          j        |	|
                   }||t          j        ||          <   ||d	z   |d
z
  |                    d          fD ]$}t          t!          | ||||          |           %dS )zDHelper for tests of cross_val_predict with multiclass classificationr   Fr  r   r  r  r   rp  Treturn_inverser   r   r  r  N)r%   r   finfor  minr  r   r   r   r  r   r]   r  ix_r  rF   r+   )r#  rd   r   r  r   	float_mindefault_valuesr  r  y_encr+  r,  
fold_preds
i_cols_fitr  s                  rU   "check_cross_val_predict_multiclassr     s   	5	)	)	)B $$(I&& N
 7	QSVV~f5RZ   y4000HAuxx5)) D DtCjjnnQuXuU|44)WS&))!D'22
YuU|,,
9CRVD*5566 !a%Q0 
 
c1bB???AU	
 	
 	
 	

 
rW   c                 J   t          dd          }t          j        t          j                  j        }||dd}j        d         }g }t          |          D ]}	t          t          dd|	f                             }
|
dk    r|d	k    rt          |          f}nt          |          |
f}|	                    t          j
        |||         t          j        
                     fdt          j        d                   D             }t          j        |d          }|                    ||          D ]\  }}t          |                               ||         ||                   }  t          | |          ||                   }t          |          D ]o}	t          j        ||         dd|	f                   }||	         j        dk    r||	         ||	         |<   It          j        ||          }||	         ||	         |<   pӉdz   dz
                      d          fD ]s}t)          | ||||          }t          |          t          |          k    sJ t          t          |                    D ]}t+          ||         ||                    tdS )zCheck the output of cross_val_predict for 2D targets using
    Estimators which provide a predictions as a list with one
    element per class.
    r   Fr  r   r  r   Nr   r  rp  c                     g | ]<}t          j        d d |f         d          d         d d t           j        f         =S )NTr  r   )r   r   r   )r  r  r   s     rU   
<listcomp>z6check_cross_val_predict_multilabel.<locals>.<listcomp>7  sX        		!AAAqD'$///2111bj=A  rW   r+  r  r  )r%   r   r  r  r  rZ   r   r   r   r  r  concatenater  r   r]   r  r   r   r  r  r+   rF   )r#  rd   r   r  r   r  r  	n_targetsexpected_predsi_coln_classes_in_labelr  
y_enc_colsr  r+  r,  r  	fold_colsidxr  cv_predict_outputr  s     `                   rU   "check_cross_val_predict_multilabelr    s   
 
5	)	)	)B $$(I&& N
 
INy!! 
 
 Qqqq%x[!1!122""v1D'D'DQ	IIQ!34IGI~f5RZHHH	
 	
 	
 	

   qwqz""  J N:A...Exx5)) 
? 
?tCjjnnQuXuU|44)WS&))!D'22
9%% 	? 	?E	%,qqq%x"899Ie$)Q...8.?u%d++fT9---7->u%c**	? !a%Q0 E E-c1bBOOO$%%^)<)<<<<<s,--.. 	E 	EA-a0.2CDDDD	EE ErW   c                 \    t          dd          \  }}dD ]}t          | |||           d S )Nr   r   )rR  r   r  r   r  )r   r  )r#  rd   r   r  s       rU   *check_cross_val_predict_with_method_binaryr  P  sJ     ;;;DAqM : :&sAq&9999: :rW   c                     t                      }|j        |j        }}t          ||d          \  }}dD ]}t	          | |||           d S )Nr   r   r  )r   rr  rs  rC   r  )r#  rw  rd   r   r  s        rU   .check_cross_val_predict_with_method_multiclassr  X  s`    ;;D9dkqA1aa(((DAqM > >*31f====> >rW   c                  z    t          t          d                     t          t          d                     d S )Nr  r  )r  r   r  rv   rW   rU   "test_cross_val_predict_with_methodr  `  sF    ./A/U/U/UVVV2+...    rW   c                      t                      } | j        | j        }}t          ||d          \  }}dD ]%}t	          dd          }t          ||||           &d S )Nr   r   r  log_lossr   )lossr   )r   rr  rs  rC   r   r  )rw  rd   r   r  r#  s        rU   &test_cross_val_predict_method_checkingr  g  st     ;;D9dkqA1aa(((DAqM > >!<<<*31f====> >rW   c                      t                      } | j        | j        }}t          ||d          \  }}t	          t          dd          dddgid	
          }dD ]}t          ||||           d S )Nr   r   r  r  r*  r   r?  r   r   r   r  )r   rr  rs  rC   r"   r   r  )rw  rd   r   r#  r  s        rU   /test_gridsearchcv_cross_val_predict_with_methodr  r  s    ;;D9dkqA1aa(((DAq
;???#QxUV  C N > >*31f====> >rW   c                      d} d}t          | d|dd          \  }}t          t          dd	                    }d
D ]}t          ||||           d S )Nr  r   r   r   r  r  r  rR  r  r   r  r   )r  r   )r   r  r  )r   r6   r   r  )n_samprR  rd   r   r#  r  s         rU   1test_cross_val_predict_with_method_multilabel_ovrr  }  s     FI)1	aVX  DAq 0RSTTT
U
UC8 A A&sAq@@@@@A ArW   c                       e Zd Zd ZdS )RFWithDecisionFunctionc                     |                      |          }d}t          |t                    s
J |            d |D             }|S )Nz?This helper should only be used on multioutput-multiclass tasksc                 L    g | ]!}|j         d          dk    r|dddf         n|"S )r   r   Nr   r  )r  r  s     rU   r  z<RFWithDecisionFunction.decision_function.<locals>.<listcomp>  s6    CCCQWQZ1__111b5!CCCrW   )r   r   r   )rT   rd   probsmsgs       rU   r  z(RFWithDecisionFunction.decision_function  sP    ""1%%O%&&+++++CCUCCCrW   N)rr   rs   rt   r  rv   rW   rU   r  r    s#            rW   r  c                  D   d} t          dd| dd          \  }}|d d dfxx         |d d df         z  cc<   d	D ]e}t          dd
          }t          j                    5  t          j        d           t          ||||           d d d            n# 1 swxY w Y   fd S )Nr   r  r   r   r  r  r   r   r   r  r  n_estimatorsr   ignorer  )r   r  r  r  simplefilterr  )rR  rd   r   r  r#  s        rU   0test_cross_val_predict_with_method_multilabel_rfr    s&    I)Y1SU  DAq aaadGGGqAwGGGM I I$!!DDD$&& 	I 	I!(+++.sAqHHHH	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	II Is    (BB	B	c                  l   t           j                            d          } |                     ddd          }t          j        g d          }t          d          }dD ]S}t          j                    5  t          j        d	           t          ||||           d d d            n# 1 swxY w Y   Td S )
Nr   r   )r}  r   r  )r   r   r   r   r   r   r   r   r   r   r   r   r   r   r  r  r  r  )
r   r2  r3  normalr  r   r  r  r  r  )r6  rd   r   r#  r  s        rU   -test_cross_val_predict_with_method_rare_classr    s    )


"
"C

1ah
''A
;;;<<A
K
0
0
0CM B B$&& 	B 	B!(+++.sAq&AAA	B 	B 	B 	B 	B 	B 	B 	B 	B 	B 	B 	B 	B 	B 	BB Bs   5'B((B,	/B,	c                     t           j                            d          } |                     ddd          }t          j        ddgddgddgddgddgg          }dD ]e}t          dd          }t          j                    5  t          j        d	           t          ||||
           d d d            n# 1 swxY w Y   fd S )Nr   r   )r   r   r  r   )r   r  r   r  r  r  )
r   r2  r3  r  r  r  r  r  r  r  )r6  rd   r   r  r#  s        rU   ;test_cross_val_predict_with_method_multilabel_rf_rare_classr    s+    )


"
"C

1ag
&&A
1a&1a&1a&1a&1a&9::A8 I I$!!DDD$&& 	I 	I!(+++.sAqHHHH	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	II Is   (B77B;	>B;	c                    t          j        t          |          |g          }t          ||          }|                    | |          D ]\  }}	|                    | |         ||                     || |	                   }
|dk    r$t          j        t          |	          |f          }n@t          j        t          |	          |ft          j        |j                  j	                  }|
|d d |j
        f<   |||	<   |S )Nr   )r   rt  r   r  r  r]   r  r  rq  r  classes_)rd   r   r   classesr#  r  r  funcr+  r,  expected_predictions_exp_pred_tests               rU   get_expected_predictionsr    s    8SVVW$5663Dxx1~~ 3 3t%!E(### $QtW_$$Hc$ii%9::MMGTG$bh/C/I&J&J&N M *?aaao&%2T""rW   c            	         t          j        d                              dd          } t          j        d t	          d          D                       }d}t          d          }t          d          }t                      }g d	}|D ]}t          d
          }t          || |||          }	t          | |||||          }
t          |
|	           t          || |||          }	t          | |||||          }
t          |
|	           t          t          j        t	          d          d          d          }t          || |||          }	|                    |          }t          | |||||          }
t          |
|	           d S )Nr  r  r   c                     g | ]}|d z  S )r   rv   )r  r{   s     rU   r  z7test_cross_val_predict_class_subset.<locals>.<listcomp>  s    ...a!r'...rW   r   r   r  r   r  r  r  r  r   r   )r   r  r   r  r   r%   r:   r   r+   r  rH   rC   r  fit_transform)rd   r   r  kfold3kfold4lemethodsr  r#  r   r  s              rU   #test_cross_val_predict_class_subsetr    s   
	#sA&&A
..5::...//AGAFAF	BIIIG E E 444 (Q&VLLL  8q&'3 
  
 	""6DDD (Q&VLLL7q&'3 
  
 	""6DDD BIeBii,,1==='Q&VLLLQ7q&'3 
  
 	""6DDDD5E ErW   c                     t                      } | j        | j        }}t                      }t	          j        dd          }|                    d           |                                 t          j	        |j
        t          j                  t          j	        |j
        ddt          j                  	 t          |||fd	
           t          j        t                    5  t          |||fd
           d d d            n# 1 swxY w Y   d\  t!          d          D ]=}	 t#          j        |j
                    d S # t&          $ r t)          d           Y :w xY wd S # d\  t!          d          D ]<}	 t#          j        |j
                    w # t&          $ r t)          d           Y 9w xY ww xY w)NwbF)moder  s   Hello world!!!!!rp  rv   r)rZ   r   rq  c                     S r`   rv   )r#  rd   r   rl   s      rU   r   z#test_score_memmap.<locals>.<lambda>  s    U rW   r   c                     S r`   rv   )r#  rd   r   r   s      rU   r   z#test_score_memmap.<locals>.<lambda>  s     rW   rp   r   r   )r   rr  rs  r   tempfileNamedTemporaryFilewriterY  r   memmapnamer  r,   r   r   r   r   osunlinkOSErrorr   )rw  rd   r   r   tfr  rl   r   s         @@rU   test_score_memmapr    s,   ;;D9dkqA


C		$$u	=	=	=BHH !!!HHJJJYrwbj111FIbgRcDDDEQ+B+B+B+BCCCC]:&& 	I 	ICA/G/G/G/GHHHH	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I
 #q 	 	A	"'"""   c




		 	 #q 	 	A	"'"""   c




		sf   9/E* (D?E* DE* DE* -E		E%$E%*F=FF=F9	6F=8F9	9F=c                  V   t           t           fg} 	 ddlm}m} |                     ||f           n# t
          $ r Y nw xY w| D ]c\  t                      }|j        |j        }} |           |          }}fd}fd}	t          ||	          }
t          |
||           dd S )Nr   rX  c                 $    t          |           S r`   r\  r]  s    rU   r   z4test_permutation_test_score_pandas.<locals>.<lambda>,  r_  rW   c                 $    t          |           S r`   r\  ra  s    rU   r   z4test_permutation_test_score_pandas.<locals>.<lambda>-  rc  rW   rd  )rE   re  rY  rZ  r  rf  r   rr  rs  rD   r/   )rh  rY  rZ  rw  rd   r   ri  rj  rk  rl  r   r^  rb  s              @@rU   "test_permutation_test_score_pandasr    s     ]+,E,,,,,,,,fi())))   (- 1 1$
${{y$+1&&q))::a==e<<<<:::: <HHHsD%00001 1rn  c                  h   t          t           j                  } t          j        dd          }t	          | |d t	                      d d dd d d 
  
        }d|d<   t          j        t          d          5  t          d
i | d d d            n# 1 swxY w Y   | 	                                d	k    sJ d S )Nr   r   r   )
r   rd   r   r  r+  r,  rU  
parametersr  score_paramsr   r   z%Failing classifier failed as requiredr   r  rv   )
r5   FAILING_PARAMETERr   r  r   r   r   r   r2   rl   )failing_clfrd   fit_and_score_argss      rU   test_fit_and_score_failingr  2  s   #$5$GHHK
	!RA

vv   )0}%	z)P	Q	Q	Q - -,,+,,,- - - - - - - - - - - - - - - #%%%%%%s   4BBBc                  D   t          dd          \  } }t          dd          }t          t                                          |                     \  }}t          || |t                      ||ddddd d d	
          }t          di |}|d         |d         k    sJ d S )Nr  r   r  r  r  r  r?  )ri  rj  T)r   rd   r   r  r+  r,  rU  r  r  r  return_parametersr  rv   )r   r<   nextr)   r  r   r2   )rd   r   r   r+  r,  r  results          rU   test_fit_and_score_workingr  L  s    !<<<DAq
XA
.
.
.C|~~++A..//KE4

vv #C00   11011F,#5l#CCCCCCCrW   c                   &    e Zd ZddZddZddZdS )DataDependentFailingClassifierNc                     || _         d S r`   max_x_value)rT   r"  s     rU   rV   z'DataDependentFailingClassifier.__init__c  s    &rW   c                 j    || j         k                                    }|rt          d| d          d S )NzClassifier fit failed with z values too high)r"  r  r   )rT   rd   r   num_values_too_highs       rU   r]   z"DataDependentFailingClassifier.fitf  sO     4#3388:: 	S.ASSS  	 	rW   c                     dS )Nr  rv   rj   s      rU   rl   z$DataDependentFailingClassifier.scorem  s    srW   r`   rp   )rr   rs   rt   rV   r]   rl   rv   rW   rU   r  r  b  sP        ' ' ' '        rW   r  r   c                 h   t          d          }t          j        dd          }t          j        d          }|||g}d| d}d}t	          j        d	|d
          d| t          j                  }t          j        t          |          5  t          |i | d d d            d S # 1 swxY w Y   d S )Nr  r!  r   r   r  r   r   r   z8ValueError: Classifier fit failed with 1 values too highzh2 fits failed.+total of 3.+The score on these train-test partitions for these parameters will be set to r   z.+flagsr   )r  r   r  r  recompileDOTALLr   r   r   r-   )r   r  rd   r   cross_validate_argscross_validate_kwargsindividual_fit_error_messager   s           rU   -test_cross_validate_some_failing_fits_warningr0  q  sE    1Q???K
	!RA


A&1-#$[AA 	C ! jW%m4W W8TW W i  O 
&o	>	>	> F F+E/DEEEF F F F F F F F F F F F F F F F F F   B''B+.B+c                 h   t          t           j                  }t          j        dd          }t          j        d          }|||g}d| d}d}t          j        d| t
          j                  }t          j	        t          |	          5  t          |i | d d d            d S # 1 swxY w Y   d S )
Nr   r   r  r  r'  z1ValueError: Failing classifier failed as requiredz4All the 7 fits failed.+your model is misconfigured.+r(  r   )r5   r  r   r  r  r*  r+  r,  r   r   r   r-   )r   r  rd   r   r-  r.  r/  rU  s           rU   *test_cross_validate_all_failing_fits_errorr3    s'    $$5$GHHK
	!RA


A&1-#$[AA#V J.+. . i  M 
z	7	7	7 F F+E/DEEEF F F F F F F F F F F F F F F F F Fr1  c                      t          |          r`   )r   )r   rd   r   	error_msgs       rU   _failing_scorerr6    s    
Y

rW   zignore:lbfgs failed to converger   c           	         t          d          \  }}t          d                              ||          }d}t          t          |          }| dk    rKt          j        t          |          5  t          |||d	|| 
           d d d            d S # 1 swxY w Y   d S d|  }t          j	        t          |          5  t          |||d	|| 
          }t          ||            d d d            d S # 1 swxY w Y   d S )NTr   r   ri  "This scorer is supposed to fail!!!r5  r   r   r   )r   r   r   r   )r   r   r]   r   r6  r   r   r   r,   r   r   rF   )r   rd   r   r   r5  failing_scorerwarning_msgr   s           rU   #test_cross_val_score_failing_scorerr=    s   
 %%%DAq
a
(
(
(
,
,Q
2
2C4I_	BBBNg]:Y777 	 	Qa[   	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	=/:= = 	 \+[999 	1 	1$Qa[  F FK000		1 	1 	1 	1 	1 	1 	1 	1 	1 	1 	1 	1 	1 	1 	1 	1 	1 	1s$   1BBB&C44C8;C8r0  with_multimetricc           
         t          d          \  }}t          d                              ||          }d}t          t          |          }|rt          t                    }|||d}	n|}	| dk    rLt          j        t          |	          5  t          |||d
|	||            d d d            d S # 1 swxY w Y   d S d|  }
t          j        t          |
	          5  t          |||d
|	||           }|D ]C}d|v r=d|v r#||         D ]}t          |t                    sJ -t          ||         |            D	 d d d            d S # 1 swxY w Y   d S )NTr   r   r8  r9  r:  )score_1score_2score_3r   r   r   )r   r   r0  r   r   r3   _score_2)r   r   r]   r   r6  r   r   r   r   r   r-   r   r   r   ri   rF   )r   r0  r>  rd   r   r   r5  r;  non_failing_scorerr   r<  r  r   r  s                 rU   "test_cross_validate_failing_scorerrE    sz    %%%DAq
a
(
(
(
,
,Q
2
2C4I_	BBBN !();<<%)%
 
 !g]:Y777 		 		#5'   		 		 		 		 		 		 		 		 		 		 		 		 		 		 		 		 		 		=/:= = 	 \+[999 	C 	C$#5'  G  
C 
Cs??!S(( ") 8 8A#-a#7#777778
 (kBBB
C	C 	C 	C 	C 	C 	C 	C 	C 	C 	C 	C 	C 	C 	C 	C 	C 	C 	Cs%   B44B8;B8!AEEEc                     dS )Ng;pΈ^@rv   )r  jr=  s      rU   three_params_scorerrH  	  s    6rW   z:train_score, scorer, verbose, split_prg, cdt_prg, expectedr   )r   r   r   r   zS\[CV\] END .................................................... total time=   0.\ds)sc1sc2)scorersr   ze\[CV 2/3\] END  sc1: \(train=3.421, test=3.421\) sc2: \(train=3.421, test=3.421\) total time=   0.\dsr   zW\[CV 2/3; 1/1\] END ....... sc1: \(test=3.421\) sc2: \(test=3.421\) total time=   0.\dsc                    t          dd          \  }}t          dd          }	t          t                                          |                    \  }
}t          |	||||
||d d d |||          }t          d
i | |                                 \  }}|                    d          }t          |          dk    rt          j
        ||d	                   sJ d S t          j
        ||d                   sJ d S )Nr  r   r  r  r  )r   rd   r   r  r+  r,  rU  r  r  r  r0  split_progresscandidate_progress
r   r   rv   )r   r<   r  r)   r  r   r2   
readouterrr   r*  r   )capsysr1  r  rU  	split_prgcdt_prgexpectedrd   r   r   r+  r,  r  r[  r  outliness                   rU   test_fit_and_score_verbosityrW  
	  s   N !<<<DAq
XA
.
.
.C|~~++A..//KE4 

& "   (('(((  FCyyH
8}}qx(1+.......x(1+.......rW   c            	          d} d }t          j        t          |           5  t          d d d |d t          j                   d d d            d S # 1 swxY w Y   d S )Nz&scoring must return a number, got Nonec                     d S r`   rv   )r   X_tests     rU   two_params_scorerz%test_score.<locals>.two_params_scorerQ	  s    trW   r   )r   rZ  r  r  r  r   )r   r   r   r3   r   r   )rU  r[  s     rU   r2  r2  N	  s    <M   
z	7	7	7 
 
$	
 	
 	
 	

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
s    AAAc                      d } t          ddd          \  }}t          d          }|                    ||           t          |||d|           }g d	}|D ]}d
                    |          |v sJ d S )Nc                     |                      |          }t          ||          }|d         |d         |d         |d         dS )N)r   r   rI  )r   r   )r   r   tnfpfntp)re   r   )r   rd   r   rC  cms        rU   rD  zPtest_callable_multimetric_confusion_matrix_cross_validate.<locals>.custom_scorer`	  sB    Qa((hbhbhbhOOOrW   (   r   r  )r  r  r   r   r   )r   r   r^  ztest_{})r   r=   r]   r-   r   )rD  rd   r   r#  rO  score_namesr  s          rU   9test_callable_multimetric_confusion_matrix_cross_validaterf  _	  s    P P P
 KKKDAq

$
$
$CGGAqMMMQaGGGJ***K 4 4%%333334 4rW   c                  l    t          d          \  } }t          t                      | |dd           dS )z^Check that regressors with partial_fit is supported.

    Non-regression test for #22981.
    r  r   Tr   )r^  r   N)r   r.   r8   )rd   r   s     rU   *test_learning_curve_partial_fit_regressorsrh  o	  s=    
 +++DAq <>>1adqQQQQQQrW   c                    t          dddd|           \  }}t          j        |          }||         ||         }}t                      }d}t	          j        t          |          5  t          |||dt          j        	          ^}}}}d
d
d
           n# 1 swxY w Y   t          d          D ]^}	t          j
        ||	                                                   sJ t          j
        ||	                                                   sJ _t          d|j        d                   D ]^}	t          j
        ||	                                                   rJ t          j
        ||	                                                   rJ _d
S )zKChecks for fit failures in `learning_curve` and raises the required warningr  r   r  F)r  rR  r  rC   r   z#10 fits failed out of a total of 25r   r   r'  Nr   r   )r   r   r  r<   r   r   r   r.   r   r   isnanr	  rZ   any)
global_random_seedrd   r   
sorted_idxsvcr   r  r1  r2  r  s
             rU   -test_learning_curve_some_failing_fits_warningro  z	  s    '  DAq AJZ=!J-qA
%%C;O	&o	>	>	> 
 
)7A!*
 *
 *
&;
Q
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 Qxx / /xC())--/////x
3((,,......Q)!,-- 3 38K,--11333338JsO,,002222223 3s   '#BBBc                    t          d          \  }}t          |          }t                      }t          dd|           }t	          ||||dd          }d|vsJ t	          ||||dd          }d|v sJ |d         d	         }|d         d
         }t          |          |j        k    sJ t          |          |j        k    sJ t          d |D             d           t          d |D             d           t          |	                    ||                    D ]4\  }\  }	}
t          ||         |	           t          ||         |
           5dS )z<Check the behaviour of `return_indices` in `cross_validate`.Tr   r   )r  rC   r   r   F)r   r   return_indicesindicesr+  r,  c                     g | ]	}|j         
S rv   r  r  rr  s     rU   r  z6test_cross_validate_return_indices.<locals>.<listcomp>	  s    BBBBBBrW   r  c                     g | ]	}|j         
S rv   r  rt  s     rU   r  z6test_cross_validate_return_indices.<locals>.<listcomp>	  s    AAAAAArW   r  N)
r   r;   r   r%   r-   r   r  rI   r5  r  )rl  rd   r   r   r   rO  train_indicestest_indices	split_idxexpected_train_idxexpected_test_idxs              rU   "test_cross_validate_return_indicesr{  	  s   %%%DAqaA"$$I	46H	I	I	IB	1aBqQVWWWJJ&&&&	1aBqQUVVVJ
""""y)'2Mi(0L},,,,|++++BBMBBBCHHHAALAAA2FFF>GQRTU>W>W G G:	:&(9=35GHHH<	24EFFFFG GrW   c                  |   t          j        t          d          5  t          t	                      t
          t          di            ddd           n# 1 swxY w Y   t          j        t          d          5  t          t	                      t
          t          i i            ddd           dS # 1 swxY w Y   dS )z2Check that we warn about deprecating `fit_params`.z`fit_params` is deprecatedr   r   )r   rd   r   r   r  Nz1`params` and `fit_params` cannot both be provided)r   rd   r   r  r   )	r   r   FutureWarningr-   r>   rd   r   r   r   rv   rW   rU   )test_cross_validate_fit_param_deprecationr~  	  sL   	m+G	H	H	H W W!4!6!6!qQSUVVVVW W W W W W W W W W W W W W W 
M
 
 
 
 
 	)++qA"R	
 	
 	
 	

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
s#   +AAA9+B11B58B5enable_slep006	cv_methodc                     t          j        t          d          5   | t                      t          t
          g            ddd           dS # 1 swxY w Y   dS )zCheck that we raise an error if `groups` are passed to the cv method instead
    of `params` when metadata routing is enabled.
    z`groups` can only be passed ifr   )r   rd   r   r  N)r   r   r   r>   rd   r   )r  s    rU   #test_groups_with_routing_validationr  	  s     
z)I	J	J	J 
 
	)++		
 	
 	
 	

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
s   &AAAc           	          t          j        d          }t          j        t          |          5   | t                      t          t          t          g                      ddd           dS # 1 swxY w Y   dS )zMCheck that we raise an error when passing metadata that is not
    requested.z8but are not explicitly set as requested or not requestedr   )metadata)r   rd   r   r   N)	r*  escaper   r   r   r>   rd   r   r   )r  r  s     rU    test_passed_unrequested_metadatar  	  s     iRSSG	z	1	1	1 
 
	)++$$$		
 	
 	
 	

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
s   4A11A58A5c                    t                      }t          |                              dd          }t                      }t          |                              dd          }t                      }t          |                              dd	          }t          t                    }t          j
                            d
          }|                    |          }	|                    |          }
|                    d
d|          }|                    |          }|                    |          }|                    |          }t          t          t          |d                    t           t          |          t"          t                      i}t          ||||          }| t"          ur|                    |	|
            | |ft          t&          |d||          d|i | t"          urt)          |          sJ |D ]}t+          |dd|	|
           t)          |          sJ |D ]}t+          |d||           t)          |          sJ |D ]}t+          |dd||           dS )z]Check that the respective cv method is properly dispatching the metadata
    to the consumer.)registryscore_weightsscore_metadata)r   r  split_groupssplit_metadata)r  r  fit_sample_weightfit_metadatar   r   r  )	my_scorerr  r   )r  r  r  r  )r  r  )rd   r   r   r   rl   )objr  split_paramsr   r  r  )r  r  r  r  r]   N)rA   r?   set_score_requestr@   set_split_requestr>   set_fit_requestrL   rd   r   r2  r3  r4  randintr-   r   r,   r+   updater   r   rB   )r  scorer_registryr  splitter_registrysplitterestimator_registryr   r  r6  r  r  r  r  r  r  extra_paramsr   _scorer	_splitter
_estimators                       rU   test_cross_validate_routingr  	  s     kkOo666HH%0@ I  F " *;<<<NN(8 O  H ##-?@@@PP)N Q  I QI
)


"
"CHHY''MXXi((N;;q!Y//LXXi((N++88I&&L 	TFZ%P%P%PQQQf---466L !%+!	  F )))') 	 	
 	
 	

 I

	 
 y
!      )))?#####" 
 
6'#	
 	
 	
 	
 	
  !!!!!& 
 
	#		
 	
 	
 	
 	
 !"""""( 
 

6+!	
 	
 	
 	
 	

 
rW   )ru   r	  r*  rV  r  r  	functoolsr   ior   timer   numpyr   r   scipy.sparser   sklearn.baser   r   sklearn.clusterr	   sklearn.datasetsr
   r   r   r   r   r   sklearn.ensembler   sklearn.exceptionsr   sklearn.imputer   sklearn.linear_modelr   r   r   r   r   sklearn.metricsr   r   r   r   r   r   r   r   r    sklearn.metrics._scorerr!   sklearn.model_selectionr"   r#   r$   r%   r&   r'   r(   r)   r*   r+   r,   r-   r.   r/   r0   #sklearn.model_selection._validationr1   r2   r3   $sklearn.model_selection.tests.commonr4   )sklearn.model_selection.tests.test_searchr5   sklearn.multiclassr6   sklearn.neighborsr7   sklearn.neural_networkr8   sklearn.pipeliner9   sklearn.preprocessingr:   r;   sklearn.svmr<   r=   %sklearn.tests.metadata_routing_commonr>   r?   r@   rA   rB   sklearn.utilsrC   sklearn.utils._mockingrD   rE   sklearn.utils._testingrF   rG   rH   rI   sklearn.utils.fixesrJ   rK   sklearn.utils.validationrL   rN   rx   r   r   r   r  rd   r  r   r   r  r   markparametrizer   r   r  r  ru  r-  r  r  rV  filterwarningsrm  r~  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r"  r&  r0  r9  r;  rP  rS  r\  ra  rd  rf  rq  ru  rw  ry  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r   r0  r3  r6  r=  rE  rH  rW  r2  rf  rh  ro  r{  r~  usefixturesr  r  r  rv   rW   rU   <module>r     so       				 				 



                          ! ! ! ! ! ! - - - - - - - - " " " " " "                4 3 3 3 3 3 / / / / / / ( ( ( ( ( (             
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 7 6 6 6 6 6                                 "         
 A @ @ @ @ @ G G G G G G 2 2 2 2 2 2 2 2 2 2 2 2 / / / / / / % % % % % % 5 5 5 5 5 5 5 5 & & & & & & & &              " ! ! ! ! ! D D D D D D D D            ? > > > > > > > 1 1 1 1 1 1" " " " "] " " "6    (>   <" " " " " " " ",	" 	" 	" 	" 	",F 	" 	" 	"R8 R8 R8 R8 R8 R8 R8 R8n BGGBH+++,, RX,,,--
BF1II .99%< %< :9%<P) ) )1L 1L 1LhL L L& t}55.994AD 4A 4A 4A :9 654An?M ?M ?MDF9 F9 F9R> > >, FGG0 0 HG0&5 5 5&8 8 86 .99 2  2 :9 2F& & & J J J&L L L0 .99? ? :9?D$ $ $Q Q Q  H H H< .995
 5
 :95
p1W 1W 1Wh# # # # # #  .9922 22 :922j FGG2 2 HG2$T T T,
$ 
$ 
$ .99& & :9&:= := :=zS S S&% % %4K K K W W W4S S S0$ $ $NC C C., , ,,S S S(K K K\  &- - -`I I I6  ,+D +D +D\# # #L= = = .993 3 :93$
 
 
*
 
 
81E 1E 1Eh: : :> > >  > > >> > >A A A	 	 	 	 	3 	 	 	I I I"B B BI I I"     (%E %E %EP  4 FGG1 1 HG1(& & &4D D D,    ]    44F F 54F6 44F F 54F.      =>>G(<==1 1 >= ?>14 =>>G(<==-e}==+eT];;:C :C <; >= >= ?>:Cz   @ $	
  3<OPP   ?
	
  3<OPP   $
	
+ # #H/ /I# #H/@
 
 
"4 4 4 R R R3 3 3BG G G<

 

 

 )**./3DE 

 

  +*

 )**./3DE 

 

  +*

 )**./3DE S
 S
  +*S
 S
 S
rW   