
    Pi5k                       d Z ddlZddlZddlZddlZddlmZ ddlmZ ddl	Z
ddlZddlmZ ddlmZ ddlmZmZmZmZ ddlmZ dd	lmZmZmZmZmZmZ dd
lmZ ddl m!Z! ddl"m#Z#m$Z$ ddl%m&Z& ddl'm(Z(m)Z)m*Z*m+Z+ ddl,m-Z-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4m5Z5 ddl6m7Z7 ddl8m9Z9m:Z:m;Z;m<Z<m=Z=m>Z>m?Z?m@Z@mAZAmBZBmCZCmDZDmEZEmFZFmGZG ddlHmIZImJZJmKZK ddlLmMZM ddlNmOZO ddlPmQZQ ddlRmSZS ddlTmUZU ddlVmWZW ddlXmYZYmZZZ ddl[m\Z\m]Z] ddl^m_Z_m`Z`maZambZbmcZc ddldmeZe ddlfmgZgmhZhmiZimjZj ddlkmlZlmmZm ddlnmoZompZpmqZqmrZrmsZs dd ltmuZumvZv dd!lwmxZx  G d" d#e          Zy G d$ d%ey          Zz G d& d'e          Z{ G d( d)e{          Z| G d* d+ee          Z} e
j~        d,          Z e
j        g d-          Z e
j        g d.          Z e
j        d/          Zej                            d0eu          d1             Zd2 Zd3 Zd4 Zej                            d5d6d7g          ej                            d8ev          d5efd9                        Zd: Zd; Zd< Zd= Zd> Zd? Zej                            d0eu          d@             ZdA ZdB ZdC Zej                            d0eu          dD             ZdE ZdF ZdG ZdH Zej                            d0eu          dI             ZdJ ZdK ZdL Zej                            d0eu          dM             ZdN ZdO ZdP Zej                            d0eu          dQ             ZdR ZdS ZdT ZdU ZdV ZdW ZdX ZdY ZdZ Zd[ Zd\ Zd] Zd^ Zd_ Zd` Zda Zdb Zdc Zej                            d8ev          dd             Zde Zdf Zdg Zdh Zdi Zdj Zdk Zdl Zdm Z G dn doe!          Zdp Zdq Zdr Zds Zdt Zdu Zdv Zdw Zdx Z G dy dze          Zej                            d{e
j        dg          d|             Zej                            d{e
j        dg          d}             Zd~ Zej                            d          ej                            d{e
j        ddg          d                         Zej                            d          ej                            d{e
j        ddg          ej                            dd7d6g          ej                            dd6d7g          d                                                 Zd Zej                            dd6eddddfd7 e7eed          ddddfd6 e7eed          ddddfg          d             Zd Zd Zd Zd Zd Zej                            deDi feCi feBi feEi feFi feGd e
j        dg          dfg           ed7          d                         Zej                            deDi feCi feBi feEi feFi feGd e
j        dg          dfg           ed7          d                         Zej                            deDi feCi feBi feEi feFi feGd e
j        dg          dfg           ed7          d                         Zej                            deDi feCi feBi feEi feFi feGd e
j        dg          dfg           ed7          d                         Z ed7          d             Zej                            d e)             e            gddg          ej                            dg d          ej                            d ej            ei          d                                     ZdS )zTest the validation module    N)partial)sleep)issparse)config_context)BaseEstimatorClassifierMixincloneis_classifier)KMeans)load_diabetesload_digits	load_irismake_classificationmake_multilabel_classificationmake_regression)LinearDiscriminantAnalysis)RandomForestClassifier)FitFailedWarningUnsetMetadataPassedError)SimpleImputer)LogisticRegressionRidgeRidgeClassifierSGDClassifier)	accuracy_scorecheck_scoringconfusion_matrixexplained_variance_scoremake_scorermean_squared_errorprecision_recall_fscore_supportprecision_scorer2_score)_MultimetricScorer)GridSearchCV
GroupKFoldGroupShuffleSplitKFoldLeaveOneGroupOutLeaveOneOutLeavePGroupsOutShuffleSplitStratifiedKFoldcross_val_predictcross_val_scorecross_validatelearning_curvepermutation_test_scorevalidation_curve)_check_is_permutation_fit_and_score_score)OneTimeSplitter)FailingClassifier)OneVsRestClassifier)KNeighborsClassifier)MLPRegressor)Pipeline)LabelEncoderscale)SVC	LinearSVC)ConsumingClassifierConsumingScorerConsumingSplitter	_Registrycheck_recorded_metadata)shuffle)_atol_for_type_convert_to_numpy_get_namespace_device_dtype_ids)yield_namespace_device_dtype_combinations)CheckingClassifierMockDataFrame)_array_api_for_testsassert_allcloseassert_almost_equalassert_array_almost_equalassert_array_equal)COO_CONTAINERSCSR_CONTAINERS)_num_samplesc                   4    e Zd ZdZd ZddZd Zd	dZd ZdS )
MockImprovingEstimatorz+Dummy classifier to test the learning curvec                 0    || _         d| _        d | _        d S Nr   )n_max_train_sizestrain_sizesX_subset)selfrY   s     /home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/sklearn/model_selection/tests/test_validation.py__init__zMockImprovingEstimator.__init__j   s    !2    Nc                 8    || _         |j        d         | _        | S rX   r[   shaperZ   r\   r[   y_subsets      r]   fitzMockImprovingEstimator.fito        #>!,r_   c                     t           NNotImplementedErrorr\   Xs     r]   predictzMockImprovingEstimator.predictt       !!r_   c                     |                      |          rdt          | j                  | j        z  z
  S t          | j                  | j        z  S )Ng       @)_is_training_datafloatrZ   rY   r\   rl   Ys      r]   scorezMockImprovingEstimator.scorew   sP    !!!$$ 	Dt/0043IIII)**T-CCCr_   c                     || j         u S rh   r[   rk   s     r]   rp   z(MockImprovingEstimator._is_training_data~       DM!!r_   rh   NN	__name__
__module____qualname____doc__r^   re   rm   rt   rp    r_   r]   rV   rV   g   ss        55  
   
" " "D D D D" " " " "r_   rV   c                   2     e Zd ZdZd fd	Zd ZddZ xZS )!MockIncrementalImprovingEstimatorz*Dummy classifier that provides partial_fitNc                 f    t                                          |           d | _        || _        d S rh   )superr^   xexpected_fit_params)r\   rY   r   	__class__s      r]   r^   z*MockIncrementalImprovingEstimator.__init__   s1    *+++#6   r_   c                     | j         |v S rh   r   rk   s     r]   rp   z3MockIncrementalImprovingEstimator._is_training_data   s    v{r_   c                    | xj         |j        d         z  c_         |d         | _        | j        rt	          | j                  t	          |          z
  }|r t          dt          |           d          |                                D ]c\  }}|| j        v rSt          |          t          |          k    r3t          d| dt          |           dt          |           d          bd S d S )Nr   zExpected fit parameter(s) z
 not seen.zFit parameter z has length z; expected .)	rZ   rb   r   r   setAssertionErrorlistitemsrT   )r\   rl   yparamsmissingkeyvalues          r]   partial_fitz-MockIncrementalImprovingEstimator.partial_fit   s.   AGAJ&1# 	$233c&kkAG $JgJJJ   %llnn  
U$222|8 8!!__8% 8% )9 9 9,u:M:M 9 9&21oo9 9 9  	 	 r_   rh   )rz   r{   r|   r}   r^   rp   r   __classcell__r   s   @r]   r   r      sf        447 7 7 7 7 7
         r_   r   c                   4    e Zd ZdZd	dZd Zd Zd
dZd ZdS )MockEstimatorWithParameterz-Dummy classifier to test the validation curve      ?c                 "    d | _         || _        d S rh   )r[   param)r\   r   s     r]   r^   z#MockEstimatorWithParameter.__init__   s    


r_   c                 8    || _         |j        d         | _        | S rX   ra   rc   s      r]   re   zMockEstimatorWithParameter.fit   rf   r_   c                     t           rh   ri   rk   s     r]   rm   z"MockEstimatorWithParameter.predict   rn   r_   Nc                 N    |                      |          r| j        n	d| j        z
  S )N   )rp   r   )r\   rl   r   s      r]   rt   z MockEstimatorWithParameter.score   s'    !33A66JtzzA
NJr_   c                     || j         u S rh   rv   rk   s     r]   rp   z,MockEstimatorWithParameter._is_training_data   rw   r_   )r   rx   ry   r~   r_   r]   r   r      ss        77     
" " "K K K K" " " " "r_   r   c                   (     e Zd ZdZ fdZd Z xZS )%MockEstimatorWithSingleFitCallAllowedz<Dummy classifier that disallows repeated calls of fit methodc                     t          | d          r
J d            d| _        t                                          ||          S )Nfit_called_zfit is called the second timeT)hasattrr   r   re   )r\   r[   rd   r   s      r]   re   z)MockEstimatorWithSingleFitCallAllowed.fit   sB    4//PP1PPP/ww{{8X...r_   c                     t           rh   ri   rk   s     r]   rm   z-MockEstimatorWithSingleFitCallAllowed.predict   rn   r_   )rz   r{   r|   r}   re   rm   r   r   s   @r]   r   r      sM        FF/ / / / /
" " " " " " "r_   r   c                   P    e Zd ZdZddZ	 	 	 	 	 	 	 	 	 ddZd Zd Zdd	Zdd
Z	dS )MockClassifierz-Dummy classifier to test the cross-validationr   Fc                 "    || _         || _        d S rh   aallow_nd)r\   r   r   s      r]   r^   zMockClassifier.__init__   s     r_   Nc           	      N   || _         || _        |	| _        |
 |
|            | j        r#|                    t          |          d          }|j        dk    r| j        st          d          |P|j        d         |j        d         k    s4J d	                    |j        d         |j        d                               ||j        d         t          t          j        t                              k    sMJ d	                    |j        d         t          t          j        t                                                    |Rd}|j        d         |j        d         k    s4J |	                    |j        d         |j        d                               |md	}|j        t          j        k    sVJ |	                    |j        d         |j        d
         t          j        d         t          j        d
                               t          j        t                    | _        | S )zThe dummy arguments are to test that this fit function can
        accept non-array arguments through cross-validation, such as:
            - int
            - str (this is actually array-like)
            - object
            - function
        N   zX cannot be dr   zKMockClassifier extra fit_param sample_weight.shape[0] is {0}, should be {1}zIMockClassifier extra fit_param class_prior.shape[0] is {0}, should be {1}zRMockClassifier extra fit_param sparse_sample_weight.shape[0] is {0}, should be {1}zUMockClassifier extra fit_param sparse_param.shape is ({0}, {1}), should be ({2}, {3})r   )	dummy_int	dummy_str	dummy_objr   reshapelenndim
ValueErrorrb   formatnpuniquer   Pclasses_)r\   rl   rs   sample_weightclass_priorsparse_sample_weightsparse_paramr   r   r   callbackfmts               r]   re   zMockClassifier.fit   s   ( #""HTNNN= 	&		#a&&"%%A6Q;;t};_---$ &q)QWQZ777??Ev!'*AGAJ@ @ 877 "$Q'3ry||+<+<<<<))/0A!0Dc")TU,,FWFW)X)X =<<  +2  (-a0AGAJ>>>

$*1-qwqzA A>>> #6   %000#**"1%"1%

	3 3000 	!r_   c                 n    | j         r#|                    t          |          d          }|d d df         S )Nr   r   )r   r   r   r\   Ts     r]   rm   zMockClassifier.predict	  s6    = 	&		#a&&"%%AAwr_   c                     |S rh   r~   r   s     r]   predict_probazMockClassifier.predict_proba  s    r_   c                 @    ddt          j        | j                  z   z  S )N      ?r   )r   absr   rr   s      r]   rt   zMockClassifier.score  s    a"&..())r_   c                      | j         | j        dS )Nr   r   )r\   deeps     r]   
get_paramszMockClassifier.get_params  s    V777r_   )r   F)	NNNNNNNNNrx   )F)
rz   r{   r|   r}   r^   re   rm   r   rt   r   r~   r_   r]   r   r      s        77! ! ! ! !> > > >@  
  * * * *8 8 8 8 8 8r_   r   )      )r   r   r   r   r   r   r   r      r      r      r   r   )r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   coo_containerc                    t                      } | t                    }t          dd          D ]%}||_        t	          |t          t
                    }t          ||                    t          t
                               t          j	        t
          t
          d d d         g          }t	          |||          }t          ||                    ||                     t	          ||t
                    }t          ||                    |t
                               t	          |||          }t          ||                    ||                     'd }t          |          }t	          |t                                          t
                                          d          }t          |          }t	          |t          t
                                          d          }t          d d d d t          j        f         }t          d	
          }t	          ||t
                    }t          d
          }t          j        t                    5  t	          ||t
          d           d d d            d S # 1 swxY w Y   d S )Ni
   r   c                 ,    t          | t                    S rh   
isinstancer   r   s    r]   <lambda>z&test_cross_val_score.<locals>.<lambda>:      :a.. r_   check_Xr   cvcheck_yT)r   Fraiseerror_score)r   rl   ranger   r/   y2rQ   rt   r   column_stackrK   tolistnewaxispytestraisesr   )r   clfX_sparser   scoresmultioutput_y
list_checkX_3ds           r]   test_cross_val_scorer   "  sc   


C}QH3^^ G G a,,6399Q#3#3444 R"X77 h>>6399X}#E#EFFF h336399Xr#:#:;;; !h>>6399X}#E#EFFFF /.J
Z
0
0
0CS!((**biikka@@@F
Z
0
0
0CS!RYY[[Q777F QQQ2:D
$
'
'
'CS$++F
%
(
(
(C	z	"	" < <T27;;;;< < < < < < < < < < < < < < < < < <s   I<<J J c                      t          d          \  } }t          d          }t          |dddgi          }t          || |d	
           d S )NT
return_X_yauto)gammaCr   r   )
param_gridr   )n_jobs)r   r?   r%   r0   )rl   r   r   grids       r]   test_cross_validate_many_jobsr   K  s_     %%%DAq
F


Cq"g777D4Aa((((((r_   c            
         t          d          \  } }t                      }d}t          j        t          |          5  t          || |t          t                    t          t                    f           d d d            n# 1 swxY w Y   t          j        t          |          5  t          || |t          t                    f           d d d            n# 1 swxY w Y   t          j        t          |dz             5  t          || |d           d d d            n# 1 swxY w Y   t          j        t          |dz             5  t          || |d	           d d d            n# 1 swxY w Y   t          j        t          |          5  t          || |t          t                    gg           d d d            n# 1 swxY w Y   t          j        t          d
          5  t          || |t                                 d d d            n# 1 swxY w Y   t          t                    }dt          j         d}t          j        t          |          5  t          || ||           d d d            n# 1 swxY w Y   t          j        t          |          5  t          || |d|i           d d d            d S # 1 swxY w Y   d S )Nr   random_statez.*must be unique strings.*matchscoringzEmpty list.*r~   zDuplicate.*)f1_micror  zAn empty dict[Scoring failed. The score on this train-test partition for these parameters will be set to z. Details: 
foo)r   r   r   r   r   r0   r   r"   r   dictr!   r   nanwarnsUserWarning)rl   r   	estimatorerror_message_regexpmulticlass_scorerwarning_messages         r]   )test_cross_validate_invalid_scoring_paramr  U  s,   A...DAq  I 8 
z)=	>	>	> 
 
 11;~3N3NO		
 	
 	
 	

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
z)=	>	>	> Q Qy!Q_1M1M0OPPPPQ Q Q Q Q Q Q Q Q Q Q Q Q Q Q 
z)=)N	O	O	O 4 4y!Q33334 4 4 4 4 4 4 4 4 4 4 4 4 4 4 
z)=)M	N	N	N J Jy!Q0HIIIIJ J J J J J J J J J J J J J J 
z)=	>	>	> R Ry!Q+o2N2N1O0PQQQQR R R R R R R R R R R R R R R 
z	9	9	9 : :y!Q9999: : : : : : : : : : : : : : : $$CDD
	9;	 	 	  
k	9	9	9 C Cy!Q0ABBBBC C C C C C C C C C C C C C C 
k	9	9	9 L Ly!Q8I0JKKKKL L L L L L L L L L L L L L L L L Ls   :BB	B	+'CC"%C"D''D+.D+E00E47E4(G

GG0 HH #H &JJ
J
,KKKc                      t          d          \  } }t          dt                      fdt                      fg          }t	          || |d          }|d         }t          |t                    sJ t          d |D                       sJ d S )NTr   imputer
classifier)return_estimatorr  c              3   @   K   | ]}t          |t                    V  d S rh   )r   r<   ).0r  s     r]   	<genexpr>z7test_cross_validate_nested_estimator.<locals>.<genexpr>  s,      KK9z)X..KKKKKKr_   )r   r<   r   r   r0   r   r   all)rl   r   pipelineresults
estimatorss        r]   $test_cross_validate_nested_estimatorr    s     $'''FQ(>++,	
 H Xq!dCCCG%Jj$'''''KK
KKKKKKKKKKr_   
use_sparseFTcsr_containerc           
         t                      }t          dd          \  }}t          d          }t          dd          \  }}t	          dd          }| r ||          } ||          }|||f|||ffD ]\  }	}
}t          |d          }t          |d	          }g }g }g }g }g }|                    |	|
          D ]\  }}t          |                              |	|         |
|                   }|	                     |||	|         |
|                              |	                     |||	|         |
|                              |	                     |||	|         |
|                              |	                     |||	|         |
|                              |	                    |           t          j        |          }t          j        |          }t          j        |          }t          j        |          }t          j        |          }|||||f}t          ||	|
||           t          ||	|
||           d S )
N   r   	n_samplesr   r   linearkernelr   neg_mean_squared_errorr   r2)r(   r   r   r   r?   r   splitr	   re   appendr   array"check_cross_validate_single_metric!check_cross_validate_multi_metric)r  r  r   X_regy_regregX_clfy_clfr   rl   r   est
mse_scorer	r2_scorertrain_mse_scorestest_mse_scorestrain_r2_scorestest_r2_scoresfitted_estimatorstraintestr   s                         r]   test_cross_validater8    s    
B #Ra@@@LE5
Q


C '!DDDLE5
XA
.
.
.C %e$$e$$eS)E5#+>? $A $A	1c"30HIII
!#t444	88Aq>> 	* 	*KE4**..51U844C##JJsAeHah$G$GHHH""99S!E(AeH#E#EFFF""::c1T7AdG#D#DEEE!!))C4!D'"B"BCCC$$S))))8$455(?33(?33.11H%677 
 	+31fbAAA)#q!VR@@@@I$A $Ar_   c           	          |\  }}}}}	dD ]\  }
}|
r,t          | ||dd|          }t          |d         |           nt          | ||dd|          }t          |t                    sJ t	          |          |k    sJ t          |d         |           |
r.t          | ||dgd|          }t          |d	         |d           nt          | ||dgd|          }t          |t                    sJ t	          |          |k    sJ t          |d
         |           t          | ||dd|          }t          |d                   D ]\  }}|j                                        }t          |          r|	                                }|	|         j                                        }t          |          r|	                                }t          ||           t          |j        |	|         j                   d S )N))Tr   )Fr   r"  Tr   return_train_scorer   train_scoreF
test_scorer#  train_r2test_r2)r   r  r   r  )r0   rP   r   r  r   	enumeratecoef_copyr   toarrayrO   
intercept_)r   rl   r   r   r   r1  r2  r3  r4  r5  r;  dict_lenmse_scores_dictr2_scores_dictkr.  est_coeffitted_est_coefs                     r]   r'  r'    s    	 )@ &M &M$H 	,0#'  O &om&DFVWWWW,0#(  O /400000?##x////!/,"?QQQ  		+QD6dr  N &nZ&@/SWXXXX+QD6e  N .$/////>""h....!.";^LLLL %Q3dr  O OK899 
M 
M39>>##H 	*''))H+A.499;;O$$ 	8-5577OHo666CN,=a,@,KLLLL
M 
Mr_   c           
      @   |\  }}}}}	d }
dt          t                    dd|
f}h d}|                    ddh          }dD ]}|D ]}|rBt          | |||d|	          }t	          |d         |           t	          |d         |           nt          | |||d
|	          }t          |t                    sJ t          |                                          |r|n|k    sJ t	          |d         |           t	          |d         |           t          |d         t          j
                  sJ t          |d         t          j
                  sJ t          |d         t          j
                  sJ t          |d         t          j
                  sJ t          j        |d         dk              sJ t          j        |d         dk               sJ t          j        |d         dk              sJ t          j        |d         dk               sJ Ԑd S )Nc                 p    |                      |          }t          ||          t          ||           dS )Nr#  r"  )rm   r#   r    )r   rl   r   y_preds       r]   custom_scorerz8check_cross_validate_multi_metric.<locals>.custom_scorer!  s=    Q1f%%'9!V'D'D&D
 
 	
r_   rM  r"  >   r?  fit_time
score_timetest_neg_mean_squared_errorr>  train_neg_mean_squared_error)TFTr:  Fr?  rR  rP  rQ  r   r   )r   r#   unionr0   rP   r   r  r   keysr   ndarrayr  )r   rl   r   r   r   r1  r2  r3  r4  r5  rO  all_scoringkeys_sans_trainkeys_with_trainr;  r   
cv_resultss                    r]   r(  r(    s    	
 
 
 	)h''&>	
 	
 	K  O &++	34 O , "9 "9" !	9 !	9G! +Aw4B  
 **Z*@/RRR)=>@P    ,Aw5R  
 j$/////z(())#5J?    &j&;^LLL%89?  
 j3RZ@@@@@j)FGTTTTTj4bjAAAAAj6
CCCCC 6*Z0A5666666*Z025666666*\2a7888886*\2R7888888C!	9"9 "9r_   c                     t          ddd          \  } }t          d          }t                      t          d          t	                      t                      g}d}|D ]}t          j        t          |          5  t          || ||	           d d d            n# 1 swxY w Y   t          j        t          |          5  t          || ||	           d d d            n# 1 swxY w Y   d S )
N   r   r   r  	n_classesr   r  r!  z*The 'groups' parameter should not be None.r   )r  rl   r   r   )r   r?   r)   r+   r&   r'   r   r   r   r/   r.   )rl   r   r   	group_cvserror_messager   s         r]   #test_cross_val_score_predict_groupsrb  `  s}    qqIIIDAq
X


C 		I AM > >]:];;; 	< 	<cQ!;;;;	< 	< 	< 	< 	< 	< 	< 	< 	< 	< 	< 	< 	< 	< 	<]:];;; 	> 	>qA"====	> 	> 	> 	> 	> 	> 	> 	> 	> 	> 	> 	> 	> 	> 	>> >s$   >BB"	%B"	C$$C(	+C(	c                  6  	 t           t           fg} 	 ddlm}m} |                     ||f           n# t
          $ r Y nw xY w| D ]S\  	 t                     	t                    }}fd}	fd}t          ||          }t          |||d           Td S )Nr   	DataFrameSeriesc                 $    t          |           S rh   r   r   InputFeatureTypes    r]   r   z-test_cross_val_score_pandas.<locals>.<lambda>      Z+;<< r_   c                 $    t          |           S rh   rh  r   
TargetTypes    r]   r   z-test_cross_val_score_pandas.<locals>.<lambda>      Az!:!: r_   r   r   r   r   )
rL   pandasre  rf  r%  ImportErrorrl   r   rK   r/   
typesre  rf  X_dfy_sercheck_dfcheck_seriesr   rj  rn  s
           @@r]   test_cross_val_score_pandasry  v  s    ]+,E,,,,,,,,fi())))   (- 0 0$
$ '&q))::b>>e<<<<:::: <HHHT5Q/////0 0   3 
A A c                  $   t          d          } t                      }|j        |j        }}t	          d          }t          | |||          }t	          d          }g }|                    ||          D ]v\  }}t          j        t          |          t                    }	t          j        t          |          t                    }
d|	|<   d|
|<   |                    ||f           wt          | |||          }t          ||           d S )Nr  r_  r   r   dtyper   )r?   r   datatargetr(   r/   r$  r   zerosr   boolr%  rQ   )svmirisrl   r   kfoldscores_indicescv_masksr6  r7  
mask_train	mask_testscores_maskss               r]   test_cross_val_score_maskr    s   
X


C;;D9dkqA!HHE$S!Q5999N!HHEH{{1a(( ' 'tXc!ffD111
HSVV4000	
5	$&&&&"31:::L~|44444r_   c                     t          d          } t                      }|j        |j        }}t	          j        ||j                  }t          | ||          }t          d          } t          | ||          }t          ||           t          d           } t          | ||          }t          ||           t          d          } t          j
        t                    5  t          | ||           d d d            n# 1 swxY w Y   t          j
        t                    5  t          | |                                |           d d d            d S # 1 swxY w Y   d S )Nprecomputedr_  r  c                 6    t          j        | |j                  S rh   )r   dotr   )r   r   s     r]   r   z2test_cross_val_score_precomputed.<locals>.<lambda>  s    "&AC.. r_   )r?   r   r~  r  r   r  r   r/   rP   r   r   r   r   )r  r  rl   r   linear_kernelscore_precomputedscore_linearscore_callables           r]    test_cross_val_score_precomputedr    s   
]
#
#
#C;;D9dkqAF1acNNM']A>>
X


C"31--L/>>> 00
1
1
1C$S!Q//N/@@@ ]
#
#
#C	z	"	" # #Q"""# # # # # # # # # # # # # # #
 
z	"	" 8 8]1133Q7778 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8s$   $DD	D&$EEEc           	      8  	
 t                      }t          j        d         }t          t	          j        t                              } | t	          j        dg          t	          j        dg          t	          j        dg          ffd          } | t	          j        d                    }dd
t                      		
fd}t	          j
        |          t	          j        |d	|z            ||
	|d
}t          |t          t          |           d S )Nr   r   )r   r   rb   r   *   42c                 V    | j         k    sJ | j        k    sJ | j        k    sJ d S rh   )r   r   r   )r   	DUMMY_INT	DUMMY_OBJ	DUMMY_STRs    r]   assert_fit_paramsz:test_cross_val_score_fit_params.<locals>.assert_fit_params  sD     }	))))}	))))}	))))))r_   r   )r   r   r   r   r   r   r   r   r   )r   rl   rb   r   r   r   r   r&  eyeobjectonesfullr/   r   )r   r   r  r^  W_sparseP_sparser  
fit_paramsr  r  r  s           @@@r]   test_cross_val_score_fit_paramsr    s!   


C
IBIaLL!!I}	1#1#!67w  H }RVAYY''HIII* * * * * * * ++wy#	/:: ( %	 	J CBz222222r_   c                  4   t                      } g fd}t          j        d          5  t          |          }t	          | t
          t          |d          }d d d            n# 1 swxY w Y   t          |g d           t                    dk    sJ d S )Nc                 6                         | |f           dS )Nr   )r%  )y_test	y_predict_score_func_argss     r]   
score_funcz3test_cross_val_score_score_func.<locals>.score_func  s!     3444sr_   Trecordr   )r   r   )r   r   r   )	r   warningscatch_warningsr   r/   rl   r   rQ   r   )r   r  r   rt   r  s       @r]   test_cross_val_score_score_funcr    s   


C     
	 	-	-	- B Bj))Q7qAAAB B B B B B B B B B B B B B B uooo...  A%%%%%%s   .A&&A*-A*c                  ^   t                      } t          d          }t          || j        | j                  }t          |g dd           t          || j        | j        d          }t          |g dd           t          || j        | j        d          }t          |g dd           d S )Nr  r_  )
ףp=
?r   r  r  r   r   accuracyr   f1_weighted)r   r?   r/   r~  r  rP   )r  r   r   	zo_scores	f1_scoress        r]   3test_cross_val_score_with_score_func_classificationr    s    ;;D
X


C S$)T[99Ff&B&B&BAFFF  TYZPPPIi)E)E)EqIII  TY]SSSIi)E)E)EqIIIIIr_   c                     t          dddd          \  } }t                      }t          || |          }t          |g dd           t          || |d	          }t          |g dd           t          || |d
	          }t	          j        g d          }t          ||d           t          t                    }t          || ||	          }t          |g dd           d S )Nr  r\  r   r   r  
n_featuresn_informativer   )gGz?r  r  gGz?gq=
ףp?r   r#  r   r"  )g(\؇gzGIgGz&qg\(qg)\G)r   r   r/   rP   r   r&  r   r   )	rl   r   r+  r   	r2_scoresneg_mse_scoresexpected_neg_mser   	ev_scoress	            r]   /test_cross_val_score_with_score_func_regressionr    s   RBaVWXXXDAq
''C S!Q''Ff&D&D&DaHHH  Q4888Ii)G)G)GKKK %S!Q8PQQQNx N N NOOn.>BBB 233GQ7;;;Ii)G)G)GKKKKKr_   c                    t                      }|j        } | |          }|j        }t          d          }t	          d          }t          |||d|d          \  }}}	|dk    sJ t          |	dd	           t          |||d|dt          j        |j	                  d
          \  }
}}|
|k    sJ ||	k    sJ t          d          }t	          d          }t          |||d|dt          j        |j	                  d
          \  }
}}|
|k    sJ ||	k    sJ d }t          |          }t          |||d||d
          \  }}}	t          |dd           t          |	dd           t          j        t          j        t          |                    d          }t          |||d|d          \  }}}	|dk     sJ |	dk    sJ d S )Nr  r_  r   r  r  )n_permutationsr   r   g?        r   r   )r  r   r   groupsr   c                     | |k                                     | |k                                     z
  | j        d         z  S rX   )sumrb   )y_truerN  s     r]   custom_scorez,test_permutation_score.<locals>.custom_scoreD  s;    6!&&((Ff,<+A+A+C+CCv|TUVVr_   d   )r  r   r   r   g(\?g{Gz?r   r   皙?)r   r~  r  r?   r-   r2   rO   r   r  sizer   modaranger   )r   r  rl   r   r   r  r   rt   r   pvaluescore_group_pvalue_group
svm_sparse	cv_sparser  scorers                    r]   test_permutation_scorer    sK   ;;D	A}QHA
X


C			B2Q"Z  E66 3;;;;Q'''#9		wqv	$ 	$ 	$ KL %6!!!! H%%%J""I#9	wqv	$ 	$ 	$ KL %6!!!!W W W &&F-Q#v"1  E1f tQ'''a((( 	ryQ  !$$A2Q"Z  E66 3;;;;C<<<<<<r_   c                  t   t          j        dt           j                                      dd          } t           j        | dd d f<   t          j        ddg| j        d         dz            }t          dt          d	t           j        
          fdt                      fg          }t          || |           d S N   r|  r   r   r   r   r   r  mean)strategymissing_valuesr  )r   r  float64r   r  repeatrb   r<   r   r   r2   rl   r   ps      r]   &test_permutation_test_score_allow_nansr  Y  s    
	#RZ(((00R88AfAadG
	1a&!'!*q.))AvbfMMMN>++,	
	 	A 1a#####r_   c            
      F   t          j        d                              dd          } t          j        dgdz  dgdz  z             }t	          d          }d}t          j        t          |	          5  t          || |           d d d            n# 1 swxY w Y   d
}t          j        t          |	          5  t          || |dt          j
        d          i           d d d            n# 1 swxY w Y   t          || |dt          j
        d          i           d S )Nr  r   r   r   r   Texpected_sample_weight#Expected sample_weight to be passedr   /sample_weight.shape == \(1,\), expected \(8,\)!r   r  )r   r  r   r&  rK   r   r   r   r2   r   r  rl   r   r   err_msgs       r]   "test_permutation_test_score_paramsr  g  s   
	#r2&&A
!qA37"##A
D
9
9
9C4G	~W	5	5	5 * *sAq)))* * * * * * * * * * * * * * * AG	z	1	1	1 P PsAq/271::1NOOOOP P P P P P P P P P P P P P P31orwr{{-KLLLLLLs$   5BBB;(C//C36C3c                  t   t          j        dt           j                                      dd          } t           j        | dd d f<   t          j        ddg| j        d         dz            }t          dt          d	t           j        
          fdt                      fg          }t          || |           d S r  )r   r  r  r   r  r  rb   r<   r   r   r/   r  s      r]   test_cross_val_score_allow_nansr  v  s    
	#RZ(((00R88AfAadG
	1a&!'!*q.))AvbfMMMN>++,	
	 	A Aq!r_   c                  P   t          j        ddgddgddgddgddgddgddgddgddgddgg
          } t          j        ddgddgddgddgddgddgddgddgddgddgg
          }t          d	          }t          t          d
          }t          t          d          }t          t          d          }t          || ||          }t          || ||          }t          || ||          }t          |g d           t          |g d           t          |g d           d S )Nr   r   r   r   r   r   )n_neighborsmicro)averagemacrosamplesr   )r   r         ?r   gUUUUUU?)r   r   r  r         ?)r   r&  r:   r   r"   r/   rO   )	rl   r   r   scoring_microscoring_macroscoring_samplesscore_microscore_macroscore_sampless	            r]   test_cross_val_score_multilabelr    s   
GFFFGGFHHG	
	 	A 	
Q!Q!Q!Q!Q!Q!Q!Q!QRSUVQWX	 	A 1
-
-
-CAAAMAAAM!/9EEEO!#q!]CCCK!#q!]CCCK#CAGGGM%D%D%DEEE%D%D%DEEE'F'F'FGGGGGr_   c           
         t          d          \  }}t                      }t                      }t          j        |          }|                    ||          D ]E\  }}|                    ||         ||                    |                    ||                   ||<   Ft          ||||          }t          ||           t          |||          }t          |          t          |          k    sJ t                      }t          ||||          }t          |          t          |          k    sJ |                                }	|	|	t          j        |	          k    z  }	 | |	          }	t          ||	|          }t          t          |          t          |                     t          t          d          |          }t          |          t          |          k    sJ  G d d          }
t          j        t"                    5  t          ||| |
                       d d d            n# 1 swxY w Y   t%          d          \  }}d}t          j        t(          |	          5  t          t+          d
          ||dt          d                     d d d            d S # 1 swxY w Y   d S )NTr   r   r   )n_initc                       e Zd ZddZdS )%test_cross_val_predict.<locals>.BadCVNc              3      K   t          d          D ]0}t          j        g d          t          j        g d          fV  1d S )Nr   )r   r   r   r   )r   r   r         )r   r   r&  )r\   rl   r   r  is        r]   r$  z+test_cross_val_predict.<locals>.BadCV.split  sY      1XX H Hh|||,,bh.G.GGGGGGH Hr_   rx   )rz   r{   r|   r$  r~   r_   r]   BadCVr    s.        	H 	H 	H 	H 	H 	Hr_   r  zNumber of classes in training fold \(2\) does not match total number of classes \(3\). Results may not be appropriate for your use case.r   	liblinear)solverr   r   methodr   )r   r(   r   r   
zeros_liker$  re   rm   r.   rP   r   r*   rB  medianr   r   r   r   r   r  RuntimeWarningr   )r   rl   r   r   r.  preds2r6  r7  predsXspr  r  s               r]   test_cross_val_predictr    s<   D)))DAq	B
''C ]1Fxx1~~ , ,t%!E(###{{1T7++tc1aB///EeV,,,c1a((Eu::Q	Bc1aB///Eu::Q
&&((C33C
-

Cc3**Ec%jj#a&&111fF333Q77Eu::QH H H H H H H H
 
z	"	" 1 1#q!00001 1 1 1 1 1 1 1 1 1 1 1 1 1 1 %%%DAq	< 
 
nO	<	<	< 
 
k222"Qxx	
 	
 	
 	

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
s$   H<<I I 70J44J8;J8c            
      r   t          ddd          \  } }t          t                      | |d          }|j        dk    sJ t	          d	          \  } }t          t                      | |d          }|j        d
k    sJ | d d         } |d d         }d}t          j        t          |          5  t          t                      | |dt          d                     d d d            n# 1 swxY w Y   t          d	          \  } }t          dd          }t          || |d          }|j        dk    sJ t          j        |          }| |         ||         }} d}t          j        t          |          5  t          || |t          d          d           d d d            d S # 1 swxY w Y   d S )Nr   2   r   r^  r  r   decision_functionr  )r  Tr      r   r  zOnly 1 class/es in training fold, but 2 in overall dataset. This is not supported for decision_function with imbalanced folds. To fix this, use a cross-validation technique resulting in properly stratified foldsr   r  r  ovo)r!  decision_function_shape)i  -   zOutput shape \(599L?, 21L?\) of decision_function does not match number of classes \(7\) in fold. Irregular decision_function .*r   n_splitsr   r  )r   r.   r   rb   r   r   r   r   r   r(   r   r?   r   argsort)rl   r   r  ra  r.  indr	  s          r]   .test_cross_val_predict_decision_function_shaper    s   bqIIIDAq022AqATUUUE;%%%%DAq022AqATUUUE;("""" 	
$3$A	$3$A	1  
z	7	7	7 
 
q!,?E!HH	
 	
 	
 	

 
 
 
 
 
 
 
 
 
 
 
 
 
 

 $'''DAq
Xu
=
=
=Cc1a0CDDDE;*$$$$
*Q--CS61S6qA	  
z)=	>	>	> W W#q!q(9(9(9BUVVVVW W W W W W W W W W W W W W W W W Ws$   2.C,,C03C0<#F,,F03F0c                     t          ddd          \  } }t          t                      | |d          }|j        dk    sJ t	          d	          \  } }t          t                      | |d          }|j        d
k    sJ d S )Nr   r  r   r  r   r  r  r   Tr   r  r   r.   r   rb   r   rl   r   r  s      r]   *test_cross_val_predict_predict_proba_shaper$    s    bqIIIDAq022AqQQQE;'!!!!%%%DAq022AqQQQE;(""""""r_   c                     t          ddd          \  } }t          t                      | |d          }|j        dk    sJ t	          d	          \  } }t          t                      | |d          }|j        d
k    sJ d S )Nr   r  r   r  predict_log_probar  r!  Tr   r  r"  r#  s      r]   .test_cross_val_predict_predict_log_proba_shaper'    s    bqIIIDAq022AqATUUUE;'!!!!%%%DAq022AqATUUUE;(""""""r_   c                 F   t                      }|j        |j        }} | |          }t          j        ||d d d         g          }t          dd          }t          |||          }|j        dk    sJ t          |||          }|j        dk    sJ t          |||          }t          |j        d           t          |||          }t          |j        d           d }t          |          }t          ||
                                |
                                          }t          |	          }t          |||
                                          }t          t                      |
                                |
                                d
          }t          t                      ||
                                d
          }|d d d d t          j        f         }	d }
t          |
          }t          ||	|          }t          |j        d           d S )Nr   Fr   )fit_interceptr   )r  )r  r   c                 ,    t          | t                    S rh   r   r   s    r]   r   z4test_cross_val_predict_input_types.<locals>.<lambda><  r   r_   r   r   r  r  c                     | j         dk    S )Nr   )r   r   s    r]   r   z4test_cross_val_predict_input_types.<locals>.<lambda>S  s    1 r_   )r   r~  r  r   r   r   r.   rb   rQ   rK   r   r   r   )r   r  rl   r   r   r   r   predictionsr   r   check_3ds              r]   "test_cross_val_predict_input_typesr.  #  s   ;;D9dkqA}QHOQ$$B$L11M
e!
4
4
4C $CA..K&&&& $C=AAK((((#C155K{(&111 $C=AAK{((333 /.J
Z
0
0
0C#CQXXZZ@@K
Z
0
0
0C#CAHHJJ77K $	

	

"	  K $		

"	  K QQQ2:D$$H
X
.
.
.C#Cq11K{(&11111r_   c                  6  	 t           t           fg} 	 ddlm}m} |                     ||f           n# t
          $ r Y nw xY w| D ]S\  	 t                     	t                    }}fd}	fd}t          ||          }t          |||d           Td S )Nr   rd  c                 $    t          |           S rh   rh  ri  s    r]   r   z/test_cross_val_predict_pandas.<locals>.<lambda>e  rk  r_   c                 $    t          |           S rh   rh  rm  s    r]   r   z/test_cross_val_predict_pandas.<locals>.<lambda>f  ro  r_   rp  r   r   )
rL   rq  re  rf  r%  rr  rl   r   rK   r.   rs  s
           @@r]   test_cross_val_predict_pandasr2  Y  s    ]+,E,,,,,,,,fi())))   (- 2 2$
$&&q))::b>>e<<<<:::: <HHH#tUq111112 2rz  c                     t          dddddd          \  } }d|d<   t          d          }t          d          }t          |                    | |                    \  }}t          || ||d	          }||d                  d         dk    sJ t          j        ||d                  d d df         dk              sJ t          j        ||d                  d d ddf         dk              sJ t          j        ||d                  dk              sJ t          |	                    d
          t          j
        |j                  d           d S )Nr  r   r   r   )r  r  n_redundantr  n_clusters_per_classr   r   r  r   r  axis   )decimal)r   r   r-   r   r$  r.   r   r  rP   r  r  rb   )rl   r   r   r   r6  r7  
yhat_probas          r]   !test_cross_val_predict_unbalancedr;  k  si     DAq AaD
!
,
,
,C	!	$	$	$Brxx1~~&&KE4"31OLLLJT!W:a=A6*T!W%aaad+q0111116*T!W%aaa1f-1222226*T!W%)*****jnn!n44bgag6F6FPRSSSSSSr_   c                  0   t                      } t          j                            d          }|                    dd          }t          | |d dd          }t          |d d df         |           t          | |d dd          }t          ||           d S )	Nr  r  r   r   rm   )r   r   r  r   r   )r   r   randomRandomStaterandr.   rN   )mock_classifierrngrl   y_haty_hat_probas        r]   test_cross_val_predict_y_nonerD    s    $&&O
)


#
#CbAoqDQyQQQEAaaadGU####dq  K A{#####r_   c                    t                      }|j        |j        }}t                      }d | t	          j        |j        d                             i}t          ||||d          }t          |t	          j	        d                     d S )Nr   r   r   )r   r   )
r   r~  r  r   r   r  rb   r/   rQ   r  )r   r  rl   r   r   r  r   s          r]   &test_cross_val_score_sparse_fit_paramsrF    s}    ;;D9dkqA


C(--qwqz8J8J*K*KLJQ*;;;Aq"'!**%%%%%r_   c                     d} d}t          | dddddd          \  }}t          | |dz
  |z  z            }dD ]9}t          j        d	          5 }t	          |||t          |
          t          j        ddd          |d          \  }}}	}
}d d d            n# 1 swxY w Y   t          |          dk    rt          d|d         j
        z            |j        dk    sJ |	j        dk    sJ |
j        dk    sJ |j        dk    sJ t          |t          j        ddd                     t          |                    d          t          j        ddd                     t          |	                    d          t          j        ddd                     |
j        dk    sJ |j        dk    sJ t          j        d	          5 }t	          |||t!          ||           t          j        ddd          |          \  }}}d d d            n# 1 swxY w Y   t          |          dk    rt          d|d         j
        z            t          ||           t          ||	           ;d S )Nr  r   r   r   r   r  r  r  r4  r^  r5  r   FTTr  r  皙?r   r   )r   rZ   rF   return_timesUnexpected warning: %r)r   r   r\  r6  ffffff?r  r  r  )r   rZ   rF   )r   rV   r  r  r1   r(   r   linspacer   RuntimeErrormessagerb   rQ   rP   r  r}  r7   )r  r  rl   r   r  shuffle_trainwrZ   train_scorestest_scores	fit_timesscore_timestrain_sizes2train_scores2test_scores2s                  r]   test_learning_curver[    sK   IH  DAq 'yX\X4M'NOOI& -= -=$D111 	Q (+++KS"55%!  	 	 	 	 	 	 	 	 	 	 	 	 	 	 	  q66A::7!A$,FGGG!W,,,, G++++')))) G++++;Ar2(>(>???!,"3"3"3";";R[cSU=V=VWWW!+"2"2"2":":BKSRT<U<UVVV )++++ I---- $D111 	Q8F"H	JJJKS"55%9 9 95L-	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 q66A::7!A$,FGGG!->>>!,<<<<[-= -=s$   ?BB	"B	=HH	H	c                     t          ddddddd          \  } }t          d          }t          || d dt          j        dd	d
                    \  }}}t          |t          j        ddd
                     t          |                    d          t          j        dd	d
                     t          |                    d          t          j        dd	d
                     d S )Nr  r   r   r   rH  r\  r   rJ  r   r   )r   r   rZ   r6  rM  )r   rV   r1   r   rO  rQ   rP   r  rl   r  r  rZ   rT  rU  s         r]    test_learning_curve_unsupervisedr^    s      DAq 'r**I-;1Cb0I0I. . .*K{ {BK2r$:$:;;;l//Q/77S#r9R9RSSSk..A.66Cb8Q8QRRRRRr_   c           	          t          ddddddd          \  }}t          d          }t          |||dd           d	|                                 j        v sJ d S )
Nr  r   r   r   rH  r\  r   )r   verbosez[learning_curve])r   rV   r1   
readouterrout)capsysrl   r   r  s       r]   test_learning_curve_verboserd    s}      DAq 'r**I9aq!4444!2!2!4!4!8888888r_   c            	          t          ddddddd          \  } }t          d          }t          j        t                    5  t          || |d           d d d            d S # 1 swxY w Y   d S )Nr   r   r   rH  T)exploit_incremental_learningr   rV   r   r   r   r1   rl   r   r  s      r]   5test_learning_curve_incremental_learning_not_possibleri    s      DAq 'q))I	z	"	" K Ky!QTJJJJK K K K K K K K K K K K K K K K K Ks   A##A'*A'c                     t          ddddddd          \  } }t          d          }dD ]}t          || |dd	t          j        d
dd          |          \  }}}t          |t          j        ddd                     t          |                    d          t          j        ddd                     t          |                    d          t          j        d
dd                     d S )Nr  r   r   r   rH  r\  rI  r   TrJ  r   r   )r   rf  rZ   rF   r6  rM  r   r   r1   r   rO  rQ   rP   r  )rl   r   r  rR  rZ   rT  rU  s          r]   (test_learning_curve_incremental_learningrl    s     DAq 2"55I& W W1?)-Cb11!2
 2
 2
.\; 	;Ar2(>(>???!,"3"3"3";";R[cSU=V=VWWW!+"2"2"2":":BKSRT<U<UVVVVW Wr_   c                     t          ddddddd          \  } }t          d          }t          || d ddt          j        d	d
d                    \  }}}t          |t          j        ddd                     t          |                    d          t          j        dd
d                     t          |                    d          t          j        d	d
d                     d S )Nr  r   r   r   rH  r\  r   TrJ  r   r   )r   r   rf  rZ   r6  rM  rk  r]  s         r]   5test_learning_curve_incremental_learning_unsupervisedrn  !  s      DAq 2"55I-;	
%)KS"--. . .*K{ {BK2r$:$:;;;l//Q/77S#r9R9RSSSk..A.66Cb8Q8QRRRRRr_   c            	         t          ddddddd          \  } }t          j        ddd          }t          dd d	
          }t	          || ||dd          \  }}}t	          || |d|d	          \  }}}	t          ||           t          |                    d          |                    d                     t          |                    d          |	                    d                     d S )Nr  r   r   r   rH  r  r   r   F)max_itertolrF   r   T)rZ   r   rf  )r   rZ   rf  r6  )r   r   rO  r   r1   rQ   rP   r  )
rl   r   rZ   r  train_sizes_inctrain_scores_inctest_scores_inctrain_sizes_batchtrain_scores_batchtest_scores_batchs
             r]   <test_learning_curve_batch_and_incremental_learning_are_equalrx  9  sP     DAq +c3**KqdEBBBI9G		%): : :6O% @N		%*@ @ @<)+< (9:::1%%'9'>'>A'>'F'F   !$$&7&<&<!&<&D&D    r_   c            	         t          ddddddd          \  } }t          d          }t          j        t                    5  t          || |dddg           d d d            n# 1 swxY w Y   t          j        t                    5  t          || |dd	d
g           d d d            n# 1 swxY w Y   t          j        t                    5  t          || |dddg           d d d            n# 1 swxY w Y   t          j        t                    5  t          || |dddg           d d d            n# 1 swxY w Y   t          j        t                    5  t          || |dddg           d d d            d S # 1 swxY w Y   d S )Nr  r   r   r   rH  r\  r   r   rZ   r  r   rJ  g?   rg  rh  s      r]   0test_learning_curve_n_sample_range_out_of_boundsr|  `  s/     DAq 'r**I	z	"	" B By!Q11a&AAAAB B B B B B B B B B B B B B B	z	"	" F Fy!Q13*EEEEF F F F F F F F F F F F F F F	z	"	" F Fy!Q13*EEEEF F F F F F F F F F F F F F F	z	"	" C Cy!Q11b'BBBBC C C C C C C C C C C C C C C	z	"	" C Cy!Q11b'BBBBC C C C C C C C C C C C C C C C C CsZ   A%%A),A)	B,,B03B0C33C7:C7D::D>D>FF	Fc                  <   t          ddddddd          \  } }t          d          }d}t          j        t          |          5  t          || |dt          j        dd	d          
          \  }}}d d d            n# 1 swxY w Y   t          |ddg           d S )Nr   r   r   r   rH  zzRemoved duplicate entries from 'train_sizes'. Number of ticks will be less than the size of 'train_sizes': 2 instead of 3.r   gQ?r   rz  )	r   rV   r   r  r  r1   r   rO  rQ   )rl   r   r  r  rZ   r  s         r]   1test_learning_curve_remove_duplicate_sample_sizesr~  w  s     DAq 'q))I	G  
nO	<	<	< 
 
*q!r{4a/H/H
 
 
Q
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 {QF+++++s   -A??BBc                     t          ddddddd          \  } }t          d          }t          d          }t          || ||t	          j        d	d
d                    \  }}}t          |t	          j        ddd                     t          |                    d          t	          j        dd
d                     t          |                    d          t	          j        d	d
d                     d S )Nr  r   r   r   rH  r\  r   r  rJ  r   r   rz  r6  rM  )	r   rV   r(   r1   r   rO  rQ   rP   r  )rl   r   r  r   rZ   rT  rU  s          r]   (test_learning_curve_with_boolean_indicesr    s     DAq 'r**I				B-;1aBBKS",E,E. . .*K{ {BK2r$:$:;;;l//Q/77S#r9R9RSSSk..A.66Cb8Q8QRRRRRr_   c                  b   t          j        ddgddgddgddgd	d
gddgddgddgddgddgddgd	d
gddgddgddgg          } t          j        g d          }t          j        g d          }t          dd ddd          }t          d          }t	          || ||dt          j        ddd          |dd	  	        \  }}}t          |                    d          t          j        g d                      t          |                    d          t          j        g d!                     t          j	        t                    5  t	          || ||dt          j        ddd          |d"#           d d d            n# 1 swxY w Y   t	          || ||dt          j        ddd          |ddd$
  
        \  }}	}
t          |	                    d          |                    d                     t          |
                    d          |                    d                     d S )%Nr   r   r   r   r   r   r  r     r8        r               r\  	   r   )r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   )r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   Fpa1)rp  rq  rF   learning_rateeta0r  333333?r   T)r   r   rZ   r  rF   r   r6  )r  r  Iq?)r  r  r  r   )r   r   rZ   r  r   )r   r   rZ   r  rF   r   rf  )r   r&  r   r&   r1   rO  rP   r  r   r   r   )rl   r   r  r  r   ru  rv  rw  rr  rs  rt  s              r]    test_learning_curve_with_shuffler    s    	FFFFHHHHHFGHHHH	
	 	A& 	>>>??AXCCCDDF e5q  I 
Q			B?M		KS!,,
@ 
@ 
@<)+< Q''2I2I2I)J)J   A&&1I1I1I(J(J   
z	"	" 

 

Ca00		
 		
 		
 		


 

 

 

 

 

 

 

 

 

 

 

 

 

 

 :H		KS!,,%): : :6O% 1%%'9'>'>A'>'F'F   !$$&7&<&<!&<&D&D    s   ,FF
Fc                  N   t          j        d                              dd          } t          j        dgdz  dgdz  z             }t	          d          }d}t          j        t          |	          5  t          || |d
           d d d            n# 1 swxY w Y   d}t          j        t          |	          5  t          || |d
dt          j
        d          i           d d d            n# 1 swxY w Y   t          || |d
dt          j
        d          i           d S )Nr  r   r   r   r   Tr  r  r   r   r   z/sample_weight.shape == \(1,\), expected \(2,\)!r   )r   r   )r   r  r   r&  rK   r   r   r   r1   r   r  r  s       r]   test_learning_curve_paramsr    s   
	#r2&&A
!qA37"##A
D
9
9
9C4G	~W	5	5	5 7 7sAqg66667 7 7 7 7 7 7 7 7 7 7 7 7 7 7 AG	z	1	1	1 
 
A7ORWQZZ3P	
 	
 	
 	

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 Qw/M     s$   5BBB=)C22C69C6c                  ~   t          ddddddd          \  } }t          ddg          }d}t          j        t          |	          5  t          || |d
dt          j        ddd          d           d d d            n# 1 swxY w Y   d}t          j        t          |	          5  t          || |d
dt          j        ddd          ddt          j        d
          i           d d d            n# 1 swxY w Y   t          || |d
dt          j        ddd          ddt          j        d          i           d S )Nr  r   r   r   rH  r\  r   z9Expected fit parameter\(s\) \['sample_weight'\] not seen.r   r   TrJ  r   r   r   )r   rf  rZ   r   z2Fit parameter sample_weight has length 3; expected)r   rf  rZ   r   r   )	r   r   r   r   r   r1   r   rO  r  )rl   r   r  r  s       r]   /test_learning_curve_incremental_learning_paramsr    s     DAq 2"6GHHIJG	~W	5	5	5 	
 	
)-Cb11	
 	
 	
 	
	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 CG	~W	5	5	5 

 

)-Cb11#RWQZZ0		
 		
 		
 		


 

 

 

 

 

 

 

 

 

 

 

 

 

 

 		%)KS"--,	 	 	 	 	 	s%   +A??BB'A C33C7:C7c            	         t          ddddddd          \  } }t          j        ddd          }t          j        d          5 }t          t                      | |d|d	          \  }}d d d            n# 1 swxY w Y   t          |          dk    rt          d
|d         j	        z            t          |                    d          |           t          |                    d          d|z
             d S )Nr   r   r   rH  r   Tr  r   
param_nameparam_ranger   rL  r6  )r   r   rO  r  r  r3   r   r   rP  rQ  rP   r  )rl   r   r  rS  rT  rU  s         r]   test_validation_curver  4  sR     DAq +aB''K		 	-	-	- 
$4&((#%
 %
 %
!k
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 1vvzz3adlBCCCl//Q/77EEEk..A.66KHHHHHs   %A66A:=A:c            	          t          ddddddd          \  } }t          j        ddd          }t          t	                      | |d|d          \  }}d S )Nr   r   r   rH  r   r   r  )r   r   rO  r3   r   )rl   r   r  r  s       r]   %test_validation_curve_clone_estimatorr  O  sx      DAq +aB''K-//		  DAqqqr_   c                     d} d}t          dd          \  }}t          t          dd          ||dg dt          || 	          
          }t	          t          j        t          j        |          dd d f         d            t          t          dd          ||dg dt          |d          
          }t	          t          j        t          j        |          dd d f         d            t          t          dd          ||dg dt          |          
          }t	          t          j	        |          t          j	        |                     d S )Nr  r   r   r  r  r   r   )rJ  rJ  r  r  rN  r  )r   r   r   r   r   Tr  rF   r  )
r   r3   r?   r7   rP   r   vsplithstackr(   r&  )r  r  rl   r   scores1scores2scores3s          r]   +test_validation_curve_cv_splits_consistencyr  e  s~   IH1===DAq8!,,,		(((H	BBB  G ry7););L!!!O)LaPPQQ8!,,,		((((D111  G ry7););L!!!O)LaPPQQ8!,,,		((((###  G bhw//'1B1BCCCCCr_   c                  f   t          j        d                              dd          } t          j        dgdz  dgdz  z             }t	          d          }d}t          j        t          |	          5  t          || |d
g dd           d d d            n# 1 swxY w Y   d}t          j        t          |	          5  t          || |d
g dddt          j
        d          i           d d d            n# 1 swxY w Y   t          || |d
g dddt          j
        d          i           d S )Nr  r   r   r   r   Tr  r  r   	foo_param)r   r   r   r   )r  r  r   r  r   )r  r  r   r   )r   r  r   r&  rK   r   r   r   r3   r   r  r  s       r]   test_validation_curve_paramsr    s   
	#r2&&A
!qA37"##A
D
9
9
9C4G	~W	5	5	5 
 
"!			
 	
 	
 	

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 AG	z	1	1	1 	
 	
"!		#RWQZZ0	
 	
 	
 	
	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 		II-     s$   5BB B-C::C>C>c                  ~   t           j                            d          } t          j        d          }|                     |           t          |d          sJ t          t          j        |d          d          rJ d|d<   t          |d          rJ t          t          j        |df          d          rJ d S )Nr   r     )r   r=  r>  r  rF   r4   deleter  )rA  r  s     r]   test_check_is_permutationr    s    
)


"
"C
	#AKKNNN C((((($RYq"%5%5s;;;;;AaD$Q,,,,, %RY1v%6%6<<<<<<<r_   c                 .   t          ddddd          \  }} | |          } | |          }t          t          d                    }t          |||d	          }t          |||d	          }|                                }t          ||           d S )
Nr   r   FT)r^  n_labelsallow_unlabeledreturn_indicatorr   r  r_  r   r   )r   r9   r?   r.   rC  rP   )r  rl   r   r   y_sparseclassifr  preds_sparses           r]   (test_cross_val_predict_sparse_predictionr    s     *  DAq }QH}QH!#X"6"6"677Ggq!333E$WhRHHHL''))LlE22222r_   c           
      &   t          dd          }|j        dk    r(|dk    rt          |          fnt          |          df}n|j        }t	          j        |          }|                    ||          D ]V\  }}t          |                               ||         ||                   }  t          | |          ||                   ||<   W||dz   |dz
  |
                    d          fD ]$}	t          t          | ||	||          |           %d	S )
z@Helper for tests of cross_val_predict with binary classificationr   Fr  r   r  r   strr  N)r(   r   r   rb   r   r  r$  r	   re   getattrastyperN   r.   )
r.  rl   r   r  r   	exp_shapeexpected_predictionsr6  r7  tgs
             r]   check_cross_val_predict_binaryr    s.   	5	)	)	)B 	v{{!'+>!>!>SVVIISVVQK		G	8I..xx1~~ C CtCjjnnQuXqx00%9WS&%9%9!D'%B%BT"" !a%Q0 
 
c1bB???AU	
 	
 	
 	

 
r_   c           
         t          dd          }t          j        t          j                  j        }||dd}t          j        t          |          t          t          |                    f||         t          j                  }t          j        |d          \  }}	|	                    ||	          D ]\  }
}t          |                               ||
         |	|
                   }  t          | |          ||                   }t          j        |	|
                   }||t          j        ||          <   ||d	z   |d
z
  |                    d          fD ]$}t          t!          | ||||          |           %dS )zDHelper for tests of cross_val_predict with multiclass classificationr   Fr  r   r  r&  r   r|  Treturn_inverser   r   r  r  N)r(   r   finfor  minr  r   r   r   r$  r	   re   r  ix_r  rN   r.   )r.  rl   r   r  r   	float_mindefault_valuesr  r  y_encr6  r7  
fold_preds
i_cols_fitr  s                  r]   "check_cross_val_predict_multiclassr    s   	5	)	)	)B $$(I&& N
 7	QSVV~f5RZ   y4000HAuxx5)) D DtCjjnnQuXuU|44)WS&))!D'22
YuU|,,
9CRVD*5566 !a%Q0 
 
c1bB???AU	
 	
 	
 	

 
r_   c                 J   t          dd          }t          j        t          j                  j        }||dd}j        d         }g }t          |          D ]}	t          t          dd|	f                             }
|
dk    r|d	k    rt          |          f}nt          |          |
f}|	                    t          j
        |||         t          j        
                     fdt          j        d                   D             }t          j        |d          }|                    ||          D ]\  }}t          |                               ||         ||                   }  t          | |          ||                   }t          |          D ]o}	t          j        ||         dd|	f                   }||	         j        dk    r||	         ||	         |<   It          j        ||          }||	         ||	         |<   pӉdz   dz
                      d          fD ]s}t)          | ||||          }t          |          t          |          k    sJ t          t          |                    D ]}t+          ||         ||                    tdS )zCheck the output of cross_val_predict for 2D targets using
    Estimators which provide a predictions as a list with one
    element per class.
    r   Fr  r   r  r   Nr   r  r|  c                     g | ]<}t          j        d d |f         d          d         d d t           j        f         =S )NTr  r   )r   r   r   )r  r  r   s     r]   
<listcomp>z6check_cross_val_predict_multilabel.<locals>.<listcomp>&  sX        		!AAAqD'$///2111bj=A  r_   r6  r  r  )r(   r   r  r  r  rb   r   r   r   r%  r  concatenater$  r	   re   r  r   r   r  r  r.   rN   )r.  rl   r   r  r   r  r  	n_targetsexpected_predsi_coln_classes_in_labelr  
y_enc_colsr  r6  r7  r  	fold_colsidxr  cv_predict_outputr  s     `                   r]   "check_cross_val_predict_multilabelr    s   
 
5	)	)	)B $$(I&& N
 
INy!! 
 
 Qqqq%x[!1!122""v1D'D'DQ	IIQ!34IGI~f5RZHHH	
 	
 	
 	

   qwqz""  J N:A...Exx5)) 
? 
?tCjjnnQuXuU|44)WS&))!D'22
9%% 	? 	?E	%,qqq%x"899Ie$)Q...8.?u%d++fT9---7->u%c**	? !a%Q0 E E-c1bBOOO$%%^)<)<<<<<s,--.. 	E 	EA-a0.2CDDDD	EE Er_   c                 \    t          dd          \  }}dD ]}t          | |||           d S )Nr   r   )r^  r   r  r   r&  )r   r  )r.  rl   r   r  s       r]   *check_cross_val_predict_with_method_binaryr  ?  sJ     ;;;DAqM : :&sAq&9999: :r_   c                     t                      }|j        |j        }}t          ||d          \  }}dD ]}t	          | |||           d S )Nr   r   r  )r   r~  r  rF   r  )r.  r  rl   r   r  s        r]   .check_cross_val_predict_with_method_multiclassr  G  s`    ;;D9dkqA1aa(((DAqM > >*31f====> >r_   c                  r    t          t                                 t          t                                 d S rh   )r  r   r  r~   r_   r]   "test_cross_val_predict_with_methodr  O  s2    ./A/C/CDDD23E3G3GHHHHHr_   c                      t                      } | j        | j        }}t          ||d          \  }}dD ]%}t	          dd          }t          ||||           &d S )Nr   r   r  log_lossr   )lossr   )r   r~  r  rF   r   r  )r  rl   r   r  r.  s        r]   &test_cross_val_predict_method_checkingr  T  st     ;;D9dkqA1aa(((DAqM > >!<<<*31f====> >r_   c                      t                      } | j        | j        }}t          ||d          \  }}t	          t          d          dddgid          }d	D ]}t          ||||           d S )
Nr   r   r  r   rJ  r   r   r   r  )r   r~  r  rF   r%   r   r  )r  rl   r   r.  r  s        r]   /test_gridsearchcv_cross_val_predict_with_methodr  _  s    ;;D9dkqA1aa(((DAq
)r:::S3(OPQ
R
R
RCM > >*31f====> >r_   c                      d} d}t          | d|dd          \  }}t          t          dd	                    }d
D ]}t          ||||           d S )Nr  r   r   r   r  r  r  r^  r  r   r  r   )r  r   )r   r  r  )r   r9   r   r  )n_sampr^  rl   r   r.  r  s         r]   1test_cross_val_predict_with_method_multilabel_ovrr  h  s     FI)1	aVX  DAq 0RSTTT
U
UC8 A A&sAq@@@@@A Ar_   c                       e Zd Zd ZdS )RFWithDecisionFunctionc                     |                      |          }d}t          |t                    s
J |            d |D             }|S )Nz?This helper should only be used on multioutput-multiclass tasksc                 L    g | ]!}|j         d          dk    r|dddf         n|"S )r   r   Nr   r  )r  r  s     r]   r  z<RFWithDecisionFunction.decision_function.<locals>.<listcomp>~  s6    CCCQWQZ1__111b5!CCCr_   )r   r   r   )r\   rl   probsmsgs       r]   r  z(RFWithDecisionFunction.decision_functionz  sP    ""1%%O%&&++++&CCUCCCr_   N)rz   r{   r|   r  r~   r_   r]   r  r  v  s#            r_   r  c                  D   d} t          dd| dd          \  }}|d d dfxx         |d d df         z  cc<   d	D ]e}t          dd
          }t          j                    5  t          j        d           t          ||||           d d d            n# 1 swxY w Y   fd S )Nr   r  r   r   r  r  r   r   r   r&  r  n_estimatorsr   ignorer  )r   r  r  r  simplefilterr  )r^  rl   r   r  r.  s        r]   0test_cross_val_predict_with_method_multilabel_rfr    s&    I)Y1SU  DAq aaadGGGqAwGGGM I I$!!DDD$&& 	I 	I!(+++.sAqHHHH	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	II Is    (BB	B	c                  h   t           j                            d          } |                     ddd          }t          j        g d          }t                      }dD ]S}t          j                    5  t          j        d           t          ||||           d d d            n# 1 swxY w Y   Td S )Nr   r   )r  r   r  )r   r   r   r   r   r   r   r   r   r   r   r   r   r   r  r  )
r   r=  r>  normalr&  r   r  r  r  r  )rA  rl   r   r.  r  s        r]   -test_cross_val_predict_with_method_rare_classr    s	    )


"
"C

1ah
''A
;;;<<A


CM B B$&& 	B 	B!(+++.sAq&AAA	B 	B 	B 	B 	B 	B 	B 	B 	B 	B 	B 	B 	B 	B 	BB Bs   3'B&&B*	-B*	c                     t           j                            d          } |                     ddd          }t          j        ddgddgddgddgddgg          }dD ]e}t          dd          }t          j                    5  t          j        d	           t          ||||
           d d d            n# 1 swxY w Y   fd S )Nr   r   )r   r   r  r   )r   r&  r   r  r  r  )
r   r=  r>  r  r&  r  r  r  r  r  )rA  rl   r   r  r.  s        r]   ;test_cross_val_predict_with_method_multilabel_rf_rare_classr    s+    )


"
"C

1ag
&&A
1a&1a&1a&1a&1a&9::A8 I I$!!DDD$&& 	I 	I!(+++.sAqHHHH	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	II Is   (B77B;	>B;	c                    t          j        t          |          |g          }t          ||          }|                    | |          D ]\  }}	|                    | |         ||                     || |	                   }
|dk    r$t          j        t          |	          |f          }n@t          j        t          |	          |ft          j        |j                  j	                  }|
|d d |j
        f<   |||	<   |S )Nr   )r   r  r   r  r$  re   r  r  r}  r  r   )rl   r   r   classesr.  r  r  funcr6  r7  expected_predictions_exp_pred_tests               r]   get_expected_predictionsr    s    8SVVW$5663Dxx1~~ 3 3t%!E(### $QtW_$$Hc$ii%9::MMGTG$bh/C/I&J&J&N M *?aaao&%2T""r_   c            	         t          j        d                              dd          } t          j        d t	          d          D                       }d}t          d          }t          d          }t                      }g d	}|D ]}t                      }t          || |||
          }	t          | |||||          }
t          |
|	           t          || |||
          }	t          | |||||          }
t          |
|	           t          t          j        t	          d          d          d          }t          || |||
          }	|                    |          }t          | |||||          }
t          |
|	           d S )Nr  r  r   c                     g | ]}|d z  S )r   r~   )r  r   s     r]   r  z7test_cross_val_predict_class_subset.<locals>.<listcomp>  s    ...a!r'...r_   r   r   r  r   r  r  r   r   )r   r  r   r&  r   r(   r=   r   r.   r  rP   rF   r  fit_transform)rl   r   r  kfold3kfold4lemethodsr  r.  r,  r  s              r]   #test_cross_val_predict_class_subsetr    s   
	#sA&&A
..5::...//AGAFAF	BIIIG E E "" (Q&VLLL  8q&'3 
  
 	""6DDD (Q&VLLL7q&'3 
  
 	""6DDD BIeBii,,1==='Q&VLLLQ7q&'3 
  
 	""6DDDD5E Er_   c                     t                      } | j        | j        }}t                      }t	          j        dd          }|                    d           |                                 t          j	        |j
        t          j                  t          j	        |j
        ddt          j                  	 t          |||fd	
           t          j        t                    5  t          |||fd
           d d d            n# 1 swxY w Y   d\  t!          d          D ]=}	 t#          j        |j
                    d S # t&          $ r t)          d           Y :w xY wd S # d\  t!          d          D ]<}	 t#          j        |j
                    w # t&          $ r t)          d           Y 9w xY ww xY w)NwbF)moder  s   Hello world!!!!!r|  r~   r)rb   r  r}  c                     S rh   r~   )r.  rl   r   rt   s      r]   r   z#test_score_memmap.<locals>.<lambda>  s    U r_   r   c                     S rh   r~   )r.  rl   r   r   s      r]   r   z#test_score_memmap.<locals>.<lambda>  s     r_   rx   r   r   )r   r~  r  r   tempfileNamedTemporaryFilewritecloser   memmapnamer  r/   r   r   r   r   osunlinkOSErrorr   )r  rl   r   r   tfr  rt   r   s         @@r]   test_score_memmapr    s,   ;;D9dkqA


C		$$u	=	=	=BHH !!!HHJJJYrwbj111FIbgRcDDDEQ+B+B+B+BCCCC]:&& 	I 	ICA/G/G/G/GHHHH	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I
 #q 	 	A	"'"""   c




		 	 #q 	 	A	"'"""   c




		sf   9/E* (D?E* DE* DE* -E		E%$E%*F=FF=F9	6F=8F9	9F=c                  V   t           t           fg} 	 ddlm}m} |                     ||f           n# t
          $ r Y nw xY w| D ]c\  t                      }|j        |j        }} |           |          }}fd}fd}	t          ||	          }
t          |
||           dd S )Nr   rd  c                 $    t          |           S rh   rh  ri  s    r]   r   z4test_permutation_test_score_pandas.<locals>.<lambda>  rk  r_   c                 $    t          |           S rh   rh  rm  s    r]   r   z4test_permutation_test_score_pandas.<locals>.<lambda>  ro  r_   rp  )rL   rq  re  rf  r%  rr  r   r~  r  rK   r2   )rt  re  rf  r  rl   r   ru  rv  rw  rx  r   rj  rn  s              @@r]   "test_permutation_test_score_pandasr    s    ]+,E,,,,,,,,fi())))   (- 1 1$
${{y$+1&&q))::a==e<<<<:::: <HHHsD%00001 1rz  c                     t          t           j                  } t          j        dd          }t          j        dd          t          j        dd          }}t	          | |d t	                      ||dd d d 
  
        }d|d<   t          j        t          d	
          5  t          di | d d d            n# 1 swxY w Y   | 	                                dk    sJ d S )Nr   r   r   r   r  )
r  rl   r   r  r6  r7  r`  
parametersr  score_paramsr   r   z%Failing classifier failed as requiredr   r  r~   )
r8   FAILING_PARAMETERr   r  r  r   r   r   r5   rt   )failing_clfrl   r6  r7  fit_and_score_argss        r]   test_fit_and_score_failingr!    s*   #$5$GHHK
	!RA)Aq//29Q??4E

vv   )0}%	z)P	Q	Q	Q - -,,+,,,- - - - - - - - - - - - - - - #%%%%%%s   B77B;>B;c                  D   t          dd          \  } }t          dd          }t          t                                          |                     \  }}t          || |t                      ||ddddd d d	
          }t          di |}|d         |d         k    sJ d S )Nr  r   r  r  r   r  rJ  )rp  rq  T)r  rl   r   r  r6  r7  r`  r  r  r  return_parametersr  r~   )r   r?   nextr,   r$  r  r5   )rl   r   r   r6  r7  r   results          r]   test_fit_and_score_workingr&  7  s    !<<<DAq
XA
.
.
.C|~~++A..//KE4

vv #C00   11011F,#5l#CCCCCCCr_   c                   &    e Zd ZddZddZddZdS )DataDependentFailingClassifierNc                     || _         d S rh   max_x_value)r\   r+  s     r]   r^   z'DataDependentFailingClassifier.__init__N  s    &r_   c                 j    || j         k                                    }|rt          d| d          d S )NzClassifier fit failed with z values too high)r+  r  r   )r\   rl   r   num_values_too_highs       r]   re   z"DataDependentFailingClassifier.fitQ  sO     4#3388:: 	S.ASSS  	 	r_   c                     dS )Nr  r~   rr   s      r]   rt   z$DataDependentFailingClassifier.scoreX  s    sr_   rh   rx   )rz   r{   r|   r^   re   rt   r~   r_   r]   r(  r(  M  sP        ' ' ' '        r_   r(  r   c                 h   t          d          }t          j        dd          }t          j        d          }|||g}d| d}d}t	          j        d	|d
          d| t          j                  }t          j        t          |          5  t          |i | d d d            d S # 1 swxY w Y   d S )Nr  r*  r   r   r  r   r   r   z8ValueError: Classifier fit failed with 1 values too highzh2 fits failed.+total of 3.+The score on these train-test partitions for these parameters will be set to r   z.+flagsr   )r(  r   r  r  recompileDOTALLr   r  r   r0   )r   r  rl   r   cross_validate_argscross_validate_kwargsindividual_fit_error_messager  s           r]   -test_cross_validate_some_failing_fits_warningr9  \  sE    1Q???K
	!RA


A&1-#$[AA 	C ! jW%m4W W8TW W i  O 
&o	>	>	> F F+E/DEEEF F F F F F F F F F F F F F F F F F   B''B+.B+c                 h   t          t           j                  }t          j        dd          }t          j        d          }|||g}d| d}d}t          j        d| t
          j                  }t          j	        t          |	          5  t          |i | d d d            d S # 1 swxY w Y   d S )
Nr   r   r  r  r0  z1ValueError: Failing classifier failed as requiredz4All the 7 fits failed.+your model is misconfigured.+r1  r   )r8   r  r   r  r  r3  r4  r5  r   r   r   r0   )r   r  rl   r   r6  r7  r8  ra  s           r]   *test_cross_validate_all_failing_fits_errorr<  x  s'    $$5$GHHK
	!RA


A&1-#$[AA#V J.+. . i  M 
z	7	7	7 F F+E/DEEEF F F F F F F F F F F F F F F F F Fr:  c                      t          |          rh   )r   )r  rl   r   	error_msgs       r]   _failing_scorerr?    s    
Y

r_   zignore:lbfgs failed to converger   c           	         t          d          \  }}t          d                              ||          }d}t          t          |          }| dk    rKt          j        t          |          5  t          |||d	|| 
           d d d            d S # 1 swxY w Y   d S d|  }t          j	        t          |          5  t          |||d	|| 
          }t          ||            d d d            d S # 1 swxY w Y   d S )NTr   r   rp  "This scorer is supposed to fail!!!r>  r   r   r   )r   r   r   r  )r   r   re   r   r?  r   r   r   r/   r  r  rN   )r   rl   r   r   r>  failing_scorerwarning_msgr   s           r]   #test_cross_val_score_failing_scorerrF    s   
 %%%DAq
a
(
(
(
,
,Q
2
2C4I_	BBBNg]:Y777 	 	Qa[   	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	=/:= = 	 \+[999 	1 	1$Qa[  F FK000		1 	1 	1 	1 	1 	1 	1 	1 	1 	1 	1 	1 	1 	1 	1 	1 	1 	1s$   1BBB&C44C8;C8r;  with_multimetricc           
         t          d          \  }}t          d                              ||          }d}t          t          |          }|rt          t                    }|||d}	n|}	| dk    rLt          j        t          |	          5  t          |||d
|	||            d d d            d S # 1 swxY w Y   d S d|  }
t          j        t          |
	          5  t          |||d
|	||           }|D ]C}d|v r=d|v r#||         D ]}t          |t                    sJ -t          ||         |            D	 d d d            d S # 1 swxY w Y   d S )NTr   r   rA  rB  rC  )score_1score_2score_3r   r   r   )r   r   r;  r   r  r6   _score_2)r   r   re   r   r?  r   r    r   r   r   r0   r  r  r   rq   rN   )r   r;  rG  rl   r   r   r>  rD  non_failing_scorerr   rE  r  r   r  s                 r]   "test_cross_validate_failing_scorerrN    sz    %%%DAq
a
(
(
(
,
,Q
2
2C4I_	BBBN !();<<%)%
 
 !g]:Y777 		 		#5'   		 		 		 		 		 		 		 		 		 		 		 		 		 		 		 		 		 		=/:= = 	 \+[999 	C 	C$#5'  G  
C 
Cs??!S(( ") 8 8A#-a#7#777#778
 (kBBB
C	C 	C 	C 	C 	C 	C 	C 	C 	C 	C 	C 	C 	C 	C 	C 	C 	C 	Cs%   B44B8;B8!AEEEc                     dS )Ng;pΈ^@r~   )r  jrH  s      r]   three_params_scorerrQ    s    6r_   z:train_score, scorer, verbose, split_prg, cdt_prg, expectedr   )r   r   r   r   zS\[CV\] END .................................................... total time=   0.\ds)sc1sc2)scorersr   ze\[CV 2/3\] END  sc1: \(train=3.421, test=3.421\) sc2: \(train=3.421, test=3.421\) total time=   0.\dsr   zW\[CV 2/3; 1/1\] END ....... sc1: \(test=3.421\) sc2: \(test=3.421\) total time=   0.\dsc                    t          dd          \  }}t          dd          }	t          t                                          |                    \  }
}t          |	||||
||d d d |||          }t          d
i | |                                 \  }}|                    d          }t          |          dk    rt          j
        ||d	                   sJ d S t          j
        ||d                   sJ d S )Nr  r   r  r  r   )r  rl   r   r  r6  r7  r`  r  r  r  r;  split_progresscandidate_progress
r   r   r~   )r   r?   r$  r,   r$  r  r5   ra  r   r3  r   )rc  r<  r  r`  	split_prgcdt_prgexpectedrl   r   r   r6  r7  r   rb  r  outliness                   r]   test_fit_and_score_verbosityr^    s   N !<<<DAq
XA
.
.
.C|~~++A..//KE4 

& "   (('(((  FCyyH
8}}qx(1+.......x(1+.......r_   c            	          d} d }t          j        t          |           5  t          d d d |d t          j                   d d d            d S # 1 swxY w Y   d S )Nz&scoring must return a number, got Nonec                     d S rh   r~   )r  X_tests     r]   two_params_scorerz%test_score.<locals>.two_params_scorer<	  s    tr_   r   )r  ra  r  r  r  r   )r   r   r   r6   r   r  )ra  rb  s     r]   r=  r=  9	  s    <M   
z	7	7	7 
 
$	
 	
 	
 	

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
s    AAAc                      d } t          ddd          \  }}t          d          }|                    ||           t          |||d|           }g d	}|D ]}d
                    |          |v sJ d S )Nc                     |                      |          }t          ||          }|d         |d         |d         |d         dS )N)r   r   rR  )r   r   )r   r   tnfpfntp)rm   r   )r   rl   r   rN  cms        r]   rO  zPtest_callable_multimetric_confusion_matrix_cross_validate.<locals>.custom_scorerK	  sB    Qa((hbhbhbhOOOr_   (   r   r  )r  r  r   r   r   )r   r   re  ztest_{})r   r@   re   r0   r   )rO  rl   r   r.  rZ  score_namesr  s          r]   9test_callable_multimetric_confusion_matrix_cross_validaterm  J	  s    P P P
 KKKDAq

$
$
$CGGAqMMMQaGGGJ***K 4 4%%333334 4r_   c                  l    t          d          \  } }t          t                      | |dd           dS )z^Check that regressors with partial_fit is supported.

    Non-regression test for #22981.
    r  r   Tr   )rf  r   N)r   r1   r;   )rl   r   s     r]   *test_learning_curve_partial_fit_regressorsro  Z	  s=    
 +++DAq <>>1adqQQQQQQr_   c                    t          dddd|           \  }}t          j        |          }||         ||         }}t                      }d}t	          j        t          |          5  t          |||dt          j        	          ^}}}}d
d
d
           n# 1 swxY w Y   t          d          D ]^}	t          j
        ||	                                                   sJ t          j
        ||	                                                   sJ _t          d|j        d                   D ]^}	t          j
        ||	                                                   rJ t          j
        ||	                                                   rJ _d
S )zKChecks for fit failures in `learning_curve` and raises the required warningr  r   r   F)r  r^  r  rF   r   z#10 fits failed out of a total of 25r   r   r0  Nr   r   )r   r   r  r?   r   r  r   r1   r  r   isnanr  rb   any)
global_random_seedrl   r   
sorted_idxsvcr  r  r<  r=  r  s
             r]   -test_learning_curve_some_failing_fits_warningrv  e	  s    '  DAq AJZ=!J-qA
%%C;O	&o	>	>	> 
 
)7A!*
 *
 *
&;
Q
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 Qxx / /xC())--/////x
3((,,......Q)!,-- 3 38K,--11333338JsO,,002222223 3s   '#BBBc                    t          d          \  }}t          |          }t                      }t          dd|           }t	          ||||dd          }d|vsJ t	          ||||dd          }d|v sJ |d         d	         }|d         d
         }t          |          |j        k    sJ t          |          |j        k    sJ t          d |D             d           t          d |D             d           t          |	                    ||                    D ]4\  }\  }	}
t          ||         |	           t          ||         |
           5dS )z<Check the behaviour of `return_indices` in `cross_validate`.Tr   r   )r  rF   r   r   F)r   r   return_indicesindicesr6  r7  c                     g | ]	}|j         
S r~   r  r  ry  s     r]   r  z6test_cross_validate_return_indices.<locals>.<listcomp>	  s    BBBBBBr_   r  c                     g | ]	}|j         
S r~   r  r{  s     r]   r  z6test_cross_validate_return_indices.<locals>.<listcomp>	  s    AAAAAAr_   r  N)
r   r>   r   r(   r0   r   r  rQ   r@  r$  )rs  rl   r   r  r   rZ  train_indicestest_indices	split_idxexpected_train_idxexpected_test_idxs              r]   "test_cross_validate_return_indicesr  	  s   %%%DAqaA"$$I	46H	I	I	IB	1aBqQVWWWJJ&&&&	1aBqQUVVVJ
""""y)'2Mi(0L},,,,|++++BBMBBBCHHHAALAAA2FFF>GQRTU>W>W G G:	:&(9=35GHHH<	24EFFFFG Gr_   zfunc, extra_argsalphar   )r  r  )enable_metadata_routingc           	          t          j        t          d          5   | dt                      t          t
          g d| ddd           dS # 1 swxY w Y   dS )zCheck that we raise an error if `groups` are passed to the cv method instead
    of `params` when metadata routing is enabled.
    z`groups` can only be passed ifr   )r  rl   r   r  Nr~   )r   r   r   rA   rl   r   )r  
extra_argss     r]   #test_groups_with_routing_validationr  	  s      
z)I	J	J	J 
 
 	
)++		
 	

 	
 	
 	

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
s   $AAAc                 `    t          ddd          \  }} | dt                      ||d| dS )zTest that no errors are raised when passing `params=None`, which is the
    default value.
    Non-regression test for: https://github.com/scikit-learn/scikit-learn/issues/30447
    r  r   r   r]  )r  rl   r   Nr~   )r   rA   )r  r  rl   r   s       r]   test_cross_validate_params_noner  	  sH    " JJJDAqDA&((AAAjAAAAAr_   c           
      R   t          j        d          }t          j        t          |          5   | dt                      t          t          t          g           d| ddd           n# 1 swxY w Y   | t          k    rdS t          j        d          }t          j        t          |          5   | dt                      
                    d                              d          t          t          t          g           d| ddd           dS # 1 swxY w Y   dS )	zMCheck that we raise an error when passing metadata that is not
    requested.z[metadata] are passed but are not explicitly set as requested or not requested for ConsumingClassifier.fit, which is used withinr   )metadata)r  rl   r   r   Nz[metadata] are passed but are not explicitly set as requested or not requested for ConsumingClassifier.score, which is used withinTr~   )r3  escaper   r   r   rA   rl   r   r  r.   set_fit_requestset_partial_fit_request)r  r  r  s      r]    test_passed_unrequested_metadatar  	  s     i	F G 
/w	?	?	? 
 
 	
)++$$$		
 	

 	
 	
 	

 
 
 
 
 
 
 
 
 
 
 
 
 
 
    i	H G 
/w	?	?	? 	
 	
 	
)++_d_++$$d$33$$$	
 	
 	
 	
 	
	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
s$   2A..A25A25ADD #D c                    t                      }t          |                              dd          }t                      }t          |                              dd          }t                      }t          |                              dd	          }t          t                    }t          j
                            d
          }	|	                    |          }
|	                    |          }|	                    d
d|          }|	                    |          }|	                    |          }|	                    |          }t          t          t          |d                    t           t          |          t"          t          |          t$          t          |          t&          t          |          t(          t                      i}t          ||||          }| t(          ur|                    |
|            | |ft          t,          |d||          |d|i | t(          urt/          |          sJ |D ]}t1          |d| j        d|
|           t/          |          sJ |D ]}t1          |d| j        ||           t/          |          sJ |D ]}t1          |d| j        d||           dS )z]Check that the respective cv method is properly dispatching the metadata
    to the consumer.registryscore_weightsscore_metadatar   r  split_groupssplit_metadata)r  r  fit_sample_weightfit_metadatar   r   r  )	my_scorerr  r   )r  r  r  r  )r  r  )rl   r   r   r   rt   objr  parentsplit_paramsr   r  r$  )r  r  r  r  r  re   N)rD   rB   set_score_requestrC   set_split_requestrA   r  rT   rl   r   r=  r>  r?  randintr0   r  r/   r1   r3   r2   r.   updater   r   rE   rz   )r  r  scorer_registryr  splitter_registrysplitterestimator_registryr  r  rA  r  r  r  r  r  r  scoring_argsr   _scorer	_splitter
_estimators                        r]   !test_validation_functions_routingr  
  sd     kkOo666HH%0@ I  F " *;<<<NN(8 O  H ##-?@@@PP)N Q  I QI
)


"
"CHHY''MXXi((N;;q!Y//LXXi((N++88I&&L 	TFZ%P%P%PQQQf---V,,,$v...V 4 4 4466L !%+!	  F $$$') 	 	
 	
 	

 	D

	 
 t
       $$$?#####" 
 
=6'#	
 	
 	
 	
 	
  !!!!!& 
 
	=#	
 	
 	
 	
 	
 !"""""( 
 

=6+!	
 	
 	
 	
 	

 
r_   c                     t          t                    } t          j                            d          }|                    |           }|                    |           }t                      }t          |                              dd          }t          |t          t          t                      dt          ||                     t          |          sJ |D ]}t          |d	d
d||           dS )zTest that learning_curve routes metadata to the estimator correctly while
    partial_fitting it with `exploit_incremental_learning=True`.r   r  r  r  r  T)r  r  )rl   r   r   rf  r   r   r1   r  N)rT   rl   r   r=  r>  r?  rD   rA   r  r1   r   rC   r  r   rE   )r  rA  r  r  r  r  r  s          r]   8test_learning_curve_exploit_incremental_learning_routingr  k
  s"   
 QI
)


"
"C++88I&&L"##  )N     

%)&7lSSS    !"""""( 
 

 #6+!	
 	
 	
 	
 	

 
r_   r  r   r   )idsr   )Nr   r   znamespace, device_, dtype_namec                 F   t          ||          }t          |           rt          ddddd          \  }}nt          dddd          \  }}|                    |          }|                    |          }	|                    ||          }
|                    |	|          }t          d	          5  t          | |
||
          }ddd           n# 1 swxY w Y   t          | ||	|
          }t          t          ||          |t          |                     dS )zpTest that `cross_val_predict` functions correctly with the array API
    with both a classifier and a regressor.i  r   r   r  )r  r  r^  r  r   r  )deviceT)array_api_dispatchr   N)atol)rM   r
   r   r   r  asarrayr   r.   rN   rH   rG   )r  r   	namespacedevice_
dtype_namexprl   r   X_npy_npX_xpy_xppred_xppred_nps                 r]   +test_cross_val_predict_array_api_compliancer  
  s   " 
i	1	1BY 
"qAQUW
 
 
11 q
 
 
1 88JD88JD::d7:++D::d7:++D	4	0	0	0 B B#ItTbAAAB B B B B B B B B B B B B B B  	4"===G'2&&nZ6P6P     s   6CCC)r}   r  r3  r  r  	functoolsr   timer   numpyr   r   scipy.sparser   sklearnr   sklearn.baser   r   r	   r
   sklearn.clusterr   sklearn.datasetsr   r   r   r   r   r   sklearn.discriminant_analysisr   sklearn.ensembler   sklearn.exceptionsr   r   sklearn.imputer   sklearn.linear_modelr   r   r   r   sklearn.metricsr   r   r   r   r   r    r!   r"   r#   sklearn.metrics._scorerr$   sklearn.model_selectionr%   r&   r'   r(   r)   r*   r+   r,   r-   r.   r/   r0   r1   r2   r3   #sklearn.model_selection._validationr4   r5   r6   $sklearn.model_selection.tests.commonr7   )sklearn.model_selection.tests.test_searchr8   sklearn.multiclassr9   sklearn.neighborsr:   sklearn.neural_networkr;   sklearn.pipeliner<   sklearn.preprocessingr=   r>   sklearn.svmr?   r@   %sklearn.tests.metadata_routing_commonrA   rB   rC   rD   rE   sklearn.utilsrF   sklearn.utils._array_apirG   rH   rI   rJ   sklearn.utils._mockingrK   rL   sklearn.utils._testingrM   rN   rO   rP   rQ   sklearn.utils.fixesrR   rS   sklearn.utils.validationrT   rV   r   r   r   r   r  rl   r&  r   r   r  r   markparametrizer   r   r  r  r  r8  r'  r(  rb  ry  r  r  r  r  r  r  r  r  r  r  r  r  r  r$  r'  r.  r2  r;  rD  rF  r[  r^  rd  ri  rl  rn  rx  r|  r~  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r!  r&  r(  r  r9  r<  r?  filterwarningsrF  rN  rQ  r^  r=  rm  ro  rv  r  r  r  r  r  r  r  r~   r_   r]   <module>r     s       				 				                    ! ! ! ! ! ! " " " " " " M M M M M M M M M M M M " " " " " "                E D D D D D 3 3 3 3 3 3 I I I I I I I I ( ( ( ( ( (           
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 7 6 6 6 6 6                                 "         
 A @ @ @ @ @ G G G G G G 2 2 2 2 2 2 2 2 2 2 2 2 / / / / / / % % % % % % 5 5 5 5 5 5 5 5 & & & & & & & &              " ! ! ! ! !            E D D D D D D D              ? > > > > > > > 1 1 1 1 1 1" " " " "] " " "6    (>   <" " " " " " " ",	" 	" 	" 	" 	",F 	" 	" 	"S8 S8 S8 S8 S8_m S8 S8 S8p BGGBH:::;; RX;;;<<
BF1II .99%< %< :9%<P) ) )1L 1L 1LhL L L& t}55.994AD 4A 4A 4A :9 654An?M ?M ?MDF9 F9 F9R> > >,0 0 0&5 5 5&8 8 86 .99 3  3 :9 3F& & & J J J&L L L0 .99? ? :9?D$ $ $M M M  H H H< .995
 5
 :95
p-W -W -W`	# 	# 	#	# 	# 	# .9922 22 :922j2 2 2$T T T,
$ 
$ 
$ .99& & :9&:= := :=zS S S&9 9 9K K K W W W4S S S0$ $ $NC C C., , ,,S S S(M M M`  &- - -`I I I6  ,+D +D +D\# # #L= = = .993 3 :93$
 
 
*
 
 
81E 1E 1Eh: : :> > >I I I
> > >> > >A A A	 	 	 	 	3 	 	 	I I I"B B BI I I"     (%E %E %EP  41 1 1(& & &6D D D,    ]    44F F 54F6 44F F 54F.      =>>G(<==1 1 >= ?>14 =>>G(<==-e}==+eT];;:C :C <; >= >= ?>:Cz   @ $	
  3<OPP   ?
	
  3<OPP   $
	
+ # #H/ /I# #H/@
 
 
"4 4 4 R R R3 3 3BG G G: 		"	B		$	'("(A3--PPQ
 
 ---
 
 .-
 

 		"	B		$	'("(A3--PPQ
 
 ---B B .-
 
B 		"	B		$	'("(A3--PPQ
 
 ---"
 "
 .-
 
"
J 		"	B		$	'("(A3--PPQ
 
 ---Y
 Y
 .-
 
Y
x ---"
 "
 .-"
R 
UWW((**+	./   
 |||,,$--//'   
   -,   r_   