
    fPi.                        d dl Z d dlZd dlZd dlZ	 d dlmZmZ n# e$ r d dlmZmZ Y nw xY wd dl	m
Z
 d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZmZ  ej        e           ej        d          k     r ede d          g dZdefdZ	 ddededz  defdZ de!e"eef                  defdZ#de!e"eef                  dedefdZ$d Z%d dZ&e'dk    r e%             dS dS )!    N)PackageNotFoundErrorversion)Path)Any)load_dataset)	evaluator)ORTModelForQuestionAnswering)__version__)r   )AutoTokenizerpipelinez1.13.1z1Please install optimum>=1.13.1. Current version: .)z5bert-large-uncased-whole-word-masking-finetuned-squadzdeepset/roberta-base-squad2z%distilbert-base-cased-distilled-squadpackage_namec                 D    	 t          |           S # t          $ r Y d S w xY wN)r   r   )r   s    /home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/onnxruntime/transformers/models/bert/eval_squad.pyget_package_versionr   7   s7    |$$$   tts    
CUDAExecutionProviderFmodel_id	onnx_pathuse_io_bindingc                    |~t          j        | d||          }t          j                            d|           }|                    |           t          j                            |d          }t          d|           nGt          j        t          j                            |          t          |          j	        ||          }||fS )a  Load onnx model given pretrained model name and optional ONNX model path. If onnx_path is None,
    the default onnx model from optimum will be used.

    Args:
        model_id (str): pretrained model name or checkpoint path
        onnx_path (Optional[str], optional): path of onnx model to evaluate. Defaults to None.

    Returns:
        model: ORTModel for the onnx model
        onnx_path: the path of onnx model
    NT)exportproviderr   r   z
model.onnxzModel is exported to onnx file:)	file_namer   r   )
r	   from_pretrainedospathjoinsave_pretrainedprintdirnamer   name)r   r   r   r   modelsave_onnx_dirs         r   load_onnx_modelr%   >   s     ,<TH^
 
 
 S(33m,,,GLL==	/;;;;,<GOOI&&9oo*)	
 
 
 )    resultscsv_filenamec                 8   t          |ddd          5 }g d}t          j        ||          }|                                 | D ]}|                    |           |                                 ddd           n# 1 swxY w Y   t          d|            dS )	zOutput a CSV file with detail of each test results.

    Args:
        results (List[Dict[str, Any]]): list of JSON results.
        csv_filename (str): path of output CSV file
    a asciimodenewlineencoding)pretrained_model_namer   r   disable_fused_attention
batch_sizesequence_lengthr   exactf1totalHasAns_exact	HasAns_f1HasAns_total
best_exactbest_exact_threshbest_f1best_f1_threshtotal_time_in_secondssamples_per_secondlatency_in_seconds
fieldnamesNz&Detail results are saved to csv file: )opencsv
DictWriterwriteheaderwriterowflushr    )r'   r(   csv_filecolumn_names
csv_writerresults         r   output_detailsrN   b   s     
lb7	C	C	C x
 
 
. ^HFFF
    	( 	(F'''';              > 

A<
A
ABBBBBs   AA==BBmetric_namec           	      4   t          |ddd          5 }g dt          d | D                       }|                                 t          d | D                       }|                                 t          d | D                       }|                                 g }|D ]"}|D ]}	|                    d	|	 d
|            #t	          j        ||z             }
|
                                 |D ]}i }i }|                    t          	                    |d                     | D ]t}|d         |k    rf||         r^fd|
                                D             }|s|                    |           |d         }	|d         }d	|	 d
| }||v r||         ||<   u|r3|D ]}|                    |d          ||<   |
                    |           |                                 ddd           n# 1 swxY w Y   t          d| d|            dS )a  Output a CSV file with summary of a metric on combinations of batch_size and sequence_length.

    Args:
        results (List[Dict[str, Any]]): list of JSON results.
        csv_filename (str): path of output CSV file
        metric_name (str): the metric to summarize
    r*   r+   r,   r-   )r1   r   r   r2   r   c                     h | ]
}|d          S )r    .0rM   s     r   	<setcomp>z!output_summary.<locals>.<setcomp>   s    EEE66+.EEEr&   c                     h | ]
}|d          S )r3   rR   rS   s     r   rU   z!output_summary.<locals>.<setcomp>   s    GGGVF<0GGGr&   c                     h | ]
}|d          S )r4   rR   rS   s     r   rU   z!output_summary.<locals>.<setcomp>   s     Q Q Qv(9!: Q Q Qr&   b_srB   r   c                 $    i | ]\  }}|v 	||S rR   rR   )rT   kvheader_namess      r   
<dictcomp>z"output_summary.<locals>.<dictcomp>   s)    TTT1!|BSBSq!BSBSBSr&   r3   r4   NzSummary results for z are saved to csv file: )rD   listsortappendrE   rF   rG   updatedictfromkeysitemsgetrH   rI   r    )r'   r(   rO   rJ   
model_listbatch_sizessequence_lengths	key_namesr4   r3   rL   r#   rowvaluesrM   headerskeyr]   s                    @r   output_summaryro      s    
lb7	C	C	C 4x
 
 
 EEWEEEFF
GGwGGGHH Q Q Q Q QRR	/ 	F 	FO) F F
  !DZ!D!D?!D!DEEEEF ^H	9QRRR
    	) 	)EC FMM$--	266777! : :+&%//F;4G/TTTTTTTG ,

7+++!'!5J&,->&?O=j==O==Ci''&,[&9s )$ 3 3C%zz#r22CHH##C(((i4 4 4 4 4 4 4 4 4 4 4 4 4 4 4l 

T
T
Tl
T
TUUUUUs   GG88G<?G<c                  @   t                      } t          |            dD ]&}t          |          }|rt          | d|           '| j        }| j        r;t
          j                            | j                  st          d| j                   t
          j	        
                    dd          dk    }g }t          j        |          }| j        D ]"}||_        t          |dz  d          |_        | j        t          d	           t#          j                    }t%          || j        | j        | j                  \  }	}
t#          j                    |z
  }t          d
|dd           t          |	j                   ||	j        j        k    rt          d          t/          d|	|d| j                  }t3          d          }t          d           t#          j                    }t5          d| j        dk    rd| j         dnd          }t#          j                    |z
  }t          d|dd           t          d           t#          j                    }|                    ||dd          }t#          j                    |z
  }t          d|dd           | j        |d<   ||d<   ||d<   |
|d <   | j        |d!<   ||d"<   | j        |d#<   t          |           |                    |           $t=          |d$           d%D ]}t?          || d&|           d S )'N)zonnxruntime-gpuonnxruntimeonnxtorchtransformersoptimumdatasetsevaluatez versionz Onnx model path does not exist: ORT_DISABLE_FUSED_ATTENTION01      z4Exporting onnx model. It might take a few minutes...z!Onnx model exported or loaded in z.1fz secondszTsequence length should not be larger than {ort_model.config.max_position_embeddings}zquestion-answeringT)r#   	tokenizerquestion_firstr3   zLoading dataset...squadr   zvalidation[:]
validation)splitzDataset loaded in z<Evaluating squad_v2 with ORT. It might take a few minutes...squad_v2)model_or_pipelinedatametricsquad_v2_formatzEvaluation done in r   r2   r1   r   r3   r4   r   z
detail.csv)r6   r5   r@   z.csv) parse_argumentsr    r   
model_namerr   r   r   existsRuntimeErrorenvironrf   r   r   ri   model_max_lengthmin
doc_stridetimer%   r   r   configmax_position_embeddingsr   r3   r   r   r7   computera   rN   ro   )argsr"   package_versionr1   r2   all_resultsr}   r4   
start_time	ort_modelr   latencyqa_pipelinetask_evaluatorsquad_datasetrM   rO   s                    r   mainr      s~   D	$KKKv 6 6-d33 	6T###_555 Oy K	22 KIdiIIJJJ jnn-JCPPTWWK-.CDDI0 -# -#%4	""?a#7==	9HIIIY[[
./DdiQUQ^`d`stt	9)++
*G'GGGGHHHiY-EEEuvvv 	YW[hlhw
 
 
 ##788"###Y[[
$WTXT^abTbTb4P4:4P4P4P4Phtuuu)++
*878888999LMMMY[[
'') 	 ( 
 
 )++
*9G9999:::!]z,C()*?&''{#|$3 !#'#6 f6"""";---< G G{{$8$8$8+FFFFG Gr&   c           	      H   t          j                    }|                    dddt          t          d         dt                      |                    ddd	t
          d
gd           |                    ddt
          dd           |                    ddt
          dd           |                    ddt          d d           |                    dddd           |                    dddd           |                    d           |                    |           }|S ) Nz-mz--model_nameFr   z=Checkpoint directory or pre-trained model names in the list: )requiredtypedefaulthelpz-sz--sequence_lengths+i  zFSequence lengths for onnx model inputs. It could have multiple values.)nargsr   r   r   z-bz--batch_size   zbatch size for inference.)r   r   r   z-tz--totalz+Total samples to test. 0 means all samples.z--onnxzbOptional onnx model path. If not specified, optimum will be used to export onnx model for testing.z
--providerr   zRSelect which Execution Provider to use for runs. Default is CUDAExecutionProvider.)r   r   r   z--use_io_binding
store_truezUse IO Binding for GPU.)r   actionr   )r   )argparseArgumentParseradd_argumentstrPRETRAINED_SQUAD_MODELSintset_defaults
parse_args)argvparserr   s      r   r   r     sw   $&&F
'*fMdff     U     (     ic1Cpqqq
q     'a	     *U<Voppp
u---T""DKr&   __main__)Nr   Fr   )(r   rE   r   r   importlib.metadatar   r   ImportErrorimportlib_metadatapathlibr   typingr   rv   r   rw   r   optimum.onnxruntimer	   optimum.versionr
   optimum_version	packagingversion_checkrt   r   r   parser   r   r   boolr%   r_   rc   rN   ro   r   r   __name__rR   r&   r   <module>r      s  2  



 				 A@@@@@@@@@ A A A@@@@@@@@@@A             ! ! ! ! ! !       < < < < < < : : : : : : . . . . . . 0 0 0 0 0 0 0 0=''*=-*=h*G*GGG
+\/\\\
]
]]   c     kp! !!!Dj!cg! ! ! !H&CDc3h0 &C &C &C &C &CR>VDc3h0 >V >VRU >V >V >V >VBCG CG CGL3 3 3 3l zDFFFFF s    ++