`ic1ddlZddlZddlmZmZddlmZddlmZddl m Z m Z m Z m Z ddlmZmZmZmZddlZddlmZmZddlmZejed ZeoejZd Zed Z ed Z!d e ee e fe!fde ee e fe!ffdZ"GddZ#Gdde#Z$Gdde$Z%er e%n e$Z&dS)N)cached_propertywraps)chain)median)AnyCallableOptionalUnion) Concatenate ParamSpecSelfTypeVar)counters dynamo_timed)use_experimental_benchmarker benchmarkingiPTfnreturnc tdtdtjdtjdt ffd }|S)zWraps `fn` with `dynamo_timed` context, and increments the appropriate dynamo counters. It is expected that `fn` is a method of `Benchmarker` or one of its subclasses; typing limitations prevent us from declaring this directly. selfargskwargsrc|jjdj}tdd|xxdz cc<t|d5|g|Ri|cdddS#1swxYwYdS)N.inductorz benchmarking.F)log_pt2_compile_event) __class____name__rr)rrr fn_qual_namers x/home/jaya/work/projects/VOICE-AGENT/VIET/agent-env/lib/python3.11/site-packages/torch/_inductor/runtime/benchmarking.pywrapperztime_and_count..wrapper"s.1AABKAA ;\;;<<<A<<< ,e D D D - -2d,T,,,V,, - - - - - - - - - - - - - - - - - -s A  A$'A$)rrrrrr)rr$s` r#time_and_countr%sZ 2YY-c-!&-AH------Y- Nc eZdZdeddfdZedededefdeedfde e efd ede f d Z e dded egefde de de f dZededed ede fdZdS) BenchmarkerrrNcdSN)rs r#__init__zBenchmarker.__init__-s r&r.fn_args fn_kwargsrc ld}tD]A}t|tjs||j}'|j|krt dB|t dfd}|tjdkr|j|fi|S|j|fi|S)aBenchmark `fn(*fn_args, *fn_kwargs)` and return the runtime, in milliseconds (the actual runtime calculation is dictated by the benchmarking implementation, but may be one of [mean, median, minimum, etc.]). Functions as a convenience wrapper around device-specific implementations, like `benchmark_cpu` and `benchmark_gpu`. Raises `ValueError(...)` if we can't safely infer the device type of `fn`; for example, if multiple device types are found in `fn_args` and `fn_kwargs`, or if no device types are found. Arguments: - fn: The function to benchmark. - fn_args: The function's arguments. - fn_kwargs: The function's kwargs. Keyword Arguments: - **kwargs: The benchmarking implementation's kwargs. Returns: - The runtime of `fn(*fn_args, **fn_kwargs)`, in milliseconds. NzcCan't safely infer the device type of `fn` with multiple device types in `fn_args` and `fn_kwargs`!zCan't safely infer the device type of `fn` with no device types in `fn_args` or `fn_kwargs`! You should be calling `.benchmark_cpu` or `.benchmark_gpu` directly.ciSr*r+)rr-r.sr#z'Benchmarker.benchmark..YsBB5955r&cpu) rvalues isinstancetorchTensordevice ValueError benchmark_cpu benchmark_gpu)rrr-r.rinferred_device arg_or_kwarg _callables ``` r# benchmarkzBenchmarker.benchmark0s6!'9+;+;+=+=>>  LlEL99 &"."5$77 y8  "t 655555 el511 1 1%4%i::6:: :"t!)66v666r&dr=warmuprepcdtdttffd }||t||S)aBenchmark the CPU callable, `_callable`, and return the median runtime, in milliseconds. Arguments: - _callable: The CPU callable to benchmark. Keyword Arguments: - warmup: Optionally, the duration, in milliseconds, to run `_callable` before benchmarking starts. - rep: Optionally, the duration, in milliseconds, to run `_callable` during benchmarking. Returns: - The median runtime of `_callable`, in milliseconds. msrcg}tj} tj}tj}|||z tz||z tz|krnc|Sr*)time perf_counterappendMILLISECONDS_PER_SECOND)rDtimings run_start_tstart_tend_tr=s r#run_forz*Benchmarker.benchmark_cpu..run_forusG+--K +-- )++3JJKKK[(,CCrII  Nr&)intlistfloatr)rr=rArBrNs ` r#r9zBenchmarker.benchmark_cpuasX(  U       ggcll###r&rctr*)NotImplementedError)rrrs r#r:zBenchmarker.benchmark_gpus!!r&)r?r@)r! __module__ __qualname__r r,r%rrtupledictstrrQr>rOr9r:r+r&r#r(r(,s> t     .7.7 S#X .7sCx.7S> .7  .7  .7.7.7^.7`OR $ $ $'C0 $:= $IL $  $ $ $^ $D"D""""""^"""r&r(c veZdZedededeffdZededegefdede fdZ dS) TritonBenchmarkerrr.cZ ddlm}n"#t$r}td|d}~wwxYw|S)z"Lazily import Triton's `do_bench`.r)do_benchzrequires TritonN)triton.testingr\ ImportErrorrS)rr\es r#triton_do_benchz!TritonBenchmarker.triton_do_benchsV @ / / / / / / / @ @ @%&788a ? @s (#(r=rc tj|jj}t |D] }||vr||= d|vr|j|fi|dSd|vr|j|fi|S|j|fi|ddiS)aBenchmark the GPU callable, `_callable`, and return the runtime, in milliseconds. Arguments: - _callable: The GPU callable to benchmark. Keyword Arguments: - quantiles: Optionally, a tuple of floats denoting the requested quantiles. - return_mode: Optionally, the requested return mode. Currently, Triton's `do_bench` supports min, max, mean, and median return modes. - **kwargs: Additional kwargs passed to Triton's `do_bench`. Returns: - The runtime of `callable`, in milliseconds. If `kwargs["quantiles"]` is specified, this is the first requested quantile. Else, if `kwargs["return_mode"]` is specified, this is the requested return mode. Otherwise, this is the median. quantilesr return_moder)inspect signaturer` parametersrPkeys)rr=rdo_bench_paramskwargs r#r:zTritonBenchmarker.benchmark_gpus$"+D,@AAL&++--(( " "EO++5M & '4' <O#ORWOOO^OOOr&rZczeZdZededefdZdededeee j j e j j ffdZ dedeee j j e j j fde fdZe dded egefdedededededeee jdedee ee ffdZd S)InductorBenchmarkerrrctj}tj|}|jS)z7Get the L2 cache size, in bytes, of the current device.)r5cudacurrent_deviceget_device_properties L2_cache_size)rr7propss r#rpz!InductorBenchmarker.L2_cache_sizes5**,, 0088""r&itersc4dt|DS)z!Get `iters` pairs of CUDA events.cg|]B}tjdtjdfCS)T) enable_timing)r5rmEvent).0_s r# z7InductorBenchmarker.get_event_pairs..sX      t 44   t 44    r&)range)rrrs r#get_event_pairsz#InductorBenchmarker.get_event_pairss+  5\\     r& event_pairsc4td|DS)zIGet the minimum timing, in milliseconds, for a group of CUDA event pairs.c>g|]\}}||Sr+ elapsed_timerw start_event end_events r#ryzBInductorBenchmarker.get_event_pairs_min_timing..s:   *K((33   r&)min)rr|s r#get_event_pairs_min_timingz.InductorBenchmarker.get_event_pairs_min_timings2  .9      r&r@rNr=estimation_itersmemory_warmup_itersbenchmark_itersmax_benchmark_durationrc grad_to_nonerc tj|tjtj|jdztjd} | ||} | D]Y\} } | |D] } d| _ | | || Ztj| | }tt|t ||zd}t|D]}| ||} | D]Y\} } | |D] } d| _ | | || Ztj~ |dkrd| D}|S|dkr%| | }t||Std |d ) aBenchmark a GPU callable using a custom benchmarking implementation. Arguments: - _callable: The callable to benchmark. Keyword Arguments: - estimation_iters: Optionally, the number of iterations to run `_callable` during runtime estimation. - memory_warmup_iters: Optionally, the number of iterations to flush the L2 cache before starting benchmarking. - benchmark_iters: Optionally, the number of iterations to run `_callable` during the benchmarking. - max_benchmark_duration: Optionally, the maximum duration of the benchmarking, in milliseconds. An estimated duration is calculated based on the values of `memory_warmup_iters` and `benchmark_iters`, along with the estimated runtime of `_callable` and various other factors, and we then shrink `benchmark_iters` to fit in the allotted maximum duration. - return_mode: Return mode for benchmark results. Options are "min" (default), "all" (returns all measurements). - grad_to_none: Optionally, a list of tensors whose gradients should be cleared before each benchmark iteration. - **kwargs: Additional kwargs that may be passed to the fallback. Returns: - If return_mode="min": The minimum runtime of `_callable`, in milliseconds. - If return_mode="all": List of all runtime measurements, in milliseconds. rm)dtyper7Nrallc>g|]\}}||Sr+rrs r#ryz5InductorBenchmarker.benchmark_gpu..,s:*K((33r&rzUnsupported return_mode: z. Use 'min' or 'all'.)r5rm synchronizeemptyrprOzero_r{gradrecordrmaxrrzr8)rr=rrrrrcrrbufferr|rrxestimated_timingrx all_timingsbenchmarked_timings r#r:z!InductorBenchmarker.benchmark_gpusP           T/14EIfUUU **+;<< &1   "K'%""A!AFF LLNNN    IKKK           ::;GG %;?O%O!P!P Q QST   *++  A LLNNNN**?;; &1   "K'%""A!AFF LLNNN    IKKK             %  .9K  E ! !!%!@!@!M!M ');<< <NKNNN r&)rr@r@rrN)r!rTrUrr rOrprPrVr5rmrvr{rQrr%rrrXr r6r r:r+r&r#rkrks#D#S###_#      eEJ$ej&667 8         !%eEJ,N,N&O!P       !"#&"&( 59iiiBG$ii! i  i !$ iitEL12ii ud5k! "iii^iiir&rk)'rdrF functoolsrr itertoolsr statisticsrtypingrrr r typing_extensionsr r r rr5torch._dynamo.utilsrrtorch._inductor.configr_logginggetArtifactLoggerr!loggerrm is_availablerIrrr%r(rZrk benchmarkerr+r&r#rs2 ,,,,,,,,111111111111CCCCCCCCCCCC 66666666??????  ) )(N C C >UZ%<%<%>%>  IcNN GCLLS!V$a'( k#q&!1$%$Z"Z"Z"Z"Z"Z"Z"Z"z$O$O$O$O$O $O$O$ONIIIII+IIIZ:R?P?P?R?R r&