.`iz d dl Z d dlZd dlZd dlmZ d dlmZ d dlZd dl m
Z
m Z d dl m
Z
mZmZmZmZmZ d dlmZ d dlmZ d dlmZ ee Z G d d
e ZdS ) N)Sequence)Any)ChatCompletionRequestChatCompletionToolsParam)DeltaFunctionCall DeltaMessage
DeltaToolCall ExtractedToolCallInformation FunctionCallToolCall) init_logger)
TokenizerLike)
ToolParserc e Zd ZdZdZdef fd ZdefdZd Z ded e
e d
z de d
z fd Z
d ede
e fd
Zd ededefdZdedededee dee dee deded
z fdZ xZS )SeedOssToolParserzz tokenizerc R t | | g | _ | j | _ | j | _ d| _ d| _ d| _
d| _ d| _ d| _
d| _ d| _ d| _ | | j | j | _ | j | j | _ | j | j
| _ | j | j t- d t/ j | j }t/ j | j }t/ j | d
| t. j | _ t/ j | d
| d | d t. j | _ t/ j d
t. j | _ t/ j dt. j | _ t> d| j! j" d S )Nz
z z z
Fr zSSeed_Oss XML parser: tokenizer did not include or its closing tag.z(.*?)|z(.*?)$z*||"(,(: %1 '4 %2 (6 &4 $3 */!%* !" ##%%%(,
t7Q(R(R%&*jnnT5M&N&N#"&*..1E"F"F ( 0D4O4W 7
$"<==
i 899 (*
1 1K 1 129)
)
% "z G GK G G- G G G
)+
929)
)
% *, =ry*
*
& 8$.:Q
returnc H dt j j dd S )z Generate a unique tool call ID.call_N )uuiduuid4hexr8 s r; _generate_tool_call_idz(SeedOssToolParser._generate_tool_call_id[ s" .tz||',...r< c d| _ d| _ d| _ d| _ d| _ d| _ d| _ d| _ d| _ d| _ d| _
d| _ d| _ dS )zReset all streaming state.r FN )
current_tool_indexr# header_sentcurrent_tool_idcurrent_function_namecurrent_param_namecurrent_param_value param_countin_param in_functionaccumulated_text json_started json_closedrD s r; r z(SeedOssToolParser._reset_streaming_state_ so "# $)! ! %)""& #%
" ! r< function_call_strtoolsNc
dt dt ffd }dt dt dt dt dt f
d}| d }|d | } || }||d z d }i } | j | D ]}
|
d
r|
d
n|
d } | d } | d | }
t | | d z d }| d r
|d d }| d r
|d d } |||
|| | |
< t d
t |t j | d S )N func_namer= c i S D ]}t |d r%t |d rt |j d s8|j dk ru|j j | k ret |j d si c S |j j }t |t rd|v r
|d c S t |t r|c S i c S t d| i S )Ntypefunctionname
parameters
propertiesz+Tool '%s' is not defined in the tools list.) hasattrr[ rZ r\ r]
isinstancedictr4 warning)rX configparamsrV s r; get_arguments_configzHSeedOssToolParser._parse_xml_function_call..get_arguments_configr s }
"
" vv.. FJ// 4;FOV4T4T ;*,, 1E1R1R"6?LAA " ! #_ 7F!&$// "LF4J4J %l 3333#FD11 " %
! NNH) T T TIr< param_value
param_name param_configc | dk rd S ||vr$|i k r t d|| | S t || t rJd|| v r@t || d }nd}|dv r| S | d sT| d s?| d s*| d s| d
rG t | } n4# t t f$ r t d | || Y nw xY w| S | d s| d
rn t | }|t | z
dk r|nt | } n4# t t f$ r t d| || Y nw xY w| S |dv r;| } | dvr t d| || | dk S |dk s| d rV t j
| } | S # t t t j f$ r t d| || Y nw xY w t j | } n4# t t" f$ r t d| || Y nw xY w| S )NnullzoParsed parameter '%s' is not defined in the tool parameters for tool '%s', directly returning the string value.rZ string)rk strtextvarcharcharenumintuintlongshortunsignedz[Parsed value '%s' of parameter '%s' is not an integer in tool '%s', degenerating to string.numfloatr zXParsed value '%s' of parameter '%s' is not a float in tool '%s', degenerating to string.)booleanboolbinary)truefalsezmParsed value '%s' of parameter '%s' is not a boolean (`true` of `false`) in tool '%s', degenerating to false.r{ objectra zpParsed value '%s' of parameter '%s' is not a valid JSON object in tool '%s', will try other methods to parse it.z}Parsed value '%s' of parameter '%s' cannot be converted via Python `ast.literal_eval()` in tool '%s', degenerating to string.)lowerr4 rb r` ra rl strip
startswithrq
ValueError TypeErrorrw jsonloadsJSONDecodeErrorast literal_eval SyntaxError)rf rg rh rX
param_typefloat_param_values r; convert_param_valuezGSeedOssToolParser._parse_xml_function_call..convert_param_value s ""f,,t -- 2%%NN? #! #" <
3T::
&l: 666 j!9&!A B B H H J J P P R R
%
QQQ""%%e,,E
# ((00E
# ((00E
# ((11 E
#
((44 E
# "%k"2"2KK"I. NN8#"! #"&&u-- 3
#1F1Fw1O1O 3
#(-k(:(:% -s3D/E/EEJJ *) !2 3 3 K
#I. NN8#"! #" <<< ) / / 1 1 &777NNS#"! #f,, ))Z-B-B6-J-J)
&*j&=&= ** & 43GH W ' & % "%"2;"?"?KK"K0 NN\#"! #"sH D* *.EE 6G .G32G3I) )9J%$J%)J> >.K/.K/> r
rG r[ F) ensure_asciir\ arguments)rZ r[ ) rl ra r indexr3 findallr endswithr r r dumps)r8 rU rV re r end_index
function_namerh r]
param_dictmatch
match_textidxrg rf s ` r; _parse_xml_function_callz*SeedOssToolParser._parse_xml_function_callo s C D *` # ` #*-` #=A` #NQ` #
` # ` # ` # ` #F &++C00 )*9*5
++M:: &y1}7
3;;JGG
E%*1X;q58J ""3''C#DSD)J jq 344K%%d++
. )!""o ##D))
/ )#2# . %8%8Z }& &Jz " " !"djRW.X.X.X
r< model_outputc | j | }d |D }t | dk r|g}g }|D ]/}| | j | 0d |D }|S )Nc > g | ]}|d r|d n|d S r r .0r s r;
z9SeedOssToolParser._get_function_calls.. :
38a .E!HHeAh
r< r c > g | ]}|d r|d n|d S r r r s r; r z9SeedOssToolParser._get_function_calls.. r r< )r1 r lenextendr2 )r8 r matched_rangesraw_tool_callsraw_function_calls tool_callfunction_callss r; _get_function_callsz%SeedOssToolParser._get_function_calls s -55lCC
.: s< % --.? OO r< r z,Error in extracting tool call from response.)r r
r! r" findr r r clearappendr[ r\ r r Exceptionr4 exception) r8 r r think_end_indexresult_contentthinking_contentr r r tool_call_start_indexr s ` ` r; extract_tool_callsz$SeedOssToolParser.extract_tool_calls s{ 4 4/ "r<
"l 2 2$ 44 + / /0D E E$I I O */*:*: ;N +,<_,< = ! )N* ! 5 5n E EN>""a''3!&2| )7 J
# ) ) + + + '
+22$-$6$;)2);)E %3$7$78R$S$S ! )A-- &%#(()>??
"
' 8N9N8N)OOG/ !*oo 1%#*4
K L L L/ "r<
s 99F 3CF 6F<;F<
previous_text current_text
delta_textprevious_token_idscurrent_token_idsdelta_token_idsc
, |s|r j |vrt j | }|dk rdt j dk rL| j | j z
} | dk rt d S n j s|rt d S d S |s
| _ j r j
s}| j }
|
j k rX xj dz
c_ d _ d _ d _ d _ j | j k rd _ d S j s j |v s j |v rd _ j st | S j s j |v s j |v rFd _ j |v r4|d | j } | rt | S d S | j r| dk rd S t | S | j } j | k rd S j |v r/| j t j z nd}
g }|
} | j | }|dk rn-| | |t j z
}O j t | k rd S | j }| j | }|dk r ||d }n |||t j z } j s j |v r| j t j z }| d| }|dk r||| _ _ d _ d _
tA fd j D }|s" j j d
d t tC j j tE j d d
g S d S j
r j sC j# |vr:d _ t tC j tE d g S j sd _ j sV j$ |v rLd _ | j t j z }| j$ | }|dk r||| } % ||r|j& nd }|rYtO j D ]D\ }}|( d |j) j* k r |j) j+ j | d< nEn,# tX $ r tZ . dd Y nw xY wt tC j tE d g }d _
d _ |S | j/ } j0 s j | k rg } d} | j# | }|dk rn-| | |t j# z
}Ot | j k rz| j } | t j# z } || d } d| v rG| d }!| d |! _1 | |!z dz }"||"d }#|#2 d r
|#dd }#|# j/ }$|$dk r|#d |$ }%|% d r
|%d d }% j dk r.d j1 z dz tg j4 |% dd z dz }&n-d j1 z dz tg j4 |% dd z dz }& xj dz
c_ t tC j tE |& g S j0 r* j/ |v r"| j/ }'|d |' }( j5 s&d|(v r"|( d })|(|)dz d }( j5 s |(2 d r
|(dd }( j5 |(z }* j5 r!tg j4 j5 dd nd}+tg j4 |* dd },|,t |+ d }-d _0 d _5 t tC j tE |-dz g S |}( j5 s&d|(v r"|( d })|(|)dz d }( j5 s |(2 d r
|(dd }(|(r j5 r!tg j4 j5 dd nd}+ xj5 |(z
c_5 tg j4 j5 dd },|,t |+ d }-|-r3t tC j tE |- g S d S )Nr rH )r r FTrG r c 3 T K | ]"}| d j k V #dS )r\ N)r' rL )r toolr8 s r; zASeedOssToolParser.extract_tool_calls_streaming.. sK ( ( ((D,FF( ( ( ( ( (r< z{}r r[ )r idr[ rZ )r {)r )r r[ r\ r z0Failed to parse tool arguments during streaming.)exc_info}r "z": "z, ")6r) r r0 r r countr r r r# r rR rT rQ rI rJ rO rS r$ r* r" r( r rstripr r r r r rL rE rK anyr r r r r rV enumerater' r[ r\ r r r4 rb r rP rM r r r rN ).r8 r r r r r r r complete_calls
open_calls tool_endscontent_beforetool_starts_countr tool_startsr tool_start_idx tool_end_idx tool_text
func_startfunc_end
already_addedfunc_content_end func_content parsed_toolir resultcomplete_params param_starts param_idx param_start remainingname_end value_start
value_text
param_end_idxrf
json_fragmentend_idx value_chunkgt_idx
full_value prev_escaped full_escaped
delta_escapeds. ` r; extract_tool_calls_streamingz.SeedOssToolParser.extract_tool_calls_streaming_ s6
44#>o#U#U!$199,GG" "
"A%%#d.E*F*F*J*J!-!3!3 2" "$**4+CDD"EJ "Q +B 7 7 77 ' 2 4| 4'33334 * ' ' ) ) ) !- D$4 $**4+CDDI4222''1,''#( #$ $)!#( *l.@.@./ / 16D-t # ( # 6 6#z11#'D # 4
333 3 ( 8 - @@-;;,0)-;;%/F***4+EFFF&N & D +N C C CCt !''))2243KLL "((**b00 4#J7777 ) . .t/I J J "&7 7 74 #|33
d 2 3 3c$:N6O6O O O
"$ 3 ##D$>DDCbyy s # # # 3t122 2C 3 "c+&6&6 6 64$T%<=#(()A>RR 2 $^__5II$ D4L0M0M!MMI
) $ 11 &^^D,A B BS)F F
%>>#z::r>>1::h;N1OD.+/+F+F+H+HD('+D$'+D$ %( ( ( ( ($($;( ( ( % %M ) /66(,(B-1 ( )&*&=#'#7):)-)Cr*" *" *" &0
$ 4 P $
)>j)P)P$(!#%"&"9%6%E%E%E $
)$(! # ,
(?9(L(L#' '^^D,A B BS)F F
$->>$2I:#V#V #r))#,Z8H-H#IL&*&C&C (7*L'--' ' ' *+4T5L+M+M * *4#'88F#3#3{7K7P#P#P(3(<(F %)$;A$>{$K %*E $Q
% N%) ' &%"&"9%6%E%E%E $) #'
(ood.F G GO = B
T%5%G%G " 6#..)>DDCbyy '',,,3t 4555C 6 |$$t'777 ,T-= >I"+c$2G.H.H"HK )+,, 7Ii''#,>>##6#62;IXI2F/ '2H& $ /1 4 4$'&*&=%>&,%- '+j&=&=ad&C%D '* %* !.
%*&*&=%>&,%- '+j&=&=ad&C%D '* %* !. ! , , 1 , ,#/$1.2.E1B6C2* 2* 2*%& %& %&," $ $ $ } F
+z99 (ood.F G GG",XgX"6K 3 @{8J8J!,!1!1#!6!6&1&1*,,&? 3 6 8N8Nt8T8T 6&1!""o "&!9K!GJ 3
4#;<r s]
$ $ $ $ $ $ $ # # # # # ) ) ) ) ) )
X G G G G G
G G G G G r<