U h^@s ddlZddlZddlZddlZddlmZddlZddlZddlm Z m Z m Z m Z m Z mZmZddlmZmZddlmZeeZddZd5d d Zd6d d ZddZddddddddddddddddd d!d"d#d$d%d&d'd(d)d*gZd+gd,d+gd,d+gd-ZGd.d/d/Zd7d1d2Zd8d3d4Z dS)9N)Dict)AttributeProto GraphProto ModelProto NodeProto TensorProtohelper numpy_helper) infer_shapesinfer_shapes_path)versioncCsdd|DS)z| Convert numpy float16 to python int. :param np_list: numpy float16 list :return int_list: python int list cSs.g|]&}tt|dddddqS)HN)intbinviewzfill).0_rD/tmp/pip-unpacked-wheel-socb9apf/onnxruntime/transformers/float16.py %sz%_npfloat16_to_int..r)Znp_listrrr_npfloat16_to_intsr"\o>@cCsdd}|t|dkjddkr|t|dk}|t|dk}||krntd|d|||krtd|d||t|dkjddkr|t|dk}|t|dk}|| krtd|d| || krtd|d| t|d||||}t|| |d| |}t|||td||}t|td|| | |}t|S)a? Convert float32 numpy array to float16 without changing sign or finiteness. Positive values less than min_positive_val are mapped to min_positive_val. Positive finite values greater than max_finite_val are mapped to max_finite_val. Similar for negative values. NaN, 0, inf, and -inf are unchanged. cSst||k||kSN)np logical_and)abcrrrbetween0sz&convert_np_to_float16..betweenrzthe float32 number z will be truncated to infz-inf) rwhereshapemaxminloggerdebugfloatfloat16)Znp_arraymin_positive_valmax_finite_valr"Z positive_maxZ positive_minZ negative_maxZ negative_minrrrconvert_np_to_float16(s(  r.cCst|tstdt||jtjkrtj|_|jrntt |j||}t |}||j dd<g|jdd<|j rt j|j dd}t|||}||_ |S)aConvert tensor float to float16. Args: tensor (TensorProto): the tensor to convert. min_positive_val (float, optional): minimal positive value. Defaults to 1e-7. max_finite_val (float, optional): maximal finite value. Defaults to 1e4. Raises: ValueError: input type is not TensorProto. Returns: TensorProto: the converted tensor. 3Expected input type is an ONNX TensorProto but got Nfloat32dtype) isinstancer ValueErrortype data_typeFLOATFLOAT16 float_datar.rarrayrZ int32_dataraw_data frombuffertobytes)tensorr,r- float16_dataZint_listZ float32_listZ float16_listrrrconvert_tensor_float_to_float16Js    r@cCst|j}t|j|j|Sr)r Zto_arrayr%rZmake_tensor_value_infonamer6)r>r%rrrmake_value_info_from_tensoros rBZArrayFeatureExtractorZ BinarizerZCastMapZCategoryMapperZDictVectorizerZFeatureVectorizerZImputerZ LabelEncoderZLinearClassifierZLinearRegressorZ NormalizerZ OneHotEncoderRandomUniformLikeZ SVMClassifierZ SVMRegressorZScalerZTreeEnsembleClassifierZTreeEnsembleRegressorZZipMapZNonMaxSuppressionZTopKZRoiAlignZRangeZCumSumZMinZMaxZUpsampler)ZResizeZ GroupNormZ SkipGroupNormc@s,eZdZdZedddZedddZdS) InitializerTrackerz'Class for keeping track of initializer.) initializercCs||_g|_g|_dSr)rF fp32_nodes fp16_nodes)selfrFrrr__init__szInitializerTracker.__init__)nodecCs"|r|j|n |j|dSr)rGappendrH)rIrKis_node_blockedrrradd_nodeszInitializerTracker.add_nodeN)__name__ __module__ __qualname____doc__rrJrrNrrrrrEsrEFc )s |dkstd|tttjjks.td|dkr:in|} t|tr|} t t j t dkr|st j tj| d$} | j} t| | t | }d}W5QRXn t | }t|tstdt|d} |st t j t d krzt} W5X|dkrt}|dkrg}t|}t|}td |d |d d |d|d|d|g}g}g}g}| dk rx| |}||i}t}t}dd|jjD}dd|jjD}tt rfdd|D}fdd|D}nsg}g}t!|jjD]\}}|j|krdt|}|||j<|"|jdt|}|jj#"}|$|||_t%j&|jj'_(t)j*d|jg|gt%j&|dg}|jj+,||||"|qt!|jjD]\}}|j|krdt|}|||j<|"|jdt|}|jj#"}|$|||_t%j&|jj'_(t)j*d|g|jgd|dg}|jj+,||||"|qi}|rg}|D]n} t| tr|| jt| t-r| j.D]2}|j/t%j0kr|j|kstt1|||j<q| j+D]}|j|krqt2t3|jD]*}|j||kr||j||j|<qt2t3|jD]*}|j||krB||j||j|<qB|j4|kp|j|k}!t!|jD]P\}}||kr|!p|t56|j4gko|| 6|j4gk}"||7||"q|!r||n|j4dkr4|j8D].}#|#jdkr|#j9t%j0krt%j&|#_9q4q|j4dkrd}$|j8D],}#|#jd krJd}$|#j9t%j0krJt%j&|#_9qJ|j4d!kr|$s|j8,t):d t%j&g|j4t5ks|j4| kr|j8D]}#||#qn ||qt| t;rB|| j<| j=D]}||q| j>$t?| j>||| j@D]}t?|||}q.t| t-rztAB| j| j| j#D]}|jj'j(t%j0kr|j|krt%j&|jj'_(|||jCd"rb|jjDj(j'j(t%j0krb|j|krbt%j&|jjDj(j'_(||qbqz|}ql|ED]T}%|s|%jFrt?|%j.|||%_.|tG|%j.|%jHr|stId#J|%jFq|D]}&t!|&jD]\}}|t5|&j4ksd|| 6|&j4gkrqd|D]}'||'jkr|jj#"}|$|'|&jd$t|}||_t%j0|jj'_(|&jd%t|}t)j*d|g|gd|dg}|jj+,|||&j|<qdqqdqV|D]~}&t2t3|&jD]}|&j|}|D]}'||'jkrb|jj#"}|$|'|&jd$t|}||_t%j0|jj'_(|&jd%t|}t)j*d|g|gd|dg}|jj+,|||&j|<qPqbqPt2t3|&jD]}|&j|}(|D]}'|(|'jk r|jj#"}|$|'|&jd&t|}||_t%j0|jj'_(|&jd't|}t)j*d|g|(gd(|dg}|jj+,|||&j|< q q q q<|S))aConvert tensor float type in the input ONNX model to tensor float16. Args: model (ModelProto or str): The ONNX model or path of the model to convert. min_positive_val (float, optional): minimal positive value. Defaults to 5.96e-08. max_finite_val (float, optional): maximal finite value of float16. Defaults to 65504. keep_io_types (Union[bool, List[str]], optional): It could be boolean or a list of float32 input/output names. If True, model inputs/outputs should be left as float32. Defaults to False. disable_shape_infer (bool, optional): Skips running onnx shape/type inference. Useful if shape inference has been done. Defaults to False. op_block_list (List[str], optional): List of op types to leave as float32. Defaults to None, which will use `float16.DEFAULT_OP_BLOCK_LIST`. node_block_list (List[str], optional): List of node names to leave as float32. Defaults to None. force_fp16_initializers(bool): force converting all float initializers to float16. Default to false, which will convert only the one needed to avoid precision loss. force_fp16_inputs(Dict[str, List[int]]): Force the conversion of the inputs of some operators to float16, even if this script's preference it to keep them in float32. Raises: ValueError: input type is not ModelProto. Returns: ModelProto: converted model. rzginvalid min_positive_val. smallest positive float16 value: subnormal 5.96e-08, and normalized 6.104e-05z4invalid max_finite_val. largest float16 value: 65504Nz1.8.0)dirTz$Expected an ONNX ModelProto but got z1.2.0z"fp16 parameters: min_positive_val=z max_finite_val=z keep_io_types=z disable_shape_infer=z op_block_list=z node_block_list=z force_fp16_initializers=cSs"g|]}|jjjtjkr|jqSrr5 tensor_type elem_typerr7rArnrrrrsz,convert_float_to_float16..cSs"g|]}|jjjtjkr|jqSrrTrWrrrrscsg|]}|kr|qSrrrW keep_io_typesrrr scsg|]}|kr|qSrrrWrYrrr sZgraph_input_cast_Zgraph_input_castZCast)torAZgraph_output_cast_Zgraph_output_castrDr[)ZEyeLikeZ Multinomial RandomNormalZRandomNormalLike RandomUniformrC SequenceEmptyZ BernoulliFr2)r\r]r^ sequence_typezZinitializer is used by both fp32 and fp16 nodes. Consider add these nodes to block list:{}Z _input_cast_Z _input_castZ _output_cast_Z _output_cast )KAssertionErrorr*rZfinfor+r&r3strr parseonnx __version__tempfileNamedTemporaryFileospathdirnamerAr loadrr4r5r DEFAULT_OP_BLOCK_LISTsetr(r)rLgraphinputoutputlist enumerateadd value_infoZCopyFromrr8rUrVrZ make_noderKextendrrFr6r7rErangelenZop_typeALWAYS_FLOAT_INPUTSgetrN attributeiZmake_attributergZgraphstr@Ztensors itertoolschainZHasFieldr_valuesrHrBrGinfoformat))modelr,r-rZZdisable_shape_inferZ op_block_listZnode_block_listZforce_fp16_initializersZforce_fp16_inputsZforce_fp16_inputs_dictZ model_pathZtmpfileZshape_infer_model_pathZfunc_infer_shapequeueZvalue_info_listZ node_listZmixed_float_type_node_listZ name_mappingZgraph_io_to_skipZio_castsZ fp32_inputsZ fp32_outputsr{rXZ output_nameZ node_nameZnew_value_infoZnew_nodeZ input_nameZfp32_initializersZ next_levelqrMZuse_fp32_weightattrZ has_dtypevaluerKrtrprrYrconvert_float_to_float16s$       ,                                                $                  rcCst|tstdt||jtjkr0tdd}|jrFt|j}|j r\tj |j dd}|dkrlt dt |||}t t|t|S)zSMeasure the maximum absolute difference after converting a float tensor to float16.r/z#Expected tensor data type is float.Nr0r1zexternal data not loaded!)r3rr4r5r6r7r9rr:r;r< RuntimeErrorr.Zamaxabsr0)r>r,r-Z float32_datar?rrrfloat_to_float16_max_diffs    r)rr)rr)rrFFNNFN)rr)!r~loggingrhrftypingrZnumpyrrdrrrrrrr Zonnx.shape_inferencer r packagingr getLoggerrOr(rr.r@rBrlrxrErrrrrrsp $   " %! >