U qhR-@sddlmZddlZddlZddlZddlZddlZddlmZm Z m Z m Z m Z GdddZ dddddd d d d d ZddddddZddd dddZdS)) annotationsN) FunctionProto ModelProto NodeProto TensorProtoValueInfoProtoc@seZdZdddddZeddZdd Zd d d d dZd d d ddZdddddddddZ d d ddddZ ddZ ddddd Z dd d d!d d"dd#d$d%Z d d ddd&d'Zd(S)) ExtractorrNone)modelreturncCs<tj||_|jj|_||jj|_||jj|_ dSN) onnxZshape_inferenceZ infer_shapesr graph_build_name2obj_dict initializerwmap value_infovimap)selfr r./tmp/pip-unpacked-wheel-xnis5xre/onnx/utils.py__init__s zExtractor.__init__cCsdd|DS)NcSsi|] }|j|qSr)name).0objrrr sz2Extractor._build_name2obj_dict..r)objsrrrrszExtractor._build_name2obj_dictcsp|t}t|}||@}||}fdd|D}|fdd|D|fdd|DS)Ncsg|] }|qSrrrr)original_io_maprr !sz2Extractor._collect_new_io_core..c3s|]}j|VqdSr rrrrr #sz1Extractor._collect_new_io_core..csg|] }|qSrrr)new_io_tensors_maprrr's)rsetextend)rZ original_ioZio_names_to_extractZoriginal_io_namesZs_io_names_to_extractZio_names_to_keepZnew_io_names_to_addZnew_io_tensorsr)r#rrr_collect_new_io_cores  zExtractor._collect_new_io_core list[str]zlist[ValueInfoProto])namesr cCs||jj|Sr )r&rinputrr(rrr_collect_new_inputs)szExtractor._collect_new_inputscCs||jj|Sr )r&routputr*rrr_collect_new_outputs,szExtractor._collect_new_outputsstrzset[str]zlist[NodeProto]zset[int])node_output_namegraph_input_namesnodes reachable unreachabler c sn|kr dSfdd|D}|D]}||||q$|D]&}|jD]}|||||qPqBdS)aHelper function to find nodes which are connected to an output Arguments: node_output_name (str): The name of the output graph_input_names (set of string): The names of all inputs of the graph nodes (list of nodes): The list of all nodes of the graph reachable (set of int): The set of indexes to reachable nodes in `nodes` unreachable (set of int): The set of indexes to unreachable nodes in `nodes` Ncsg|]}|jkr|qSr)r,)rindexr/r1rrrEsz9Extractor._dfs_search_reachable_nodes..)addremover)_dfs_search_reachable_nodes) rr/r0r1r2r3Znodes_to_search node_indexrrr5rr8/s"   z%Extractor._dfs_search_reachable_nodes) input_names output_namesr cs`t|}t|jjt}ttt}|D]}|||||q.fddt|DS)Ncsg|] }|qSrr)rr9r1rrrcsz6Extractor._collect_reachable_nodes..)r$listrnoderangelenr8sorted)rr:r;Z _input_namesr2r3rrr<r_collect_reachable_nodesUs z"Extractor._collect_reachable_nodescs.fdd}g}|||}|r*|||}q|S)NcsPg}|D]BtfddjjDd}|r||kr||||jq|S)Nc3s*|]"}|jjkr|jjkr|VqdSr )rZop_typedomain)rfr>rrr"ss z[Extractor._collect_referred_local_functions..find_referred_funcs..)nextr functionsappendr%r>)r1referred_local_functions new_nodesZmatch_functionr!rErfind_referred_funcsns   zHExtractor._collect_referred_local_functions..find_referred_funcsr)rr1rKrIrJrr!r!_collect_referred_local_functionsfs    z+Extractor._collect_referred_local_functionsz.tuple[list[TensorProto], list[ValueInfoProto]])r1r cst|D]}|j|jq fddjD}fddjD}tjj}|dkrxt d|dtjj }|dkrt d|d||fS)Ncsg|]}|krj|qSr)rrtZall_tensors_namesrrrrsz8Extractor._collect_reachable_tensors..csg|]}|krj|qSrr rMrOrrrsrzlen_sparse_initializer is z, it must be 0.zlen_quantization_annotation is ) r$updater)r,rrr@rZsparse_initializer ValueErrorZquantization_annotation)rr1r>rrZlen_sparse_initializerZlen_quantization_annotationrrOr_collect_reachable_tensorss"     z$Extractor._collect_reachable_tensorszlist[TensorProto]zlist[FunctionProto])r1inputsoutputsrrlocal_functionsr c CsNd|jjd}tjj||||||d}|jj|jjd|d} tjj|f| S)NzExtracted from {})rrzonnx.utils.extract_model) ir_versionZ opset_importsZ producer_namerG) rrr helperZ make_graphr rWZ opset_importZ make_model) rr1rSrTrrrUrrmetarrr _make_models zExtractor._make_modelc CsP||}||}|||}||\}}||}|||||||} | Sr )r+r-rBrRrLrZ) rr:r;rSrTr1rrrUr rrr extract_models    zExtractor.extract_modelN)__name__ __module__ __qualname__r staticmethodrr&r+r-r8rBrLrRrZr[rrrrrs &!rTzstr | os.PathLiker'boolr ) input_path output_pathr:r; check_modelr cCs|tj|std||s&td|s2tdtj|t|}t|}| ||}t |||rxtj|dS)a5Extracts sub-model from an ONNX model. The sub-model is defined by the names of the input and output tensors *exactly*. Note: For control-flow operators, e.g. If and Loop, the _boundary of sub-model_, which is defined by the input and output tensors, should not _cut through_ the subgraph that is connected to the _main graph_ as attributes of these operators. Arguments: input_path (str | os.PathLike): The path to original ONNX model. output_path (str | os.PathLike): The path to save the extracted ONNX model. input_names (list of string): The names of the input tensors that to be extracted. output_names (list of string): The names of the output tensors that to be extracted. check_model (bool): Whether to run model checker on the extracted model. zInvalid input model path: z%Output model path shall not be empty!z'Output tensor names shall not be empty!N) ospathexistsrQr checkerrcloadrr[save)rarbr:r;rcr eZ extractedrrrr[s     r[ztarfile.TarFilezlist[tarfile.TarInfo])tarbaser cCsg}|D]r}tj||j}tj|}tj|}||sPtd|dn |s`|rptd|d| |q|S)zCheck that the content of ``tar`` will be extracted safely Args: tar: The tarball file base: The directory where the tarball will be extracted Returns: list of tarball members zThe tarball member z^ in downloading model contains directory traversal sequence which may contain harmful payload.zP in downloading model contains symbolic links which may contain harmful payload.) rdrejoinrabspath startswith RuntimeErrorissymislnkrH)rkrlresultmemberZ member_pathabs_baseZ abs_memberrrr_tar_members_filters       rv)model_tar_pathlocal_model_with_data_dir_pathr c CsHt|4}ttdr&|j|ddn|j|t||dW5QRXdS)aQSafely extracts a tar file to a specified directory. This function ensures that the extraction process mitigates against directory traversal vulnerabilities by validating or sanitizing paths within the tar file. It also provides compatibility for different versions of the tarfile module by checking for the availability of certain attributes or methods before invoking them. Args: model_tar_path: The path to the tar file to be extracted. local_model_with_data_dir_path: The directory path where the tar file contents will be extracted to. Z data_filterdata)refilter)remembersN)tarfileopenhasattr extractallrv)rwrxZmodel_with_data_zippedrrr_extract_model_safe s  r)T) __future__rrdr|Z onnx.checkerr Z onnx.helperZonnx.shape_inferencerrrrrrr[rvrrrrrs =(