o >g@s"ddlZddlZGdddZdS)Nc@s<eZdZdZeddZddZddZdd Zd d Z d S) SPieceTokenizerTcCst|SN)r)pathrN/home/comdoleger1/zen-flux-style-shape/comfy/text_encoders/spiece_tokenizer.pyfrom_pretrainedszSPieceTokenizer.from_pretrainedcCsTddl}t|r|}t|tr|j||jd|_ dS|j||jd|_ dS)Nr) model_protoadd_eos) model_filer ) sentencepiecetorch is_tensornumpytobytes isinstancebytesSentencePieceProcessorr tokenizer)selftokenizer_pathr rrr__init__ s    zSPieceTokenizer.__init__cCs,i}t|jD] }|||j|<q |Sr)rangerget_piece_size id_to_piece)routirrr get_vocabszSPieceTokenizer.get_vocabcCs|j|}d|iS)N input_ids)rencode)rstringrrrr__call__s zSPieceTokenizer.__call__cCstt|jSr)r ByteTensorlistrserialized_model_proto)rrrrserialize_modelszSPieceTokenizer.serialize_modelN) __name__ __module__ __qualname__r staticmethodrrrr r$rrrrrs   r)osr rrrrrs