biddlZddlZddlZddlZddlZddlmZddlmZddl m Z ddl m Z m Z mZmZmZddlZddlZddlZddlmZddlmcmZddlmZddlmZddlmZdd l m!Z!dd l"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-dd l.m/Z/dd l0m1Z1dd l2m3Z3m4Z4ddl5m6Z6m7Z7ddl8m9Z9ddl:m;Z;mZ>m?Z?m@Z@mAZAmBZBmCZCmDZDe3r ddlEmFZFmGZGmHZHe-rddlIZIe,r ddlJmKcmLZMGdde*ZNy)N) defaultdict) nullcontext)Path)AnyCallableLiteralOptionalUnion) PartialState)Dataset)autocast) DataLoader) AutoModelForCausalLMBaseImageProcessor DataCollatorFeatureExtractionMixinPreTrainedModelPreTrainedTokenizerBaseProcessorMixinTraineris_comet_availableis_torch_xla_availableis_wandb_available)TrainerCallback)EvalLoopOutput)is_peft_availableis_torch_fx_proxy)maybe_apply_chat_templatemaybe_extract_prompt) ORPOConfig) DPODataCollatorWithPaddingadd_bos_token_if_neededadd_eos_token_if_neededdisable_dropout_in_modelgenerate_model_cardget_comet_experiment_urllog_table_to_comet_experiment pad_to_lengthpeft_module_casting_to_bf16selective_log_softmax) PeftModelget_peft_modelprepare_model_for_kbit_trainingceZdZdZddgZ d;deeeeje fdee dee dee d eee ee e ffd eeeeeefd eegefd eeed eej0j2ej0j4j6fdeeej8ej8gej8fdeedeeegefffd ZdZddejNdejDd e#de$de#dejNf d!Z)dejdee eeejDffdeejNejNejNejNffd"Z* d?dee eeejDffd#e+d$fd%Z, d@deeejfd&ee eej8e-ffdeej8eej8ee ej8ffffd'Z.dee ejDfde fd(Z/ dj@|_ n(|j@ td|j@|_ |j@r6|j>jB|_!|j>jD|_"| td|jFtIjJdtLd}n |jF}|jNtIjJdtLd}n |jN}|jP.|j@r"tIjJdtLd|_(n|jP|_(|atS|jD|jT|j@}|jVr!d|_+tIjJdtLd|_,nd|_,|jZr t]|||_#|j8|_|jT|_*|j^ |j^n |jD|_/||_'|j`|_0||_1|jd|_2t |j>dd|_3t |j>dd|_4|jfr)|jhdk(rtIjJdtLtkd|_6d|jnd<tqjs5|jutv|jx }|jutzd!|i|jx"}|ju|j||jx }|l|jutv|jx }|jutzd!|i|jx"}|ju|j||jx }dddt~||||||||| || | # t|jd$r%|jj|jt|d%s td&y#1swYwxYw)'NzSYou passed model_kwargs to the ORPOTrainer. But your model is already instantiated. torch_dtypeautozoInvalid `torch_dtype` passed to the ORPOConfig. Expected a string with either `torch.dtype` or 'auto', but got .FzvPEFT is not installed and you passed a `peft_config` in the trainer's kwargs, please install it to use the PEFT modelsis_loaded_in_8bitis_loaded_in_4bitgradient_checkpointing_kwargsuse_gradient_checkpointingenable_input_require_gradsc&|jdyNTrequires_grad_moduleinputoutputs S/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/trl/trainer/orpo_trainer.pymake_inputs_require_gradz6ORPOTrainer.__init__..make_inputs_require_grads--d3Tc&|jdyrJrKrMs rQrRz6ORPOTrainer.__init__..make_inputs_require_grads))$/rSz`generate_during_eval=True` requires Weights and Biases or Comet to be installed. Please install `wandb` or `comet-ml` to resolve.zMWhen no model is provided, you need to pass the parameter is_encoder_decoder.z>processing_class must be specified to tokenize a ORPO dataset.z`max_length` is not set in the ORPOConfig's init it will default to `512` by default, but you should do it yourself in the future.iz`max_prompt_length` is not set in the ORPOConfig's init it will default to `128` by default, but you should do it yourself in the future.zWhen using an encoder decoder architecture, you should set `max_completion_length` in the ORPOConfig's init it will default to `128` by default, but you should do it yourself in the future.) pad_token_idlabel_pad_token_idis_encoder_decoderzWhen using DPODataCollatorWithPadding, you should set `remove_unused_columns=False` in your TrainingArguments we have set it for you, but you should do it yourself in the future.output_router_logitsrouter_aux_loss_coefga-You set `output_router_logits` to `True` in the model config, but `router_aux_loss_coef` is set to `0.0`, meaning the auxiliary loss will not be used. Either set `router_aux_loss_coef` to a value greater than `0.0`, or set `output_router_logits` to `False` if you don't want to use the auxiliary loss.c ttSN)rlistrSrQz&ORPOTrainer.__init__..1s ;t3DrSestimate_tokens)num_proc tokenizer) fn_kwargsra) r4r5r6r7r8r9r:r?r;r<r=add_model_tags acceleratorzXYour `Trainer` does not have an `accelerator` object. Consider upgrading `transformers`.)Emodel_init_kwargs isinstancestr ValueErrorgetgetattrtorchdtyperfrom_pretrained_peft_has_been_casted_to_bf16rr-merge_and_unloadhasattrr]inspect signaturer/ parametersgradient_checkpointingrFrHget_input_embeddingsregister_forward_hookr.bf16r+generate_during_evalrrconfigrXdecoder_start_token_idrV max_lengthwarningswarn UserWarningmax_prompt_lengthmax_completion_lengthr#rWremove_unused_columnsuse_dpo_data_collatordisable_dropoutr& padding_valuetruncation_moder9betaaux_loss_enabled aux_loss_coefr_stored_metricswarnings_issuedr main_process_firstmapr dataset_num_procr tokenize_rowsuper__init__r4rd _tag_namesAttributeError)selfr4r5r6r7r8r9r:r;r<r=r>r?rfrA_support_gc_kwargsprepare_model_kwargsrRr|r __class__s rQrzORPOTrainer.__init__vs"  ! ! ) " E3'rs s $ 6 6 +// >K&k3/K64I")%"=K&(K1U$JKVJWWXY4?!-0 eS !(88TBSTE.3* "{'>I  [%<%+..0u159WUL_af=g%,9&&5%%&EFQQ:# )EdFaFa'b$%LPLnLn()HI7VAUV,,5">?4464..0FFG_`#5+6EyyWU,?G+E2592  ( (u:;0020**,BBC[\  $ $.@.BFXFZD   &+ll&E&ED #  $ $ ,lm m&*&=&=D #  " "*/,,*M*MD ' % 9 9D   #]^ ^ ?? " MMe  JJ  ! ! ) MMe  !$  $ 6 6   % % -$2I2I MMe  *-D &)-)C)CD &  6-::#'#:#:#'#:#:M ))-2* \ *.D &).D &    $U +$$($=$=!"&"9"9373E3E3QT//WgWtWt!2#33 0II ' 6Le T$U\\3I3O  T%7%73%> MM   ++DE48/0^ . . 0 c)--.BTMbMb-cM)--)kCS5T_c_t_t.M*--d.?.?$J_J_-`M'+//0DtOdOd/e +//-*,<=!22 0  ,//0A0ADLaLa/b  c ''%-!+!*G   4::/ 0 JJ % %doo 6t]+ j ,E c cs C[[c|j||zd}|j|dd}|dt|d}|dt|d}tj||g}tj|d}t|t|k7r t dt|} ||dd| k7r| dz} |dd| }|dd| } t|t| k7r t d|d| d}|d| d}t || || S) a Llama tokenizer does satisfy `enc(a + b) = enc(a) + enc(b)`. It does ensure `enc(a + b) = enc(a) + enc(a + b)[len(enc(a)):]`. Reference: https://github.com/EleutherAI/lm-evaluation-harness/pull/531#issuecomment-1595586257 Fadd_special_tokens input_idsNattention_maskzBPrompt input ids and answer input ids should have the same length.r!z@Prompt input ids and attention mask should have the same length.)prompt_input_idsprompt_attention_maskrr)r9lennp concatenatearrayridict) rpromptanswerfull_tokenizedranswer_input_idsanswer_attention_maskfull_concat_input_idsfull_input_idsresponse_token_ids_start_idxrs rQbuild_tokenized_answerz"ORPOTrainer.build_tokenized_answeresu..vSX.Y00E0RS^_)+6s;K7L7NO ./? @EUAVAX Y!#0@BR/S T."=> ~ #&;"< <ab b (++;'<$ ~k:;X >_` `)+67S7TU ./? @A]A^ _-"7&0   rSreturnc  i}|d}|d}|d}|jst|tstdt ||j |d}|j D cic] \}} d||  }}} t|tstdt ||j||} t|tstd t ||j||} t|d } t| d } t| d }t| |} |j D] \}} | d | ||<tt| d | d Dcgc] \}}||k7 c}}}t| |z }|d kDs|d kDr td t|j j| || | || \}} } t|j j | | \} } t#t| dt| d}| | |fD]}t|d |z|j$kDs"|j&dk(rdD]}||d |j(||<N|j&dk(rdD]}|||j( d ||<{td|j&| | fD]J}t|d |z|j$kDs"dD]$}||d |j$|j(z ||<&LdDcic]}|| d|| |z}}dDcic]}|| d|| |z}}|dd d |d<|j*gt| d z|dd t| d |dd d |d<|j*gt| d z|dd t| d |||dj D],\}}|j D]\}}|dk(r ||||<.n|j |d|j,d} |j |d|j,d} |j |d|j(d}| d|d<| d|d<|d|d <|d|d<|bt/|drV|j1t3j4|d|d<|j1t3j4|d|d <t7r}|D]x}d|vs |jr |j*}n1|j9d!r |j:}n|j9d"rd#}||g|j$t||z zz||<z|Scc} }wcc}}wcc}wcc}w)$a/Tokenize a single row from a ORPO specific dataset. At this stage, we don't convert to PyTorch tensors yet; we just handle the truncation in case the prompt + chosen or prompt + rejected responses is/are too long. First we truncate the prompt; if we're still too long, we truncate the chosen/rejected. We also create the labels for the chosen/rejected responses, which are of length equal to the sum of the length of the prompt and the chosen/rejected response, with label_pad_token_id for the prompt tokens. rchosenrejectedz prompt should be an str but got Frprompt_z chosen should be an str but got z"rejected should be an str but got rNr!zdChosen and rejected prompt_input_ids might only differ on the last token due to tokenizer merge ops.r keep_start)rrkeep_endzUnknown truncation mode: )rrlabels)chosen_ rejected_token_type_idsT) truncationr|r chosen_labelsrejected_labelsrr%prepare_decoder_input_ids_from_labels)rrejected_decoder_input_idschosen_decoder_input_ids _input_ids_attention_maskr)rXrgrhrityper9itemsrrminsumzipabsr$ bos_token_idr% eos_token_idmaxr|rrrWrrqrrltensorrendswithr)rfeaturer4batchrrr prompt_tokenskv chosen_tokensrejected_tokensprompt_len_input_idschosen_prompt_len_input_idsrejected_prompt_len_input_idsabnum_diff_tokens num_diff_lenlonger_response_length answer_tokenschosen_sequence_tokensrejected_sequence_tokenstokstype_keytokens pad_values rQrzORPOTrainer.tokenize_rows"":&&& fc* #CDL>!RSS 11&U1SM:G:M:M:OP$!Qwqc]A-PMPfc* #CDL>!RSS 77GMh, #Ed8nEU!VWW"99&(KO$'}5G'H#I *-mf+g a(h hMl&GH=71#/-2BBB& "&Qp(KL?WQC=1OA4FFF( $(0Fk/RST/U "8 ,''ZM"456Z7 "8 ,-Us=AS3T/U V2J+1VWX1Y $X .''^O$678^9 $X ./Y_EW5X1Y Z 25!eg  54 )- 5$Hf#33 .4EQCz*+5  5!114D4N4Ncg2M#33Td6P6Pei4O!114D4J4J_c2M&3;%?E/ "'6{'CE# $(5k(BE$ %-:;K-LE) * WU4[%\6;6a6a <<.?(@A7b7235:4_4_ <<o(>?5`501 " # Vq=D$;$; $ 7 7IZZ - $ 2 2IZZ 12 !I 8ykT__s5QR8}5T&UUa V CQ0q\&(s,U!U :UUrrXrWrdevicec i}|r-t|djd|djd}n,t|djd|djd}|D]}|jdst||tj s3d|vs|r|}n'|j dr|}n|j d rd }|jdd } t||| || <|D]}|jd st||tj s3d|vs|r|}n'|j dr|}n|j d rd }|jd d } t j|| t||| fd j||| <|rP|djddj||d<|djddj||d<|S)aConcatenate the chosen and rejected inputs into a single tensor. Args: batch: A batch of data. Must contain the keys 'chosen_input_ids' and 'rejected_input_ids', which are tensors of shape (batch_size, sequence_length). is_encoder_decoder: Whether the model is an encoder-decoder model. label_pad_token_id: The label pad token id. padding_value: The padding value to use for the concatenated inputs_ids. device: The device for the concatenated inputs. Returns: A dictionary containing the concatenated inputs under the key 'concatenated_input_ids'. rr!rchosen_input_idsrejected_input_idsrrrrr concatenated)rrdimrrrconcatenated_input_idsrconcatenated_attention_mask) rshape startswithrgrlTensorrreplacer*cattorepeat) rrXrWrrconcatenated_batchr|rrconcatenated_keys rQconcatenated_inputszORPOTrainer.concatenated_inputs1s4  U?399!V>\>\]^>_`JU#56<>f>M < ="!rSpolicy_chosen_logpspolicy_rejected_logpscV||z tjtj| tjtj| z z }tj|}|j |z}|j |j |jjjz}|j |j |jjjz}|||tj|tj|fS)anCompute ORPO's odds ratio (OR) loss for a batch of policy and reference model log probabilities. Args: policy_chosen_logps: Log probabilities of the policy model for the chosen responses. Shape: (batch_size,) policy_rejected_logps: Log probabilities of the policy model for the rejected responses. Shape: (batch_size,) Returns: A tuple of three tensors: (losses, chosen_rewards, rejected_rewards). The losses tensor contains the ORPO loss for each example in the batch. The chosen_rewards and rejected_rewards tensors contain the rewards for the chosen and rejected responses, respectively. The log odds ratio of the chosen responses over the rejected responses ratio for logging purposes. The `log(sigmoid(log_odds_chosen))` for logging purposes. ) rllog1pexpF logsigmoidrrrerdetachmean)rrrlog_oddsratiolosseschosen_rewardsrejected_rewardss rQodds_ratio_losszORPOTrainer.odds_ratio_lossus*(*?? KK#677 85;; RgHhGh;i i  X&U"&9&<&.cross_entropy_losss**SbS! ,779QR335**,H[[V\\"%56F[[_FYYv}}-FFF+DKrSr!N)rrXrWr)rrXrWrrerr _shift_rightrrr rlr r aux_loss)rr4rr len_chosen model_kwargsoutputs all_logitsrrrchosen_nll_loss all_logps chosen_logpsrejected_logps chosen_logitsrejected_logitss` rQconcatenated_forwardz ORPOTrainer.concatenated_forwards1"55 #66#66,,##** 6 ?+11!4 &&$T%6%67IJ_7`%a    37L/ 0 7 8 -.KL    ^^    " "'(=>DDFF'(@AGGIF/0MNN[[1!4fd>U>UVF,Z -Df[jFYZ((  4 5!#66#66 ) !*- ":;/&&&{ {CRC':;M(crc1)<=O&{ 3M(5O   .-Racjcscst tnm_o^^rS train_eval)trainevalci}|j||}|dd\}}}} } |jr|d} |j||\} } }}}| | jz }| |kDj }|dk(rdnd}|j j | j||d<|j j |j||d<|j j |j||d<|j j | |z j||d <|j j |jj||d <|j j |jj||d <|j j | jjj||d <|j j |jjj||d <|j j | jj||d<|j j |jj||d<|j j |jj||d<trtj|jD]\}}|j||<|jr||j zz }||fS)zXCompute the ORPO loss and other metrics for the given batch of inputs for train or test.Nr)eval_rzrewards/chosenzrewards/rejectedzrewards/accuracieszrewards/marginszlogps/rejectedz logps/chosenzlogits/rejectedz logits/chosennll_losslog_odds_ratiolog_odds_chosen)r&rrrfloatregather_for_metricsrrxm mark_stepritemr)rr4rr'metricsforward_outputrrpolicy_chosen_logitspolicy_rejected_logitspolicy_nll_lossrrrrr.r/rreward_accuraciesprefixrrs rQget_batch_loss_metricsz"ORPOTrainer.get_batch_loss_metricssP225%@ 2A    ! "   %a(HTXThTh !6U Q 0./.+.>>EEG&&0b-1-=-=-P-PQ_-`-e-e-g6(.)*/3/?/?/R/RSc/d/i/i/k6(*+,151A1A1T1TUf1g1l1l1n6(,-..2.>.>.Q.Q - -/ $& 6(/*+.2-=-=-P-PQf-g-n-n-p-u-u-w6(.)*+/+;+;+N+NOb+c+j+j+l+q+q+s6(,'(.2.>.>.Q.Q " ) ) + 0 0 2/ $& 6(/*+-1,<,<,O,O ' ' ) . . 0- $& 6(-()(,'7'7'J'J?'['b'b'd'i'i'k6((#$-1-=-=-P-PQ_-`-g-g-i-n-n-p6(.)*.2.>.>.Q.QRa.b.i.i.k.p.p.r6(/*+ ! # LLNMMO "DAqGAJ "  D&&1 1DW}rSinputsc\|jr)t|jjjn t }|5|j ||d\}}dddj|jj}|jd|r||fS|S#1swYIxYw)Nr(r') ror rerrrr<rr5 store_metrics)rr4r=return_outputsnum_items_in_batchcompute_loss_context_managerrr5s rQ compute_losszORPOTrainer.compute_lossHs7;6X6XHT%%,,11 2^i^k %* [ 77vRY7ZMD' [wwtyy''( 7w7 '? "  [ [s B""B+c|jr)t|jjjn t }|5|j |d|d|jd|jj}dddt|j|jj}|jj|d}|S#1swYSxYw)zRGenerate samples from the model and reference model for the given batch of inputs.rrT)rrr| do_samplerVN)skip_special_tokens) ror rerrrgenerater|r9rVr* batch_decode)rr4rgenerate_context_manager policy_outputpolicy_output_decodeds rQgenerate_from_modelzORPOTrainer.generate_from_model`s 7;6X6XHT%%,,11 2^i^k !& !NN 23$%<=??!22?? +M &mT__dF[F[FhFhi $ 5 5 B B=fj B k$$  s ;CCprediction_loss_only ignore_keyscl|jstjd|&t|drt |j dg}ng}|j r)t|jjjn t}tj5|5|j||d\}}dddddd|jd|rj!ddfS|d|dd}|j#D cgc] \} } | |vs | } } } tj$| |jj } tj&| j(d |jj } j!| | fS#1swYxYw#1swYxYwcc} } w) Na!prediction_step is only implemented for DPODataCollatorWithPadding, and you passed a datacollator that is different than DPODataCollatorWithPadding - you might see unexpected behavior. Alternatively, you can implement your own prediction_step method if you are using a custom data collatorrzkeys_to_ignore_at_inferencer)r?eval_logits/choseneval_logits/rejected)rRrSrr)rr}r~rqrkrzror rerrrrlno_gradr<r@rrrzerosr) rr4r=rNrOprediction_context_managerrr5 logits_dictrrrrs rQprediction_stepzORPOTrainer.prediction_stepws)) MM{   uh'%ell4QSUV  7;6X6XHT%%,,11 2^i^k #]]_ Z8 Z 77vRX7YMD' Z Z 7v6 KKM4. .#**>"?$+,B$C !, 1 1 3L1q 7K!LLfT-=-=-D-DEV\\!_T5E5E5L5LM vv..% Z Z Z ZMs0F$ F8F$ F0F0F! F$$F-r5cv|jD]&\}}|j||j|(yr\)rrappend)rr5r'keyvalues rQr@zORPOTrainer.store_metricss;!--/ @JC   ,S 1 8 8 ? @rS dataloader descriptionmetric_key_prefixc |jrQt|j}tjt ||j j}|jj|}|j|} |j| } |j|j| } tjddgt| d| D cgc]\} } | | t| dgc} } } d|j j vr+t#j$dt#j&| id |j j vr t)d | t*|Y|||||}|Scc} } w) z Overriding built-in evaluation loop to store metrics for each batch. Prediction/evaluation loop, shared by `Trainer.evaluate()` and `Trainer.predict()`. Works both with or without labels. )rPromptPolicyrN)columnsdatawandbgame_log)rdcomet_mlz game_log.csv)nametable)ryrdatasetrandomsampleranger5eval_batch_sizeselectr6_prepare_inputsrMr4pd DataFramer report_torelogTabler)revaluation_loop)rr]r^rNrOr_ num_samplesrandom_indicesrandom_batch_dataset random_batchrLrpolriinitial_outputrs rQrvzORPOTrainer.evaluation_loopsR  $ $j001K#]]5+=AZAZ[N$.#5#5#<#<^#L --.BCL// =L$($<$?@TYY000-' 0  %9;HY %sE5 logs start_timecd|vrdnd}|j|jD]9\}}tj|j j ||<;|j|=t |||S)a1 Log `logs` on the various objects watching training, including stored metrics. Args: logs (`dict[str, float]`): The values to log. start_time (`float` or `None`, *optional*, defaults to `None`): Start time of the training. rr(r))rrrlrrr4rrt)rr}r~r'r[r5rs rQrtzORPOTrainer.logs|!'$WF  00<BBD Q*RXZ [  ) 3 3IOO D )238)<)B)B)D c12g &(,(C(C f %    $KL L&&'8D'@$BSBST  rSc|jj*t|jjj}n(|jjj dd}|j |t|!||y)N/r) model_name) r5 hub_model_idr output_dirrhsplitcreate_model_cardr_save_checkpoint)rr4trialrrs rQrzORPOTrainer._save_checkpointsl 99 ! ! )dii22388J//55c:2>J *5  .rSr dataset_nametagsc |jsyt|jjdr^tj j |jjjs!|jjj}nd}| t}nt|tr|h}n t|}t|jjdr|jd|j|jtjd}t!|||j"||t%r.t&j(t&j(j+ndt-d|dd }|j/tj j1|j2j4d y) a Creates a draft of a model card using the information available to the `Trainer`. Args: model_name (`str` or `None`, *optional*, defaults to `None`): Name of the model. dataset_name (`str` or `None`, *optional*, defaults to `None`): Name of the dataset used for training. tags (`str`, `list[str]` or `None`, *optional*, defaults to `None`): Tags to be associated with the model card. N _name_or_pathunsloth_versionunslotha @article{hong2024orpo, title = {{ORPO: Monolithic Preference Optimization without Reference Model}}, author = {Jiwoo Hong and Noah Lee and James Thorne}, year = 2024, eprint = {arXiv:2403.07691} }ORPOz@ORPO: Monolithic Preference Optimization without Reference Modelz 2403.07691) base_modelrrrr wandb_url comet_url trainer_nametrainer_citation paper_titlepaper_idz README.md)is_world_process_zerorqr4rzospathisdirrsetrgrhaddupdatertextwrapdedentr'rrrerunget_urlr(savejoinr5r)rrrrrcitation model_cards rQrzORPOTrainer.create_model_card sG"))+  4::$$o 6rww}}TZZM^M^MlMl?m**88JJ <5D c "6Dt9D 4::$$&7 8 HHY  DOO$??$ )!!**%-?-AeiiF[eii'')ae.0%Z!    TYY%9%9;GHrS) NNNNNNNN)NNNNNr\)FrrN)FrF)r()FN)NNr))NNN):__name__ __module__ __qualname____doc__rr r rrModulerhr"rr rrrrrrr]rtuplerloptim Optimizer lr_schedulerLambdaLRrrrrr staticmethod LongTensorboolintrr FloatTensorrr r&rr<rrDrMrXr0r@rrvrtrrr __classcell__)rs@rQr1r1Ns#JJCG%)04+/EI >B59Vbhl&*FJmoryy#=>?mz"m - m  ( m uWd3<.@%@AB m# )+=?UWee f mXb/&9:;mD12m%++//1I1I1R1RRSm(0%,, 9UW\WcWc9c0d'emd^m"(N+;T+A"BCm^/ bY8E/299:T4U+VYbfYv$)"&)- A"CtU%5%55667A" A" A" A" & A" c5### $ A"A"Fa"..a %00a u  %"3"3U5F5FHYHY[`[l[ll m a@"'"&#( (9!!(9  (9(9 (9 ! (9    (9(9TO_YYO_'+CtU=M=M7M1N,N'OO_ u  %"3"3U5F5FHYHYY ZO_j07 5CtU%5%556675O, 5v _bii/0S% c 1223 u||U5<<c5<<6G1H#HII J 0%S%:J:J5J0K%PS%8,0 (/_bii/0(/S% c 1223(/# (/ d3i( (/T@T#u*%5@7?C[@jn@04+/!' 000'tn 0 d3i( 0  0 0d-S%Z(-huo-QU-$!0/%)&*,0 <ISM<Ism<ICcD() <IrSr1)Orrrrkrr} collectionsr contextlibrpathlibrtypingrrrr r numpyrpandasrqrltorch.nnrtorch.nn.functional functionalr accelerater datasetsr r torch.utils.datar transformersrrrrrrrrrrrtransformers.trainer_callbackrtransformers.trainer_utilsrtransformers.utilsrr data_utilsrr orpo_configr"utilsr#r$r%r&r'r(r)r*r+r,peftr-r.r/retorch_xla.core.xla_modelcore xla_modelr2r1r^rSrQrs #":: #'    :5CH#   OO))wI'wIrS