biddlZddlZddlZddlZddlZddlmZddlmZddl m Z ddl m Z m Z mZmZmZddlZddlZddlZddlmZddlmcmZddlmZddlmZddlmZdd l m!Z!dd l"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,dd l-m.Z.dd l/m0Z0dd l1m2Z2m3Z3ddl4m5Z5m6Z6ddl7m8Z8ddl9m:Z:m;Z;mZ>m?Z?m@Z@mAZAmBZBmCZCe2r ddlDmEZEmFZFmGZGe,rddlHZHGdde*ZIy)N) defaultdict) nullcontext)Path)AnyCallableLiteralOptionalUnion) PartialState)Dataset)autocast) DataLoader) AutoModelForCausalLMBaseImageProcessor DataCollatorFeatureExtractionMixinPreTrainedModelPreTrainedTokenizerBaseProcessorMixinTraineris_comet_availableis_wandb_available)TrainerCallback)EvalLoopOutput)is_peft_availableis_torch_fx_proxy)maybe_apply_chat_templatemaybe_extract_prompt) CPOConfig) DPODataCollatorWithPaddingadd_bos_token_if_neededadd_eos_token_if_neededdisable_dropout_in_modelgenerate_model_cardget_comet_experiment_urllog_table_to_comet_experiment pad_to_lengthpeft_module_casting_to_bf16selective_log_softmax) PeftModelget_peft_modelprepare_model_for_kbit_trainingceZdZdZddgZ d;deeeeje fdee dee dee d eee ee e ffd eeeeeefd eegefd eeed eej0j2ej0j4j6fdeeej8ej8gej8fdeedeeegefffd ZdZddejNdejDd e#de$de#dejNf d!Z)dejdee eeejDffdeejNejNejNejNffd"Z* d?dee eeejDffd#e+d$fd%Z, d@deeejfd&ee eej8e-ffdeej8eej8ee ej8ffffd'Z.dee ejDfde fd(Z/ dj@|_ n(|j@ td|j@|_ |j@r6|j>jB|_!|j>jD|_"| td|jFtIjJdtLd}n |jF}|jNtIjJdtLd}n |jN}||kstd|d|d|jP)|j@rtIjJdtLd}n |jP}|atS|jD|jT|j@}|jVr!d|_+tIjJdtLd|_,nd|_,|jZr t]|||_#|j8|_|jT|_*|j^ |j^n |jD|_/||_'|j`|_0||_(||_1|jddvr7|jfdkDr(tIjJd|jdd tL|jdd!k(r td"|jh|_4|jf|_3|jd|_2|jj|_5t |j>d#d|_6t |j>d$d%|_7|jlr)|jnd%k(rtIjJd&tL|jdd'k(r|jp|_8tsd(|_:d|jvd)<tyj{5|j}t~|j*}|j}td+|i|j,}|E|j}t~|j*}|j}td+|i|j,}|j}|j|j*}|'|j}|j|j*}dddt||||||||| || | - d|_Et|jd.r%|jj|jt|d/s td0y#1swY~xYw)1NzRYou passed model_kwargs to the CPOTrainer. But your model is already instantiated. torch_dtypeautoznInvalid `torch_dtype` passed to the CPOConfig. Expected a string with either `torch.dtype` or 'auto', but got .FzvPEFT is not installed and you passed a `peft_config` in the trainer's kwargs, please install it to use the PEFT modelsis_loaded_in_8bitis_loaded_in_4bitgradient_checkpointing_kwargsuse_gradient_checkpointingenable_input_require_gradsc&|jdyNTrequires_grad_moduleinputoutputs R/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/trl/trainer/cpo_trainer.pymake_inputs_require_gradz5CPOTrainer.__init__..make_inputs_require_grads--d3Tc&|jdyrIrJrLs rPrQz5CPOTrainer.__init__..make_inputs_require_grads))$/rRz`generate_during_eval=True` requires Weights and Biases or Comet to be installed. Please install `wandb` or `comet-ml` to resolve.zMWhen no model is provided, you need to pass the parameter is_encoder_decoder.z=processing_class must be specified to tokenize a CPO dataset.z`max_length` is not set in the CPOConfig's init it will default to `512` by default, but you should do it yourself in the future.iz`max_prompt_length` is not set in the CPOConfig's init it will default to `128` by default, but you should do it yourself in the future.zmax_prompt_length (z+) should be strictly less than max_length (z).zWhen using an encoder decoder architecture, you should set `max_completion_length` in the CPOConfig's init it will default to `128` by default, but you should do it yourself in the future.) pad_token_idlabel_pad_token_idis_encoder_decoderzWhen using DPODataCollatorWithPadding, you should set `remove_unused_columns=False` in your TrainingArguments we have set it for you, but you should do it yourself in the future.)hingeiporzYou are using the z loss type that does not support label smoothing. The `label_smoothing` parameter will be ignored. Set `label_smoothing` to `0.0` to remove this warning.kto_pairzKSupport for kto_pair has been removed in CPOTrainer. Please use KTOTrainer.output_router_logitsrouter_aux_loss_coefa-You set `output_router_logits` to `True` in the model config, but `router_aux_loss_coef` is set to `0.0`, meaning the auxiliary loss will not be used. Either set `router_aux_loss_coef` to a value greater than `0.0`, or set `output_router_logits` to `False` if you don't want to use the auxiliary loss.simpoc ttSN)rlistrRrPz%CPOTrainer.__init__..Bs ;t3DrRestimate_tokens)num_proc tokenizer) fn_kwargsre) r3r4r5r6r7r8r9r>r:r;r<add_model_tags acceleratorzXYour `Trainer` does not have an `accelerator` object. Consider upgrading `transformers`.)Jmodel_init_kwargs isinstancestr ValueErrorgetgetattrtorchdtyperfrom_pretrained_peft_has_been_casted_to_bf16rr,merge_and_unloadhasattrrainspect signaturer. parametersgradient_checkpointingrErGget_input_embeddingsregister_forward_hookr-bf16r*generate_during_evalrrconfigrWdecoder_start_token_idrU max_lengthwarningswarn UserWarningmax_prompt_lengthmax_completion_lengthr"rVremove_unused_columnsuse_dpo_data_collatordisable_dropoutr% padding_valuetruncation_moder8 loss_typelabel_smoothingbeta cpo_alphaaux_loss_enabled aux_loss_coef simpo_gammar_stored_metricswarnings_issuedr main_process_firstmaprdataset_num_procr tokenize_rowsuper__init__model_accepts_loss_kwargsr3rh _tag_namesAttributeError)selfr3r4r5r6r7r8r9r:r;r<r=r>rjr@_support_gc_kwargsprepare_model_kwargsrQrrr __class__s rPrzCPOTrainer.__init__rsn"  ! ! ) " E3'qr r $ 6 6 +// >K&k3/K64I")%"=K&(K1U$IJUIVVWX4?!-0 eS !(88TBSTE.3* "{'>I  [%<%+..0u159WUL_af=g%,9&&5%%&EFQQ:# )EdFaFa'b$%LPLnLn()HI7VAUV,,5">?4464..0FFG_`#5+6EyyWU,?G+E2592  ( (u:;0020**,BBC[\  $ $.@.BFXFZD   &+ll&E&ED #  $ $ ,lm m&*&=&=D #  " "*/,,*M*MD ' % 9 9D   #\] ] ?? " MMe  JJ  ! ! ) MMe  !$  $ 6 6  :-%&7%88cdncooqr   % % -$2I2I MMe  %( !$($>$> !  6-::#'#:#:#'#:#:M ))-2* \ *.D &).D &    $U +$$($=$=!"&"9"9373E3E3QT//WgWtWt!2#33%:" 0 >>- -$2F2F2J MM$T^^$45vv  >>Z 'jk kII #33 ' 6Le T$U\\3I3O  T%7%73%> MM   >>W $#//D *+DE48/0^ . . 0 c)--.BTMbMb-cM)--)kCS5T_c_t_t.M'+//0DtOdOd/e +//-*,<=!22 0  *--d.?.?$J_J_-`M'+//0A0ADLaLa/b # c& ''%-!+!*G  "*/& 4::/ 0 JJ % %doo 6t]+ j ,U c cs C^""^+c|j||zd}|j|dd}|dt|d}|dt|d}tj||g}tj|d}t|t|k7r t dt|} ||dd| k7r| dz} |dd| }|dd| } t|t| k7r t d|d| d}|d| d}t || || S) a Llama tokenizer does satisfy `enc(a + b) = enc(a) + enc(b)`. It does ensure `enc(a + b) = enc(a) + enc(a + b)[len(enc(a)):]`. Reference: https://github.com/EleutherAI/lm-evaluation-harness/pull/531#issuecomment-1595586257 Fadd_special_tokens input_idsNattention_maskzBPrompt input ids and answer input ids should have the same length.r z@Prompt input ids and attention mask should have the same length.)prompt_input_idsprompt_attention_maskrr)r8lennp concatenatearrayrmdict) rpromptanswerfull_tokenizedranswer_input_idsanswer_attention_maskfull_concat_input_idsfull_input_idsresponse_token_ids_start_idxrs rPbuild_tokenized_answerz!CPOTrainer.build_tokenized_answer~su..vSX.Y00E0RS^_)+6s;K7L7NO ./? @EUAVAX Y!#0@BR/S T."=> ~ #&;"< <ab b (++;'<$ ~k:;X >_` `)+67S7TU ./? @A]A^ _-"7&0   rRreturnc  i}|d}|d}|d}|jst|tstdt ||j |d}|j D cic] \}} d||  }}} t|tstdt ||j||} t|tstd t ||j||} t|d } t| d } t| d }t| |} |j D] \}} | d | ||<tt| d | d Dcgc] \}}||k7 c}}}t| |z }|d kDs|d kDr td t|j j| || | || \}} } t|j j | | \} } t#t| dt| d}| | |fD]}t|d |z|j$kDs"|j&dk(rdD]}||d |j(||<N|j&dk(rdD]}|||j( d ||<{td|j&| | fD]J}t|d |z|j$kDs"dD]$}||d |j$|j(z ||<&LdDcic]}|| d|| |z}}dDcic]}|| d|| |z}}|dd d |d<|j*gt| d z|dd t| d |dd d |d<|j*gt| d z|dd t| d |||dj D],\}}|j D]\}}|dk(r ||||<.|S|j |d|j,d} |j |d|j,d} |j |d|j(d}| d|d<| d|d<|d|d <|d|d<|bt/|drV|j1t3j4|d|d<|j1t3j4|d|d <|Scc} }wcc}}wcc}wcc}w)!a.Tokenize a single row from a CPO specific dataset. At this stage, we don't convert to PyTorch tensors yet; we just handle the truncation in case the prompt + chosen or prompt + rejected responses is/are too long. First we truncate the prompt; if we're still too long, we truncate the chosen/rejected. We also create the labels for the chosen/rejected responses, which are of length equal to the sum of the length of the prompt and the chosen/rejected response, with label_pad_token_id for the prompt tokens. rchosenrejectedz prompt should be an str but got Frprompt_z chosen should be an str but got z"rejected should be an str but got rNr zdChosen and rejected prompt_input_ids might only differ on the last token due to tokenizer merge ops.r keep_start)rrkeep_endzUnknown truncation mode: )rrlabels)chosen_ rejected_token_type_idsT) truncationrr chosen_labelsrejected_labelsrr%prepare_decoder_input_ids_from_labels)rrejected_decoder_input_idschosen_decoder_input_ids)rWrkrlrmtyper8itemsrrminsumzipabsr# bos_token_idr$ eos_token_idmaxrrrrVrrurrptensor)rfeaturer3batchrrr prompt_tokenskv chosen_tokensrejected_tokensprompt_len_input_idschosen_prompt_len_input_idsrejected_prompt_len_input_idsabnum_diff_tokens num_diff_lenlonger_response_length answer_tokenschosen_sequence_tokensrejected_sequence_tokenstokstype_keytokenss rPrzCPOTrainer.tokenize_rowsd"":&&& fc* #CDL>!RSS 11&U1SM:G:M:M:OP$!Qwqc]A-PMPfc* #CDL>!RSS 77GMh, #Ed8nEU!VWW"99&(KO$'}5G'H#I *-mf+g a(h hMl&GH=71#/-2BBB& "&Qp(KL?WQC=1OA4FFF( $(0Fk/RST/U "8 ,''ZM"456Z7 "8 ,-Us=AS3T/U V2J+1VWX1Y $X .''^O$678^9 $X ./Y_EW5X1Y Z 25!eg  54 )- 5$Hf#33 .4EQCz*+5  5D /!114D4N4Ncg2M#33Td6P6Pei4O!114D4J4J_c2M&3;%?E/ "'6{'CE# $(5k(BE$ %-:;K-LE) * WU4[%\6;6a6a <<.?(@A7b7235:4_4_ <<o(>?5`501 oQ0q\&(s,S!S :SSrrWrVrdevicec i}|r-t|djd|djd}n,t|djd|djd}|D]}|jdst||tj s3d|vs|r|}n'|j dr|}n|j d rd }|jdd } t||| || <|D]}|jd st||tj s3d|vs|r|}n'|j dr|}n|j d rd }|jd d } t j|| t||| fd j||| <|rP|djddj||d<|djddj||d<|S)aConcatenate the chosen and rejected inputs into a single tensor. Args: batch: A batch of data. Must contain the keys 'chosen_input_ids' and 'rejected_input_ids', which are tensors of shape (batch_size, sequence_length). is_encoder_decoder: Whether the model is an encoder-decoder model. label_pad_token_id: The label pad token id. padding_value: The padding value to use for the concatenated inputs_ids. device: The device for the concatenated inputs. Returns: A dictionary containing the concatenated inputs under the key 'concatenated_input_ids'. rr rchosen_input_idsrejected_input_idsrr _input_ids_attention_maskr concatenated) pad_valuerdimrrrconcatenated_input_idsrconcatenated_attention_mask) rshape startswithrkrpTensorendswithreplacer)cattorepeat) rrWrVrrconcatenated_batchrrrconcatenated_keys rPconcatenated_inputszCPOTrainer.concatenated_inputs@s4  U?399!V>\>\]^>_`JU#56<>f>M < ="!rRpolicy_chosen_logpspolicy_rejected_logpscT||z j|jj}|jdk(r|j|j z }||z }t j|j |z d|jz zt j|j |z|jzz }n|jdk(ret j|j |z d|jz zt j|j |z|jzz }nv|jdk(r&tjd|j |zz }nA|jdk(r|dd|j zz z dz}ntd|jd|j |j|jjjz}|j |j|jjjz}|||fS) aCompute the CPO loss for a batch of policy and reference model log probabilities. Args: policy_chosen_logps: Log probabilities of the policy model for the chosen responses. Shape: (batch_size,) policy_rejected_logps: Log probabilities of the policy model for the rejected responses. Shape: (batch_size,) Returns: A tuple of three tensors: (losses, chosen_rewards, rejected_rewards). The losses tensor contains the CPO loss for each example in the batch. The chosen_rewards and rejected_rewards tensors contain the rewards for the chosen and rejected responses, respectively. r^r sigmoidrXrYrzUnknown loss type: z7. Should be one of ['sigmoid', 'hinge', 'ipo', 'simpo']) rrirrrrF logsigmoidrrprelurmdetach)rrrlogitsgamma_logratioslosseschosen_rewardsrejected_rewardss rPcpo_losszCPOTrainer.cpo_losss$&(==AA$BRBRBYBYZ >>W $"..:Oo-Fdii&011Q9M9M5MN,, zF23d6J6JJK ^^y (dii&011Q9M9M5MN,, zF23d6J6JJK ^^w &ZZDII$6 67F ^^u $qA M22q8F%dnn%55lm &9&<&.cross_entropy_losss**SbS! ,779QR335**,H[[V\\"%56F[[_FYYv}}-FFF+DKrRr]N)rYr^)r rWrV)rrWrVrrirr _shift_rightrrrrrprrrraux_loss)rr3rr len_chosen model_kwargsoutputs all_logitsrrnll_loss all_logps chosen_logpsrejected_logps chosen_logitsrejected_logitss` rPconcatenated_forwardzCPOTrainer.concatenated_forwards"55 #66#66,,##** 6 ?+11!4 &&$T%6%67IJ_7`%a    37L/ 0 7 8 -.KL    ^^  $$9:@@B >>Q ||C(++D,<,<,C,CDH)*[j*A6+:CVWH((  4 5!^^/??#66#66 ) !*- ":;/";J/ $Z[1   .-RZ\c\l\lm mnm_hWWrR train_eval)trainevalc^i}|j||}|dd\}}}} } |jr|d} |j||\} } }| j|j| zz}| |kDj }|dk(rdnd}|j j| jj||d<|j j|jj||d<|j j|jj||d<|j j| |z jj||d <|j j|jjj||d <|j j|jjj||d <|j j| jjjj||d <|j j|jjjj||d <|j j| jjj||d<|jr||j zz }||fS)zWCompute the CPO loss and other metrics for the given batch of inputs for train or test.Nr/eval_rzrewards/chosenzrewards/rejectedzrewards/accuracieszrewards/marginszlogps/rejectedz logps/chosenzlogits/rejectedz logits/chosenr&) r,rr meanrfloatrigather_for_metricsitemrr)rr3rr-metricsforward_outputrrpolicy_chosen_logitspolicy_rejected_logitspolicy_nll_lossr!r r r rreward_accuraciesprefixs rPget_batch_loss_metricsz!CPOTrainer.get_batch_loss_metrics/s 225%@ 2A    ! "   %a(H37==  !4 0 0 {{}t~~??+.>>EEG&&0b-1-=-=-P-PQ_-`-e-e-g-l-l-n6(.)*/3/?/?/R/RSc/d/i/i/k/p/p/r6(*+,151A1A1T1TUf1g1l1l1n1s1s1u6(,-.    / /AQ0Q R W W Y ^ ^ ` 6(/*+    / /0E F M M O T T V [ [ ] 6(.)*    / /0C D K K M R R T Y Y [ 6(,'(    / /0F0M0M0O0T0T0V W \ \ ^ c c e 6(/*+    / /0D0K0K0M0R0R0T U Z Z \ a a c 6(-()(,'7'7'J'J?'['b'b'd'i'i'k'p'p'r6((#$  D&&1 1DW}rRinputsc|jr)t|jjjn t }|5|j ||d\}}ddd|jd|r|fSS#1swY$xYw)Nr.r-)rsr rirrrr> store_metrics)rr3r?return_outputsnum_items_in_batchcompute_loss_context_managerrr7s rP compute_losszCPOTrainer.compute_losses7;6X6XHT%%,,11 2^i^k %* [ 77vRY7ZMD' [ 7w7 '? "  [ [s A==Bc|jr)t|jjjn t }|5|j |d|d|jd|jj}dddt|j|jj}|jj|d}|S#1swYSxYw)zRGenerate samples from the model and reference model for the given batch of inputs.rrT)rrr do_samplerUN)skip_special_tokens) rsr rirrrgeneraterr8rUr) batch_decode)rr3rgenerate_context_manager policy_outputpolicy_output_decodeds rPgenerate_from_modelzCPOTrainer.generate_from_modelzs 7;6X6XHT%%,,11 2^i^k !& !NN 23$%<=??!22?? +M &mT__dF[F[FhFhi $ 5 5 B B=fj B k$$  s ;CCprediction_loss_only ignore_keysc*|&t|drt|jdg}ng}|jr)t |j j jn t}tj5|5|j||d\}}dddddd|jd|rjddfS|d|dd}|jD cgc] \} } | |vs | } } } tj| |j j } tj | j"d |j j } j| | fS#1swYxYw#1swYxYwcc} } w) Nr~keys_to_ignore_at_inferencer/rAeval_logits/choseneval_logits/rejected)rTrUrr)ruror~rsr rirrrrpno_gradr>rBrrrzerosr) rr3r?rPrQprediction_context_managerrr7 logits_dictrrrrs rPprediction_stepzCPOTrainer.prediction_steps  uh'%ell4QSUV  7;6X6XHT%%,,11 2^i^k #]]_ Z8 Z 77vRX7YMD' Z Z 7v6 KKM4. .#**>"?$+,B$C !, 1 1 3L1q 7K!LLfT-=-=-D-DEV\\!_T5E5E5L5LM vv..% Z Z Z ZMs0<F?E7F- F:F7F <FF r7cv|jD]&\}}|j||j|(yr`)rrappend)rr7r-keyvalues rPrBzCPOTrainer.store_metricss;!--/ @JC   ,S 1 8 8 ? @rR dataloader descriptionmetric_key_prefixc |jrQt|j}tjt ||j j}|jj|}|j|} |j| } |j|j| } tjddgt| d| D cgc]\} } | | t| dgc} } } d|j j vr+t#j$dt#j&| id |j j vr t)d | t*|Y|||||}|Scc} } w) z Overriding built-in evaluation loop to store metrics for each batch. Prediction/evaluation loop, shared by `Trainer.evaluate()` and `Trainer.predict()`. Works both with or without labels. )rPromptPolicyrN)columnsdatawandbgame_log)rfcomet_mlz game_log.csv)nametable)r}rdatasetrandomsampleranger4eval_batch_sizeselectr5_prepare_inputsrOr3pd DataFramer report_torglogTabler(revaluation_loop)rr_r`rPrQra num_samplesrandom_indicesrandom_batch_dataset random_batchrNrpolrkinitial_outputrs rPrxzCPOTrainer.evaluation_loopsR  $ $j001K#]]5+=AZAZ[N$.#5#5#<#<^#L --.BCL// =L$($<$?@TYY000-' 0  %9;HY %sE5 logs start_timecd|vrdnd}|j|jD]9\}}tj|j j ||<;|j|=t |||S)a1 Log `logs` on the various objects watching training, including stored metrics. Args: logs (`dict[str, float]`): The values to log. start_time (`float` or `None`, *optional*, defaults to `None`): Start time of the training. rr.r/)rrrprr3r6rrv)rrrr-r]r7rs rPrvzCPOTrainer.logs|!'$WF  00<BBD Q*RXZ [  ) 3 3IOO D )238)<)B)B)D c12g &(,(C(C f %    $KL L&&'8D'@$BSBST  rRc|jj*t|jjj}n(|jjj dd}|j |t|!||y)N/r) model_name) r4 hub_model_idr output_dirrjsplitcreate_model_cardr_save_checkpoint)rr3trialrrs rPrzCPOTrainer._save_checkpointsl 99 ! ! )dii22388J//55c:2>J *5  .rRr dataset_nametagsc |jsyt|jjdr^tj j |jjjs!|jjj}nd}| t}nt|tr|h}n t|}t|jjdr|jd|j|jtjd}t!|||j"||t%r.t&j(t&j(j+ndt-d|dd }|j/tj j1|j2j4d y) a Creates a draft of a model card using the information available to the `Trainer`. Args: model_name (`str` or `None`, *optional*, defaults to `None`): Name of the model. dataset_name (`str` or `None`, *optional*, defaults to `None`): Name of the dataset used for training. tags (`str`, `list[str]` or `None`, *optional*, defaults to `None`): Tags to be associated with the model card. N _name_or_pathunsloth_versionunslothay @inproceedings{xu2024contrastive, title = {{Contrastive Preference Optimization: Pushing the Boundaries of LLM Performance in Machine Translation}}, author = {Haoran Xu and Amr Sharaf and Yunmo Chen and Weiting Tan and Lingfeng Shen and Benjamin Van Durme and Kenton Murray and Young Jin Kim}, year = 2024, booktitle = {Forty-first International Conference on Machine Learning, {ICML} 2024, Vienna, Austria, July 21-27, 2024}, publisher = {OpenReview.net}, url = {https://openreview.net/forum?id=51iwkioZpn} }CPOzeContrastive Preference Optimization: Pushing the Boundaries of LLM Performance in Machine Translationz 2401.08417) base_modelrrrr wandb_url comet_url trainer_nametrainer_citation paper_titlepaper_idz README.md)is_world_process_zerorur3r~ospathisdirrsetrkrladdupdatertextwrapdedentr&rrrgrunget_urlr'savejoinr4r)rrrrrcitation model_cards rPrzCPOTrainer.create_model_cardsG"))+  4::$$o 6rww}}TZZM^M^MlMl?m**88JJ <5D c "6Dt9D 4::$$&7 8 HHY  DOO$??$ )!!**%-?-AeiiF[eii'')ae.0%!    TYY%9%9;GHrR) NNNNNNNN)NNNNNr`)FrrN)FrF)r.)FN)NNr/)NNN):__name__ __module__ __qualname____doc__rr r rrModulerlr!rr rrrrrrrartuplerpoptim Optimizer lr_schedulerLambdaLRrrrrr staticmethod LongTensorboolintrr FloatTensorr rr,rr>rrFrOrZr4rBrrxrvr rr __classcell__)rs@rPr0r0Js#JJCG$(04+/EI >B59Vbhl&*FJJoryy#=>?Jy!J - J  ( J uWd3<.@%@AB J# )+=?UWee f JXb/&9:;JD12J%++//1I1I1R1RRSJ(0%,, 9UW\WcWc9c0d'eJd^J"(N+;T+A"BCJX/ bO8E/299:T4U+VObfOb$)"&)- A"CtU%5%55667A" A" A" A" & A" c5### $ A"A"F38"..38 %0038 u  %"3"3U5F5FF G 38j"'"&#( (9!!(9  (9(9 (9 ! (9    (9(9TIXYYIX'+CtU=M=M7M1N,N'OIX u  %"3"3U5F5FHYHYY ZIX^07 4CtU%5%556674O, 4t _bii/0S% c 1223 u||U5<<c5<<6G1H#HII J *%S%:J:J5J0K%PS%8,0 #/_bii/0#/S% c 1223#/# #/ d3i( #/J@T#u*%5@7?C[@jn@04+/!' 000'tn 0 d3i( 0  0 0d-S%Z(-huo-QU-$!0/%)&*,0 =ISM=Ism=ICcD() =IrRr0)Jrvrrmrr collectionsr contextlibrpathlibrtypingrrrr r numpyrpandasrsrptorch.nnrtorch.nn.functional functionalr accelerater datasetsr r torch.utils.datar transformersrrrrrrrrrrtransformers.trainer_callbackrtransformers.trainer_utilsrtransformers.utilsrr data_utilsrr cpo_configr!utilsr"r#r$r%r&r'r(r)r*r+peftr,r-r.rgr0rbrRrPrs #":: #'   :5CH!   OOQIQIrR