biZ vddlZddlZddlmZddlmZddlmZmZddl Z ddl Z ddl m Z mZddededefd Zdd e j$d e j$d eede j$fd Zdd e j$d e j$dede j$fdZdd e j$d e j$dede j$fdZGddZGddZ ddeeefdeeee j6e j6fdee j8dee j:dee j<de j$f dZy) N)Mapping)contextmanager)OptionalUnion)is_torch_npu_availableis_torch_xpu_availablenestedsepreturncXdtdtdtddffd i}|d||S)z>Flatten dictionary and concatenate nested keys with separator.nestprefixintor Nc|jD]D\}}|vrtdd|dt|tr|||zz|=||||z<Fy)Nz separator 'z' not allowed to be in key '')items ValueError isinstancer)r rrkvrecurser s C/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/trl/core.pyrzflatten_dict..recursesmJJL %DAqax ;se3OPQsRS!TUU!W%6A:+T2#$VaZ  %)dictstr)r r flatrs ` @r flatten_dictrs<%d%C%t%% D FB Krvaluesmaskaxisc|(||zj||j|z S||zj|jz S)z,Compute mean of tensor with a masked values.)r!)sum)rr r!s r masked_meanr$+sN  """-d0CCC ""$txxz11runbiasedct||}||z }t|dz|}|r-|j}|dk(r td||dz z }||z}|S)z.Compute variance of tensor with masked values.rzThe sum of the mask is zero, which can happen when `mini_batch_size=1`;try increase the `mini_batch_size` or `gradient_accumulation_steps`)r$r#r)rr r%meancentered_valuesvariancemask_sumbessel_corrections r masked_varr.3sq vt $DtmO?A-t4H88: q=V  %1 5// Or shift_meanct||t||}}||z tj|dzz}|s||z }|S)z!Whiten values with masked values.g:0yE>)r$r.torchrsqrt)rr r/r)varwhiteneds r masked_whitenr5FsFFD):fd+C#D S4Z!88H D Orc,eZdZdZdedefdZdefdZy) LengthSamplerz Samples a length min_value max_valuec8tt|||_yN)listranger)selfr8r9s r__init__zLengthSampler.__init__Ts5I67 rr cTtjj|jSr;)nprandomchoicer)r>s r__call__zLengthSampler.__call__Wsyy ,,rN)__name__ __module__ __qualname____doc__intr?rDrrr7r7Os&8#8#8-#-rr7c*eZdZdZeedZy) PPODecoratorsFc#>Kd|jrtrGtjtj j tjytrGtjtjj tjytjjrGtjtjj tjyyywr;) optimize_device_cachergccollectr1xpu empty_cachernpucuda is_available)clss rempty_device_cachez PPODecorators.empty_device_cache^s   $ $%'  %%' ')  %%' ((*  &&( + %sDDN)rErFrGrN classmethodrrWrJrrrLrL[s"!rrLshape generatordevicedtypelayoutc D|}|d}|xstj}|xstjd}|t|ts|jj n|djj }||j k7r1|dk(r,d}|dk7rKt jd|d|d|dtn&||j k7r|d k(rtd |d |d t|trt|d k(r|d}t|trcd|d dz}t|Dcgc]}tj||||||!} }tj| dj|} | Stj|||||j|} | Scc}w)zA helper function to create random tensors on the desired `device` with the desired `dtype`. When passing a list of generators, you can seed each batch size individually. If CPU generators are passed, the tensor is always created on the CPU. rcpuNmpszBThe passed generator was created on 'cpu' even though a tensor on zB was expected. Tensors will be created on 'cpu' and then moved to zk. Note that one can probably slighly speed up this function by passing a generator that was created on the z device.rTzCannot generate a z! tensor from a generator of type .r()r()rZr[r\r])dim)r1stridedr[rr<typewarningswarn UserWarningrlenr=randncatto) rYrZr[r\r] rand_device batch_sizegen_device_typeilatentss r randn_tensorrqqsKqJ  $u}}F  *u||E*F7A)T7R)**//XabcXdXkXkXpXp fkk )o.FK XY_X`aKKQ(Sfflemmuw   +60I1&9Z[jZkklmn n)T"s9~':aL )T"uQRy :&  KK1kQV_e f  ))G+..v6 N++eyTYbhillmst N s$$F)/r;)T)NNNN) rOrecollections.abcr contextlibrtypingrrnumpyrAr1 transformersrrrrrTensorboolr$r.r5r7rLtupler< Generatorr[r\r]rqrJrrr|sr #%" GC$"2 2ELL22Z_ZfZf2u||5<<4SXS_S_&%,,ellX]XdXd - -0JN%)#'%) .  .d5??3U__DEF. U\\ ". EKK . U\\ " .  \\ .r