
    hj7a              *       >   U d Z ddlmZ ddlmZ ddlZddd	Z ej         ej        d
d          g ej	         ej
        d                              Z ej        d ej        d ej        d                    d           ej        d ej        d ej        d                    d           ej        d ej        d ej        d           ej        d                    d           ej        d ej        d ej        d           ej        d d!d"#                    d           ej        d$ ej        d ej        d           ej        d d!ej        j        d%&                    d           ej        d' ej        d ej        d           ej        d                    d           ej        d( ej        d ej        d           ej        d d!d)d* ej        d+          ,                    d           ej        d- ej        d ej        d./                    d           ej        d0 ej        d ej        d./           ej        d d!d)d* ej        d1/          d2d3i4                    d           ej        d5 ej        d6 ej        d7           ej        d8d9d: ej        d;          d<e=                    d           ej        d> ej        d6 ej        d7           ej        d: ej        d;          d?d@eA                    d           ej        dB ej        d6 ej        d7           ej        d: ej        d;          dCdDd?d@eE                    d           ej        dF ej        d6 ej        d7           ej        d8d9d: ej        d;          d?d@eG                    d           ej        dH ej        dI ej        d7           ej        d8dJdKdLd@dDeM                    d           ej        dN ej        d ej        d           ej         ej        dOP          Q                    d           ej        dR ej        d ej        d           ej        d d!dSd*d)dLdLdTdTdU ej         ej         ej        dVW          X          Y          Z                    d           ej        d[ ej        d ej        d           ej        d d!dSd*d) ej         ej        d) ej         ej         ej        dVW          X          Y          \          g]          ^                    d          gZeej                 ed_<    ej        e e            d`ea          ZdbZ ej!        "                    dcdde          df             Z#ej!        "                    dcdde          dg             Z$ej!        "                    dcdde          dh             Z%ej!        "                    dcdde          di             Z&ej!        "                    dcdde          ej!        j'        dj                         Z(dS )kzTests for tunings.tune().   )types   )pytest_helper    Nv1beta1z?https://us-central1-autopush-aiplatform.sandbox.googleapis.com/)api_versionbase_urlbleuztest prompt template)nameprompt_templategs://sararob_test/output_uri_prefixgcs_destination)metricsoutput_configtest_dataset_gcs_urigemini-2.5-flashzVgs://cloud-samples-data/ai-platform/generative_ai/gemini-1_5/text/sft_train_data.jsonlgcs_uri)
base_modeltraining_datasetzDparameter is only supported in Gemini Enterprise Agent Platform mode)r   
parametersexception_if_mldevtest_tune_pretuned_modelzFprojects/801452371447/locations/us-central1/models/9030969596621881344Vgs://cloud-samples-data/ai-platform/generative_ai/gemini-2_0/text/sft_train_data.jsonl+test_tune_pretuned_model_with_checkpoint_id3)pre_tuned_model_checkpoint_idr   r   configtest_tune_simple_dpozModel display name   PREFERENCE_TUNING)tuned_model_display_nameepoch_countmethodtest_tune_dpo_with_beta      ?)r&   r'   r(   beta*test_non_pretuned_model_with_checkpoint_id#test_dataset_gcs_uri_all_parameters      ?ADAPTER_SIZE_ONEz[gs://cloud-samples-data/ai-platform/generative_ai/gemini-1_5/text/sft_validation_data.jsonl)r&   r'   learning_rate_multiplieradapter_sizevalidation_dataset$test_dataset_vertex_dataset_resourcezHprojects/801452371447/locations/us-central1/datasets/5779918772206829568)vertex_dataset_resource,test_dataset_dataset_resource_all_parameterszHprojects/801452371447/locations/us-central1/datasets/1168232753779441664testlabelkeytestlabelvalue)r&   r'   r0   r1   r2   labelstest_tune_distillationz#meta/llama3_1@llama-3.1-8b-instructzEgs://nathreya-oss-tuning-sdk-test/distillation-openai-opposites.jsonlDISTILLATIONzdeepseek-ai/deepseek-v3.1-maas   zIgs://nathreya-oss-tuning-sdk-test/distillation-val-openai-opposites.jsonlzDgs://nathreya-oss-tuning-sdk-test/ayushagra-distillation-test-folder)r(   base_teacher_modelr'   r2   
output_urihttp_optionstest_tune_oss_sftzigs://nathreya-oss-tuning-sdk-test/ayushagra-distillation-test-folder/postprocess/node-0/checkpoints/finalz=gs://nathreya-oss-tuning-sdk-test/ayushagra-distillation-test)r'   r2   custom_base_modelr=   r>   test_tune_oss_sft_hyperparamsgMb0?TUNING_MODE_FULL)r'   r2   learning_ratetuning_moder@   r=   r>   test_tune_oss_distillation)r(   r<   r'   r2   r@   r=   r>   &test_tune_oss_distillation_hyperparamszqwen/qwen3@qwen3-4bz!deepseek-ai/deepseek-r1-0528-maasg-C6?   )r(   r<   rC   
batch_sizer=   rD   r>   test_tune_encryption_speczZprojects/vertex-sdk-dev/locations/us-central1/keyRings/ayush-test-key/cryptoKeys/ayush-key)kms_key_name)encryption_spectest_tune_reinforcementREINFORCEMENT_TUNINGd   i   
test-model)autorater_model)autorater_config)autorater_scorer)r&   r'   r(   r1   r0   rH   samples_per_promptevaluate_intervalcheckpoint_intervalmax_output_tokensreward_config!test_tune_reinforcement_composite)weightrW   )weighted_reward_configs)r&   r'   r(   r1   r0   composite_reward_config
test_tableztunings.tune)fileglobals_for_filetest_methodr\   )pytest_asynciozconfig.getoption('--private')zSkipping in pre-public tests)reasonc                 >   | j         j        r| j                            dt	          j        d          t	          j        ddddt	          j        d          t          	          
          }|j	        t          j
        j        k    sJ |j        t          k    sJ dS dS zTests tuning with eval config.r   r   r   tuning job with eval configr$   r.   r/   [gs://cloud-samples-data/ai-platform/generative_ai/gemini-2_0/text/sft_validation_data.jsonlr&   r'   r0   r1   r2   evaluation_configr!   N)_api_clientvertexaituningstunegenai_typesTuningDatasetCreateTuningJobConfigTuningValidationDatasetrg   stateJobStateJOB_STATE_PENDINGclient
tuning_jobs     h/var/www/app.qaans.com/backend/venv/lib/python3.11/site-packages/google/genai/tests/tunings/test_tune.pytest_eval_configrw   b  s       =$$#"0  :R  S  S  S.#@#&)(@s   .	 	 	 %  J {3EEEEE'+<<<<<#= =" =<    c                 2   | j         j        rt          j        t          j        dddd          t          j        d          t          j        d          gt          j        t          j        d	
                    t          j        dd                    }| j        	                    dt          j
        d          t          j        ddddt          j        d          |                    }|j        t          j        j        k    sJ dS dS )&Tests tuning with eval config metrics.prompt-relevanceXHow well does the response address the prompt?: PROMPT: {request}
 RESPONSE: {response}
T:You are a cat. Make all evaluations from this perspective.r   r   return_raw_outputjudge_model_system_instructionr
   )r   rouge_1r   r   r   r$   rO   sampling_countrP   r   r   rQ   r   r   r   rd   r.   r/   re   rf   r!   N)rh   ri   rl   EvaluationConfigMetricOutputConfigGcsDestinationAutoraterConfigrj   rk   rm   rn   ro   rp   rq   rr   rt   rg   ru   s      rv   test_eval_config_with_metricsr   }  sq      $F!2' |"&/k	   F+++I...	
 ".'6"6  
 
 

 %4(
 
 
!  * $$#"0  :R  S  S  S.#@#&)(@s   .	 	 	 %  J {3EEEEEI$F $FH FErx   c                    | j         j        rgt          j        t          j        t          j        ddt          j        d                              t          j        t          j        d                    t          j        t          j        d	
                    gt          j	        t          j
        d                    t          j        dd          t          j        dd                    }| j                            dt          j        d          t          j        ddddt          j        d          |                    }|j        t          j        j        k    sJ dS dS ) rz   r|   r}   T)r   )metric_prompt_templatesystem_instructioncustom_output_format_config)pointwise_metric_spec)use_effective_order)	bleu_specrouge1)
rouge_type)
rouge_specr   r   r   r$   rO   r   r*   i   )temperaturerV   )r   r   rQ   inference_generation_configr   r   r   rd   r.   r/   re   rf   r!   N)rh   ri   rl   r   UnifiedMetricPointwiseMetricSpecCustomOutputFormatConfigBleuSpec	RougeSpecr   r   r   GenerationConfigrj   rk   rm   rn   ro   rp   rq   rr   r   s      rv   %test_eval_config_with_unified_metricsr     s      4F!2%&1&EC S.9.R(,/ / /' ' '   %%.4HHH   %&0HEEE  %
, ".'6"6  
 
 

 %4(
 
 
 %0$@"%
 %
 %
A$ $ $J $$#"0  :R  S  S  S.#@#&)(@s   .	 	 	 %  J {3EEEEEi4F 4Fh FErx   c                    | j         j        rt          j        dddddddiddid	d
diigt          j        t          j        d                    t          j        dd                    }| j                            dt          j	        d          t          j
        ddddt          j        d          |                    }|j        t          j        j        k    sJ dS dS )rz   r{   r|   Tr}   r~   r   r
   r   r   r   r   r   r   r$   rO   r   r   r   r   r   rd   r.   r/   re   rf   r!   N)rh   ri   rl   r   r   r   r   rj   rk   rm   rn   ro   rp   rq   rr   r   s      rv   "test_eval_config_with_metrics_dictr     sh      )F!2 +#%)2n	  VY)4
 ".'6"6  
 
 

 %4(
 
 
+  4 $$#"0  :R  S  S  S.#@#&)(@s   .	 	 	 %  J {3EEEEES)F )FR FErx   c                 4  K   | j         j        r| j        j                            dt          j        d          t          j        ddddt          j        d          t          	          
           d{V }|j
        t
          j        j        k    sJ dS dS rc   )rh   ri   aiorj   rk   rl   rm   rn   ro   rg   rp   rq   rr   rs   s     rv   test_eval_config_asyncr     s         Fz)..#"0  :R  S  S  S.#@#&)(@s   .	 	 	 /        J {3EEEEE!F F  FErx   ))__doc__ r   rl   r   pytestVERTEX_HTTP_OPTIONSr   r   r   r   rg   TestTableItemCreateTuningJobParametersrm   rn   TuningMethodr%   ro   EncryptionSpec%SingleReinforcementTuningRewardConfig"ReinforcementTuningAutoraterScorerr   (CompositeReinforcementTuningRewardConfig<CompositeReinforcementTuningRewardConfigWeightedRewardConfigr\   list__annotations__setup__file__globals
pytestmarkpytest_pluginsmarkskipifrw   r   r   r   asyncior    rx   rv   <module>r      s  "     $ $ $ $ $ $        Q  
 /+.8NOOO ++*222
 
 
  		 	 	   M#8;8)6[6p  
 
 
 b	 	 	  M'8;8_6[6p  
 
 
 b	 	 	  M:8;8_6[6p   5;4.1  
 
 
 b    M#8;8)6[6p   5;4)=*  

 

 

 b    M&8;8)6[6p   5;4)="/A	  
 
 
 b     M98;8)6[6p   5;4.1  
 
 
 b    M28;8)6[6p   5;4)=),/#F;#Fy$ $ $  
 
 
 b#  &  M38;8)6[6(r  
 
 
 b	 	 	  M;8;8)6[6(r   5;4)=),/#F;#F,v$ $ $ '(89	 	 	
 
 
  b%  (  M%8;8<6[6_   5;4%#C#F;#Fg$ $ $ b0	 	 	
 
 
  b%  (  M 8;8<6[6_   5;4#F;#Fg$ $ $ #NZ0  
 
 
 b#  &  M,8;8<6[6_   5;4#F;#Fg$ $ $ %. #NZ0
 
 

 
 
" b'  *  M)8;8<6[6_   5;4%#C#F;#Fg$ $ $ #NZ0
 
 

 
 
" b'  *  M58;8,6[6_   5;4%#F"Z.0  
 
 
 b#  &  M(8;8)6[6p   5;4 : :!}! ! !  

 

 

 b    M&8;8)6[6p   5;4)=-/),#$"%$'"&OkO%S[%S)D)D,8* * *& & &    
 
 
2 b7  :  M08;8)6[6p   5;4)=-/),(\(\``#&*[+*[1_1_5P[5P8D6& 6& 6&2" 2" 2"+ + +	 	 	-) ) )  
 
 
6 b;  _n1
D,- n n n`	 !] 	WYY	  
 % #)   = =	 =. #)   &F &F	 &FR #)   6F 6F	 6Fr #)   +F +F	 +F^ #)    F F 	 
F F Frx   