From 9d19993e552b8d7c141902ed804fb72653fc5a8b Mon Sep 17 00:00:00 2001 From: sherwin6180 Date: Thu, 26 Jun 2025 12:32:35 -0400 Subject: [PATCH] added support for running eval_instruct.py --- Evaluation/HumanEval/eval.sh | 6 +++--- Evaluation/HumanEval/eval_instruct.sh | 12 ++++++++++++ .../utils/__pycache__/utils.cpython-39.pyc | Bin 3570 -> 3611 bytes Evaluation/HumanEval/utils/utils.py | 5 +++++ 4 files changed, 20 insertions(+), 3 deletions(-) create mode 100755 Evaluation/HumanEval/eval_instruct.sh diff --git a/Evaluation/HumanEval/eval.sh b/Evaluation/HumanEval/eval.sh index d1020f9..8d3843f 100755 --- a/Evaluation/HumanEval/eval.sh +++ b/Evaluation/HumanEval/eval.sh @@ -1,4 +1,4 @@ -MODEL_NAME_OR_PATH="/scratch/shared_dir/xinyu/deepseek-1.3b" +MODEL_NAME_OR_PATH="/scratch/shared_dir/xinyu/deepseek-6.7b-instruct" DATASET_ROOT="data/" -LANGUAGE="scala" -CUDA_VISIBLE_DEVICES=1,2,3 python -m accelerate.commands.launch --config_file test_config.yaml eval_pal.py --logdir ${MODEL_NAME_OR_PATH} --language ${LANGUAGE} --dataroot ${DATASET_ROOT} +LANGUAGE="rust" +CUDA_VISIBLE_DEVICES=4,5,6 python -m accelerate.commands.launch --config_file test_config.yaml eval_pal.py --logdir ${MODEL_NAME_OR_PATH} --language ${LANGUAGE} --dataroot ${DATASET_ROOT} diff --git a/Evaluation/HumanEval/eval_instruct.sh b/Evaluation/HumanEval/eval_instruct.sh new file mode 100755 index 0000000..7ee480f --- /dev/null +++ b/Evaluation/HumanEval/eval_instruct.sh @@ -0,0 +1,12 @@ +LANG="rust" +OUTPUT_DIR="output" +MODEL="/scratch/shared_dir/xinyu/deepseek-6.7b-instruct" +MODEL_NAME=$(basename "$MODEL") + +mkdir -p "$OUTPUT_DIR" + +CUDA_VISIBLE_DEVICES=0,1 python eval_instruct.py \ + --model "$MODEL" \ + --output_path "$OUTPUT_DIR/${LANG}.${MODEL_NAME}.jsonl" \ + --language $LANG \ + --temp_dir $OUTPUT_DIR diff --git a/Evaluation/HumanEval/utils/__pycache__/utils.cpython-39.pyc b/Evaluation/HumanEval/utils/__pycache__/utils.cpython-39.pyc index 448c3196056db8fd50a6d3636041092adf49a50f..357fab0d4aafac05a590ea0e576153150aa85cf5 100644 GIT binary patch delta 980 zcmZ8f%}*0S6z|*JZri0b&=y+?)dGTD5lJLsLQpV#h!WArMc0o_+nJWevfIpV6*Q(X zawZ`Y6N&u?-}DR>VlgcsmbLs~ zc{?5md_m*riV0&%JkGn#p?ggtlQ;>rWRhrU22hDgB-xTkCpo^Z7&7TY4^>E7WOR|t zde8_G5Ia=~lodaQ(e9Q)dPr78RdRxyTn`u_k|Vw13=>!i=>5Ccz@n$#T633eYgZAL zE_so1WAor87`zGTgrVGSIPTL(*~+aHs#adtHNFKSs-ZC9-}wMJP!od368xL=5Jve8 zc@!f2g?#5qr*L~veG7(GZdfI=R<)?Z{Nxle(gMctrAxr>PqFqP&p;R+JkwQ{Y7;Dt=g7p6;|kZzM*tOerH?Rk-GLF$1;Ag=eo?INJEX; zG5$DIge3nI>W4o5BXq8r$1IgqEkak>DKsLpt0h{;x_X@H$U1{Cgm4yN7y(-|LJoD^ zs#RfPr~wv4msYX~HH$U7^vKXqHPdkm%w>a^aaw>E^c|i|E#fI-MVqX#QS?iqf!}Ik zs(;HAU3FQZdpUH*F?f= z@i;iQ0EETXAPp)ojOZ|@N0|TbTo9%wJ1%4X6qTZF*VwdhvIPYEx=%SG>)#89Csu6I nsL{)Sb90LPRcvY?D1q2Nl#l92dn|d;w?H~LJVE*%&4jgINf6_| delta 865 zcmZ8eO;6NN6z!dU%(Tov7-sky7#RsjC89>7i3*|#AEKBTHd=_2()So98QRHfQH&vO zNQ_JJ7RER$cdkstxO3s!f6$n?a7Dt3B^$lB8b#krPkZ0J=k)aS^}u0i+|A`I1y}HS zWqmHMDBr0#zS^*u#w&ckJHEWqG9BP4o?`h{igmHVj^?Orh)%f1x?3viVZA%Lqcgx= ztSM_+(xYx)OJn`4gk6ITu)!V0NwYFLf%go9CB?4n9<)lawctK-mpoB#MvivV4c8rg z$#Wa7ZORXDc5fesz$l;{QIH+#Lzs}hdI2o?M!kQd8;@hCzXiuy+w{F^qwe!iB*VSL z=tDS95B&)HKX4tvm;fwdcxoG>D6eXx24B_*NubFRIqK&xM=V_X5VRNH8D3j9fMkF~Ymn2(-({nukvL$*Q2&Z`Sq6SyGuJ zm?yY}5L;msG^>vT!G6d4C*wHlHMr|7(@~S#x^UbvV(c^{a7OLVN#DfR4+8 diff --git a/Evaluation/HumanEval/utils/utils.py b/Evaluation/HumanEval/utils/utils.py index 5e66ba7..08e5fa6 100644 --- a/Evaluation/HumanEval/utils/utils.py +++ b/Evaluation/HumanEval/utils/utils.py @@ -39,6 +39,11 @@ languge_settings = { 'scala': { 'full_name': "Scala", 'indent': 4, + }, + 'rust': { + 'full_name': 'Rust', + 'indent': 4, + 'main': 'fn main()' } }