From 9d19993e552b8d7c141902ed804fb72653fc5a8b Mon Sep 17 00:00:00 2001
From: sherwin6180 <sherwin6180@gmail.com>
Date: Thu, 26 Jun 2025 12:32:35 -0400
Subject: [PATCH] added support for running eval_instruct.py

---
 Evaluation/HumanEval/eval.sh                  |   6 +++---
 Evaluation/HumanEval/eval_instruct.sh         |  12 ++++++++++++
 .../utils/__pycache__/utils.cpython-39.pyc    | Bin 3570 -> 3611 bytes
 Evaluation/HumanEval/utils/utils.py           |   5 +++++
 4 files changed, 20 insertions(+), 3 deletions(-)
 create mode 100755 Evaluation/HumanEval/eval_instruct.sh

diff --git a/Evaluation/HumanEval/eval.sh b/Evaluation/HumanEval/eval.sh
index d1020f9..8d3843f 100755
--- a/Evaluation/HumanEval/eval.sh
+++ b/Evaluation/HumanEval/eval.sh
@@ -1,4 +1,4 @@
-MODEL_NAME_OR_PATH="/scratch/shared_dir/xinyu/deepseek-1.3b"
+MODEL_NAME_OR_PATH="/scratch/shared_dir/xinyu/deepseek-6.7b-instruct"
 DATASET_ROOT="data/"
-LANGUAGE="scala"
-CUDA_VISIBLE_DEVICES=1,2,3 python -m accelerate.commands.launch --config_file test_config.yaml eval_pal.py --logdir ${MODEL_NAME_OR_PATH} --language ${LANGUAGE} --dataroot ${DATASET_ROOT}
+LANGUAGE="rust"
+CUDA_VISIBLE_DEVICES=4,5,6 python -m accelerate.commands.launch --config_file test_config.yaml eval_pal.py --logdir ${MODEL_NAME_OR_PATH} --language ${LANGUAGE} --dataroot ${DATASET_ROOT}
diff --git a/Evaluation/HumanEval/eval_instruct.sh b/Evaluation/HumanEval/eval_instruct.sh
new file mode 100755
index 0000000..7ee480f
--- /dev/null
+++ b/Evaluation/HumanEval/eval_instruct.sh
@@ -0,0 +1,12 @@
+LANG="rust"
+OUTPUT_DIR="output"
+MODEL="/scratch/shared_dir/xinyu/deepseek-6.7b-instruct"
+MODEL_NAME=$(basename "$MODEL")
+
+mkdir -p "$OUTPUT_DIR"
+
+CUDA_VISIBLE_DEVICES=0,1 python eval_instruct.py \
+    --model "$MODEL" \
+    --output_path "$OUTPUT_DIR/${LANG}.${MODEL_NAME}.jsonl" \
+    --language $LANG \
+    --temp_dir $OUTPUT_DIR
diff --git a/Evaluation/HumanEval/utils/__pycache__/utils.cpython-39.pyc b/Evaluation/HumanEval/utils/__pycache__/utils.cpython-39.pyc
index 448c3196056db8fd50a6d3636041092adf49a50f..357fab0d4aafac05a590ea0e576153150aa85cf5 100644
GIT binary patch
delta 980
zcmZ8f%}*0S6z|*JZri0b&=y+?)dGTD5lJLsLQpV#h!WArMc0o_+nJWevfIpV6*Q(X
zawZ`Y6N&u?<jU1}_D|3~nRt<l2acf8c`H$LCi|OxAHTP6esA}>-}DR>VlgcsmbLs~
zc{?5md_m*riV0&%JkGn#p?ggtlQ;>rWRhrU22hDgB-xTkCpo^Z7&7TY4^>E7WOR|t
zde8_G5Ia=~lodaQ(e9Q)dPr78RdRxyTn`u_k|Vw13=>!i=>5Ccz@n$#T633eYgZAL
zE_so1WAor87`zGTgrVGSIPTL(*~+aHs#adtHNFKSs-ZC9-}wMJP!od368xL=5Jve8
zc@!f2g?#5qr*L~veG7(GZdfI=R<)?Z{Nxle(gMctrAxr>PqFqP&p;R+JkwQ{<QZj9
z?ZAvUpHl|=d}<bL<a-?z>Y7;Dt=g7p6;|kZzM*tOerH?Rk-GLF$1;Ag=eo?INJEX;
zG5$DIge3nI>W4o5BXq8r$1IgqEkak>DKsLpt0h{;x_X@H$U1{Cgm4yN7y(-|LJoD^
zs#RfPr~wv4msYX~HH$U7^vKXqHPdkm%w>a^aaw>E^c|i|E#fI-MVqX#QS?iqf!}Ik
zs(;HAU3FQZ<eC*~QC9HpiuEV?=kQ&a=Hrn)q4%&h05QI#%|knHX}#iN@3qNGGgxI7
zVGiM{08evVyKX+Q89AKApMiMMQjHd@M%~2vE+22}g#~`UZ7V+_h<eZqF<P$`N|a4t
zwIo(k%0e|?GokSe&qlM!fBR?iB5i{&MpH1)%h7rq^<_5&=t1W9cb$KY>dpUH*F?f=
z@i;iQ0EETXAPp)ojOZ|@N0|TbTo9%wJ1%4X6qTZF*VwdhvIPYEx=%SG>)#89Csu6I
nsL{)Sb90LPRcvY?D1q2Nl#l92dn|d;w?H~LJVE*%&4jgINf6_|

delta 865
zcmZ8eO;6NN6z!dU%(Tov7-sky7#RsjC89>7i3*|#AEKBTHd=_2()So98QRHfQH&vO
zNQ_JJ7RER$cdkstxO3s!f6$n?a7Dt3B^$lB8b#krPkZ0J=k)aS^}u0i+|A`I1y}HS
zWqmHMDBr0#zS^*u#w&ckJHEWqG9BP4o?`h{igmHVj^?Orh)%f1x?3viVZA%Lqcgx=
ztSM_+(xYx)OJn`4gk6ITu)!V0NwYFLf%go9CB?4n9<)lawctK-mpoB#MvivV4c8rg
z$#Wa7ZORXDc5fesz$l;{QIH+#Lzs}hdI2o?M!kQd8;@hCzXiuy+w{F^qwe!iB*VSL
z=tDS95B&)HKX4tvm;fwdcxoG>D6eXx24<j$E?Lt?PbSnd)x?iGYdorwa#Roes_$;_
zGxD8Q0(<X^wwLOQVMvq`v29xwIij@_bjX+KRVd1%^e_y|a%Q4z)083VKI2<rgbH(J
z%i~SbwevE_6blo?Br9<`<WW@jYv@U4DvPJ6F-8qNj9d}LD)(8etp@CgxJbt-)ch3?
zO(w>B_*NubFRIqK&xM=V_X5VRNH8D3j9fMkF~Ymn2(-({nukvL$*Q2&Z`Sq6SyGuJ
zm?yY}5L;msG^>vT!G6d4C*wHlHMr|<Hmjr`$mO;QEXwV+S7YapXzOuW@MgpHxR@lh
zBB^O>7(@~S#x^UbvV(<x{fh-0dm)2t4=l*%*=C+16Za5oT_pbQNHb?Y`;T=S2P9r$
z7D0gwZVP(B06`raW^Gek#wT$JAy&gkh~&p$mO(?zP-l^V-kwm7#rpSyGuJi(w%OoQ
TpyOQV$d9>c^{a7OLVN#DfR4+8

diff --git a/Evaluation/HumanEval/utils/utils.py b/Evaluation/HumanEval/utils/utils.py
index 5e66ba7..08e5fa6 100644
--- a/Evaluation/HumanEval/utils/utils.py
+++ b/Evaluation/HumanEval/utils/utils.py
@@ -39,6 +39,11 @@ languge_settings = {
     'scala': {
         'full_name': "Scala",
         'indent': 4,
+    },
+    'rust': {
+        'full_name': 'Rust',
+        'indent': 4,
+        'main': 'fn main()'
     }
 }