diff --git a/Evaluation/HumanEval/eval.sh b/Evaluation/HumanEval/eval.sh index d1020f9..8d3843f 100755 --- a/Evaluation/HumanEval/eval.sh +++ b/Evaluation/HumanEval/eval.sh @@ -1,4 +1,4 @@ -MODEL_NAME_OR_PATH="/scratch/shared_dir/xinyu/deepseek-1.3b" +MODEL_NAME_OR_PATH="/scratch/shared_dir/xinyu/deepseek-6.7b-instruct" DATASET_ROOT="data/" -LANGUAGE="scala" -CUDA_VISIBLE_DEVICES=1,2,3 python -m accelerate.commands.launch --config_file test_config.yaml eval_pal.py --logdir ${MODEL_NAME_OR_PATH} --language ${LANGUAGE} --dataroot ${DATASET_ROOT} +LANGUAGE="rust" +CUDA_VISIBLE_DEVICES=4,5,6 python -m accelerate.commands.launch --config_file test_config.yaml eval_pal.py --logdir ${MODEL_NAME_OR_PATH} --language ${LANGUAGE} --dataroot ${DATASET_ROOT} diff --git a/Evaluation/HumanEval/eval_instruct.sh b/Evaluation/HumanEval/eval_instruct.sh new file mode 100755 index 0000000..7ee480f --- /dev/null +++ b/Evaluation/HumanEval/eval_instruct.sh @@ -0,0 +1,12 @@ +LANG="rust" +OUTPUT_DIR="output" +MODEL="/scratch/shared_dir/xinyu/deepseek-6.7b-instruct" +MODEL_NAME=$(basename "$MODEL") + +mkdir -p "$OUTPUT_DIR" + +CUDA_VISIBLE_DEVICES=0,1 python eval_instruct.py \ + --model "$MODEL" \ + --output_path "$OUTPUT_DIR/${LANG}.${MODEL_NAME}.jsonl" \ + --language $LANG \ + --temp_dir $OUTPUT_DIR diff --git a/Evaluation/HumanEval/utils/__pycache__/utils.cpython-39.pyc b/Evaluation/HumanEval/utils/__pycache__/utils.cpython-39.pyc index 448c319..357fab0 100644 Binary files a/Evaluation/HumanEval/utils/__pycache__/utils.cpython-39.pyc and b/Evaluation/HumanEval/utils/__pycache__/utils.cpython-39.pyc differ diff --git a/Evaluation/HumanEval/utils/utils.py b/Evaluation/HumanEval/utils/utils.py index 5e66ba7..08e5fa6 100644 --- a/Evaluation/HumanEval/utils/utils.py +++ b/Evaluation/HumanEval/utils/utils.py @@ -39,6 +39,11 @@ languge_settings = { 'scala': { 'full_name': "Scala", 'indent': 4, + }, + 'rust': { + 'full_name': 'Rust', + 'indent': 4, + 'main': 'fn main()' } }