From 41bd79ecb104626a3cf287babda593ab46d82125 Mon Sep 17 00:00:00 2001 From: stack-heap-overflow Date: Tue, 7 May 2024 14:00:53 +0800 Subject: [PATCH] Update README.md --- README.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 8138350..31df55d 100644 --- a/README.md +++ b/README.md @@ -198,7 +198,7 @@ model_name = "deepseek-ai/DeepSeek-V2" tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True) # `max_memory` should be set based on your devices max_memory = {i: "75GB" for i in range(8)} -model = AutoModelForCausalLM.from_pretrained(model_path, trust_remote_code=True, device_map="auto", torch_dtype=torch.bfloat16, max_memory=max_memory) +model = AutoModelForCausalLM.from_pretrained(model_name, trust_remote_code=True, device_map="auto", torch_dtype=torch.bfloat16, max_memory=max_memory) model.generation_config = GenerationConfig.from_pretrained(model_name) model.generation_config.pad_token_id = model.generation_config.eos_token_id @@ -219,7 +219,7 @@ model_name = "deepseek-ai/DeepSeek-V2-Chat" tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True) # `max_memory` should be set based on your devices max_memory = {i: "75GB" for i in range(8)} -model = AutoModelForCausalLM.from_pretrained(model_path, trust_remote_code=True, device_map="auto", torch_dtype=torch.bfloat16, max_memory=max_memory) +model = AutoModelForCausalLM.from_pretrained(model_name, trust_remote_code=True, device_map="auto", torch_dtype=torch.bfloat16, max_memory=max_memory) model.generation_config = GenerationConfig.from_pretrained(model_name) model.generation_config.pad_token_id = model.generation_config.eos_token_id @@ -232,9 +232,11 @@ outputs = model.generate(input_tensor.to(model.device), max_new_tokens=100) result = tokenizer.decode(outputs[0][input_tensor.shape[1]:], skip_special_tokens=True) print(result) ``` + The complete chat template can be found within `tokenizer_config.json` located in the huggingface model repository. An example of chat template is as belows: + ```bash <｜begin▁of▁sentence｜>User: {user_message_1} @@ -242,7 +244,9 @@ Assistant: {assistant_message_1}<｜end▁of▁sentence｜>User: {user_message_2 Assistant: ``` + You can also add an optional system message: + ```bash <｜begin▁of▁sentence｜>{system_message}