Update generate.py

inference/generate.py

@@ -12,19 +12,13 @@ from model import Transformer, ModelArgs
 
 
 def sample(logits, temperature: float = 1.0):
-    """
+    try:
-    Samples a token from the logits using temperature scaling.
+        logits = logits / max(temperature, 1e-5)
-
+        probs = torch.softmax(logits, dim=-1)
-    Args:
+        return probs.div_(torch.empty_like(probs).exponential_(1)).argmax(dim=-1)
-        logits (torch.Tensor): The logits tensor for token predictions.
+    except Exception as e:
-        temperature (float, optional): Temperature for scaling logits. Defaults to 1.0.
+        print(f"Erro ao amostrar token: {e}")
-
+        raise
-    Returns:
-        torch.Tensor: The sampled token.
-    """
-    logits = logits / max(temperature, 1e-5)
-    probs = torch.softmax(logits, dim=-1)
-    return probs.div_(torch.empty_like(probs).exponential_(1)).argmax(dim=-1)
 
 
 @torch.inference_mode()
@@ -35,47 +29,40 @@ def generate(
     eos_id: int,
     temperature: float = 1.0
 ) -> List[List[int]]:
-    """
+    try:
-    Generates new tokens based on the given prompt tokens using the specified model.
+        prompt_lens = [len(t) for t in prompt_tokens]
-
+        assert max(prompt_lens) <= model.max_seq_len, f"Prompt é maior do que o comprimento máximo do modelo ({model.max_seq_len})"
-    Args:
+        total_len = min(model.max_seq_len, max_new_tokens + max(prompt_lens))
-        model (Transformer): The transformer model used for token generation.
+        tokens = torch.full((len(prompt_tokens), total_len), -1, dtype=torch.long, device="cuda")
-        prompt_tokens (List[List[int]]): A list of lists containing the prompt tokens for each sequence.
+        
-        max_new_tokens (int): The maximum number of new tokens to generate.
+        for i, t in enumerate(prompt_tokens):
-        eos_id (int): The end-of-sequence token ID.
+            tokens[i, :len(t)] = torch.tensor(t, dtype=torch.long, device="cuda")
-        temperature (float, optional): The temperature value for sampling. Defaults to 1.0.
+            
-
+        prev_pos = 0
-    Returns:
+        finished = torch.tensor([False] * len(prompt_tokens), device="cuda")
-        List[List[int]]: A list of lists containing the generated tokens for each sequence.
+        prompt_mask = tokens != -1
-    """
+        
-    prompt_lens = [len(t) for t in prompt_tokens]
+        for cur_pos in range(min(prompt_lens), total_len):
-    assert max(prompt_lens) <= model.max_seq_len
+            logits = model.forward(tokens[:, prev_pos:cur_pos], prev_pos)
-    total_len = min(model.max_seq_len, max_new_tokens + max(prompt_lens))
+            next_token = sample(logits, temperature) if temperature > 0 else logits.argmax(dim=-1)
-    tokens = torch.full((len(prompt_tokens), total_len), -1, dtype=torch.long, device="cuda")
+            next_token = torch.where(prompt_mask[:, cur_pos], tokens[:, cur_pos], next_token)
-    for i, t in enumerate(prompt_tokens):
+            tokens[:, cur_pos] = next_token
-        tokens[i, :len(t)] = torch.tensor(t, dtype=torch.long, device="cuda")
+            finished |= torch.logical_and(~prompt_mask[:, cur_pos], next_token == eos_id)
-    prev_pos = 0
+            prev_pos = cur_pos
-    finished = torch.tensor([False] * len(prompt_tokens), device="cuda")
+            if finished.all():
-    prompt_mask = tokens != -1
+                break
-    for cur_pos in range(min(prompt_lens), total_len):
+        
-        logits = model.forward(tokens[:, prev_pos:cur_pos], prev_pos)
+        completion_tokens = []
-        if temperature > 0:
+        for i, toks in enumerate(tokens.tolist()):
-            next_token = sample(logits, temperature)
+            toks = toks[prompt_lens[i]:prompt_lens[i] + max_new_tokens]
-        else:
+            if eos_id in toks:
-            next_token = logits.argmax(dim=-1)
+                toks = toks[:toks.index(eos_id)]
-        next_token = torch.where(prompt_mask[:, cur_pos], tokens[:, cur_pos], next_token)
+            completion_tokens.append(toks)
-        tokens[:, cur_pos] = next_token
+        
-        finished |= torch.logical_and(~prompt_mask[:, cur_pos], next_token == eos_id)
+        return completion_tokens
-        prev_pos = cur_pos
+    except Exception as e:
-        if finished.all():
+        print(f"Erro ao gerar texto: {e}")
-            break
+        raise
-    completion_tokens = []
-    for i, toks in enumerate(tokens.tolist()):
-        toks = toks[prompt_lens[i]:prompt_lens[i]+max_new_tokens]
-        if eos_id in toks:
-            toks = toks[:toks.index(eos_id)]
-        completion_tokens.append(toks)
-    return completion_tokens
 
 
 def main(
@@ -86,100 +73,110 @@ def main(
     max_new_tokens: int = 100,
     temperature: float = 1.0,
 ) -> None:
-    """
+    try:
-    Main function to load the model and perform interactive or batch text generation.
+        if not os.path.exists(ckpt_path):
-
+            raise FileNotFoundError(f"O caminho para o checkpoint '{ckpt_path}' não existe.")
-    Args:
+        
-        ckpt_path (str): Path to the model checkpoint directory.
+        if not os.path.exists(config):
-        config (str): Path to the model configuration file.
+            raise FileNotFoundError(f"O arquivo de configuração '{config}' não foi encontrado.")
-        input_file (str, optional): Path to a file containing input prompts. Defaults to "".
+        
-        interactive (bool, optional): Whether to run in interactive mode. Defaults to True.
+        world_size = int(os.getenv("WORLD_SIZE", "1"))
-        max_new_tokens (int, optional): Maximum number of new tokens to generate. Defaults to 100.
+        rank = int(os.getenv("RANK", "0"))
-        temperature (float, optional): Temperature for sampling. Defaults to 1.0.
+        local_rank = int(os.getenv("LOCAL_RANK", "0"))
-    """
+        
-    world_size = int(os.getenv("WORLD_SIZE", "1"))
+        if world_size > 1:
-    rank = int(os.getenv("RANK", "0"))
+            dist.init_process_group("nccl")
-    local_rank = int(os.getenv("LOCAL_RANK", "0"))
+        
-    if world_size > 1:
+        global print
-        dist.init_process_group("nccl")
+        if rank != 0:
-    global print
+            print = lambda *_, **__: None
-    if rank != 0:
+        
-        print = lambda *_, **__: None
+        torch.cuda.set_device(local_rank)
-    torch.cuda.set_device(local_rank)
+        torch.set_default_dtype(torch.bfloat16)
-    torch.set_default_dtype(torch.bfloat16)
+        torch.set_num_threads(8)
-    torch.set_num_threads(8)
+        torch.manual_seed(965)
-    torch.manual_seed(965)
+        
-    with open(config) as f:
+        with open(config) as f:
-        args = ModelArgs(**json.load(f))
+            args = ModelArgs(**json.load(f))
-    print(args)
+        print(args)
-    with torch.device("cuda"):
+        
-        model = Transformer(args)
+        with torch.device("cuda"):
-    tokenizer = AutoTokenizer.from_pretrained(ckpt_path)
+            model = Transformer(args)
-    tokenizer.decode(generate(model, [tokenizer.encode("DeepSeek")], 2, -1, 1.)[0])
+        
-    load_model(model, os.path.join(ckpt_path, f"model{rank}-mp{world_size}.safetensors"))
+        tokenizer = AutoTokenizer.from_pretrained(ckpt_path)
-
+        
-    if interactive:
+        tokenizer.decode(generate(model, [tokenizer.encode("DeepSeek")], 2, -1, 1.)[0])
-        messages = []
+        
-        while True:
+        load_model(model, os.path.join(ckpt_path, f"model{rank}-mp{world_size}.safetensors"))
-            if world_size == 1:
+        
-                prompt = input(">>> ")
+        if interactive:
-            elif rank == 0:
+            messages = []
-                prompt = input(">>> ")
+            while True:
-                objects = [prompt]
+                try:
-                dist.broadcast_object_list(objects, 0)
+                    if world_size == 1:
-            else:
+                        prompt = input(">>> ")
-                objects = [None]
+                    elif rank == 0:
-                dist.broadcast_object_list(objects, 0)
+                        prompt = input(">>> ")
-                prompt = objects[0]
+                        objects = [prompt]
-            if prompt == "/exit":
+                        dist.broadcast_object_list(objects, 0)
-                break
+                    else:
-            elif prompt == "/clear":
+                        objects = [None]
-                messages.clear()
+                        dist.broadcast_object_list(objects, 0)
-                continue
+                        prompt = objects[0]
-            messages.append({"role": "user", "content": prompt})
+                    
-            prompt_tokens = tokenizer.apply_chat_template(messages, add_generation_prompt=True)
+                    if prompt == "/exit":
-            completion_tokens = generate(model, [prompt_tokens], max_new_tokens, tokenizer.eos_token_id, temperature)
+                        break
-            completion = tokenizer.decode(completion_tokens[0], skip_special_tokens=True)
+                    elif prompt == "/clear":
-            print(completion)
+                        messages.clear()
-            messages.append({"role": "assistant", "content": completion})
+                        continue
-    else:
+                    
-        with open(input_file) as f:
+                    messages.append({"role": "user", "content": prompt})
-            prompts = [line.strip() for line in f.readlines()]
+                    prompt_tokens = tokenizer.apply_chat_template(messages, add_generation_prompt=True)
-        assert len(prompts) <= args.max_batch_size
+                    completion_tokens = generate(model, [prompt_tokens], max_new_tokens, tokenizer.eos_token_id, temperature)
-        prompt_tokens = [tokenizer.apply_chat_template([{"role": "user", "content": prompt}], add_generation_prompt=True) for prompt in prompts]
+                    completion = tokenizer.decode(completion_tokens[0], skip_special_tokens=True)
-        completion_tokens = generate(model, prompt_tokens, max_new_tokens, tokenizer.eos_token_id, temperature)
+                    print(completion)
-        completions = tokenizer.batch_decode(completion_tokens, skip_special_tokens=True)
+                    messages.append({"role": "assistant", "content": completion})
-        for prompt, completion in zip(prompts, completions):
+                
-            print("Prompt:", prompt)
+                except Exception as e:
-            print("Completion:", completion)
+                    print(f"Erro durante a interação: {e}")
-            print()
+                    continue
-
+        
-    if world_size > 1:
+        else:
-        dist.destroy_process_group()
+            with open(input_file) as f:
+                prompts = [line.strip() for line in f.readlines()]
+            
+            assert len(prompts) <= args.max_batch_size
+            prompt_tokens = [tokenizer.apply_chat_template([{"role": "user", "content": prompt}], add_generation_prompt=True) for prompt in prompts]
+            completion_tokens = generate(model, prompt_tokens, max_new_tokens, tokenizer.eos_token_id, temperature)
+            completions = tokenizer.batch_decode(completion_tokens, skip_special_tokens=True)
+            
+            for prompt, completion in zip(prompts, completions):
+                print("Prompt:", prompt)
+                print("Completion:", completion)
+                print()
+    
+    except FileNotFoundError as e:
+        print(f"Erro de arquivo: {e}")
+    
+    except Exception as e:
+        print(f"Erro na execução principal: {e}")
+    
+    finally:
+        if world_size > 1:
+            dist.destroy_process_group()
 
 
 if __name__ == "__main__":
-    """
+    try:
-    Command-line interface for distributed text generation.
+        parser = ArgumentParser()
-
+        parser.add_argument("--ckpt-path", type=str, required=True)
-    Arguments:
+        parser.add_argument("--config", type=str, required=True)
-        --ckpt-path (str): Path to the model checkpoint directory.
+        parser.add_argument("--input-file", type=str, default="")
-        --config (str): Path to the model configuration file.
+        parser.add_argument("--interactive", action="store_true")
-        --input-file (str, optional): File containing prompts for batch processing.
+        parser.add_argument("--max-new-tokens", type=int, default=200)
-        --interactive (bool, optional): Enable interactive mode for generating text.
+        parser.add_argument("--temperature", type=float, default=0.2)
-        --max-new-tokens (int, optional): Maximum number of new tokens to generate. Defaults to 200.
+        args = parser.parse_args()
-        --temperature (float, optional): Temperature for sampling. Defaults to 0.2.
+        assert args.input_file or args.interactive
-
+        main(args.ckpt_path, args.config, args.input_file, args.interactive, args.max_new_tokens, args.temperature)
-    Raises:
+    except Exception as e:
-        AssertionError: If neither input-file nor interactive mode is specified.
+        print(f"Erro na execução do script: {e}")
-    """
-    parser = ArgumentParser()
-    parser.add_argument("--ckpt-path", type=str, required=True)
-    parser.add_argument("--config", type=str, required=True)
-    parser.add_argument("--input-file", type=str, default="")
-    parser.add_argument("--interactive", action="store_true")
-    parser.add_argument("--max-new-tokens", type=int, default=200)
-    parser.add_argument("--temperature", type=float, default=0.2)
-    args = parser.parse_args()
-    assert args.input_file or args.interactive
-    main(args.ckpt_path, args.config, args.input_file, args.interactive, args.max_new_tokens, args.temperature)