mirror of
https://github.com/deepseek-ai/DeepSeek-VL.git
synced 2025-04-19 18:19:03 -04:00
add gradio demo.
This commit is contained in:
haoy945
parent
8d4d9a6ccf
commit
a5bbf5f8bb
466
deepseek_vl/serve/app_deepseek.py
Executable file
466
deepseek_vl/serve/app_deepseek.py
Executable file
@ -0,0 +1,466 @@
|
||||
# -*- coding:utf-8 -*-
|
||||
|
||||
import gradio as gr
|
||||
import torch
|
||||
import base64
|
||||
from io import BytesIO
|
||||
|
||||
from app_modules.gradio_utils import (cancel_outputing, delete_last_conversation, reset_state,
|
||||
reset_textbox, transfer_input, wrap_gen_fn)
|
||||
from app_modules.overwrites import reload_javascript
|
||||
from app_modules.presets import CONCURRENT_COUNT, description, description_top, title
|
||||
from app_modules.utils import (configure_logger, is_variable_assigned,
|
||||
strip_stop_words)
|
||||
from deepseek_vl.serve.inference import convert_conversation_to_prompts, deepseek_generate, load_model
|
||||
from deepseek_vl.utils.conversation import SeparatorStyle
|
||||
|
||||
|
||||
def load_models():
|
||||
models = {
|
||||
"DeepSeek-VL 7B": "/hf3fs-jd/prod/deepseek/shared/liuwen/ckpts/deepseek-vl-7b-chat",
|
||||
}
|
||||
|
||||
for model_name in models:
|
||||
models[model_name] = load_model(models[model_name])
|
||||
|
||||
return models
|
||||
|
||||
|
||||
logger = configure_logger()
|
||||
models = load_models()
|
||||
MODELS = sorted(list(models.keys()))
|
||||
|
||||
|
||||
def generate_prompt_with_history(text, image, history, vl_chat_processor, tokenizer, max_length=2048):
|
||||
"""
|
||||
Generate a prompt with history for the deepseek application.
|
||||
|
||||
Args:
|
||||
text (str): The text prompt.
|
||||
image (str): The image prompt.
|
||||
history (list): List of previous conversation messages.
|
||||
tokenizer: The tokenizer used for encoding the prompt.
|
||||
max_length (int): The maximum length of the prompt.
|
||||
|
||||
Returns:
|
||||
tuple: A tuple containing the generated prompt, image list, conversation, and conversation copy. If the prompt could not be generated within the max_length limit, returns None.
|
||||
"""
|
||||
|
||||
sft_format = "deepseek"
|
||||
user_role_ind = 0
|
||||
bot_role_ind = 1
|
||||
|
||||
# Initialize conversation
|
||||
conversation = vl_chat_processor.new_chat_template()
|
||||
|
||||
if history:
|
||||
conversation.messages = history
|
||||
|
||||
if image is not None:
|
||||
if '<image_placeholder>' not in text:
|
||||
text = '<image_placeholder>' + '\n' + text # append the <image_placeholder> in a new line after the text prompt
|
||||
text = (text, image)
|
||||
|
||||
conversation.append_message(conversation.roles[user_role_ind], text)
|
||||
conversation.append_message(conversation.roles[bot_role_ind], "")
|
||||
|
||||
# Create a copy of the conversation to avoid history truncation in the UI
|
||||
conversation_copy = conversation.copy()
|
||||
logger.info("=" * 80)
|
||||
logger.info(get_prompt(conversation))
|
||||
|
||||
rounds = len(conversation.messages) // 2
|
||||
|
||||
for _ in range(rounds):
|
||||
current_prompt = get_prompt(conversation)
|
||||
current_prompt = current_prompt.replace("</s>", "") if sft_format == "deepseek" else current_prompt
|
||||
|
||||
if torch.tensor(tokenizer.encode(current_prompt)).size(-1) <= max_length:
|
||||
return conversation_copy
|
||||
|
||||
if len(conversation.messages) % 2 != 0:
|
||||
gr.Error("The messages between user and assistant are not paired.")
|
||||
return
|
||||
|
||||
try:
|
||||
for _ in range(2): # pop out two messages in a row
|
||||
conversation.messages.pop(0)
|
||||
except IndexError:
|
||||
gr.Error("Input text processing failed, unable to respond in this round.")
|
||||
return None
|
||||
|
||||
gr.Error("Prompt could not be generated within max_length limit.")
|
||||
return None
|
||||
|
||||
|
||||
def to_gradio_chatbot(conv):
|
||||
"""Convert the conversation to gradio chatbot format."""
|
||||
ret = []
|
||||
for i, (role, msg) in enumerate(conv.messages[conv.offset :]):
|
||||
if i % 2 == 0:
|
||||
if type(msg) is tuple:
|
||||
msg, image = msg
|
||||
if isinstance(image, str):
|
||||
with open(image, 'rb') as f:
|
||||
data = f.read()
|
||||
img_b64_str = base64.b64encode(data).decode()
|
||||
image_str = f'<video src="data:video/mp4;base64,{img_b64_str}" controls width="426" height="240"></video>'
|
||||
msg = msg.replace('\n'.join(['<image_placeholder>'] * 4), image_str)
|
||||
else:
|
||||
max_hw, min_hw = max(image.size), min(image.size)
|
||||
aspect_ratio = max_hw / min_hw
|
||||
max_len, min_len = 800, 400
|
||||
shortest_edge = int(min(max_len / aspect_ratio, min_len, min_hw))
|
||||
longest_edge = int(shortest_edge * aspect_ratio)
|
||||
W, H = image.size
|
||||
if H > W:
|
||||
H, W = longest_edge, shortest_edge
|
||||
else:
|
||||
H, W = shortest_edge, longest_edge
|
||||
image = image.resize((W, H))
|
||||
buffered = BytesIO()
|
||||
image.save(buffered, format="JPEG")
|
||||
img_b64_str = base64.b64encode(buffered.getvalue()).decode()
|
||||
img_str = f'<img src="data:image/png;base64,{img_b64_str}" alt="user upload image" />'
|
||||
msg = msg.replace('<image_placeholder>', img_str)
|
||||
ret.append([msg, None])
|
||||
else:
|
||||
ret[-1][-1] = msg
|
||||
return ret
|
||||
|
||||
|
||||
def to_gradio_history(conv):
|
||||
"""Convert the conversation to gradio history state."""
|
||||
return conv.messages[conv.offset :]
|
||||
|
||||
|
||||
def get_prompt(conv) -> str:
|
||||
"""Get the prompt for generation."""
|
||||
system_prompt = conv.system_template.format(system_message=conv.system_message)
|
||||
if conv.sep_style == SeparatorStyle.DeepSeek:
|
||||
seps = [conv.sep, conv.sep2]
|
||||
if system_prompt == "" or system_prompt is None:
|
||||
ret = ""
|
||||
else:
|
||||
ret = system_prompt + seps[0]
|
||||
for i, (role, message) in enumerate(conv.messages):
|
||||
if message:
|
||||
if type(message) is tuple: # multimodal message
|
||||
message, _ = message
|
||||
ret += role + ": " + message + seps[i % 2]
|
||||
else:
|
||||
ret += role + ":"
|
||||
return ret
|
||||
else:
|
||||
return conv.get_prompt
|
||||
|
||||
|
||||
@wrap_gen_fn
|
||||
def predict(
|
||||
text,
|
||||
image,
|
||||
chatbot,
|
||||
history,
|
||||
top_p,
|
||||
temperature,
|
||||
repetition_penalty,
|
||||
max_length_tokens,
|
||||
max_context_length_tokens,
|
||||
model_select_dropdown,
|
||||
):
|
||||
"""
|
||||
Function to predict the response based on the user's input and selected model.
|
||||
|
||||
Parameters:
|
||||
user_text (str): The input text from the user.
|
||||
user_image (str): The input image from the user.
|
||||
chatbot (str): The chatbot's name.
|
||||
history (str): The history of the chat.
|
||||
top_p (float): The top-p parameter for the model.
|
||||
temperature (float): The temperature parameter for the model.
|
||||
max_length_tokens (int): The maximum length of tokens for the model.
|
||||
max_context_length_tokens (int): The maximum length of context tokens for the model.
|
||||
model_select_dropdown (str): The selected model from the dropdown.
|
||||
|
||||
Returns:
|
||||
generator: A generator that yields the chatbot outputs, history, and status.
|
||||
"""
|
||||
print("running the prediction function")
|
||||
try:
|
||||
tokenizer, vl_gpt, vl_chat_processor = models[model_select_dropdown]
|
||||
|
||||
if text == "":
|
||||
yield chatbot, history, "Empty context."
|
||||
return
|
||||
except KeyError:
|
||||
yield [[text, "No Model Found"]], [], "No Model Found"
|
||||
return
|
||||
|
||||
conversation = generate_prompt_with_history(
|
||||
text, image, history, vl_chat_processor, tokenizer, max_length=max_context_length_tokens
|
||||
)
|
||||
prompts = convert_conversation_to_prompts(conversation)
|
||||
|
||||
stop_words = conversation.stop_str
|
||||
gradio_chatbot_output = to_gradio_chatbot(conversation)
|
||||
|
||||
full_response = ""
|
||||
with torch.no_grad():
|
||||
for x in deepseek_generate(
|
||||
prompts=prompts,
|
||||
vl_gpt=vl_gpt,
|
||||
vl_chat_processor=vl_chat_processor,
|
||||
tokenizer=tokenizer,
|
||||
stop_words=stop_words,
|
||||
max_length=max_length_tokens,
|
||||
temperature=temperature,
|
||||
repetition_penalty=repetition_penalty,
|
||||
top_p=top_p,
|
||||
):
|
||||
full_response += x
|
||||
response = strip_stop_words(full_response, stop_words)
|
||||
conversation.update_last_message(response)
|
||||
gradio_chatbot_output[-1][1] = response
|
||||
yield gradio_chatbot_output, to_gradio_history(conversation), "Generating..."
|
||||
|
||||
print("flushed result to gradio")
|
||||
torch.cuda.empty_cache()
|
||||
|
||||
if is_variable_assigned("x"):
|
||||
print(f"{model_select_dropdown}:\n{text}\n{'-' * 80}\n{x}\n{'=' * 80}")
|
||||
print(
|
||||
f"temperature: {temperature}, top_p: {top_p}, repetition_penalty: {repetition_penalty}, max_length_tokens: {max_length_tokens}"
|
||||
)
|
||||
|
||||
yield gradio_chatbot_output, to_gradio_history(conversation), "Generate: Success"
|
||||
|
||||
|
||||
def retry(
|
||||
text,
|
||||
image,
|
||||
chatbot,
|
||||
history,
|
||||
top_p,
|
||||
temperature,
|
||||
repetition_penalty,
|
||||
max_length_tokens,
|
||||
max_context_length_tokens,
|
||||
model_select_dropdown,
|
||||
):
|
||||
if len(history) == 0:
|
||||
yield (chatbot, history, "Empty context")
|
||||
return
|
||||
|
||||
chatbot.pop()
|
||||
history.pop()
|
||||
text = history.pop()[-1]
|
||||
if type(text) is tuple:
|
||||
text, image = text
|
||||
|
||||
yield from predict(
|
||||
text,
|
||||
image,
|
||||
chatbot,
|
||||
history,
|
||||
top_p,
|
||||
temperature,
|
||||
repetition_penalty,
|
||||
max_length_tokens,
|
||||
max_context_length_tokens,
|
||||
model_select_dropdown,
|
||||
)
|
||||
|
||||
|
||||
def build_demo(MODELS):
|
||||
with open("deepseek_vl/serve/assets/custom.css", "r", encoding="utf-8") as f:
|
||||
customCSS = f.read()
|
||||
|
||||
with gr.Blocks(theme=gr.themes.Soft()) as demo:
|
||||
history = gr.State([])
|
||||
input_text = gr.State()
|
||||
input_image = gr.State()
|
||||
|
||||
with gr.Row():
|
||||
gr.HTML(title)
|
||||
status_display = gr.Markdown("Success", elem_id="status_display")
|
||||
gr.Markdown(description_top)
|
||||
|
||||
with gr.Row(equal_height=True):
|
||||
with gr.Column(scale=4):
|
||||
with gr.Row():
|
||||
chatbot = gr.Chatbot(
|
||||
elem_id="deepseek_chatbot",
|
||||
show_share_button=True,
|
||||
likeable=True,
|
||||
bubble_full_width=False,
|
||||
height=600,
|
||||
)
|
||||
with gr.Row():
|
||||
with gr.Column(scale=4):
|
||||
text_box = gr.Textbox(show_label=False, placeholder="Enter text", container=False)
|
||||
with gr.Column(
|
||||
min_width=70,
|
||||
):
|
||||
submitBtn = gr.Button("Send")
|
||||
with gr.Column(
|
||||
min_width=70,
|
||||
):
|
||||
cancelBtn = gr.Button("Stop")
|
||||
with gr.Row():
|
||||
emptyBtn = gr.Button(
|
||||
"🧹 New Conversation",
|
||||
)
|
||||
retryBtn = gr.Button("🔄 Regenerate")
|
||||
delLastBtn = gr.Button("🗑️ Remove Last Turn")
|
||||
|
||||
with gr.Column():
|
||||
image_box = gr.Image(type="pil")
|
||||
|
||||
with gr.Tab(label="Parameter Setting") as parameter_row:
|
||||
top_p = gr.Slider(
|
||||
minimum=-0,
|
||||
maximum=1.0,
|
||||
value=0.95,
|
||||
step=0.05,
|
||||
interactive=True,
|
||||
label="Top-p",
|
||||
)
|
||||
temperature = gr.Slider(
|
||||
minimum=0,
|
||||
maximum=1.0,
|
||||
value=0.1,
|
||||
step=0.1,
|
||||
interactive=True,
|
||||
label="Temperature",
|
||||
)
|
||||
repetition_penalty = gr.Slider(
|
||||
minimum=0.0,
|
||||
maximum=2.0,
|
||||
value=1.1,
|
||||
step=0.1,
|
||||
interactive=True,
|
||||
label="Repetition penalty",
|
||||
)
|
||||
max_length_tokens = gr.Slider(
|
||||
minimum=0,
|
||||
maximum=4096,
|
||||
value=2048,
|
||||
step=8,
|
||||
interactive=True,
|
||||
label="Max Generation Tokens",
|
||||
)
|
||||
max_context_length_tokens = gr.Slider(
|
||||
minimum=0,
|
||||
maximum=4096,
|
||||
value=4096,
|
||||
step=128,
|
||||
interactive=True,
|
||||
label="Max History Tokens",
|
||||
)
|
||||
model_select_dropdown = gr.Dropdown(
|
||||
label="Select Models",
|
||||
choices=MODELS,
|
||||
multiselect=False,
|
||||
value=MODELS[0],
|
||||
interactive=True,
|
||||
)
|
||||
|
||||
examples_list = [
|
||||
[
|
||||
'deepseek_vl/serve/examples/rap.jpeg',
|
||||
'Can you write me a master rap song that rhymes very well based on this image?',
|
||||
],
|
||||
[
|
||||
'deepseek_vl/serve/examples/app.png',
|
||||
'What is this app about?',
|
||||
],
|
||||
[
|
||||
'deepseek_vl/serve/examples/pipeline.png',
|
||||
'Help me write a python code based on the image.',
|
||||
],
|
||||
[
|
||||
'deepseek_vl/serve/examples/chart.png',
|
||||
'Could you help me to re-draw this picture with python codes?',
|
||||
],
|
||||
[
|
||||
'deepseek_vl/serve/examples/mirror.png',
|
||||
'How many people are there in the image. Why?',
|
||||
],
|
||||
[
|
||||
'deepseek_vl/serve/examples/puzzle.png',
|
||||
'Can this 2 pieces combine together?',
|
||||
],
|
||||
]
|
||||
gr.Examples(examples=examples_list, inputs=[image_box, text_box])
|
||||
gr.Markdown(description)
|
||||
|
||||
input_widgets = [
|
||||
input_text,
|
||||
input_image,
|
||||
chatbot,
|
||||
history,
|
||||
top_p,
|
||||
temperature,
|
||||
repetition_penalty,
|
||||
max_length_tokens,
|
||||
max_context_length_tokens,
|
||||
model_select_dropdown,
|
||||
]
|
||||
output_widgets = [chatbot, history, status_display]
|
||||
|
||||
transfer_input_args = dict(
|
||||
fn=transfer_input,
|
||||
inputs=[text_box, image_box],
|
||||
outputs=[input_text, input_image, text_box, image_box, submitBtn],
|
||||
show_progress=True,
|
||||
)
|
||||
|
||||
predict_args = dict(
|
||||
fn=predict,
|
||||
inputs=input_widgets,
|
||||
outputs=output_widgets,
|
||||
show_progress=True,
|
||||
)
|
||||
|
||||
retry_args = dict(
|
||||
fn=retry,
|
||||
inputs=input_widgets,
|
||||
outputs=output_widgets,
|
||||
show_progress=True,
|
||||
)
|
||||
|
||||
reset_args = dict(fn=reset_textbox, inputs=[], outputs=[text_box, status_display])
|
||||
|
||||
predict_events = [
|
||||
text_box.submit(**transfer_input_args).then(**predict_args),
|
||||
submitBtn.click(**transfer_input_args).then(**predict_args),
|
||||
]
|
||||
|
||||
emptyBtn.click(reset_state, outputs=output_widgets, show_progress=True)
|
||||
emptyBtn.click(**reset_args)
|
||||
retryBtn.click(**retry_args)
|
||||
|
||||
delLastBtn.click(
|
||||
delete_last_conversation,
|
||||
[chatbot, history],
|
||||
output_widgets,
|
||||
show_progress=True,
|
||||
)
|
||||
|
||||
cancelBtn.click(cancel_outputing, [], [status_display], cancels=predict_events)
|
||||
|
||||
return demo
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
demo = build_demo(MODELS)
|
||||
demo.title = "DeepSeek-VL Chatbot"
|
||||
|
||||
reload_javascript()
|
||||
demo.queue(concurrency_count=CONCURRENT_COUNT).launch(
|
||||
share=False,
|
||||
favicon_path="deepseek_vl/serve/assets/favicon.ico",
|
||||
inbrowser=False,
|
||||
server_name="0.0.0.0",
|
||||
server_port=8122,
|
||||
)
|
||||
75
deepseek_vl/serve/app_modules/gradio_utils.py
Executable file
75
deepseek_vl/serve/app_modules/gradio_utils.py
Executable file
@ -0,0 +1,75 @@
|
||||
from functools import wraps
|
||||
|
||||
import gradio as gr
|
||||
|
||||
|
||||
def wrap_gen_fn(gen_fn):
|
||||
@wraps(gen_fn)
|
||||
def wrapped_gen_fn(prompt, *args, **kwargs):
|
||||
try:
|
||||
yield from gen_fn(prompt, *args, **kwargs)
|
||||
except gr.Error as g_err:
|
||||
raise g_err
|
||||
except Exception as e:
|
||||
raise gr.Error(f'Failed to generate text: {e}') from e
|
||||
|
||||
return wrapped_gen_fn
|
||||
|
||||
|
||||
def delete_last_conversation(chatbot, history):
|
||||
if len(history) % 2 != 0:
|
||||
gr.Error("history length is not even")
|
||||
return (
|
||||
chatbot,
|
||||
history,
|
||||
"Delete Done",
|
||||
)
|
||||
|
||||
if len(chatbot) > 0:
|
||||
chatbot.pop()
|
||||
|
||||
if len(history) > 0 and len(history) % 2 == 0:
|
||||
history.pop()
|
||||
history.pop()
|
||||
|
||||
return (
|
||||
chatbot,
|
||||
history,
|
||||
"Delete Done",
|
||||
)
|
||||
|
||||
|
||||
def reset_state():
|
||||
return [], [], None, "Reset Done"
|
||||
|
||||
|
||||
def reset_textbox():
|
||||
return gr.update(value=""), ""
|
||||
|
||||
|
||||
def cancel_outputing():
|
||||
return "Stop Done"
|
||||
|
||||
|
||||
def transfer_input(input_text, input_image):
|
||||
print("transferring input text and input image")
|
||||
return (
|
||||
input_text,
|
||||
input_image,
|
||||
gr.update(value=""),
|
||||
gr.update(value=None),
|
||||
gr.Button(visible=True),
|
||||
)
|
||||
|
||||
|
||||
class State:
|
||||
interrupted = False
|
||||
|
||||
def interrupt(self):
|
||||
self.interrupted = True
|
||||
|
||||
def recover(self):
|
||||
self.interrupted = False
|
||||
|
||||
|
||||
shared_state = State()
|
||||
60
deepseek_vl/serve/app_modules/overwrites.py
Executable file
60
deepseek_vl/serve/app_modules/overwrites.py
Executable file
@ -0,0 +1,60 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import List, Tuple
|
||||
|
||||
from app_modules.presets import *
|
||||
from app_modules.utils import *
|
||||
|
||||
|
||||
def compact_text_chunks(self, prompt, text_chunks: List[str]) -> List[str]:
|
||||
logging.debug("Compacting text chunks...🚀🚀🚀")
|
||||
combined_str = [c.strip() for c in text_chunks if c.strip()]
|
||||
combined_str = [f"[{index+1}] {c}" for index, c in enumerate(combined_str)]
|
||||
combined_str = "\n\n".join(combined_str)
|
||||
# resplit based on self.max_chunk_overlap
|
||||
text_splitter = self.get_text_splitter_given_prompt(prompt, 1, padding=1)
|
||||
return text_splitter.split_text(combined_str)
|
||||
|
||||
|
||||
def postprocess(self, y: List[Tuple[str | None, str | None]]) -> List[Tuple[str | None, str | None]]:
|
||||
"""
|
||||
Parameters:
|
||||
y: List of tuples representing the message and response pairs. Each message and response should be a string, which may be in Markdown format.
|
||||
Returns:
|
||||
List of tuples representing the message and response. Each message and response will be a string of HTML.
|
||||
"""
|
||||
if y is None or y == []:
|
||||
return []
|
||||
temp = []
|
||||
for x in y:
|
||||
user, bot = x
|
||||
if not detect_converted_mark(user):
|
||||
user = convert_asis(user)
|
||||
if not detect_converted_mark(bot):
|
||||
bot = convert_mdtext(bot)
|
||||
temp.append((user, bot))
|
||||
return temp
|
||||
|
||||
|
||||
with open("deepseek_vl/serve/assets/custom.js", "r", encoding="utf-8") as f, open(
|
||||
"deepseek_vl/serve/assets/Kelpy-Codos.js", "r", encoding="utf-8"
|
||||
) as f2:
|
||||
customJS = f.read()
|
||||
kelpyCodos = f2.read()
|
||||
|
||||
|
||||
def reload_javascript():
|
||||
print("Reloading javascript...")
|
||||
js = f'<script>{customJS}</script><script>{kelpyCodos}</script>'
|
||||
|
||||
def template_response(*args, **kwargs):
|
||||
res = GradioTemplateResponseOriginal(*args, **kwargs)
|
||||
res.body = res.body.replace(b'</html>', f'{js}</html>'.encode("utf8"))
|
||||
res.init_headers()
|
||||
return res
|
||||
|
||||
gr.routes.templates.TemplateResponse = template_response
|
||||
|
||||
|
||||
GradioTemplateResponseOriginal = gr.routes.templates.TemplateResponse
|
||||
77
deepseek_vl/serve/app_modules/presets.py
Executable file
77
deepseek_vl/serve/app_modules/presets.py
Executable file
@ -0,0 +1,77 @@
|
||||
# -*- coding:utf-8 -*-
|
||||
import gradio as gr
|
||||
|
||||
title = """<h1 align="left" style="min-width:200px; margin-top:0;">Chat with DeepSeek-VL </h1>"""
|
||||
description_top = """"""
|
||||
description = """"""
|
||||
CONCURRENT_COUNT = 10
|
||||
|
||||
|
||||
ALREADY_CONVERTED_MARK = "<!-- ALREADY CONVERTED BY PARSER. -->"
|
||||
|
||||
small_and_beautiful_theme = gr.themes.Soft(
|
||||
primary_hue=gr.themes.Color(
|
||||
c50="#EBFAF2",
|
||||
c100="#CFF3E1",
|
||||
c200="#A8EAC8",
|
||||
c300="#77DEA9",
|
||||
c400="#3FD086",
|
||||
c500="#02C160",
|
||||
c600="#06AE56",
|
||||
c700="#05974E",
|
||||
c800="#057F45",
|
||||
c900="#04673D",
|
||||
c950="#2E5541",
|
||||
name="small_and_beautiful",
|
||||
),
|
||||
secondary_hue=gr.themes.Color(
|
||||
c50="#576b95",
|
||||
c100="#576b95",
|
||||
c200="#576b95",
|
||||
c300="#576b95",
|
||||
c400="#576b95",
|
||||
c500="#576b95",
|
||||
c600="#576b95",
|
||||
c700="#576b95",
|
||||
c800="#576b95",
|
||||
c900="#576b95",
|
||||
c950="#576b95",
|
||||
),
|
||||
neutral_hue=gr.themes.Color(
|
||||
name="gray",
|
||||
c50="#f6f7f8",
|
||||
# c100="#f3f4f6",
|
||||
c100="#F2F2F2",
|
||||
c200="#e5e7eb",
|
||||
c300="#d1d5db",
|
||||
c400="#B2B2B2",
|
||||
c500="#808080",
|
||||
c600="#636363",
|
||||
c700="#515151",
|
||||
c800="#393939",
|
||||
# c900="#272727",
|
||||
c900="#2B2B2B",
|
||||
c950="#171717",
|
||||
),
|
||||
radius_size=gr.themes.sizes.radius_sm,
|
||||
).set(
|
||||
# button_primary_background_fill="*primary_500",
|
||||
button_primary_background_fill_dark="*primary_600",
|
||||
# button_primary_background_fill_hover="*primary_400",
|
||||
# button_primary_border_color="*primary_500",
|
||||
button_primary_border_color_dark="*primary_600",
|
||||
button_primary_text_color="white",
|
||||
button_primary_text_color_dark="white",
|
||||
button_secondary_background_fill="*neutral_100",
|
||||
button_secondary_background_fill_hover="*neutral_50",
|
||||
button_secondary_background_fill_dark="*neutral_900",
|
||||
button_secondary_text_color="*neutral_800",
|
||||
button_secondary_text_color_dark="white",
|
||||
# background_fill_primary="#F7F7F7",
|
||||
# background_fill_primary_dark="#1F1F1F",
|
||||
# block_title_text_color="*primary_500",
|
||||
block_title_background_fill_dark="*primary_900",
|
||||
block_label_background_fill_dark="*primary_900",
|
||||
input_background_fill="#F6F6F6",
|
||||
# chatbot_code_background_color_dark="*neutral_950",
|
||||
)
|
||||
200
deepseek_vl/serve/app_modules/utils.py
Executable file
200
deepseek_vl/serve/app_modules/utils.py
Executable file
@ -0,0 +1,200 @@
|
||||
# -*- coding:utf-8 -*-
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
|
||||
import mdtex2html
|
||||
from markdown import markdown
|
||||
from pygments import highlight
|
||||
from pygments.formatters import HtmlFormatter
|
||||
from pygments.lexers import ClassNotFound, get_lexer_by_name, guess_lexer
|
||||
|
||||
from app_modules.presets import ALREADY_CONVERTED_MARK
|
||||
|
||||
logger = logging.getLogger('gradio_logger')
|
||||
|
||||
|
||||
def configure_logger():
|
||||
logger = logging.getLogger('gradio_logger')
|
||||
logger.setLevel(logging.DEBUG)
|
||||
|
||||
timestr = time.strftime("%Y%m%d-%H%M%S")
|
||||
file_handler = logging.FileHandler(f'deepseek_vl/serve/logs/{timestr}_gradio_log.log')
|
||||
console_handler = logging.StreamHandler()
|
||||
|
||||
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
|
||||
console_handler.setFormatter(formatter)
|
||||
file_handler.setFormatter(formatter)
|
||||
|
||||
console_handler.setLevel(logging.INFO)
|
||||
file_handler.setLevel(logging.INFO)
|
||||
|
||||
logger.addHandler(console_handler)
|
||||
logger.addHandler(file_handler)
|
||||
|
||||
return logger
|
||||
|
||||
|
||||
def strip_stop_words(x, stop_words):
|
||||
for w in stop_words:
|
||||
if w in x:
|
||||
return x[: x.index(w)].strip()
|
||||
return x.strip()
|
||||
|
||||
|
||||
def format_output(history, text, x):
|
||||
updated_history = history + [[text, x]]
|
||||
a = [[y[0], convert_to_markdown(y[1])] for y in updated_history]
|
||||
return a, updated_history
|
||||
|
||||
|
||||
def markdown_to_html_with_syntax_highlight(md_str): # deprecated
|
||||
def replacer(match):
|
||||
lang = match.group(1) or "text"
|
||||
code = match.group(2)
|
||||
|
||||
try:
|
||||
lexer = get_lexer_by_name(lang, stripall=True)
|
||||
except ValueError:
|
||||
lexer = get_lexer_by_name("text", stripall=True)
|
||||
|
||||
formatter = HtmlFormatter()
|
||||
highlighted_code = highlight(code, lexer, formatter)
|
||||
|
||||
return f'<pre><code class="{lang}">{highlighted_code}</code></pre>'
|
||||
|
||||
code_block_pattern = r"```(\w+)?\n([\s\S]+?)\n```"
|
||||
md_str = re.sub(code_block_pattern, replacer, md_str, flags=re.MULTILINE)
|
||||
|
||||
html_str = markdown(md_str)
|
||||
return html_str
|
||||
|
||||
|
||||
def normalize_markdown(md_text: str) -> str: # deprecated
|
||||
lines = md_text.split("\n")
|
||||
normalized_lines = []
|
||||
inside_list = False
|
||||
|
||||
for i, line in enumerate(lines):
|
||||
if re.match(r"^(\d+\.|-|\*|\+)\s", line.strip()):
|
||||
if not inside_list and i > 0 and lines[i - 1].strip() != "":
|
||||
normalized_lines.append("")
|
||||
inside_list = True
|
||||
normalized_lines.append(line)
|
||||
elif inside_list and line.strip() == "":
|
||||
if i < len(lines) - 1 and not re.match(r"^(\d+\.|-|\*|\+)\s", lines[i + 1].strip()):
|
||||
normalized_lines.append(line)
|
||||
continue
|
||||
else:
|
||||
inside_list = False
|
||||
normalized_lines.append(line)
|
||||
|
||||
return "\n".join(normalized_lines)
|
||||
|
||||
|
||||
def convert_mdtext(md_text):
|
||||
code_block_pattern = re.compile(r"```(.*?)(?:```|$)", re.DOTALL)
|
||||
inline_code_pattern = re.compile(r"`(.*?)`", re.DOTALL)
|
||||
code_blocks = code_block_pattern.findall(md_text)
|
||||
non_code_parts = code_block_pattern.split(md_text)[::2]
|
||||
|
||||
result = []
|
||||
for non_code, code in zip(non_code_parts, code_blocks + [""]):
|
||||
if non_code.strip():
|
||||
non_code = normalize_markdown(non_code)
|
||||
if inline_code_pattern.search(non_code):
|
||||
result.append(markdown(non_code, extensions=["tables"]))
|
||||
else:
|
||||
result.append(mdtex2html.convert(non_code, extensions=["tables"]))
|
||||
if code.strip():
|
||||
code = f"\n```{code}\n\n```"
|
||||
code = markdown_to_html_with_syntax_highlight(code)
|
||||
result.append(code)
|
||||
result = "".join(result)
|
||||
result += ALREADY_CONVERTED_MARK
|
||||
return result
|
||||
|
||||
|
||||
def convert_asis(userinput):
|
||||
return f'<p style=\"white-space:pre-wrap;\">{html.escape(userinput)}</p>{ALREADY_CONVERTED_MARK}'
|
||||
|
||||
|
||||
def is_stop_word_or_prefix(s: str, stop_words: list) -> bool:
|
||||
return any(s.endswith(stop_word) for stop_word in stop_words)
|
||||
|
||||
|
||||
def detect_converted_mark(userinput):
|
||||
return bool(userinput.endswith(ALREADY_CONVERTED_MARK))
|
||||
|
||||
|
||||
def detect_language(code):
|
||||
first_line = "" if code.startswith("\n") else code.strip().split("\n", 1)[0]
|
||||
language = first_line.lower() if first_line else ""
|
||||
code_without_language = code[len(first_line) :].lstrip() if first_line else code
|
||||
return language, code_without_language
|
||||
|
||||
|
||||
def convert_to_markdown(text):
|
||||
text = text.replace("$", "$")
|
||||
text = text.replace("\r\n", "\n")
|
||||
|
||||
def replace_leading_tabs_and_spaces(line):
|
||||
new_line = []
|
||||
|
||||
for char in line:
|
||||
if char == "\t":
|
||||
new_line.append("	")
|
||||
elif char == " ":
|
||||
new_line.append(" ")
|
||||
else:
|
||||
break
|
||||
return "".join(new_line) + line[len(new_line) :]
|
||||
|
||||
markdown_text = ""
|
||||
lines = text.split("\n")
|
||||
in_code_block = False
|
||||
|
||||
for line in lines:
|
||||
if in_code_block is False and line.startswith("```"):
|
||||
in_code_block = True
|
||||
markdown_text += f"{line}\n"
|
||||
elif in_code_block is True and line.startswith("```"):
|
||||
in_code_block = False
|
||||
markdown_text += f"{line}\n"
|
||||
elif in_code_block:
|
||||
markdown_text += f"{line}\n"
|
||||
else:
|
||||
line = replace_leading_tabs_and_spaces(line)
|
||||
line = re.sub(r"^(#)", r"\\\1", line)
|
||||
markdown_text += f"{line} \n"
|
||||
|
||||
return markdown_text
|
||||
|
||||
|
||||
def add_language_tag(text):
|
||||
def detect_language(code_block):
|
||||
try:
|
||||
lexer = guess_lexer(code_block)
|
||||
return lexer.name.lower()
|
||||
except ClassNotFound:
|
||||
return ""
|
||||
|
||||
code_block_pattern = re.compile(r"(```)(\w*\n[^`]+```)", re.MULTILINE)
|
||||
|
||||
def replacement(match):
|
||||
code_block = match.group(2)
|
||||
if match.group(2).startswith("\n"):
|
||||
language = detect_language(code_block)
|
||||
return f"```{language}{code_block}```" if language else f"```\n{code_block}```"
|
||||
else:
|
||||
return match.group(1) + code_block + "```"
|
||||
|
||||
text2 = code_block_pattern.sub(replacement, text)
|
||||
return text2
|
||||
|
||||
|
||||
def is_variable_assigned(var_name: str) -> bool:
|
||||
return var_name in locals()
|
||||
79
deepseek_vl/serve/assets/Kelpy-Codos.js
Executable file
79
deepseek_vl/serve/assets/Kelpy-Codos.js
Executable file
@ -0,0 +1,79 @@
|
||||
// ==UserScript==
|
||||
// @name Kelpy Codos
|
||||
// @namespace https://github.com/Keldos-Li/Kelpy-Codos
|
||||
// @version 1.0.5
|
||||
// @author Keldos; https://keldos.me/
|
||||
// @description Add copy button to PRE tags before CODE tag, for Chuanhu ChatGPT especially.
|
||||
// Based on Chuanhu ChatGPT version: ac04408 (2023-3-22)
|
||||
// @license GPL-3.0
|
||||
// @grant none
|
||||
// ==/UserScript==
|
||||
|
||||
(function () {
|
||||
"use strict";
|
||||
|
||||
function addCopyButton(pre) {
|
||||
var code = pre.querySelector("code");
|
||||
if (!code) {
|
||||
return; // 如果没有找到 <code> 元素，则不添加按钮
|
||||
}
|
||||
var firstChild = code.firstChild;
|
||||
if (!firstChild) {
|
||||
return; // 如果 <code> 元素没有子节点，则不添加按钮
|
||||
}
|
||||
var button = document.createElement("button");
|
||||
button.textContent = "\uD83D\uDCCE"; // 使用 📎 符号作为“复制”按钮的文本
|
||||
button.style.position = "relative";
|
||||
button.style.float = "right";
|
||||
button.style.fontSize = "1em"; // 可选：调整按钮大小
|
||||
button.style.background = "none"; // 可选：去掉背景颜色
|
||||
button.style.border = "none"; // 可选：去掉边框
|
||||
button.style.cursor = "pointer"; // 可选：显示指针样式
|
||||
button.addEventListener("click", function () {
|
||||
var range = document.createRange();
|
||||
range.selectNodeContents(code);
|
||||
range.setStartBefore(firstChild); // 将范围设置为第一个子节点之前
|
||||
var selection = window.getSelection();
|
||||
selection.removeAllRanges();
|
||||
selection.addRange(range);
|
||||
|
||||
try {
|
||||
var success = document.execCommand("copy");
|
||||
if (success) {
|
||||
button.textContent = "\u2714";
|
||||
setTimeout(function () {
|
||||
button.textContent = "\uD83D\uDCCE"; // 恢复按钮为“复制”
|
||||
}, 2000);
|
||||
} else {
|
||||
button.textContent = "\u2716";
|
||||
}
|
||||
} catch (e) {
|
||||
console.error(e);
|
||||
button.textContent = "\u2716";
|
||||
}
|
||||
|
||||
selection.removeAllRanges();
|
||||
});
|
||||
code.insertBefore(button, firstChild); // 将按钮插入到第一个子元素之前
|
||||
}
|
||||
|
||||
function handleNewElements(mutationsList, observer) {
|
||||
for (var mutation of mutationsList) {
|
||||
if (mutation.type === "childList") {
|
||||
for (var node of mutation.addedNodes) {
|
||||
if (node.nodeName === "PRE") {
|
||||
addCopyButton(node);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var observer = new MutationObserver(handleNewElements);
|
||||
observer.observe(document.documentElement, {
|
||||
childList: true,
|
||||
subtree: true,
|
||||
});
|
||||
|
||||
document.querySelectorAll("pre").forEach(addCopyButton);
|
||||
})();
|
||||
BIN
deepseek_vl/serve/assets/avatar.png
Executable file
BIN
deepseek_vl/serve/assets/avatar.png
Executable file
Binary file not shown.
|
After Width: | Height: | Size: 61 KiB |
334
deepseek_vl/serve/assets/custom.css
Executable file
334
deepseek_vl/serve/assets/custom.css
Executable file
@ -0,0 +1,334 @@
|
||||
:root {
|
||||
--chatbot-color-light: #f3f3f3;
|
||||
--chatbot-color-dark: #121111;
|
||||
}
|
||||
|
||||
/* status_display */
|
||||
#status_display {
|
||||
display: flex;
|
||||
min-height: 2.5em;
|
||||
align-items: flex-end;
|
||||
justify-content: flex-end;
|
||||
}
|
||||
#status_display p {
|
||||
font-size: 0.85em;
|
||||
font-family: monospace;
|
||||
color: var(--body-text-color-subdued);
|
||||
}
|
||||
|
||||
/* usage_display */
|
||||
#usage_display {
|
||||
height: 1em;
|
||||
}
|
||||
#usage_display p {
|
||||
padding: 0 1em;
|
||||
font-size: 0.85em;
|
||||
font-family: monospace;
|
||||
color: var(--body-text-color-subdued);
|
||||
}
|
||||
/* list */
|
||||
ol:not(.options),
|
||||
ul:not(.options) {
|
||||
padding-inline-start: 2em !important;
|
||||
}
|
||||
|
||||
/* Thank @Keldos-Li for fixing it */
|
||||
/* Light mode (default) */
|
||||
#deepseek_chatbot {
|
||||
background-color: var(--chatbot-color-light) !important;
|
||||
color: #000000 !important;
|
||||
}
|
||||
[data-testid="bot"] {
|
||||
background-color: #ffffff !important;
|
||||
}
|
||||
[data-testid="user"] {
|
||||
background-color: #95ec69 !important;
|
||||
}
|
||||
|
||||
/* Dark mode */
|
||||
.dark #deepseek_chatbot {
|
||||
background-color: var(--chatbot-color-dark) !important;
|
||||
color: #ffffff !important;
|
||||
}
|
||||
.dark [data-testid="bot"] {
|
||||
background-color: #2c2c2c !important;
|
||||
}
|
||||
.dark [data-testid="user"] {
|
||||
background-color: #26b561 !important;
|
||||
}
|
||||
|
||||
#deepseek_chatbot {
|
||||
height: 100%;
|
||||
min-height: 800px;
|
||||
flex-grow: 1;
|
||||
overflow: auto;
|
||||
}
|
||||
|
||||
[class*="message"] {
|
||||
border-radius: var(--radius-xl) !important;
|
||||
border: none;
|
||||
padding: var(--spacing-xl) !important;
|
||||
font-size: var(--text-md) !important;
|
||||
line-height: var(--line-md) !important;
|
||||
min-height: calc(var(--text-md) * var(--line-md) + 2 * var(--spacing-xl));
|
||||
min-width: calc(var(--text-md) * var(--line-md) + 2 * var(--spacing-xl));
|
||||
}
|
||||
[data-testid="bot"] {
|
||||
max-width: 85%;
|
||||
border-bottom-left-radius: 0 !important;
|
||||
}
|
||||
[data-testid="user"] {
|
||||
max-width: 85%;
|
||||
width: auto !important;
|
||||
border-bottom-right-radius: 0 !important;
|
||||
}
|
||||
/* Table */
|
||||
table {
|
||||
margin: 1em 0;
|
||||
border-collapse: collapse;
|
||||
empty-cells: show;
|
||||
}
|
||||
td,
|
||||
th {
|
||||
border: 1.2px solid var(--border-color-primary) !important;
|
||||
padding: 0.2em;
|
||||
}
|
||||
thead {
|
||||
background-color: rgba(175, 184, 193, 0.2);
|
||||
}
|
||||
thead th {
|
||||
padding: 0.5em 0.2em;
|
||||
}
|
||||
/* Inline code */
|
||||
#deepseek_chatbot code {
|
||||
display: inline;
|
||||
white-space: break-spaces;
|
||||
border-radius: 6px;
|
||||
margin: 0 2px 0 2px;
|
||||
padding: 0.2em 0.4em 0.1em 0.4em;
|
||||
background-color: rgba(175, 184, 193, 0.2);
|
||||
}
|
||||
/* Code block */
|
||||
#deepseek_chatbot pre code {
|
||||
display: block;
|
||||
overflow: auto;
|
||||
white-space: pre;
|
||||
background-color: #1c1d1e !important;
|
||||
border-radius: 10px;
|
||||
padding: 1.4em 1.2em 0em 1.4em;
|
||||
margin: 1.2em 2em 1.2em 0.5em;
|
||||
color: #fdf8f8;
|
||||
box-shadow: 6px 6px 16px hsla(0, 0%, 0%, 0.2);
|
||||
}
|
||||
/* Hightlight */
|
||||
#deepseek_chatbot .highlight {
|
||||
background-color: transparent;
|
||||
}
|
||||
#deepseek_chatbot .highlight .hll {
|
||||
background-color: #49483e;
|
||||
}
|
||||
#deepseek_chatbot .highlight .c {
|
||||
color: #75715e;
|
||||
} /* Comment */
|
||||
#deepseek_chatbot .highlight .err {
|
||||
color: #960050;
|
||||
background-color: #1e0010;
|
||||
} /* Error */
|
||||
#deepseek_chatbot .highlight .k {
|
||||
color: #66d9ef;
|
||||
} /* Keyword */
|
||||
#deepseek_chatbot .highlight .l {
|
||||
color: #ae81ff;
|
||||
} /* Literal */
|
||||
#deepseek_chatbot .highlight .n {
|
||||
color: #f8f8f2;
|
||||
} /* Name */
|
||||
#deepseek_chatbot .highlight .o {
|
||||
color: #f92672;
|
||||
} /* Operator */
|
||||
#deepseek_chatbot .highlight .p {
|
||||
color: #f8f8f2;
|
||||
} /* Punctuation */
|
||||
#deepseek_chatbot .highlight .ch {
|
||||
color: #75715e;
|
||||
} /* Comment.Hashbang */
|
||||
#deepseek_chatbot .highlight .cm {
|
||||
color: #75715e;
|
||||
} /* Comment.Multiline */
|
||||
#deepseek_chatbot .highlight .cp {
|
||||
color: #75715e;
|
||||
} /* Comment.Preproc */
|
||||
#deepseek_chatbot .highlight .cpf {
|
||||
color: #75715e;
|
||||
} /* Comment.PreprocFile */
|
||||
#deepseek_chatbot .highlight .c1 {
|
||||
color: #75715e;
|
||||
} /* Comment.Single */
|
||||
#deepseek_chatbot .highlight .cs {
|
||||
color: #75715e;
|
||||
} /* Comment.Special */
|
||||
#deepseek_chatbot .highlight .gd {
|
||||
color: #f92672;
|
||||
} /* Generic.Deleted */
|
||||
#deepseek_chatbot .highlight .ge {
|
||||
font-style: italic;
|
||||
} /* Generic.Emph */
|
||||
#deepseek_chatbot .highlight .gi {
|
||||
color: #a6e22e;
|
||||
} /* Generic.Inserted */
|
||||
#deepseek_chatbot .highlight .gs {
|
||||
font-weight: bold;
|
||||
} /* Generic.Strong */
|
||||
#deepseek_chatbot .highlight .gu {
|
||||
color: #75715e;
|
||||
} /* Generic.Subheading */
|
||||
#deepseek_chatbot .highlight .kc {
|
||||
color: #66d9ef;
|
||||
} /* Keyword.Constant */
|
||||
#deepseek_chatbot .highlight .kd {
|
||||
color: #66d9ef;
|
||||
} /* Keyword.Declaration */
|
||||
#deepseek_chatbot .highlight .kn {
|
||||
color: #f92672;
|
||||
} /* Keyword.Namespace */
|
||||
#deepseek_chatbot .highlight .kp {
|
||||
color: #66d9ef;
|
||||
} /* Keyword.Pseudo */
|
||||
#deepseek_chatbot .highlight .kr {
|
||||
color: #66d9ef;
|
||||
} /* Keyword.Reserved */
|
||||
#deepseek_chatbot .highlight .kt {
|
||||
color: #66d9ef;
|
||||
} /* Keyword.Type */
|
||||
#deepseek_chatbot .highlight .ld {
|
||||
color: #e6db74;
|
||||
} /* Literal.Date */
|
||||
#deepseek_chatbot .highlight .m {
|
||||
color: #ae81ff;
|
||||
} /* Literal.Number */
|
||||
#deepseek_chatbot .highlight .s {
|
||||
color: #e6db74;
|
||||
} /* Literal.String */
|
||||
#deepseek_chatbot .highlight .na {
|
||||
color: #a6e22e;
|
||||
} /* Name.Attribute */
|
||||
#deepseek_chatbot .highlight .nb {
|
||||
color: #f8f8f2;
|
||||
} /* Name.Builtin */
|
||||
#deepseek_chatbot .highlight .nc {
|
||||
color: #a6e22e;
|
||||
} /* Name.Class */
|
||||
#deepseek_chatbot .highlight .no {
|
||||
color: #66d9ef;
|
||||
} /* Name.Constant */
|
||||
#deepseek_chatbot .highlight .nd {
|
||||
color: #a6e22e;
|
||||
} /* Name.Decorator */
|
||||
#deepseek_chatbot .highlight .ni {
|
||||
color: #f8f8f2;
|
||||
} /* Name.Entity */
|
||||
#deepseek_chatbot .highlight .ne {
|
||||
color: #a6e22e;
|
||||
} /* Name.Exception */
|
||||
#deepseek_chatbot .highlight .nf {
|
||||
color: #a6e22e;
|
||||
} /* Name.Function */
|
||||
#deepseek_chatbot .highlight .nl {
|
||||
color: #f8f8f2;
|
||||
} /* Name.Label */
|
||||
#deepseek_chatbot .highlight .nn {
|
||||
color: #f8f8f2;
|
||||
} /* Name.Namespace */
|
||||
#deepseek_chatbot .highlight .nx {
|
||||
color: #a6e22e;
|
||||
} /* Name.Other */
|
||||
#deepseek_chatbot .highlight .py {
|
||||
color: #f8f8f2;
|
||||
} /* Name.Property */
|
||||
#deepseek_chatbot .highlight .nt {
|
||||
color: #f92672;
|
||||
} /* Name.Tag */
|
||||
#deepseek_chatbot .highlight .nv {
|
||||
color: #f8f8f2;
|
||||
} /* Name.Variable */
|
||||
#deepseek_chatbot .highlight .ow {
|
||||
color: #f92672;
|
||||
} /* Operator.Word */
|
||||
#deepseek_chatbot .highlight .w {
|
||||
color: #f8f8f2;
|
||||
} /* Text.Whitespace */
|
||||
#deepseek_chatbot .highlight .mb {
|
||||
color: #ae81ff;
|
||||
} /* Literal.Number.Bin */
|
||||
#deepseek_chatbot .highlight .mf {
|
||||
color: #ae81ff;
|
||||
} /* Literal.Number.Float */
|
||||
#deepseek_chatbot .highlight .mh {
|
||||
color: #ae81ff;
|
||||
} /* Literal.Number.Hex */
|
||||
#deepseek_chatbot .highlight .mi {
|
||||
color: #ae81ff;
|
||||
} /* Literal.Number.Integer */
|
||||
#deepseek_chatbot .highlight .mo {
|
||||
color: #ae81ff;
|
||||
} /* Literal.Number.Oct */
|
||||
#deepseek_chatbot .highlight .sa {
|
||||
color: #e6db74;
|
||||
} /* Literal.String.Affix */
|
||||
#deepseek_chatbot .highlight .sb {
|
||||
color: #e6db74;
|
||||
} /* Literal.String.Backtick */
|
||||
#deepseek_chatbot .highlight .sc {
|
||||
color: #e6db74;
|
||||
} /* Literal.String.Char */
|
||||
#deepseek_chatbot .highlight .dl {
|
||||
color: #e6db74;
|
||||
} /* Literal.String.Delimiter */
|
||||
#deepseek_chatbot .highlight .sd {
|
||||
color: #e6db74;
|
||||
} /* Literal.String.Doc */
|
||||
#deepseek_chatbot .highlight .s2 {
|
||||
color: #e6db74;
|
||||
} /* Literal.String.Double */
|
||||
#deepseek_chatbot .highlight .se {
|
||||
color: #ae81ff;
|
||||
} /* Literal.String.Escape */
|
||||
#deepseek_chatbot .highlight .sh {
|
||||
color: #e6db74;
|
||||
} /* Literal.String.Heredoc */
|
||||
#deepseek_chatbot .highlight .si {
|
||||
color: #e6db74;
|
||||
} /* Literal.String.Interpol */
|
||||
#deepseek_chatbot .highlight .sx {
|
||||
color: #e6db74;
|
||||
} /* Literal.String.Other */
|
||||
#deepseek_chatbot .highlight .sr {
|
||||
color: #e6db74;
|
||||
} /* Literal.String.Regex */
|
||||
#deepseek_chatbot .highlight .s1 {
|
||||
color: #e6db74;
|
||||
} /* Literal.String.Single */
|
||||
#deepseek_chatbot .highlight .ss {
|
||||
color: #e6db74;
|
||||
} /* Literal.String.Symbol */
|
||||
#deepseek_chatbot .highlight .bp {
|
||||
color: #f8f8f2;
|
||||
} /* Name.Builtin.Pseudo */
|
||||
#deepseek_chatbot .highlight .fm {
|
||||
color: #a6e22e;
|
||||
} /* Name.Function.Magic */
|
||||
#deepseek_chatbot .highlight .vc {
|
||||
color: #f8f8f2;
|
||||
} /* Name.Variable.Class */
|
||||
#deepseek_chatbot .highlight .vg {
|
||||
color: #f8f8f2;
|
||||
} /* Name.Variable.Global */
|
||||
#deepseek_chatbot .highlight .vi {
|
||||
color: #f8f8f2;
|
||||
} /* Name.Variable.Instance */
|
||||
#deepseek_chatbot .highlight .vm {
|
||||
color: #f8f8f2;
|
||||
} /* Name.Variable.Magic */
|
||||
#deepseek_chatbot .highlight .il {
|
||||
color: #ae81ff;
|
||||
} /* Literal.Number.Integer.Long */
|
||||
1
deepseek_vl/serve/assets/custom.js
Executable file
1
deepseek_vl/serve/assets/custom.js
Executable file
@ -0,0 +1 @@
|
||||
// custom javascript here
|
||||
BIN
deepseek_vl/serve/assets/favicon.ico
Executable file
BIN
deepseek_vl/serve/assets/favicon.ico
Executable file
Binary file not shown.
|
After Width: | Height: | Size: 15 KiB |
BIN
deepseek_vl/serve/examples/app.png
Normal file
BIN
deepseek_vl/serve/examples/app.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 81 KiB |
BIN
deepseek_vl/serve/examples/chart.png
Normal file
BIN
deepseek_vl/serve/examples/chart.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 153 KiB |
BIN
deepseek_vl/serve/examples/mirror.png
Normal file
BIN
deepseek_vl/serve/examples/mirror.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 266 KiB |
BIN
deepseek_vl/serve/examples/pipeline.png
Normal file
BIN
deepseek_vl/serve/examples/pipeline.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 37 KiB |
BIN
deepseek_vl/serve/examples/puzzle.png
Normal file
BIN
deepseek_vl/serve/examples/puzzle.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 190 KiB |
BIN
deepseek_vl/serve/examples/rap.jpeg
Executable file
BIN
deepseek_vl/serve/examples/rap.jpeg
Executable file
Binary file not shown.
|
After Width: | Height: | Size: 56 KiB |
139
deepseek_vl/serve/inference.py
Executable file
139
deepseek_vl/serve/inference.py
Executable file
@ -0,0 +1,139 @@
|
||||
from threading import Thread
|
||||
from typing import List
|
||||
|
||||
import torch
|
||||
import transformers
|
||||
from transformers import (AutoModelForCausalLM, StoppingCriteria, StoppingCriteriaList,
|
||||
TextIteratorStreamer)
|
||||
|
||||
from deepseek_vl.models import VLChatProcessor, MultiModalityCausalLM
|
||||
from deepseek_vl.utils.conversation import Conversation
|
||||
|
||||
|
||||
def load_model(model_path):
|
||||
vl_chat_processor: VLChatProcessor = VLChatProcessor.from_pretrained(model_path)
|
||||
tokenizer = vl_chat_processor.tokenizer
|
||||
vl_gpt: MultiModalityCausalLM = AutoModelForCausalLM.from_pretrained(model_path, trust_remote_code=True)
|
||||
vl_gpt = vl_gpt.to(torch.bfloat16).cuda().eval()
|
||||
return tokenizer, vl_gpt, vl_chat_processor
|
||||
|
||||
|
||||
def convert_conversation_to_prompts(conversation: Conversation):
|
||||
prompts = []
|
||||
messages = conversation.messages
|
||||
|
||||
for i in range(0, len(messages), 2):
|
||||
prompt = {
|
||||
"role": messages[i][0],
|
||||
"content": messages[i][1][0] if isinstance(messages[i][1], tuple) else messages[i][1],
|
||||
"images": [messages[i][1][1]] if isinstance(messages[i][1], tuple) else [],
|
||||
}
|
||||
response = {"role": messages[i + 1][0], "content": messages[i + 1][1]}
|
||||
prompts.extend([prompt, response])
|
||||
|
||||
return prompts
|
||||
|
||||
|
||||
class StoppingCriteriaSub(StoppingCriteria):
|
||||
def __init__(self, stops=[], encounters=1):
|
||||
super().__init__()
|
||||
self.stops = [stop.to("cuda") for stop in stops]
|
||||
|
||||
def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor, **kwargs):
|
||||
for stop in self.stops:
|
||||
if input_ids.shape[-1] < len(stop):
|
||||
continue
|
||||
if torch.all((stop == input_ids[0][-len(stop) :])).item():
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def deepseek_generate(
|
||||
prompts: list,
|
||||
vl_gpt: torch.nn.Module,
|
||||
vl_chat_processor,
|
||||
tokenizer: transformers.PreTrainedTokenizer,
|
||||
stop_words: list,
|
||||
max_length: int = 256,
|
||||
temperature: float = 1.0,
|
||||
top_p: float = 1.0,
|
||||
repetition_penalty=1.1,
|
||||
):
|
||||
prompts = prompts
|
||||
pil_images = list()
|
||||
for message in prompts:
|
||||
if "images" not in message:
|
||||
continue
|
||||
for pil_img in message["images"]:
|
||||
pil_images.append(pil_img)
|
||||
|
||||
prepare_inputs = vl_chat_processor(
|
||||
conversations=prompts,
|
||||
images=pil_images,
|
||||
force_batchify=True
|
||||
).to(vl_gpt.device)
|
||||
|
||||
return generate(
|
||||
vl_gpt,
|
||||
tokenizer,
|
||||
prepare_inputs,
|
||||
max_length,
|
||||
temperature,
|
||||
repetition_penalty,
|
||||
top_p,
|
||||
stop_words,
|
||||
)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def generate(
|
||||
vl_gpt,
|
||||
tokenizer,
|
||||
prepare_inputs,
|
||||
max_gen_len: int = 256,
|
||||
temperature: float = 0,
|
||||
repetition_penalty=1.1,
|
||||
top_p: float = 0.95,
|
||||
stop_words: List[str] = [],
|
||||
):
|
||||
"""Stream the text output from the multimodality model with prompt and image inputs."""
|
||||
inputs_embeds = vl_gpt.prepare_inputs_embeds(**prepare_inputs)
|
||||
|
||||
streamer = TextIteratorStreamer(tokenizer)
|
||||
|
||||
stop_words_ids = [
|
||||
torch.tensor(tokenizer.encode(stop_word)) for stop_word in stop_words
|
||||
]
|
||||
stopping_criteria = StoppingCriteriaList([StoppingCriteriaSub(stops=stop_words_ids)])
|
||||
|
||||
generation_config = dict(
|
||||
inputs_embeds=inputs_embeds,
|
||||
attention_mask=prepare_inputs.attention_mask,
|
||||
pad_token_id=tokenizer.eos_token_id,
|
||||
bos_token_id=tokenizer.bos_token_id,
|
||||
eos_token_id=tokenizer.eos_token_id,
|
||||
max_new_tokens=max_gen_len,
|
||||
do_sample=True,
|
||||
use_cache=True,
|
||||
streamer=streamer,
|
||||
stopping_criteria=stopping_criteria,
|
||||
)
|
||||
|
||||
if temperature > 0:
|
||||
generation_config.update(
|
||||
{
|
||||
"do_sample": True,
|
||||
"top_p": top_p,
|
||||
"temperature": temperature,
|
||||
"repetition_penalty": repetition_penalty,
|
||||
}
|
||||
)
|
||||
else:
|
||||
generation_config["do_sample"] = False
|
||||
|
||||
thread = Thread(target=vl_gpt.language_model.generate, kwargs=generation_config)
|
||||
thread.start()
|
||||
|
||||
yield from streamer
|
||||
Loading…
Reference in New Issue
Block a user