Fine tune the Janus-Pro-7B model using ms swift,

And add a proxy service interface,
Integrate Swift with fastapi_cient py
The client can directly use fastapi_cient Py doesn't need to be changed
This commit is contained in:
furanger@sina.com 2025-02-25 17:40:40 +08:00
parent 5376490061
commit c74d76954f
2 changed files with 110 additions and 0 deletions

42
demo/README.md Normal file
View File

@ -0,0 +1,42 @@
## 0 . Fine tuning restrictions
Fine tuning supports training for image understanding but not image generation
## 1. install ms-swift
use ms-swift Fine tune the Janus-Pro-7B model,
First, install ms-swift
----------------------------------------------
pip install git+https://github.com/modelscope/ms-swift.git
cd ms-swift
pip install -e .
--------------------------------------------------------
## 2. Datasets
The dataset format is
{"messages": [{"role": "user", "content": "<image>Does the construction worker in this picture comply with the safety regulations for high-altitude operations"}, {"role": "assistant", "content": "In the high-altitude work area, people entering the construction site must wear safety helmets, and high-altitude workers should wear safety belts. The other end of the safety belt must be hung higher than the human body, which is called high hanging and low use. The high-altitude workers in the picture did not wear safety belts, which does not meet the safety standards for high-altitude operations."}], "images": ["root/train/train_images/wpd-36.jpg"]}
## 3. Fine tuning
lora Fine tuning
swift sft --model_type deepseek_janus_pro --model <Janus-Pro-7B model path> --dataset <dataset path> --target_modules all-linear
full Fine tuning
swift sft --model_type deepseek_janus_pro --model <Janus-Pro-7B model path> --dataset <dataset path> --train_type full
## 4. swift model Service
swift deploy --ckpt_dir <swift mod path>
## 5. swift model Proxy Service
fastapi_swift.py
## 6. Client API fastapi_client. py
Submit questions and receive responses to the swift model Proxy Service using fastapi_client. py
## Other1.
The parameters(seed、top_p、temperature ) are no longer useful, but in order to maintain interface reuse, they are retained
## Other2.
If the Swift model needs to change the directory, the configuration file needs to be changed
Adapterconfig.json Modify 'base_madel_name_or_path'
Args.json modifies 'model'
Specify the Janus-Pro-7B directory for classical gravity

68
demo/fastapi_swift.py Normal file
View File

@ -0,0 +1,68 @@
from fastapi import FastAPI, File, Form, UploadFile, HTTPException
from fastapi.responses import JSONResponse, StreamingResponse
from PIL import Image
import numpy as np
import io
import hashlib
import traceback
import json
import requests
app = FastAPI()
understand_image_and_question_url = "http://localhost:8000/v1/chat/completions"
@app.post("/understand_image_and_question/")
async def understand_image_and_question(
file: UploadFile = File(...),
question: str = Form(...),
seed: int = Form(42),
top_p: float = Form(0.95),
temperature: float = Form(0.1)
):
# images file max size 8mb
maxfilesize = 8 * 1024 * 1024
image_data = await file.read(maxfilesize)
try:
# Upload file directory
imagedirectory = "./uploads/"
# Need to match version with Swift service
JanusVersion = "Janus-Pro-7B"
#JanusVersion = "Janus-Pro-1B"
file = Image.open(io.BytesIO(image_data))
hash_obj = hashlib.md5()
hash_obj.update(image_data)
file_hash = hash_obj.hexdigest()
filename = imagedirectory + file_hash + ".png"
file.save(filename, format='PNG')
file.close()
outjson = {"model": JanusVersion,
"messages": [{"role": "user",
"content": "<image>" + question} ],
"images": [filename]}
outjson = json.dumps(outjson,ensure_ascii=False)
response = requests.post(understand_image_and_question_url, data=outjson, stream=False)
response_data = response.json()
return response_data
except Exception as e:
print("-----------------------------------------------")
error_type = type(e).__name__
error_msg = str(e)
print(error_type)
print(error_msg)
traceback.print_exc()
print("-----------------------------------------------")
return "images file bad"
if __name__ == "__main__":
import uvicorn
uvicorn.run(app, host="0.0.0.0", port=8000)