url = "http://:8000/v1/chat/completions" headers = {"Content-Type": "application/json", "Authorization": "Bearer token"}

model = "mistralai/Mistral-Small-24B-Instruct-2501"

def load_system_prompt(repo_id: str, filename: str) -> str: file_path = hf_hub_download(repo_id=repo_id, filename=filename) with open(file_path, "r") as file: system_prompt = file.read() today = datetime.today().strftime("%Y-%m-%d") yesterday = (datetime.today() - timedelta(days=1)).strftime("%Y-%m-%d") model_name = repo_id.split("/")[-1] return system_prompt.format(name=model_name, today=today, yesterday=yesterday)

SYSTEM_PROMPT = load_system_prompt(model, "SYSTEM_PROMPT.txt")

tools = [ { "type": "function", "function": { "name": "get_current_weather", "description": "获取给定地点的当前天气", "parameters": { "type": "object", "properties": { "city": { "type": "string", "description": "要查询天气的城市，例如 'San Francisco'", }, "state": { "type": "string", "description": "州的缩写，例如 'CA' 代表加利福尼亚州", }, "unit": { "type": "string", "description": "温度单位", "enum": ["celsius", "fahrenheit"], }, }, "required": ["city", "state", "unit"], }, }, }, { "type": "function", "function": { "name": "rewrite", "description": "重写给定文本以提高清晰度", "parameters": { "type": "object", "properties": { "text": { "type": "string", "description": "要重写的输入文本", } }, }, }, }, ]

messages = [ {"role": "system", "content": SYSTEM_PROMPT}, { "role": "user", "content": "你能把下面的文章写得更简洁吗？\n\nOpenAI是一个人工智能研究实验室，由非营利组织OpenAI Incorporated及其营利性子公司OpenAI Limited Partnership组成。", }, { "role": "assistant", "content": "", "tool_calls": [ { "id": "bbc5b7ede", "type": "function", "function": { "name": "rewrite", "arguments": '{"text": "OpenAI is an artificial intelligence research laboratory consisting of the non-profit OpenAI Incorporated and its for-profit subsidiary corporation OpenAI Limited Partnership."}', }, } ], }, { "role": "tool", "content": '{"action":"rewrite","outcome":"OpenAI is a FOR-profit company."}', "tool_call_id": "bbc5b7ede", "name": "rewrite", }, { "role": "assistant", "content": "---\n\nOpenAI是一家营利性公司。", }, { "role": "user", "content": "你能告诉我达拉斯的华氏温度是多少吗？", }, ]

data = {"model": model, "messages": messages, "tools": tools}

response = requests.post(url, headers=headers, data=json.dumps(data)) import ipdb; ipdb.set_trace() print(response.json()["choices"][0]["message"]["tool_calls"])

[{'id': '8PdihwL6d', 'type': 'function', 'function': {'name': 'get_current_weather', 'arguments': '{"city": "Dallas", "state": "TX", "unit": "fahrenheit"}'}}]

</details>

#### 离线使用
```py
from vllm import LLM
from vllm.sampling_params import SamplingParams
from datetime import datetime, timedelta

SYSTEM_PROMPT = "你是一个对话式代理，总是直截了当地回答问题，每次准确回答后都以一只猫的ASCII画结尾。"

user_prompt = "请给我5种法语中非正式的“再见”表达方式。"

messages = [
    {
        "role": "system",
        "content": SYSTEM_PROMPT
    },
    {
        "role": "user",
        "content": user_prompt
    },
]

# 注意，在GPU上运行此模型需要超过60GB的GPU显存
llm = LLM(model=model_name, tokenizer_mode="mistral", tensor_parallel_size=8)

sampling_params = SamplingParams(max_tokens=512, temperature=0.15)
outputs = llm.chat(messages, sampling_params=sampling_params)

print(outputs[0].outputs[0].text)
# 当然，以下是法语中5种非正式的“再见”表达方式：
#
# 1. À plus tard
# 2. À plus
# 3. Salut
# 4. À toute
# 5. Bisous
#
# ```
#  /\_/\
# ( o.o )
#  > ^ <
# ```

Transformers

如果你想使用Hugging Face transformers生成文本，可以这样做：

from transformers import pipeline
import torch

messages = [
    {"role": "user", "content": "请给我5种法语中非正式的“再见”表达方式。"},
]
chatbot = pipeline("text-generation", model="mistralai/Mistral-Small-24B-Instruct-2501", max_new_tokens=256, torch_dtype=torch.bfloat16)
chatbot(messages)

Ollama

Ollama 可以在MacOS、Windows和Linux上本地运行此模型。

ollama run mistral-small

4位量化（默认别名）：

ollama run mistral-small:24b-instruct-2501-q4_K_M

8位量化：

ollama run mistral-small:24b-instruct-2501-q8_0

FP16：

ollama run mistral-small:24b-instruct-2501-fp16

📄 许可证

本模型采用Apache 2.0许可证。如果你想了解更多关于我们如何处理你的个人数据的信息，请阅读我们的 隐私政策。