url = "http://:8000/v1/chat/completions" headers = {"Content-Type": "application/json", "Authorization": "Bearer token"}

model = "mistralai/Mistral-Small-24B-Instruct-2501"

def load_system_prompt(repo_id: str, filename: str) -> str: file_path = hf_hub_download(repo_id=repo_id, filename=filename) with open(file_path, "r") as file: system_prompt = file.read() today = datetime.today().strftime("%Y-%m-%d") yesterday = (datetime.today() - timedelta(days=1)).strftime("%Y-%m-%d") model_name = repo_id.split("/")[-1] return system_prompt.format(name=model_name, today=today, yesterday=yesterday)

SYSTEM_PROMPT = load_system_prompt(model, "SYSTEM_PROMPT.txt")

tools = [ { "type": "function", "function": { "name": "get_current_weather", "description": "獲取給定地點的當前天氣", "parameters": { "type": "object", "properties": { "city": { "type": "string", "description": "要查詢天氣的城市，例如 'San Francisco'", }, "state": { "type": "string", "description": "州的縮寫，例如 'CA' 代表加利福尼亞州", }, "unit": { "type": "string", "description": "溫度單位", "enum": ["celsius", "fahrenheit"], }, }, "required": ["city", "state", "unit"], }, }, }, { "type": "function", "function": { "name": "rewrite", "description": "重寫給定文本以提高清晰度", "parameters": { "type": "object", "properties": { "text": { "type": "string", "description": "要重寫的輸入文本", } }, }, }, }, ]

messages = [ {"role": "system", "content": SYSTEM_PROMPT}, { "role": "user", "content": "你能把下面的文章寫得更簡潔嗎？\n\nOpenAI是一個人工智能研究實驗室，由非營利組織OpenAI Incorporated及其營利性子公司OpenAI Limited Partnership組成。", }, { "role": "assistant", "content": "", "tool_calls": [ { "id": "bbc5b7ede", "type": "function", "function": { "name": "rewrite", "arguments": '{"text": "OpenAI is an artificial intelligence research laboratory consisting of the non-profit OpenAI Incorporated and its for-profit subsidiary corporation OpenAI Limited Partnership."}', }, } ], }, { "role": "tool", "content": '{"action":"rewrite","outcome":"OpenAI is a FOR-profit company."}', "tool_call_id": "bbc5b7ede", "name": "rewrite", }, { "role": "assistant", "content": "---\n\nOpenAI是一家營利性公司。", }, { "role": "user", "content": "你能告訴我達拉斯的華氏溫度是多少嗎？", }, ]

data = {"model": model, "messages": messages, "tools": tools}

response = requests.post(url, headers=headers, data=json.dumps(data)) import ipdb; ipdb.set_trace() print(response.json()["choices"][0]["message"]["tool_calls"])

[{'id': '8PdihwL6d', 'type': 'function', 'function': {'name': 'get_current_weather', 'arguments': '{"city": "Dallas", "state": "TX", "unit": "fahrenheit"}'}}]

</details>

#### 離線使用
```py
from vllm import LLM
from vllm.sampling_params import SamplingParams
from datetime import datetime, timedelta

SYSTEM_PROMPT = "你是一個對話式代理，總是直截了當地回答問題，每次準確回答後都以一隻貓的ASCII畫結尾。"

user_prompt = "請給我5種法語中非正式的“再見”表達方式。"

messages = [
    {
        "role": "system",
        "content": SYSTEM_PROMPT
    },
    {
        "role": "user",
        "content": user_prompt
    },
]

# 注意，在GPU上運行此模型需要超過60GB的GPU顯存
llm = LLM(model=model_name, tokenizer_mode="mistral", tensor_parallel_size=8)

sampling_params = SamplingParams(max_tokens=512, temperature=0.15)
outputs = llm.chat(messages, sampling_params=sampling_params)

print(outputs[0].outputs[0].text)
# 當然，以下是法語中5種非正式的“再見”表達方式：
#
# 1. À plus tard
# 2. À plus
# 3. Salut
# 4. À toute
# 5. Bisous
#
# ```
#  /\_/\
# ( o.o )
#  > ^ <
# ```

Transformers

如果你想使用Hugging Face transformers生成文本，可以這樣做：

from transformers import pipeline
import torch

messages = [
    {"role": "user", "content": "請給我5種法語中非正式的“再見”表達方式。"},
]
chatbot = pipeline("text-generation", model="mistralai/Mistral-Small-24B-Instruct-2501", max_new_tokens=256, torch_dtype=torch.bfloat16)
chatbot(messages)

Ollama

Ollama 可以在MacOS、Windows和Linux上本地運行此模型。

ollama run mistral-small

4位量化（默認別名）：

ollama run mistral-small:24b-instruct-2501-q4_K_M

8位量化：

ollama run mistral-small:24b-instruct-2501-q8_0

FP16：

ollama run mistral-small:24b-instruct-2501-fp16

📄 許可證

本模型採用Apache 2.0許可證。如果你想了解更多關於我們如何處理你的個人數據的信息，請閱讀我們的 隱私政策。