Rank1 32b

from vllm import LLM, SamplingParams
import math

# Initialize the model with vLLM
model = LLM(
    model="jhu-clsp/rank1-32b",
    tensor_parallel_size=1,  # Number of GPUs
    trust_remote_code=True,
    max_model_len=16000,     # Context length
    gpu_memory_utilization=0.9,
    dtype="float16",
)

# Set up sampling parameters
sampling_params = SamplingParams(
    temperature=0,
    max_tokens=8192,
    logprobs=20,
    stop=["</think> true", "</think> false"],
    skip_special_tokens=False
)

# Prepare the prompt
def create_prompt(query, document):
    return (
        "Determine if the following passage is relevant to the query. "
        "Answer only with 'true' or 'false'.\n"
        f"Query: {query}\n"
        f"Passage: {document}\n"
        "<think>"
    )

# Example usage
query = "What are the effects of climate change?"
document = "Climate change leads to rising sea levels, extreme weather events, and disruptions to ecosystems. These effects are caused by increasing greenhouse gas concentrations in the atmosphere due to human activities."

# Generate prediction
prompt = create_prompt(query, document)
outputs = model.generate([prompt], sampling_params)

# Extract score
output = outputs[0].outputs[0]
text = output.text
final_logits = output.logprobs[-1]

# Get token IDs for "true" and "false" tokens
from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained("jhu-clsp/rank1-32b")
true_token = tokenizer(" true", add_special_tokens=False).input_ids[0]
false_token = tokenizer(" false", add_special_tokens=False).input_ids[0]

# Calculate relevance score (probability of "true")
true_logit = final_logits[true_token].logprob
false_logit = final_logits[false_token].logprob
true_score = math.exp(true_logit)
false_score = math.exp(false_logit)
relevance_score = true_score / (true_score + false_score)

print(f"Reasoning chain: {text}")
print(f"Relevance score: {relevance_score}")