Fixing bug with max tokens
This commit is contained in:
parent
fbca788cbf
commit
588b0a434d
@ -165,10 +165,16 @@ class Pipeline:
|
||||
}
|
||||
else:
|
||||
reasoning_config = {}
|
||||
|
||||
# If budget_tokens is greater than max_tokens, adjust max_tokens to MAX_COMBINED_TOKENS
|
||||
max_tokens = body.get("max_tokens", MAX_COMBINED_TOKENS)
|
||||
if max_tokens < budget_tokens and budget_tokens > 0:
|
||||
max_tokens = MAX_COMBINED_TOKENS
|
||||
|
||||
payload = {"modelId": model_id,
|
||||
"messages": processed_messages,
|
||||
"system": [{'text': system_message['content'] if system_message else 'you are an intelligent ai assistant'}],
|
||||
"inferenceConfig": {"temperature": 1, "maxTokens": body.get("max_tokens", MAX_COMBINED_TOKENS)},
|
||||
"inferenceConfig": {"temperature": 1, "maxTokens": max_tokens },
|
||||
"additionalModelRequestFields": reasoning_config
|
||||
}
|
||||
if body.get("stream", False):
|
||||
|
Loading…
x
Reference in New Issue
Block a user