Support protocol thread configuration
This commit is contained in:
@@ -62,6 +62,7 @@ def generate_once(
|
||||
timeout: int,
|
||||
num_ctx: int,
|
||||
num_predict: int,
|
||||
num_thread: int | None = None,
|
||||
) -> OllamaGeneration:
|
||||
payload = {
|
||||
"model": model,
|
||||
@@ -74,6 +75,10 @@ def generate_once(
|
||||
"num_predict": num_predict,
|
||||
},
|
||||
}
|
||||
if num_thread is not None:
|
||||
if type(num_thread) is not int or num_thread <= 0:
|
||||
raise ValueError("Protocol thread count must be a positive integer.")
|
||||
payload["options"]["num_thread"] = num_thread
|
||||
started = time.perf_counter()
|
||||
try:
|
||||
response = requests.post(generate_url(endpoint), json=payload, timeout=timeout)
|
||||
|
||||
Reference in New Issue
Block a user