> ## Documentation Index
> Fetch the complete documentation index at: https://togetherai-migration.mintlify.site/llms.txt
> Use this file to discover all available pages before exploring further.
# Create Chat Completion
> Query a chat model.
## OpenAPI
````yaml POST /chat/completions
openapi: 3.1.0
info:
title: Together APIs
description: The Together REST API. Please see https://docs.together.ai for more details.
version: 2.0.0
termsOfService: https://www.together.ai/terms-of-service
contact:
name: Together Support
url: https://www.together.ai/contact
license:
name: MIT
url: https://github.com/togethercomputer/openapi/blob/main/LICENSE
servers:
- url: https://api.together.xyz/v1
security:
- bearerAuth: []
paths:
/chat/completions:
post:
tags:
- Chat
summary: Create chat completion
description: Query a chat model.
operationId: chat-completions
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/ChatCompletionRequest'
responses:
'200':
description: '200'
content:
application/json:
schema:
$ref: '#/components/schemas/ChatCompletionResponse'
text/event-stream:
schema:
$ref: '#/components/schemas/ChatCompletionStream'
'400':
description: BadRequest
content:
application/json:
schema:
$ref: '#/components/schemas/ErrorData'
'401':
description: Unauthorized
content:
application/json:
schema:
$ref: '#/components/schemas/ErrorData'
'404':
description: NotFound
content:
application/json:
schema:
$ref: '#/components/schemas/ErrorData'
'429':
description: RateLimit
content:
application/json:
schema:
$ref: '#/components/schemas/ErrorData'
'503':
description: Overloaded
content:
application/json:
schema:
$ref: '#/components/schemas/ErrorData'
'504':
description: Timeout
content:
application/json:
schema:
$ref: '#/components/schemas/ErrorData'
deprecated: false
components:
schemas:
ChatCompletionRequest:
type: object
required:
- model
- messages
properties:
messages:
type: array
description: A list of messages comprising the conversation so far.
items:
type: object
properties:
role:
type: string
description: >-
The role of the messages author. Choice between: system, user,
assistant, or tool.
enum:
- system
- user
- assistant
- tool
content:
description: >-
The content of the message, which can either be a simple
string or a structured format.
type: string
oneOf:
- type: string
description: A plain text message.
- type: array
description: A structured message with mixed content types.
items:
type: object
oneOf:
- type: object
properties:
type:
type: string
enum:
- text
text:
type: string
required:
- type
- text
- type: object
properties:
type:
type: string
enum:
- image_url
image_url:
type: object
properties:
url:
type: string
description: The URL of the image
required:
- url
- type: object
title: Video
properties:
type:
type: string
enum:
- video_url
video_url:
type: object
properties:
url:
type: string
description: The URL of the video
required:
- url
required:
- type
- video_url
required:
- role
- content
model:
description: >
The name of the model to query.
[See all of Together AI's
chat
models](https://docs.together.ai/docs/serverless-models#chat-models)
example: meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo
anyOf:
- type: string
enum:
- Qwen/Qwen2.5-72B-Instruct-Turbo
- Qwen/Qwen2.5-7B-Instruct-Turbo
- meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo
- meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo
- meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo
- type: string
max_tokens:
type: integer
description: The maximum number of tokens to generate.
stop:
type: array
description: >-
A list of string sequences that will truncate (stop) inference text
output. For example, "" will stop generation as soon as the
model generates the given token.
items:
type: string
temperature:
type: number
description: >-
A decimal number from 0-1 that determines the degree of randomness
in the response. A temperature less than 1 favors more correctness
and is appropriate for question answering or summarization. A value
closer to 1 introduces more randomness in the output.
format: float
top_p:
type: number
description: >-
A percentage (also called the nucleus parameter) that's used to
dynamically adjust the number of choices for each predicted token
based on the cumulative probabilities. It specifies a probability
threshold below which all less likely tokens are filtered out. This
technique helps maintain diversity and generate more fluent and
natural-sounding text.
format: float
top_k:
type: integer
description: >-
An integer that's used to limit the number of choices for the next
predicted word or token. It specifies the maximum number of tokens
to consider at each step, based on their probability of occurrence.
This technique helps to speed up the generation process and can
improve the quality of the generated text by focusing on the most
likely options.
format: int32
context_length_exceeded_behavior:
type: string
enum:
- truncate
- error
default: error
description: >-
Defined the behavior of the API when max_tokens exceed the maximum
context length of the model. When set to 'error', API will return
400 with appropriate error message. When set to 'truncate', override
the max_tokens with maximum context length of the model.
repetition_penalty:
type: number
description: >-
A number that controls the diversity of generated text by reducing
the likelihood of repeated sequences. Higher values decrease
repetition.
stream:
type: boolean
description: >-
If true, stream tokens as Server-Sent Events as the model generates
them instead of waiting for the full model response. The stream
terminates with `data: [DONE]`. If false, return a single JSON
object containing the results.
logprobs:
type: integer
minimum: 0
maximum: 1
description: >-
Integer (0 or 1) that controls whether log probabilities of
generated tokens are returned. Log probabilities help assess model
confidence in token predictions.
echo:
type: boolean
description: >-
If true, the response will contain the prompt. Can be used with
`logprobs` to return prompt logprobs.
'n':
type: integer
description: The number of completions to generate for each prompt.
minimum: 1
maximum: 128
min_p:
type: number
description: >-
A number between 0 and 1 that can be used as an alternative to top_p
and top-k.
format: float
presence_penalty:
type: number
description: >-
A number between -2.0 and 2.0 where a positive value increases the
likelihood of a model talking about new topics.
format: float
frequency_penalty:
type: number
description: >-
A number between -2.0 and 2.0 where a positive value decreases the
likelihood of repeating tokens that have already been mentioned.
format: float
logit_bias:
type: object
additionalProperties:
type: number
format: float
description: >-
Adjusts the likelihood of specific tokens appearing in the generated
output.
example:
'105': 21.4
'1024': -10.5
seed:
type: integer
description: Seed value for reproducibility.
example: 42
function_call:
oneOf:
- type: string
enum:
- none
- auto
- type: object
required:
- name
properties:
name:
type: string
response_format:
type: object
description: An object specifying the format that the model must output.
properties:
type:
description: The type of the response format.
type: string
example: json
schema:
description: The schema of the response format.
type: object
additionalProperties: true
tools:
type: array
description: >-
A list of tools the model may call. Currently, only functions are
supported as a tool. Use this to provide a list of functions the
model may generate JSON inputs for.
items:
$ref: '#/components/schemas/ToolsPart'
tool_choice:
description: >-
Controls which (if any) function is called by the model. By default
uses `auto`, which lets the model pick between generating a message
or calling a function.
oneOf:
- type: string
example: tool_name
- $ref: '#/components/schemas/ToolChoice'
safety_model:
type: string
description: >-
The name of the moderation model used to validate tokens. Choose
from the available moderation models found
[here](https://docs.together.ai/docs/inference-models#moderation-models).
example: safety_model_name
ChatCompletionResponse:
type: object
properties:
id:
type: string
choices:
$ref: '#/components/schemas/ChatCompletionChoicesData'
usage:
$ref: '#/components/schemas/UsageData'
created:
type: integer
model:
type: string
object:
type: string
enum:
- chat.completion
required:
- choices
- id
- created
- model
- object
ChatCompletionStream:
oneOf:
- $ref: '#/components/schemas/ChatCompletionEvent'
- $ref: '#/components/schemas/StreamSentinel'
ErrorData:
type: object
required:
- error
properties:
error:
type: object
properties:
message:
type: string
nullable: false
type:
type: string
nullable: false
param:
type: string
nullable: true
default: null
code:
type: string
nullable: true
default: null
required:
- type
- message
ToolsPart:
type: object
properties:
type:
type: string
example: tool_type
function:
type: object
properties:
description:
type: string
example: A description of the function.
name:
type: string
example: function_name
parameters:
type: object
additionalProperties: true
description: A map of parameter names to their values.
ToolChoice:
type: object
required:
- id
- type
- function
- index
properties:
index:
type: number
id:
type: string
type:
type: string
enum:
- function
function:
type: object
required:
- name
- arguments
properties:
name:
type: string
example: function_name
arguments:
type: string
ChatCompletionChoicesData:
type: array
items:
type: object
properties:
text:
type: string
index:
type: integer
seed:
type: integer
finish_reason:
$ref: '#/components/schemas/FinishReason'
message:
$ref: '#/components/schemas/ChatCompletionMessage'
logprobs:
allOf:
- nullable: true
- $ref: '#/components/schemas/LogprobsPart'
UsageData:
type: object
properties:
prompt_tokens:
type: integer
completion_tokens:
type: integer
total_tokens:
type: integer
required:
- prompt_tokens
- completion_tokens
- total_tokens
nullable: true
ChatCompletionEvent:
type: object
required:
- data
properties:
data:
$ref: '#/components/schemas/ChatCompletionChunk'
StreamSentinel:
type: object
required:
- data
properties:
data:
title: stream_signal
type: string
enum:
- '[DONE]'
FinishReason:
type: string
enum:
- stop
- eos
- length
- tool_calls
- function_call
ChatCompletionMessage:
type: object
required:
- role
- content
properties:
content:
type: string
nullable: true
role:
type: string
enum:
- assistant
tool_calls:
type: array
items:
$ref: '#/components/schemas/ToolChoice'
function_call:
type: object
deprecated: true
required:
- arguments
- name
properties:
arguments:
type: string
name:
type: string
LogprobsPart:
type: object
properties:
token_ids:
type: array
items:
type: number
description: List of token IDs corresponding to the logprobs
tokens:
type: array
items:
type: string
description: List of token strings
token_logprobs:
type: array
items:
type: number
description: List of token log probabilities
ChatCompletionChunk:
type: object
required:
- id
- object
- created
- choices
- model
properties:
id:
type: string
object:
type: string
enum:
- chat.completion.chunk
created:
type: integer
system_fingerprint:
type: string
model:
type: string
example: mistralai/Mixtral-8x7B-Instruct-v0.1
choices:
title: ChatCompletionChoices
type: array
items:
type: object
required:
- index
- delta
- finish_reason
properties:
index:
type: integer
finish_reason:
$ref: '#/components/schemas/FinishReason'
nullable: true
logprobs:
type: number
nullable: true
seed:
type: integer
nullable: true
delta:
title: ChatCompletionChoiceDelta
type: object
required:
- role
properties:
token_id:
type: integer
role:
type: string
enum:
- system
- user
- assistant
- function
- tool
content:
type: string
nullable: true
tool_calls:
type: array
items:
$ref: '#/components/schemas/ToolChoice'
function_call:
type: object
deprecated: true
nullable: true
properties:
arguments:
type: string
name:
type: string
required:
- arguments
- name
usage:
allOf:
- $ref: '#/components/schemas/UsageData'
- nullable: true
securitySchemes:
bearerAuth:
type: http
scheme: bearer
x-bearer-format: bearer
x-default: default
````