diff --git a/src/together/types/chat_completions.py b/src/together/types/chat_completions.py index 0efd34f..b3104b2 100644 --- a/src/together/types/chat_completions.py +++ b/src/together/types/chat_completions.py @@ -185,12 +185,53 @@ class ChatCompletionResponse(BaseModel): usage: UsageData | None = None +class ChatCompletionDeltaToolCalls(ToolCalls): + """One tool call fragment inside a streaming delta. + + Streaming splits a single tool call across several chunks, so every + fragment carries an ``index`` naming the call it belongs to. The + non-streaming :class:`ToolCalls` has no such field, so the index is + declared on a streaming-only subclass. Putting it on the shared class + instead would add an ``index`` key to non-streaming + :class:`ChatCompletionMessage` dumps, which is why the subclass exists. + """ + + index: int | None = None + + +class ChatCompletionDeltaContent(DeltaContent): + """Streaming delta for chat completion chunks. + + The API returns an explicit ``null`` for ``choices[n].delta.tool_calls`` on + text-only chunks, and for ``function.name`` / ``function.arguments`` inside + tool-call fragments, where the OpenAI streaming format either omits the + field or sends an empty string, never ``null`` + (https://github.com/togethercomputer/together-python/issues/160). + + Declaring these as typed optional fields makes ``null`` and *missing* parse + identically (to ``None``) and validates the items into + :class:`ChatCompletionDeltaToolCalls`, matching the non-streaming + :class:`ChatCompletionMessage`, so ``model_dump(exclude_none=True)`` + produces OpenAI-shaped deltas with the nulls omitted. + + ``role`` is declared for the same reason. It is sent on the first chunk of + a response and left out of later ones, so while it was undeclared a present + role and an absent one behaved differently for callers. It is typed as + ``str`` rather than :class:`MessageRole` on purpose: chunks are parsed one + at a time inside the streaming generator, so an unrecognised role value + would otherwise raise part way through and end the stream. + """ + + role: str | None = None + tool_calls: List[ChatCompletionDeltaToolCalls] | None = None + + class ChatCompletionChoicesChunk(BaseModel): index: int | None = None logprobs: float | None = None seed: int | None = None finish_reason: FinishReason | None = None - delta: DeltaContent | None = None + delta: ChatCompletionDeltaContent | None = None class ChatCompletionChunk(BaseModel): diff --git a/tests/unit/test_chat_completion_stream_types.py b/tests/unit/test_chat_completion_stream_types.py new file mode 100644 index 0000000..6b9588d --- /dev/null +++ b/tests/unit/test_chat_completion_stream_types.py @@ -0,0 +1,220 @@ +"""Regression tests for https://github.com/togethercomputer/together-python/issues/160 + +The Together API returns an explicit ``null`` where the OpenAI streaming format +either leaves the field out or sends an empty string, in three known places: + +1. ``choices[n].delta.tool_calls`` (text-only chunks, left out by OpenAI) +2. ``choices[n].delta.tool_calls[n].function.arguments`` (first tool-call chunk, + where only the name is given; OpenAI sends an empty string here so that + consumers can concatenate every fragment without a special case) +3. ``choices[n].delta.tool_calls[n].function.name`` (continuation chunks that + stream the JSON arguments incrementally, left out by OpenAI) + +``choices[n].delta.role`` has the same shape of problem from the other +direction: it is sent on the first chunk and left out of the rest, so while it +was undeclared a present role and an absent one behaved differently. + +These tests pin down that the parsed models normalize ``null`` to be +indistinguishable from a missing field, so OpenAI-compatible consumers do not +need Together-specific special cases. +""" + +from together.types import ChatCompletionChunk, ChatCompletionResponse +from together.types.chat_completions import FunctionCall, ToolCalls + + +def _chunk(delta: dict) -> ChatCompletionChunk: + """Build a chunk the way the SDK does: ChatCompletionChunk(**line.data).""" + return ChatCompletionChunk( + **{ + "id": "884581f24f0cfdd0-SJC", + "object": "chat.completion.chunk", + "created": 1725561260, + "model": "meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo", + "choices": [{"index": 0, "delta": delta, "finish_reason": None}], + } + ) + + +# Wire payloads as observed in issue #160 +TEXT_ONLY_DELTA_WITH_NULL = { + "role": "assistant", + "content": "Hello", + "tool_calls": None, +} +TEXT_ONLY_DELTA_OMITTED = {"role": "assistant", "content": "Hello"} +FIRST_TOOL_CALL_DELTA = { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "index": 0, + "id": "call_f7g2h8i9j0", + "type": "function", + "function": {"name": "get_current_weather", "arguments": None}, + } + ], +} +CONTINUATION_TOOL_CALL_DELTA = { + "tool_calls": [ + { + "index": 0, + "function": {"name": None, "arguments": '{"location": "San Fra'}, + } + ] +} + + +def _has_no_none_values(obj: object) -> bool: + if obj is None: + return False + if isinstance(obj, dict): + return all(_has_no_none_values(v) for v in obj.values()) + if isinstance(obj, list): + return all(_has_no_none_values(v) for v in obj) + return True + + +def test_null_tool_calls_parses_like_omitted_tool_calls() -> None: + """`tool_calls: null` (text-only chunks) must behave exactly like a + missing `tool_calls` field: attribute exists and is None in both cases.""" + with_null = _chunk(TEXT_ONLY_DELTA_WITH_NULL).choices[0].delta + omitted = _chunk(TEXT_ONLY_DELTA_OMITTED).choices[0].delta + + assert with_null is not None and omitted is not None + assert with_null.tool_calls is None + assert omitted.tool_calls is None # was AttributeError before the fix + assert with_null.content == omitted.content == "Hello" + + +def test_tool_call_delta_items_are_typed_models() -> None: + """Streaming tool-call fragments parse into the same ToolCalls/FunctionCall + models used by the non-streaming ChatCompletionMessage.""" + delta = _chunk(FIRST_TOOL_CALL_DELTA).choices[0].delta + assert delta is not None and delta.tool_calls is not None + + (tool_call,) = delta.tool_calls + assert isinstance(tool_call, ToolCalls) + assert isinstance(tool_call.function, FunctionCall) + assert tool_call.id == "call_f7g2h8i9j0" + assert tool_call.type == "function" + assert tool_call.function.name == "get_current_weather" + # null arguments on the first chunk normalizes to None (absent) + assert tool_call.function.arguments is None + + +def test_null_function_name_on_continuation_chunks() -> None: + """`function.name: null` on argument-continuation chunks normalizes to + None while the incremental arguments fragment is preserved verbatim.""" + delta = _chunk(CONTINUATION_TOOL_CALL_DELTA).choices[0].delta + assert delta is not None and delta.tool_calls is not None + + (tool_call,) = delta.tool_calls + assert tool_call.function is not None + assert tool_call.function.name is None + assert tool_call.function.arguments == '{"location": "San Fra' + + +def test_exclude_none_dump_produces_openai_shaped_deltas() -> None: + """model_dump(exclude_none=True) must omit every API-provided null, + including the ones nested inside tool_calls[n].function.""" + for wire_delta in ( + TEXT_ONLY_DELTA_WITH_NULL, + TEXT_ONLY_DELTA_OMITTED, + FIRST_TOOL_CALL_DELTA, + CONTINUATION_TOOL_CALL_DELTA, + ): + delta = _chunk(wire_delta).choices[0].delta + assert delta is not None + dumped = delta.model_dump(exclude_none=True) + assert _has_no_none_values(dumped), f"None survived in {dumped!r}" + + text_only = _chunk(TEXT_ONLY_DELTA_WITH_NULL).choices[0].delta + assert text_only is not None + assert "tool_calls" not in text_only.model_dump(exclude_none=True) + + +def test_role_parses_the_same_whether_sent_or_left_out() -> None: + """`delta.role` is sent on the first chunk and left out of later ones, so + both must give the same attribute rather than one raising AttributeError.""" + first = _chunk(TEXT_ONLY_DELTA_WITH_NULL).choices[0].delta + later = _chunk(CONTINUATION_TOOL_CALL_DELTA).choices[0].delta + + assert first is not None and later is not None + assert first.role == "assistant" + assert later.role is None # was AttributeError before the fix + # An unrecognised role must not end the stream, which is why the field is + # typed as str rather than the MessageRole enum. + assert _chunk({"role": "some_future_role"}).choices[0].delta.role == ( + "some_future_role" + ) + + +def test_tool_call_index_is_typed_on_the_streaming_model_only() -> None: + """Streaming splits one tool call across chunks, so `index` says which call + a fragment belongs to. + + It is declared on a streaming-only subclass. Declaring it on the shared + ToolCalls instead would add an `index` key to non-streaming message dumps, + so this pins the separation in both directions. + """ + delta = _chunk(FIRST_TOOL_CALL_DELTA).choices[0].delta + assert delta is not None and delta.tool_calls is not None + + (tool_call,) = delta.tool_calls + assert isinstance(tool_call, ToolCalls) + assert tool_call.index == 0 + assert "index" in type(tool_call).model_fields + assert "index" not in ToolCalls.model_fields + assert delta.model_dump(exclude_none=True)["tool_calls"][0]["index"] == 0 + + +def test_undeclared_wire_fields_are_still_preserved() -> None: + """Fields the SDK does not declare must keep flowing through, as they did + before the fix (extra="allow"), so new API fields are never dropped.""" + delta = ( + _chunk({"role": "assistant", "content": "Hi", "future_field": 7}) + .choices[0] + .delta + ) + assert delta is not None + assert delta.future_field == 7 # type: ignore[attr-defined] + assert delta.model_dump(exclude_none=True)["future_field"] == 7 + + +def test_non_streaming_tool_calls_unchanged() -> None: + """Non-streaming responses keep parsing tool_calls into typed models.""" + response = ChatCompletionResponse( + **{ + "id": "884581f24f0cfdd0-SJC", + "object": "chat.completion", + "created": 1725561260, + "model": "meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo", + "choices": [ + { + "index": 0, + "finish_reason": "tool_calls", + "message": { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_f7g2h8i9j0", + "type": "function", + "function": { + "name": "get_current_weather", + "arguments": '{"location": "San Francisco, CA"}', + }, + } + ], + }, + } + ], + } + ) + assert response.choices is not None + message = response.choices[0].message + assert message is not None and message.tool_calls is not None + assert isinstance(message.tool_calls[0], ToolCalls) + assert message.tool_calls[0].function is not None + assert message.tool_calls[0].function.name == "get_current_weather"