File: test_beta_messages.py

package info (click to toggle)
anthropic-sdk-python 0.75.0-1
  • links: PTS, VCS
  • area: main
  • in suites: sid
  • size: 4,252 kB
  • sloc: python: 29,737; sh: 177; makefile: 5
file content (444 lines) | stat: -rw-r--r-- 15,187 bytes parent folder | download
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
from __future__ import annotations

import os
import json
import inspect
from typing import Any, Set, Dict, TypeVar, cast
from unittest import TestCase

import httpx
import pytest
from respx import MockRouter

from anthropic import Anthropic, AsyncAnthropic
from anthropic._compat import PYDANTIC_V1
from anthropic.types.beta.beta_message import BetaMessage
from anthropic.lib.streaming._beta_types import ParsedBetaMessageStreamEvent
from anthropic.resources.messages.messages import DEPRECATED_MODELS
from anthropic.lib.streaming._beta_messages import TRACKS_TOOL_INPUT, BetaMessageStream, BetaAsyncMessageStream

from .helpers import get_response, to_async_iter

base_url = os.environ.get("TEST_API_BASE_URL", "http://127.0.0.1:4010")
api_key = "my-anthropic-api-key"

sync_client = Anthropic(base_url=base_url, api_key=api_key, _strict_response_validation=True)
async_client = AsyncAnthropic(base_url=base_url, api_key=api_key, _strict_response_validation=True)

_T = TypeVar("_T")

# Expected message fixtures
EXPECTED_BASIC_MESSAGE = {
    "id": "msg_4QpJur2dWWDjF6C758FbBw5vm12BaVipnK",
    "model": "claude-3-opus-latest",
    "role": "assistant",
    "stop_reason": "end_turn",
    "type": "message",
    "content": [{"type": "text", "text": "Hello there!"}],
    "usage": {"input_tokens": 11, "output_tokens": 6},
}

EXPECTED_BASIC_EVENT_TYPES = [
    "message_start",
    "content_block_start",
    "content_block_delta",
    "text",
    "content_block_delta",
    "text",
    "content_block_delta",
    "text",
    "content_block_stop",
    "message_delta",
]

EXPECTED_TOOL_USE_MESSAGE = {
    "id": "msg_019Q1hrJbZG26Fb9BQhrkHEr",
    "model": "claude-sonnet-4-20250514",
    "role": "assistant",
    "stop_reason": "tool_use",
    "type": "message",
    "content": [
        {"type": "text", "text": "I'll check the current weather in Paris for you."},
        {
            "type": "tool_use",
            "id": "toolu_01NRLabsLyVHZPKxbKvkfSMn",
            "name": "get_weather",
            "input": {"location": "Paris"},
        },
    ],
    "usage": {
        "input_tokens": 377,
        "output_tokens": 65,
        "cache_creation_input_tokens": 0,
        "cache_read_input_tokens": 0,
        "service_tier": "standard",
    },
}

EXPECTED_TOOL_USE_EVENT_TYPES = [
    "message_start",
    "content_block_start",
    "content_block_delta",
    "text",
    "content_block_delta",
    "text",
    "content_block_stop",
    "content_block_start",
    "content_block_delta",
    "input_json",
    "content_block_delta",
    "input_json",
    "content_block_delta",
    "input_json",
    "content_block_delta",
    "input_json",
    "content_block_delta",
    "input_json",
    "content_block_stop",
    "message_delta",
]

EXPECTED_INCOMPLETE_MESSAGE = {
    "id": "msg_01UdjYBBipA9omjYhicnevgq",
    "model": "claude-3-7-sonnet-20250219",
    "role": "assistant",
    "stop_reason": "max_tokens",
    "type": "message",
    "content": [
        {
            "type": "text",
            "text": "I'll create a comprehensive tax guide for someone with multiple W2s and save it in a file called taxes.txt. Let me do that for you now.",
        },
        {
            "type": "tool_use",
            "id": "toolu_01EKqbqmZrGRXy18eN7m9kvY",
            "name": "make_file",
            "input": {
                "filename": "taxes.txt",
                "lines_of_text": [
                    "# COMPREHENSIVE TAX GUIDE FOR INDIVIDUALS WITH MULTIPLE W-2s",
                    "",
                    "## INTRODUCTION",
                    "",
                ],
            },
        },
    ],
    "usage": {
        "input_tokens": 450,
        "output_tokens": 124,
        "cache_creation_input_tokens": 0,
        "cache_read_input_tokens": 0,
        "service_tier": "standard",
    },
}

EXPECTED_INCOMPLETE_EVENT_TYPES = [
    "message_start",
    "content_block_start",
    "content_block_delta",
    "text",
    "content_block_delta",
    "text",
    "content_block_delta",
    "text",
    "content_block_delta",
    "text",
    "content_block_delta",
    "text",
    "content_block_stop",
    "content_block_start",
    "content_block_delta",
    "input_json",
    "content_block_delta",
    "input_json",
    "content_block_delta",
    "input_json",
    "content_block_delta",
    "input_json",
    "message_delta",
]


def assert_message_matches(message: BetaMessage, expected: Dict[str, Any]) -> None:
    actual_message_json = message.model_dump_json(
        indent=2, exclude_none=True, exclude={"content": {"__all__": {"__json_buf"}}}
    )

    test_case = TestCase()
    test_case.maxDiff = None
    test_case.assertEqual(expected, json.loads(actual_message_json))


def assert_basic_response(events: list[ParsedBetaMessageStreamEvent], message: BetaMessage) -> None:
    assert_message_matches(message, EXPECTED_BASIC_MESSAGE)
    assert [e.type for e in events] == EXPECTED_BASIC_EVENT_TYPES


def assert_tool_use_response(events: list[ParsedBetaMessageStreamEvent], message: BetaMessage) -> None:
    assert_message_matches(message, EXPECTED_TOOL_USE_MESSAGE)
    assert [e.type for e in events] == EXPECTED_TOOL_USE_EVENT_TYPES


def assert_incomplete_partial_input_response(events: list[ParsedBetaMessageStreamEvent], message: BetaMessage) -> None:
    assert_message_matches(message, EXPECTED_INCOMPLETE_MESSAGE)
    assert [e.type for e in events] == EXPECTED_INCOMPLETE_EVENT_TYPES


class TestSyncMessages:
    @pytest.mark.respx(base_url=base_url)
    def test_basic_response(self, respx_mock: MockRouter) -> None:
        respx_mock.post("/v1/messages").mock(
            return_value=httpx.Response(200, content=get_response("basic_response.txt"))
        )

        with sync_client.beta.messages.stream(
            max_tokens=1024,
            messages=[
                {
                    "role": "user",
                    "content": "Say hello there!",
                }
            ],
            model="claude-3-opus-latest",
        ) as stream:
            assert isinstance(cast(Any, stream), BetaMessageStream)

            assert_basic_response([event for event in stream], stream.get_final_message())

    @pytest.mark.respx(base_url=base_url)
    def test_tool_use(self, respx_mock: MockRouter) -> None:
        respx_mock.post("/v1/messages").mock(
            return_value=httpx.Response(200, content=get_response("tool_use_response.txt"))
        )

        with sync_client.beta.messages.stream(
            max_tokens=1024,
            messages=[
                {
                    "role": "user",
                    "content": "Say hello there!",
                }
            ],
            model="claude-sonnet-4-20250514",
        ) as stream:
            assert isinstance(cast(Any, stream), BetaMessageStream)

            assert_tool_use_response([event for event in stream], stream.get_final_message())

    @pytest.mark.respx(base_url=base_url)
    def test_context_manager(self, respx_mock: MockRouter) -> None:
        respx_mock.post("/v1/messages").mock(
            return_value=httpx.Response(200, content=get_response("basic_response.txt"))
        )

        with sync_client.beta.messages.stream(
            max_tokens=1024,
            messages=[
                {
                    "role": "user",
                    "content": "Say hello there!",
                }
            ],
            model="claude-3-opus-latest",
        ) as stream:
            assert not stream.response.is_closed

        # response should be closed even if the body isn't read
        assert stream.response.is_closed

    @pytest.mark.respx(base_url=base_url)
    def test_deprecated_model_warning_stream(self, respx_mock: MockRouter) -> None:
        for deprecated_model in DEPRECATED_MODELS:
            respx_mock.post("/v1/messages").mock(
                return_value=httpx.Response(200, content=get_response("basic_response.txt"))
            )

            with pytest.warns(DeprecationWarning, match=f"The model '{deprecated_model}' is deprecated"):
                with sync_client.beta.messages.stream(
                    max_tokens=1024,
                    messages=[{"role": "user", "content": "Hello"}],
                    model=deprecated_model,
                ) as stream:
                    # Consume the stream to ensure the warning is triggered
                    stream.until_done()


class TestAsyncMessages:
    @pytest.mark.asyncio
    @pytest.mark.respx(base_url=base_url)
    async def test_basic_response(self, respx_mock: MockRouter) -> None:
        respx_mock.post("/v1/messages").mock(
            return_value=httpx.Response(200, content=to_async_iter(get_response("basic_response.txt")))
        )

        async with async_client.beta.messages.stream(
            max_tokens=1024,
            messages=[
                {
                    "role": "user",
                    "content": "Say hello there!",
                }
            ],
            model="claude-opus-4-0",
        ) as stream:
            assert isinstance(cast(Any, stream), BetaAsyncMessageStream)

            assert_basic_response([event async for event in stream], await stream.get_final_message())

    @pytest.mark.asyncio
    @pytest.mark.respx(base_url=base_url)
    async def test_context_manager(self, respx_mock: MockRouter) -> None:
        respx_mock.post("/v1/messages").mock(
            return_value=httpx.Response(200, content=to_async_iter(get_response("basic_response.txt")))
        )

        async with async_client.beta.messages.stream(
            max_tokens=1024,
            messages=[
                {
                    "role": "user",
                    "content": "Say hello there!",
                }
            ],
            model="claude-3-opus-latest",
        ) as stream:
            assert not stream.response.is_closed

        # response should be closed even if the body isn't read
        assert stream.response.is_closed

    @pytest.mark.asyncio
    @pytest.mark.respx(base_url=base_url)
    async def test_deprecated_model_warning_stream(self, respx_mock: MockRouter) -> None:
        for deprecated_model in DEPRECATED_MODELS:
            respx_mock.post("/v1/messages").mock(
                return_value=httpx.Response(200, content=to_async_iter(get_response("basic_response.txt")))
            )

            with pytest.warns(DeprecationWarning, match=f"The model '{deprecated_model}' is deprecated"):
                async with async_client.beta.messages.stream(
                    max_tokens=1024,
                    messages=[{"role": "user", "content": "Hello"}],
                    model=deprecated_model,
                ) as stream:
                    # Consume the stream to ensure the warning is triggered
                    await stream.get_final_message()

    @pytest.mark.asyncio
    @pytest.mark.respx(base_url=base_url)
    async def test_tool_use(self, respx_mock: MockRouter) -> None:
        respx_mock.post("/v1/messages").mock(
            return_value=httpx.Response(200, content=to_async_iter(get_response("tool_use_response.txt")))
        )

        async with async_client.beta.messages.stream(
            max_tokens=1024,
            messages=[
                {
                    "role": "user",
                    "content": "Say hello there!",
                }
            ],
            model="claude-sonnet-4-20250514",
        ) as stream:
            assert isinstance(cast(Any, stream), BetaAsyncMessageStream)

            assert_tool_use_response([event async for event in stream], await stream.get_final_message())

    @pytest.mark.asyncio
    @pytest.mark.respx(base_url=base_url)
    async def test_incomplete_response(self, respx_mock: MockRouter) -> None:
        respx_mock.post("/v1/messages").mock(
            return_value=httpx.Response(
                200, content=to_async_iter(get_response("incomplete_partial_json_response.txt"))
            )
        )

        async with async_client.beta.messages.stream(
            max_tokens=1024,
            messages=[
                {
                    "role": "user",
                    "content": "Say hello there!",
                }
            ],
            model="claude-sonnet-4-20250514",
        ) as stream:
            assert isinstance(cast(Any, stream), BetaAsyncMessageStream)

            assert_incomplete_partial_input_response(
                [event async for event in stream], await stream.get_final_message()
            )


@pytest.mark.parametrize("sync", [True, False], ids=["sync", "async"])
def test_stream_method_definition_in_sync(sync: bool) -> None:
    client: Anthropic | AsyncAnthropic = sync_client if sync else async_client

    sig = inspect.signature(client.beta.messages.stream)
    generated_sig = inspect.signature(client.beta.messages.create)

    errors: list[str] = []

    for name, generated_param in generated_sig.parameters.items():
        if name == "stream":
            # intentionally excluded
            continue

        if name == "output_format":
            continue

        custom_param = sig.parameters.get(name)
        if not custom_param:
            errors.append(f"the `{name}` param is missing")
            continue

        if custom_param.annotation != generated_param.annotation:
            errors.append(
                f"types for the `{name}` param are do not match; generated={repr(generated_param.annotation)} custom={repr(custom_param.annotation)}"
            )
            continue

    if errors:
        raise AssertionError(
            f"{len(errors)} errors encountered with the {'sync' if sync else 'async'} client `messages.stream()` method:\n\n"
            + "\n\n".join(errors)
        )


# go through all the ContentBlock types to make sure the type alias is up to date
# with any type that has an input property of type object
def test_tracks_tool_input_type_alias_is_up_to_date() -> None:
    # only run this on Pydantic v2
    if PYDANTIC_V1:
        pytest.skip("This test is only applicable for Pydantic v2")
    from typing import get_args

    from pydantic import BaseModel

    from anthropic.types.beta.beta_content_block import BetaContentBlock

    # Get the content block union type
    content_block_union = get_args(BetaContentBlock)[0]

    # Get all types from BetaContentBlock union
    content_block_types = get_args(content_block_union)

    # Types that should have an input property
    types_with_input: Set[Any] = set()

    # Check each type to see if it has an input property in its model_fields
    for block_type in content_block_types:
        if issubclass(block_type, BaseModel) and "input" in block_type.model_fields:
            types_with_input.add(block_type)

    # Get the types included in TRACKS_TOOL_INPUT
    tracked_types = TRACKS_TOOL_INPUT

    # Make sure all types with input are tracked
    for block_type in types_with_input:
        assert block_type in tracked_types, (
            f"ContentBlock type {block_type.__name__} has an input property, "
            f"but is not included in TRACKS_TOOL_INPUT. You probably need to update the TRACKS_TOOL_INPUT type alias."
        )