SchemathesisでOpenAPIスキーマからプロパティベーステスト(PBT)を行ってみる

uv add fastapi "uvicorn[standard]" schemathesis pytest

import uuid
from typing import Optional

from fastapi import Depends, FastAPI, Header, HTTPException
from pydantic import BaseModel, Field

app = FastAPI(title="Booking API", version="1.0.0")

BOOKINGS: dict[str, dict] = {}

def verify_token(authorization: Optional[str] = Header(None)) -> bool:
    if not authorization or authorization != "Bearer secret-token":
        raise HTTPException(status_code=401, detail="Invalid or missing token")
    return True

class BookingRequest(BaseModel):
    guest_name: str = Field(min_length=2, max_length=100)
    room_type: str
    nights: int = Field(gt=0, le=365)

class BookingResponse(BaseModel):
    booking_id: str
    guest_name: str
    room_type: str
    nights: int
    status: str
    price_per_night: float
    total_price: float

@app.post("/bookings", response_model=BookingResponse, responses={400: {"description": "Invalid booking"}})  # type: ignore[misc]
def create_booking(booking: BookingRequest, _: bool = Depends(verify_token)) -> BookingResponse:
    # Calculate price based on room type
    room_prices = {"standard": 99.99, "deluxe": 149.99, "suite": 299.99}

    price_per_night = room_prices[booking.room_type]
    total_price = price_per_night * booking.nights

    booking_id = str(uuid.uuid4())
    booking_data = {
        "booking_id": booking_id,
        "guest_name": booking.guest_name,
        "room_type": booking.room_type,
        "nights": booking.nights,
        "status": "confirmed",
        "price_per_night": price_per_night,
        "total_price": total_price,
    }
    BOOKINGS[booking_id] = booking_data
    return BookingResponse(**booking_data)

@app.get(  # type: ignore[misc]
    "/bookings/{booking_id}", response_model=BookingResponse, responses={404: {"description": "Booking not found"}}
)
def get_booking(booking_id: str, _: bool = Depends(verify_token)) -> BookingResponse:
    if booking_id not in BOOKINGS:
        raise HTTPException(status_code=404, detail="Booking not found")
    return BookingResponse(**BOOKINGS[booking_id])

@app.get("/health")  # type: ignore[misc]
def health_check() -> dict:
    return {"status": "healthy"}

uvicorn app:app --reload
INFO:     Will watch for changes in these directories: ['/tmp/pytest-schemathesis']
INFO:     Uvicorn running on http://127.0.0.1:8000 (Press CTRL+C to quit)
INFO:     Started reloader process [79391] using WatchFiles
INFO:     Started server process [79393]
INFO:     Waiting for application startup.
INFO:     Application startup complete.

import schemathesis
from hypothesis import settings

# from_url を使って、OpenAPIスキーマの場所を指定する
schema = schemathesis.openapi.from_url("http://127.0.0.1:8000/openapi.json")

# @schema.parametrize() デコレータをテスト関数につけるだけでSchemathesisがスキーマ内の各エンドポイントに対してテストケースを自動生成する
@schema.parametrize()
@settings(max_examples=500)
def test_api(case):
    """
    APIがスキーマ定義に準拠しているかをテストする。

    Args:
        case: Schemathesisが生成した個々のテストケース。
              リクエストのメソッド、パス、ヘッダー、ボディなどが含まれる。
    """
    # case.call_and_validate() は、以下の処理をまとめて実行してくれる便利なメソッド
    case.call_and_validate(headers={"Authorization": "Bearer secret-token"})

headers={"Authorization": "Bearer secret-token"}

$ pytest test_api.py -v
Test session starts (platform: darwin, Python 3.11.4, pytest 8.4.2, pytest-sugar 1.1.1)
...                                                                                                                                                                                                             
―――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――― test_api[POST /bookings] ―――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――
  + Exception Group Traceback (most recent call last):
  |   File "/tmp/venv/lib/python3.11/site-packages/schemathesis/generation/hypothesis/builder.py", line 226, in test_api
  |     def test_wrapper(*args: Any, **kwargs: Any) -> Any:
  |                ^^^
  |   File "/tmp/venv/lib/python3.11/site-packages/hypothesis/core.py", line 1988, in wrapped_test
  |     _raise_to_user(errors, state.settings, [], " in explicit examples")
  |   File "/tmp/venv/lib/python3.11/site-packages/hypothesis/core.py", line 1582, in _raise_to_user
  |     raise the_error_hypothesis_found
  |   File "/tmp/venv/lib/python3.11/site-packages/schemathesis/generation/hypothesis/builder.py", line 228, in test_wrapper
  |     return test_function(*args, **kwargs)
  |            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  |   File "/tmp/pytest-schemathesis/test_api.py", line 20, in test_api
  |     case.call_and_validate(headers={"Authorization": "Bearer secret-token"})
  |   File "/tmp/venv/lib/python3.11/site-packages/schemathesis/generation/case.py", line 476, in call_and_validate
  |     self.validate_response(
  |   File "/tmp/venv/lib/python3.11/site-packages/schemathesis/generation/case.py", line 450, in validate_response
  |     raise FailureGroup(_failures, message) from None
  | schemathesis.core.failures.FailureGroup: Schemathesis found 2 distinct failures
  | 
  | - Server error
  | 
  | - Undocumented HTTP status code
  | 
  |     Received: 500
  |     Documented: 200, 400, 422
  | 
  | [500] Internal Server Error:
  | 
  |     `Internal Server Error`
  | 
  | Reproduce with: 
  | 
  |     curl -X POST -H 'Authorization: [Filtered]' -H 'Content-Type: application/json' -d '{"guest_name": "00", "nights": 1, "room_type": ""}' http://127.0.0.1:8000/bookings
  | 
  |  (2 sub-exceptions)
  +-+---------------- 1 ----------------
    | schemathesis.core.failures.ServerError: Server error
    +---------------- 2 ----------------
    | schemathesis.openapi.checks.UndefinedStatusCode: Undocumented HTTP status code
    | 
    | Received: 500
    | Documented: 200, 400, 422
    +------------------------------------

 ../../venv/lib/python3.11/site-packages/schemathesis/generation/hypothesis/builder.py::test_api[POST /bookings] ⨯                                                                                                   33% ███▍      
 ../../venv/lib/python3.11/site-packages/schemathesis/generation/hypothesis/builder.py::test_api[GET /bookings/{booking_id}] ✓                                                                                       67% ██████▋   
 ../../venv/lib/python3.11/site-packages/schemathesis/generation/hypothesis/builder.py::test_api[GET /health] ✓                                                                                                     100% ██████████
===================================================================================================== short test summary info =====================================================================================================
FAILED test_api.py::test_api[POST /bookings]

Results (12.78s):
       2 passed
       1 failed
         - ../../venv/lib/python3.11/site-packages/schemathesis/generation/hypothesis/builder.py:225 test_api[POST /bookings]

―――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――― test_api[POST /bookings] ―――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――
...
  | - Server error
  | 
  | - Undocumented HTTP status code
  | 
  |     Received: 500
  |     Documented: 200, 400, 422
  | 
  | [500] Internal Server Error:
  | 
  |     `Internal Server Error`
  | 
  | Reproduce with: 
  | 
  |     curl -X POST -H 'Authorization: [Filtered]' -H 'Content-Type: application/json' -d '{"guest_name": "00", "nights": 1, "room_type": ""}' http://127.0.0.1:8000/bookings
...

$ curl -X POST -H 'Authorization: [Filtered]' -H 'Content-Type: application/json' -d '{"guest_name": "00", "nights": 1, "room_type": ""}' http://127.0.0.1:8000/bookings
{"detail":"Internal Server Error"}%   

room_prices = {"standard": 99.99, "deluxe": 149.99, "suite": 299.99}

price_per_night = room_prices[booking.room_type]

$ diff --git a/pytest-schemathesis/app.py b/pytest-schemathesis/app.py
--- a/pytest-schemathesis/app.py	(revision 12231fe1bfc5ed8caf1f5d6430210d8c50cdc0aa)
+++ b/pytest-schemathesis/app.py	(date 1762762281090)
@@ -1,4 +1,5 @@
 import uuid
+from enum import Enum
 from typing import Optional

 from fastapi import Depends, FastAPI, Header, HTTPException
@@ -15,9 +16,15 @@
     return True

+class RoomType(str, Enum):
+    standard = "standard"
+    deluxe = "deluxe"
+    suite = "suite"
+
+
 class BookingRequest(BaseModel):
     guest_name: str = Field(min_length=2, max_length=100)
-    room_type: str
+    room_type: RoomType
     nights: int = Field(gt=0, le=365)

$ pytest test_api.py -v                                                                   
Test session starts (platform: darwin, Python 3.11.4, pytest 8.4.2, pytest-sugar 1.1.1)
...

collected 3 items

 ../../venv/lib/python3.11/site-packages/schemathesis/generation/hypothesis/builder.py::test_api[POST /bookings] ✓                                                                                                   33% ███▍      
 ../../venv/lib/python3.11/site-packages/schemathesis/generation/hypothesis/builder.py::test_api[GET /bookings/{booking_id}] ✓                                                                                       67% ██████▋   
 ../../venv/lib/python3.11/site-packages/schemathesis/generation/hypothesis/builder.py::test_api[GET /health] ✓                                                                                                     100% ██████████

Results (20.08s):
       3 passed