diff --git a/docs/devel_doc/openapi.json b/docs/devel_doc/openapi.json index 3be087e33..e142f5769 100644 --- a/docs/devel_doc/openapi.json +++ b/docs/devel_doc/openapi.json @@ -14349,7 +14349,14 @@ "anyOf": [ { "items": { - "$ref": "#/components/schemas/FeedbackCategory" + "anyOf": [ + { + "$ref": "#/components/schemas/FeedbackCategory" + }, + { + "$ref": "#/components/schemas/PositiveFeedbackCategory" + } + ] }, "type": "array" }, @@ -14358,11 +14365,16 @@ } ], "title": "Categories", - "description": "List of feedback categories that describe issues with the LLM response (for negative feedback).", + "description": "List of positive or negative feedback categories.", "examples": [ [ "incorrect", "incomplete" + ], + [ + "helpful", + "accurate", + "resolved_issue" ] ] } @@ -14375,7 +14387,7 @@ "llm_response" ], "title": "FeedbackRequest", - "description": "Model representing a feedback request.\n\nAttributes:\n conversation_id: The required conversation ID (UUID).\n user_question: The required user question.\n llm_response: The required LLM response.\n sentiment: The optional sentiment.\n user_feedback: The optional user feedback.\n categories: The optional list of feedback categories (multi-select for negative feedback).", + "description": "Model representing a feedback request.\n\nAttributes:\n conversation_id: The required conversation ID (UUID).\n user_question: The required user question.\n llm_response: The required LLM response.\n sentiment: The optional sentiment.\n user_feedback: The optional user feedback.\n categories: The optional list of positive or negative feedback categories.", "examples": [ { "conversation_id": "12345678-abcd-0000-0123-456789abcdef", @@ -14393,6 +14405,15 @@ "sentiment": -1, "user_question": "What is the capital of France?" }, + { + "categories": [ + "helpful" + ], + "conversation_id": "12345678-abcd-0000-0123-456789abcdef", + "llm_response": "The API is easy to use.", + "sentiment": 1, + "user_question": "How do I use the API?" + }, { "categories": [ "incomplete", @@ -18215,6 +18236,19 @@ "title": "PgvectorVectorStoreProviderConfig", "description": "Storage config for a pgvector dynamic vector-store provider." }, + "PositiveFeedbackCategory": { + "type": "string", + "enum": [ + "helpful", + "accurate", + "clear", + "relevant", + "actionable", + "resolved_issue" + ], + "title": "PositiveFeedbackCategory", + "description": "Enum representing predefined categories for positive AI responses.\n\nThese categories cover qualities commonly associated with a useful answer." + }, "PostgreSQLDatabaseConfiguration": { "properties": { "host": { diff --git a/docs/devel_doc/openapi.md b/docs/devel_doc/openapi.md index ff872ff8c..4131376a0 100644 --- a/docs/devel_doc/openapi.md +++ b/docs/devel_doc/openapi.md @@ -323,6 +323,7 @@ Lightspeed Core Stack (LCS) service API specification. * [OpenIdConnectSecurityScheme](#openidconnectsecurityscheme) * [PasswordOAuthFlow](#passwordoauthflow) * [PostgreSQLDatabaseConfiguration](#postgresqldatabaseconfiguration) + * [PositiveFeedbackCategory](#positivefeedbackcategory) * [PromptCreateRequest](#promptcreaterequest) * [PromptDeleteResponse](#promptdeleteresponse) * [PromptResourceResponse](#promptresourceresponse) @@ -6403,9 +6404,6 @@ These categories help provide structured feedback about AI inference quality when users provide negative feedback (thumbs down). Multiple categories can be selected to provide comprehensive feedback about response issues. - - - ## FeedbackRequest @@ -6417,7 +6415,7 @@ Attributes: llm_response: The required LLM response. sentiment: The optional sentiment. user_feedback: The optional user feedback. - categories: The optional list of feedback categories (multi-select for negative feedback). + categories: The optional list of positive or negative feedback categories. | Field | Type | Description | @@ -6427,7 +6425,7 @@ Attributes: | llm_response | string | Response from LLM | | sentiment | | User sentiment, if provided must be -1 or 1 | | user_feedback | | Feedback on the LLM response. | -| categories | | List of feedback categories that describe issues with the LLM response (for negative feedback). | +| categories | array of FeedbackCategory or PositiveFeedbackCategory | List of positive or negative feedback categories. | ## FeedbackResponse @@ -7797,6 +7795,10 @@ Useful resources: | ca_cert_path | | Path to CA certificate | +## PositiveFeedbackCategory + +Enum representing predefined categories for positive feedback. + ## PromptCreateRequest diff --git a/docs/models/requests.json b/docs/models/requests.json index 4a56726b3..32365245a 100644 --- a/docs/models/requests.json +++ b/docs/models/requests.json @@ -137,7 +137,7 @@ }, "FeedbackRequest": { "additionalProperties": false, - "description": "Model representing a feedback request.\n\nAttributes:\n conversation_id: The required conversation ID (UUID).\n user_question: The required user question.\n llm_response: The required LLM response.\n sentiment: The optional sentiment.\n user_feedback: The optional user feedback.\n categories: The optional list of feedback categories (multi-select for negative feedback).", + "description": "Model representing a feedback request.\n\nAttributes:\n conversation_id: The required conversation ID (UUID).\n user_question: The required user question.\n llm_response: The required LLM response.\n sentiment: The optional sentiment.\n user_feedback: The optional user feedback.\n categories: The optional list of positive or negative feedback categories.", "examples": [ { "conversation_id": "12345678-abcd-0000-0123-456789abcdef", @@ -155,6 +155,15 @@ "sentiment": -1, "user_question": "What is the capital of France?" }, + { + "categories": [ + "helpful" + ], + "conversation_id": "12345678-abcd-0000-0123-456789abcdef", + "llm_response": "The API is easy to use.", + "sentiment": 1, + "user_question": "How do I use the API?" + }, { "categories": [ "incomplete", @@ -215,13 +224,28 @@ }, "categories": { "type": "array", + "items": { + "anyOf": [ + { + "$ref": "`#/components/schemas/`FeedbackCategory" + }, + { + "$ref": "`#/components/schemas/`PositiveFeedbackCategory" + } + ] + }, "nullable": true, "default": null, - "description": "List of feedback categories that describe issues with the LLM response (for negative feedback).", + "description": "List of positive or negative feedback categories.", "examples": [ [ "incorrect", "incomplete" + ], + [ + "helpful", + "accurate", + "resolved_issue" ] ], "title": "Categories" @@ -1850,6 +1874,19 @@ "title": "OpenAITopLogProb", "type": "object" }, + "PositiveFeedbackCategory": { + "description": "Enum representing predefined categories for positive AI responses.\n\nThese categories cover qualities commonly associated with a useful answer.", + "enum": [ + "helpful", + "accurate", + "clear", + "relevant", + "actionable", + "resolved_issue" + ], + "title": "PositiveFeedbackCategory", + "type": "string" + }, "PromptCreateRequest": { "additionalProperties": false, "description": "Request body to create a stored prompt template in OGX.\n\nAttributes:\n prompt: Prompt text with variable placeholders.\n variables: Variable names allowed in the template.", diff --git a/docs/models/requests.md b/docs/models/requests.md index baf8562e7..6de9f2349 100644 --- a/docs/models/requests.md +++ b/docs/models/requests.md @@ -102,7 +102,7 @@ Attributes: llm_response: The required LLM response. sentiment: The optional sentiment. user_feedback: The optional user feedback. - categories: The optional list of feedback categories (multi-select for negative feedback). + categories: The optional list of positive or negative feedback categories. | Field | Type | Description | @@ -112,7 +112,7 @@ Attributes: | llm_response | string | Response from LLM | | sentiment | integer | User sentiment, if provided must be -1 or 1 | | user_feedback | string | Feedback on the LLM response. | -| categories | array | List of feedback categories that describe issues with the LLM response (for negative feedback). | +| categories | array | List of positive or negative feedback categories. | ## FeedbackStatusUpdateRequest @@ -839,6 +839,16 @@ The top log probability for a token from an OpenAI-compatible chat completion re | logprob | number | The log probability of the token. | +## PositiveFeedbackCategory + + +Enum representing predefined categories for positive AI responses. + +These categories cover qualities commonly associated with a useful answer. + + + + ## PromptCreateRequest diff --git a/src/models/api/requests/feedback.py b/src/models/api/requests/feedback.py index 0bfcd6865..684333a81 100644 --- a/src/models/api/requests/feedback.py +++ b/src/models/api/requests/feedback.py @@ -4,7 +4,7 @@ from pydantic import BaseModel, Field, field_validator, model_validator -from models.common import FeedbackCategory +from models.common import FeedbackCategory, PositiveFeedbackCategory from utils import suid @@ -17,7 +17,7 @@ class FeedbackRequest(BaseModel): llm_response: The required LLM response. sentiment: The optional sentiment. user_feedback: The optional user feedback. - categories: The optional list of feedback categories (multi-select for negative feedback). + categories: The optional list of positive or negative feedback categories. """ conversation_id: str = Field( @@ -51,14 +51,14 @@ class FeedbackRequest(BaseModel): examples=["I'm not satisfied with the response because it is too vague."], ) - # Optional list of predefined feedback categories for negative feedback - categories: Optional[list[FeedbackCategory]] = Field( + # Optional list of predefined feedback categories for positive and negative feedback + categories: Optional[list[FeedbackCategory | PositiveFeedbackCategory]] = Field( default=None, - description=( - "List of feedback categories that describe issues with the LLM response " - "(for negative feedback)." - ), - examples=[["incorrect", "incomplete"]], + description="List of positive or negative feedback categories.", + examples=[ + ["incorrect", "incomplete"], + ["helpful", "accurate", "resolved_issue"], + ], ) # provides examples for /docs endpoint @@ -80,6 +80,13 @@ class FeedbackRequest(BaseModel): "sentiment": -1, "categories": ["incorrect"], }, + { + "conversation_id": "12345678-abcd-0000-0123-456789abcdef", + "user_question": "How do I use the API?", + "llm_response": "The API is easy to use.", + "sentiment": 1, + "categories": ["helpful"], + }, { "conversation_id": "12345678-abcd-0000-0123-456789abcdef", "user_question": "How do I deploy a web app?", @@ -135,8 +142,9 @@ def check_sentiment(cls, value: Optional[int]) -> Optional[int]: @field_validator("categories") @classmethod def validate_categories( - cls, value: Optional[list[FeedbackCategory]] - ) -> Optional[list[FeedbackCategory]]: + cls, + value: Optional[list[FeedbackCategory | PositiveFeedbackCategory]], + ) -> Optional[list[FeedbackCategory | PositiveFeedbackCategory]]: """Normalize and deduplicate a feedback categories list. Converts an empty list to None for consistency and removes duplicate diff --git a/src/models/common/__init__.py b/src/models/common/__init__.py index deb19bff5..a6c34302c 100644 --- a/src/models/common/__init__.py +++ b/src/models/common/__init__.py @@ -6,7 +6,7 @@ ConversationTurn, Message, ) -from models.common.feedback import FeedbackCategory +from models.common.feedback import FeedbackCategory, PositiveFeedbackCategory from models.common.health import ( HealthStatus, ProviderHealthStatus, @@ -56,6 +56,7 @@ "MCPServerAuthInfo", "MCPServerInfo", "Message", + "PositiveFeedbackCategory", "ProviderHealthStatus", "RAGChunk", "RAGContext", diff --git a/src/models/common/feedback.py b/src/models/common/feedback.py index eb353181c..f23f09a3e 100644 --- a/src/models/common/feedback.py +++ b/src/models/common/feedback.py @@ -17,3 +17,17 @@ class FeedbackCategory(StrEnum): OUTDATED_INFORMATION = "outdated_information" # "This information is from several years ago and no longer accurate" # pylint: disable=line-too-long UNSAFE = "unsafe" # "This response could be harmful or dangerous if followed" OTHER = "other" # "The response has issues not covered by other categories" + + +class PositiveFeedbackCategory(StrEnum): + """Enum representing predefined categories for positive AI responses. + + These categories cover qualities commonly associated with a useful answer. + """ + + HELPFUL = "helpful" # "The response is useful" + ACCURATE = "accurate" # "The information provided is correct" + CLEAR = "clear" # "The response is easy to understand" + RELEVANT = "relevant" # "This answer addresses my question" + ACTIONABLE = "actionable" # "The response provides steps I can follow" + RESOLVED_ISSUE = "resolved_issue" # "This response solved my problem" diff --git a/tests/unit/app/endpoints/test_feedback.py b/tests/unit/app/endpoints/test_feedback.py index 3b0b18c6f..115ead8c9 100644 --- a/tests/unit/app/endpoints/test_feedback.py +++ b/tests/unit/app/endpoints/test_feedback.py @@ -138,13 +138,14 @@ async def test_assert_feedback_enabled_disabled_full_config_chain( "feedback_request_data", [ {**VALID_BASE, "sentiment": 1}, + {**VALID_BASE, "sentiment": 1, "categories": ["helpful"]}, { **VALID_BASE, "sentiment": -1, "categories": ["incorrect", "incomplete"], }, ], - ids=["no_categories", "with_negative_categories"], + ids=["no_categories", "with_positive_category", "with_negative_categories"], ) @pytest.mark.asyncio async def test_feedback_endpoint_handler( diff --git a/tests/unit/models/requests/test_feedback_request.py b/tests/unit/models/requests/test_feedback_request.py index 8472b8742..2a8c1760b 100644 --- a/tests/unit/models/requests/test_feedback_request.py +++ b/tests/unit/models/requests/test_feedback_request.py @@ -4,7 +4,7 @@ from pydantic import ValidationError from models.api.requests import FeedbackRequest -from models.common import FeedbackCategory +from models.common import FeedbackCategory, PositiveFeedbackCategory class TestFeedbackRequest: @@ -110,6 +110,74 @@ def test_with_single_category(self) -> None: ) assert fr.categories == [FeedbackCategory.INCORRECT] + def test_mixed_categories_are_deduplicated_and_serialize_as_strings(self) -> None: + """Test positive and negative categories retain order and serialize as strings.""" + wire_categories = [ + "helpful", + "incorrect", + "accurate", + "helpful", + "incorrect", + ] + fr = FeedbackRequest.model_validate( + { + "conversation_id": "123e4567-e89b-12d3-a456-426614174000", + "user_question": "What is Docker?", + "llm_response": "Docker is a container platform.", + "categories": wire_categories, + "sentiment": None, + } + ) + assert fr.categories == [ + PositiveFeedbackCategory.HELPFUL, + FeedbackCategory.INCORRECT, + PositiveFeedbackCategory.ACCURATE, + ] + assert fr.categories is not None + assert all( + isinstance(category, (FeedbackCategory, PositiveFeedbackCategory)) + for category in fr.categories + ) + assert fr.model_dump(mode="json")["categories"] == wire_categories[:3] + + def test_all_positive_feedback_categories_are_valid(self) -> None: + """Test each predefined positive category is accepted independent of sentiment.""" + positive_categories = [ + "helpful", + "accurate", + "clear", + "relevant", + "actionable", + "resolved_issue", + ] + fr = FeedbackRequest.model_validate( + { + "conversation_id": "123e4567-e89b-12d3-a456-426614174000", + "user_question": "What is Docker?", + "llm_response": "Docker is a container platform.", + "categories": positive_categories, + "sentiment": -1, + } + ) + assert fr.categories is not None + assert fr.model_dump(mode="json")["categories"] == positive_categories + assert all( + isinstance(category, PositiveFeedbackCategory) for category in fr.categories + ) + + def test_unknown_feedback_category_is_rejected(self) -> None: + """Test category strings outside the fixed enums are rejected.""" + with pytest.raises(ValidationError): + FeedbackRequest.model_validate( + { + "conversation_id": "123e4567-e89b-12d3-a456-426614174000", + "user_question": "What is Docker?", + "llm_response": "Docker is a container platform.", + "categories": ["domain_issue"], + "sentiment": 1, + } + ) + def test_categories_with_duplicates(self) -> None: """Test that duplicate categories are removed.""" fr = FeedbackRequest( @@ -179,13 +247,14 @@ def test_mixed_feedback_types(self) -> None: def test_all_feedback_categories(self) -> None: """Test that all defined feedback categories are valid.""" all_categories = list(FeedbackCategory) - - fr = FeedbackRequest( - conversation_id="123e4567-e89b-12d3-a456-426614174000", - user_question="Test question", - llm_response="Test response", - categories=all_categories, - sentiment=1, + fr = FeedbackRequest.model_validate( + { + "conversation_id": "123e4567-e89b-12d3-a456-426614174000", + "user_question": "Test question", + "llm_response": "Test response", + "categories": all_categories, + "sentiment": 1, + } ) assert fr.categories is not None assert len(fr.categories) == len(all_categories) @@ -203,6 +272,15 @@ def test_categories_invalid_type(self) -> None: sentiment=1, ) + with pytest.raises(ValidationError): + FeedbackRequest( + conversation_id="123e4567-e89b-12d3-a456-426614174000", + user_question="Test question", + llm_response="Test response", + categories=[None], # pyright: ignore[reportArgumentType] + sentiment=1, + ) + def test_empty_user_feedback_not_sufficient(self) -> None: """Test that an empty string for user_feedback is treated as no feedback.""" with pytest.raises( diff --git a/tests/unit/utils/dumpers/test_models_dumper.py b/tests/unit/utils/dumpers/test_models_dumper.py index 2dce7605a..99da5e83c 100644 --- a/tests/unit/utils/dumpers/test_models_dumper.py +++ b/tests/unit/utils/dumpers/test_models_dumper.py @@ -10179,6 +10179,7 @@ def test_dump_models(tmpdir: Path) -> None: "PgvectorVectorStoreProvider", "PgvectorVectorStoreProviderConfig", "PostgreSQLDatabaseConfiguration", + "PositiveFeedbackCategory", "PromptCreateRequest", "PromptDeleteResponse", "PromptResourceResponse",