From 6332f3d0f9537af38f59dc7be41d9a7869fe44bb Mon Sep 17 00:00:00 2001 From: RapidPoseidon Date: Mon, 26 Oct 2026 15:25:59 +0000 Subject: [PATCH 1/2] feat(job): require a capability below 10 responses per datapoint Checks the caller's LowResponseCount capability (userinfo claims) before any upload and raises LowResponseCountError with the backend's message. Ranking is not gated. Validation errors now render one 'field: message' line each. Co-Authored-By: Claude Opus 5.5 Co-Authored-By: luca@rapidata.ai <25279790+LucStr@users.noreply.github.com> --- docs/error_handling.md | 10 ++ docs/job_definition_parameters.md | 4 +- src/rapidata/__init__.py | 1 + src/rapidata/_skill/SKILL.md | 2 +- src/rapidata/_skill/reference.md | 2 +- src/rapidata/rapidata_client/__init__.py | 7 +- .../rapidata_client/exceptions/__init__.py | 2 + .../exceptions/low_response_count_error.py | 25 +++ .../exceptions/rapidata_error.py | 20 ++- .../rapidata_client/job/_response_minimum.py | 77 +++++++++ .../job/rapidata_job_definition.py | 11 +- .../job/rapidata_job_manager.py | 28 +++- .../job/test_response_minimum.py | 151 ++++++++++++++++++ 13 files changed, 323 insertions(+), 17 deletions(-) create mode 100644 src/rapidata/rapidata_client/exceptions/low_response_count_error.py create mode 100644 src/rapidata/rapidata_client/job/_response_minimum.py create mode 100644 tests/rapidata_client/job/test_response_minimum.py diff --git a/docs/error_handling.md b/docs/error_handling.md index 0139eda429..c27089d717 100644 --- a/docs/error_handling.md +++ b/docs/error_handling.md @@ -200,6 +200,16 @@ except FailedUploadException as e: Note that this only re-uploads the datapoints; it does not create the job definition. Prefer `exception.retry()` unless you specifically need the lower-level control. +## Fewer than 10 responses per datapoint + +Job definitions collect at least 10 responses per datapoint. Asking for fewer, in a `create_*_job_definition` call or by setting `responses_per_datapoint` on an existing definition, raises `LowResponseCountError` before anything is uploaded: + +``` +Requested 5 responses per datapoint (minimum 10). At least 10 responses per datapoint are required. To collect fewer, email info@rapidata.ai with your use case and we will enable it for your organization. +``` + +Once your organization is enabled, the same call goes through; a new `RapidataClient` is not needed. Ranking jobs (`responses_per_comparison`) are not affected. + ## Too many open files If uploads fail with `OSError: [Errno 24] Too many open files`, the process has hit its file-descriptor limit — the upload cache, worker pool, and HTTP connections all consume descriptors. When the SDK detects this it appends a hint to the `FailedUploadException` message pointing at the relevant knobs. diff --git a/docs/job_definition_parameters.md b/docs/job_definition_parameters.md index ab20ce42f8..5e5c76c476 100644 --- a/docs/job_definition_parameters.md +++ b/docs/job_definition_parameters.md @@ -101,10 +101,12 @@ datapoints=[ The minimum number of responses to collect for each datapoint. The actual number may slightly exceed this due to concurrent labelers. +The minimum is `10`. Fewer raises `LowResponseCountError` before any datapoint is uploaded. If your use case needs fewer, email [info@rapidata.ai](mailto:info@rapidata.ai) with it and we will enable it for your organization. Ranking jobs are not affected: their `responses_per_comparison` can stay below 10. + **Best Practices:** - Use 15-25 for ambiguous or subjective tasks -- Use 5-10 for clear-cut decisions +- Use 10 for clear-cut decisions, with `confidence_threshold` or `quorum_threshold` to stop early once the answer is clear ```python responses_per_datapoint=15 diff --git a/src/rapidata/__init__.py b/src/rapidata/__init__.py index 2c4e255abd..48439b0eec 100644 --- a/src/rapidata/__init__.py +++ b/src/rapidata/__init__.py @@ -95,6 +95,7 @@ ContextManager, FailedUploadException, FailedUpload, + LowResponseCountError, RevisionConflictError, SampleUpload, rapidata_config, diff --git a/src/rapidata/_skill/SKILL.md b/src/rapidata/_skill/SKILL.md index b75bb0e02e..018ee40553 100644 --- a/src/rapidata/_skill/SKILL.md +++ b/src/rapidata/_skill/SKILL.md @@ -98,7 +98,7 @@ The file is just one transport. To move the token over any transport (key-value Follow these for every job definition and audience example. Design each task so a labeler can answer it in about 10 seconds, 15 at most: labelers also answer validation tasks in the same session, mostly on phones, and within that time they read the instruction and the context and look at the media. A task they can't take in at a glance gets guessed. If a task needs longer, contact Rapidata (info@rapidata.ai) before launching it. The limits the SDK enforces are caps, not targets. -1. **Keep `responses_per_datapoint` at 10 or more.** 10 is the default; never lower it to cut cost. With 3 responses, a single labeler who misread the task can flip the result. To spend less, label fewer datapoints, or add `confidence_threshold` / `quorum_threshold` when the task has one correct answer. +1. **Keep `responses_per_datapoint` at 10 or more.** 10 is the default and the minimum; never lower it to cut cost. With 3 responses, a single labeler who misread the task can flip the result. To spend less, label fewer datapoints, or add `confidence_threshold` / `quorum_threshold` when the task has one correct answer. Below 10, `create_*_job_definition` and the `job_def.responses_per_datapoint` setter raise `LowResponseCountError` before any upload, unless the organization was enabled for it. Don't work around it: tell the user to email info@rapidata.ai with their use case. Ranking's `responses_per_comparison` is not affected. 2. **Ask one short question about one criterion.** Aim for under 80 characters, like the instructions in this guide. To judge several aspects (motion, on-screen text, sound, …), create one job definition per aspect. - ✅ `"Which video matches the description better?"` - ❌ `"Which video is better overall? Consider the motion, timing, camera, style, on-screen text and sound."` diff --git a/src/rapidata/_skill/reference.md b/src/rapidata/_skill/reference.md index b1c77160b5..ef12754e91 100644 --- a/src/rapidata/_skill/reference.md +++ b/src/rapidata/_skill/reference.md @@ -12,7 +12,7 @@ The new job-definition API exposes **classification**, **comparison**, **locate* | `instruction` | str | Task description shown to labelers (max 250 characters — longer raises `ValueError`) | | `datapoints` | list | Data to label (URLs or local paths) | | `data_type` | `"media"` \| `"text"` | `"media"` (default, covers image/video/audio) or `"text"`. **Text assets are NOT translated** unless `TranslateTextAssetsSetting` is added (source assumed English) — otherwise labelers see them verbatim | -| `responses_per_datapoint` | int | Responses per item (default 10; never set it below 10 — see **Task design rules** in the main guide) | +| `responses_per_datapoint` | int | Responses per item (default and minimum 10; fewer raises `LowResponseCountError` before any upload unless the organization was enabled for it via info@rapidata.ai — see **Task design rules** in the main guide) | | `contexts` | list[str] \| None | Text context per datapoint: one sentence under 150 characters with only what the instruction asks about (see **Task design rules** in the main guide). Max 400 characters each; contexts over the limit are always shortened against the instruction before upload — set `rapidata_config.upload.contextShortening = True` to shorten every context, or use `client.context` to shorten manually | | `media_contexts` | list[list[str]] \| None | Reference images per datapoint; each entry is a list of at most 10 image URLs/paths (one inner list per datapoint). Shown in a collapsible drawer unless `PinContextMediaSetting` is added (only when there is enough space — check the preview); for a compare of two image edits, `ReferenceOverlaySetting` lays a single image context over both options instead | | `confidence_threshold` | float \| None | Confidence-based early stopping threshold (0-1); cannot combine with `quorum_threshold` | diff --git a/src/rapidata/rapidata_client/__init__.py b/src/rapidata/rapidata_client/__init__.py index 1362e0af78..74aaf09410 100644 --- a/src/rapidata/rapidata_client/__init__.py +++ b/src/rapidata/rapidata_client/__init__.py @@ -95,6 +95,11 @@ DeviceFilter, DeviceType, ) -from .exceptions import FailedUploadException, FailedUpload, RevisionConflictError +from .exceptions import ( + FailedUploadException, + FailedUpload, + LowResponseCountError, + RevisionConflictError, +) from .config import rapidata_config, logger, managed_print from .config.upload_config import CompressionConfig diff --git a/src/rapidata/rapidata_client/exceptions/__init__.py b/src/rapidata/rapidata_client/exceptions/__init__.py index ef135a59e8..bb20736ceb 100644 --- a/src/rapidata/rapidata_client/exceptions/__init__.py +++ b/src/rapidata/rapidata_client/exceptions/__init__.py @@ -2,6 +2,7 @@ from .asset_warning import AssetWarning from .failed_upload import FailedUpload from .failed_upload_exception import FailedUploadException +from .low_response_count_error import LowResponseCountError from .rapidata_error import RapidataError from .revision_conflict_error import RevisionConflictError @@ -10,6 +11,7 @@ "AssetWarning", "FailedUpload", "FailedUploadException", + "LowResponseCountError", "RapidataError", "RevisionConflictError", ] diff --git a/src/rapidata/rapidata_client/exceptions/low_response_count_error.py b/src/rapidata/rapidata_client/exceptions/low_response_count_error.py new file mode 100644 index 0000000000..6844eaeed6 --- /dev/null +++ b/src/rapidata/rapidata_client/exceptions/low_response_count_error.py @@ -0,0 +1,25 @@ +from __future__ import annotations + +MIN_RESPONSES_PER_DATAPOINT = 10 +LOW_RESPONSE_COUNT_MESSAGE = ( + "At least 10 responses per datapoint are required. To collect fewer, email " + "info@rapidata.ai with your use case and we will enable it for your organization." +) + + +class LowResponseCountError(ValueError): + """Raised before any upload when a job asks for fewer than 10 responses per + datapoint and the organization has not been enabled for it. + + Attributes: + requested (int): The responses per datapoint that were asked for. + minimum (int): The minimum without the capability (10). + """ + + def __init__(self, requested: int): + self.requested = requested + self.minimum = MIN_RESPONSES_PER_DATAPOINT + super().__init__( + f"Requested {requested} responses per datapoint (minimum {self.minimum}). " + f"{LOW_RESPONSE_COUNT_MESSAGE}" + ) diff --git a/src/rapidata/rapidata_client/exceptions/rapidata_error.py b/src/rapidata/rapidata_client/exceptions/rapidata_error.py index 3786f01e12..52689d3490 100644 --- a/src/rapidata/rapidata_client/exceptions/rapidata_error.py +++ b/src/rapidata/rapidata_client/exceptions/rapidata_error.py @@ -1,7 +1,6 @@ from __future__ import annotations from typing import Optional, Any -import json class RapidataError(Exception): @@ -44,6 +43,20 @@ def _trace_id_from_details(details: Any) -> str | None: return raw return None + @staticmethod + def _format_errors(errors: Any) -> list[str]: + """One ``field: message`` line per validation message.""" + if isinstance(errors, dict): + lines = [] + for field, messages in errors.items(): + if not isinstance(messages, list): + messages = [messages] + lines.extend(f"{field}: {m}" if field else str(m) for m in messages) + return lines + if isinstance(errors, list): + return [str(e) for e in errors] + return [str(errors)] + def get_reason(self) -> str: """Get a concise reason string suitable for grouping and display. @@ -76,10 +89,7 @@ def __str__(self): error_parts.append(f"{self.message or 'Unknown error'}") if errors: - if isinstance(errors, dict): - error_parts.append(f"Reasons: {json.dumps({'errors': errors})}") - else: - error_parts.append(f"Reasons: {errors}") + error_parts.extend(self._format_errors(errors)) if self.trace_id: error_parts.append(f"Trace Id: {self.trace_id}") diff --git a/src/rapidata/rapidata_client/job/_response_minimum.py b/src/rapidata/rapidata_client/job/_response_minimum.py new file mode 100644 index 0000000000..8886bbef78 --- /dev/null +++ b/src/rapidata/rapidata_client/job/_response_minimum.py @@ -0,0 +1,77 @@ +from __future__ import annotations + +import json +import weakref +from typing import TYPE_CHECKING, Any + +from rapidata.rapidata_client.config import logger +from rapidata.rapidata_client.job._revision_body import _as_dict +from rapidata.rapidata_client.exceptions.low_response_count_error import ( + MIN_RESPONSES_PER_DATAPOINT, + LowResponseCountError, +) + +if TYPE_CHECKING: + from rapidata.api_client.models.i_order_workflow_input import IOrderWorkflowInput + from rapidata.service.openapi_service import OpenAPIService + +_LOW_RESPONSE_COUNT_CAPABILITY = "LowResponseCount" + +# Only grants are cached: an organization enabled mid-session works without a new client. +_granted: weakref.WeakKeyDictionary[OpenAPIService, bool] = weakref.WeakKeyDictionary() + + +def ensure_response_count_allowed( + openapi_service: OpenAPIService, + workflow: IOrderWorkflowInput, + responses: int, +) -> None: + """Raise LowResponseCountError before anything is uploaded. + + Only simple workflows are gated, as on the backend; ranking is not. + """ + if responses >= MIN_RESPONSES_PER_DATAPOINT: + return + if _as_dict(workflow).get("_t") != "SimpleWorkflow": + return + if _low_response_count_allowed(openapi_service) is False: + raise LowResponseCountError(responses) + + +def _low_response_count_allowed(openapi_service: OpenAPIService) -> bool | None: + """None when the lookup failed, which leaves the decision to the backend.""" + if _granted.get(openapi_service): + return True + + from rapidata.rapidata_client.api.rapidata_api_client import ( + suppress_rapidata_error_logging, + ) + + try: + with suppress_rapidata_error_logging(): + response = openapi_service.api_client.call_api( + "GET", + f"https://auth.{openapi_service.environment}/connect/userinfo", + _request_timeout=5, + ) + if response.status != 200: + raise RuntimeError(f"userinfo returned {response.status}") + userinfo: dict[str, Any] = json.loads(response.read().decode("utf-8")) + except Exception as e: + logger.debug("Capability lookup failed, the backend decides: %s", e) + return None + + allowed = _LOW_RESPONSE_COUNT_CAPABILITY in _claim_values( + userinfo, "capabilities" + ) or "admin" in _claim_values(userinfo, "platform_relations") + if allowed: + _granted[openapi_service] = True + return allowed + + +def _claim_values(userinfo: dict[str, Any], claim: str) -> list[str]: + # A claim with a single value comes back as a bare string. + value = userinfo.get(claim) + if isinstance(value, str): + return [value] + return value if isinstance(value, list) else [] diff --git a/src/rapidata/rapidata_client/job/rapidata_job_definition.py b/src/rapidata/rapidata_client/job/rapidata_job_definition.py index b8413fe2a2..f525b8bd2d 100644 --- a/src/rapidata/rapidata_client/job/rapidata_job_definition.py +++ b/src/rapidata/rapidata_client/job/rapidata_job_definition.py @@ -29,6 +29,9 @@ _with_instruction, _with_responses, ) +from rapidata.rapidata_client.job._response_minimum import ( + ensure_response_count_allowed, +) import webbrowser import urllib.parse from colorama import Fore @@ -214,7 +217,10 @@ def answer_options(self, value: list[str]) -> None: @property def responses_per_datapoint(self) -> int: - """The maximum number of responses collected per datapoint.""" + """The maximum number of responses collected per datapoint. + + Setting fewer than 10 raises LowResponseCountError unless your organization was enabled for it. + """ return self._effective( "responses_per_datapoint", _get_responses(self._snapshot.referee) ) @@ -224,6 +230,9 @@ def responses_per_datapoint(self, value: int) -> None: if isinstance(value, bool) or not isinstance(value, int) or value < 1: raise ValueError("responses_per_datapoint must be a positive integer") _with_responses(self._snapshot.referee, value) + ensure_response_count_allowed( + self._openapi_service, self._snapshot.workflow, value + ) self._pending["responses_per_datapoint"] = value @property diff --git a/src/rapidata/rapidata_client/job/rapidata_job_manager.py b/src/rapidata/rapidata_client/job/rapidata_job_manager.py index c052941572..c4c9d55d99 100644 --- a/src/rapidata/rapidata_client/job/rapidata_job_manager.py +++ b/src/rapidata/rapidata_client/job/rapidata_job_manager.py @@ -9,6 +9,9 @@ from rapidata.rapidata_client.job._job_creation_state_machine import ( JobDefinitionCreationMachine, ) +from rapidata.rapidata_client.job._response_minimum import ( + ensure_response_count_allowed, +) from typing import Sequence, Literal, TYPE_CHECKING, cast from rapidata.rapidata_client.datapoints._datapoints_validator import ( DatapointsValidator, @@ -79,6 +82,10 @@ def _create_general_job_definition( self._warn_unsupported_settings(workflow, settings) + ensure_response_count_allowed( + self._openapi_service, workflow._to_model(), responses_per_datapoint + ) + self.__context_manager._apply_context_shortening( datapoints=datapoints, question=workflow._get_instruction(), @@ -181,7 +188,8 @@ def create_classification_job_definition( datapoints (list[str]): The list of datapoints for the classification - each datapoint will be labeled. data_type (str, optional): The data type of the datapoints. Defaults to "media" (any form of image, video or audio). \n Other option: "text". - responses_per_datapoint (int, optional): The number of responses that will be collected per datapoint. Defaults to 10. + responses_per_datapoint (int, optional): The number of responses that will be collected per datapoint. Defaults to 10.\n + Fewer than 10 raises LowResponseCountError before any upload unless your organization was enabled for it (email info@rapidata.ai with your use case). contexts (list[str], optional): The list of contexts for the classification. Defaults to None.\n If provided has to be the same length as datapoints and will be shown in addition to the instruction and options. (Therefore will be different for each datapoint) Will be match up with the datapoints using the list index. @@ -264,7 +272,8 @@ def create_compare_job_definition( datapoints (list[list[str]]): Outher list is the datapoints, inner list is the options for the comparison - each datapoint will be labeled. data_type (str, optional): The data type of the datapoints. Defaults to "media" (any form of image, video or audio). \n Other option: "text". - responses_per_datapoint (int, optional): The number of responses that will be collected per datapoint. Defaults to 10. + responses_per_datapoint (int, optional): The number of responses that will be collected per datapoint. Defaults to 10.\n + Fewer than 10 raises LowResponseCountError before any upload unless your organization was enabled for it (email info@rapidata.ai with your use case). contexts (list[str], optional): The list of contexts for the comparison. Defaults to None.\n If provided has to be the same length as datapoints and will be shown in addition to the instruction. (Therefore will be different for each datapoint) Will be matched up with the datapoints using the list index. @@ -449,7 +458,8 @@ def create_free_text_job_definition( datapoints (list[str]): The list of datapoints for the free text - each datapoint will be labeled. data_type (str, optional): The data type of the datapoints. Defaults to "media" (any form of image, video or audio). \n Other option: "text". - responses_per_datapoint (int, optional): The number of responses that will be collected per datapoint. Defaults to 10. + responses_per_datapoint (int, optional): The number of responses that will be collected per datapoint. Defaults to 10.\n + Fewer than 10 raises LowResponseCountError before any upload unless your organization was enabled for it (email info@rapidata.ai with your use case). contexts (list[str], optional): The list of contexts for the free text. Defaults to None.\n If provided has to be the same length as datapoints and will be shown in addition to the instruction. (Therefore will be different for each datapoint) Will be matched up with the datapoints using the list index. @@ -505,7 +515,8 @@ def create_select_words_job_definition( datapoints (list[str]): The list of datapoints for the select words - each datapoint will be labeled. sentences (list[str]): The list of sentences for the select words - Will be split up by spaces and shown along side each datapoint.\n Must be the same length as datapoints. - responses_per_datapoint (int, optional): The number of responses that will be collected per datapoint. Defaults to 10. + responses_per_datapoint (int, optional): The number of responses that will be collected per datapoint. Defaults to 10.\n + Fewer than 10 raises LowResponseCountError before any upload unless your organization was enabled for it (email info@rapidata.ai with your use case). settings (Sequence[RapidataSetting], optional): The list of settings for the select words. Defaults to []. Decides how the tasks should be shown. failure_tolerance (float, optional): The fraction of datapoints allowed to fail while still creating the job definition (0.0-1.0). Defaults to None, which uses rapidata_config.upload.failureTolerance (default 0.0 = strict).\n 0.0 means any failed upload aborts creation so no incomplete definition is left behind; the failed datapoints can then be retried into the same dataset via the raised FailedUploadException.retry(). 1.0 creates the definition regardless of failures, as long as at least one datapoint uploads successfully. private_metadata (list[dict[str, str]], optional): Key-value string pairs for each datapoint. Defaults to None.\n @@ -552,7 +563,8 @@ def create_locate_job_definition( name (str): The name of the job. (Will not be shown to the labeler) instruction (str): The instruction what should be located. Will be shown along side each datapoint. datapoints (list[str]): The list of datapoints for the locate - each datapoint will be labeled. - responses_per_datapoint (int, optional): The number of responses that will be collected per datapoint. Defaults to 10. + responses_per_datapoint (int, optional): The number of responses that will be collected per datapoint. Defaults to 10.\n + Fewer than 10 raises LowResponseCountError before any upload unless your organization was enabled for it (email info@rapidata.ai with your use case). contexts (list[str], optional): The list of contexts for the locate. Defaults to None.\n If provided has to be the same length as datapoints and will be shown in addition to the instruction. (Therefore will be different for each datapoint) Will be match up with the datapoints using the list index. @@ -604,7 +616,8 @@ def create_draw_job_definition( name (str): The name of the job. (Will not be shown to the labeler) instruction (str): The instruction for how the lines should be drawn. Will be shown along side each datapoint. datapoints (list[str]): The list of datapoints for the draw lines - each datapoint will be labeled. - responses_per_datapoint (int, optional): The number of responses that will be collected per datapoint. Defaults to 10. + responses_per_datapoint (int, optional): The number of responses that will be collected per datapoint. Defaults to 10.\n + Fewer than 10 raises LowResponseCountError before any upload unless your organization was enabled for it (email info@rapidata.ai with your use case). contexts (list[str], optional): The list of contexts for the comparison. Defaults to None.\n If provided has to be the same length as datapoints and will be shown in addition to the instruction. (Therefore will be different for each datapoint) Will be match up with the datapoints using the list index. @@ -659,7 +672,8 @@ def _create_timestamp_job_definition( name (str): The name of the job. (Will not be shown to the labeler) instruction (str): The instruction for the timestamp task. Will be shown along side each datapoint. datapoints (list[str]): The list of datapoints for the timestamp - each datapoint will be labeled. - responses_per_datapoint (int, optional): The number of responses that will be collected per datapoint. Defaults to 10. + responses_per_datapoint (int, optional): The number of responses that will be collected per datapoint. Defaults to 10.\n + Fewer than 10 raises LowResponseCountError before any upload unless your organization was enabled for it (email info@rapidata.ai with your use case). contexts (list[str], optional): The list of contexts for the comparison. Defaults to None.\n If provided has to be the same length as datapoints and will be shown in addition to the instruction. (Therefore will be different for each datapoint) Will be match up with the datapoints using the list index. diff --git a/tests/rapidata_client/job/test_response_minimum.py b/tests/rapidata_client/job/test_response_minimum.py new file mode 100644 index 0000000000..b2c25d35c9 --- /dev/null +++ b/tests/rapidata_client/job/test_response_minimum.py @@ -0,0 +1,151 @@ +"""Fewer than 10 responses per datapoint needs the LowResponseCount capability, checked before any upload.""" + +from __future__ import annotations + +import json +from unittest.mock import MagicMock + +import pytest + +from rapidata.api_client.models.get_job_revision_endpoint_output import ( + GetJobRevisionEndpointOutput, +) +from rapidata.rapidata_client.exceptions import LowResponseCountError, RapidataError +from rapidata.rapidata_client.job._response_minimum import ( + ensure_response_count_allowed, +) +from rapidata.rapidata_client.job.rapidata_job_definition import ( + RapidataJobDefinition, +) +from rapidata.rapidata_client.job.rapidata_job_manager import RapidataJobManager + +SIMPLE = MagicMock(to_dict=lambda: {"_t": "SimpleWorkflow"}) +RANKING = MagicMock(to_dict=lambda: {"_t": "GroupedRankingWorkflow"}) + + +def _service(userinfo: dict | Exception, status: int = 200) -> MagicMock: + service = MagicMock() + service.environment = "rapidata.ai" + if isinstance(userinfo, Exception): + service.api_client.call_api.side_effect = userinfo + else: + response = MagicMock(status=status) + response.read.return_value = json.dumps(userinfo).encode() + service.api_client.call_api.return_value = response + return service + + +@pytest.mark.parametrize( + "responses, workflow, userinfo, status, raises, lookups", + [ + (10, SIMPLE, {}, 200, False, 0), + (9, SIMPLE, {}, 200, True, 1), + (9, SIMPLE, {"capabilities": ["Faucets", "LowResponseCount"]}, 200, False, 1), + (9, SIMPLE, {"capabilities": "LowResponseCount"}, 200, False, 1), + (9, SIMPLE, {"platform_relations": ["admin"]}, 200, False, 1), + (9, SIMPLE, RuntimeError("timeout"), 200, False, 1), + (9, SIMPLE, {}, 401, False, 1), + (1, RANKING, {}, 200, False, 0), + ], + ids=[ + "at-minimum", + "not-granted", + "granted", + "granted-single-claim", + "platform-admin", + "lookup-error-fails-open", + "lookup-non-200-fails-open", + "ranking-not-gated", + ], +) +def test_gate(responses, workflow, userinfo, status, raises, lookups): + service = _service(userinfo, status) + + if raises: + with pytest.raises(LowResponseCountError) as error: + ensure_response_count_allowed(service, workflow, responses) + assert "Requested 9 responses per datapoint (minimum 10)" in str(error.value) + assert "email info@rapidata.ai with your use case" in str(error.value) + else: + ensure_response_count_allowed(service, workflow, responses) + + assert service.api_client.call_api.call_count == lookups + + +def test_grant_is_cached_per_client(): + service = _service({"capabilities": ["LowResponseCount"]}) + + ensure_response_count_allowed(service, SIMPLE, 3) + ensure_response_count_allowed(service, SIMPLE, 3) + + assert service.api_client.call_api.call_count == 1 + + +def test_create_raises_before_any_upload(): + service = _service({}) + + with pytest.raises(LowResponseCountError): + RapidataJobManager(service).create_classification_job_definition( + name="Job", + instruction="Is this a cat?", + answer_options=["yes", "no"], + datapoints=["https://example.com/a.jpg"], + responses_per_datapoint=5, + ) + + service.dataset.dataset_api.dataset_post.assert_not_called() + + +def test_revision_setter_raises(): + service = _service({}) + revision = GetJobRevisionEndpointOutput.from_dict( + { + "definitionId": "jd-1", + "revisionNumber": 1, + "state": "Completed", + "pipelineId": "pip-1", + "datasetId": "ds-1", + "workflow": { + "_t": "SimpleWorkflow", + "rapid": { + "_t": "FreeText", + "asset": {"segment": "asset"}, + "question": {"literal": "Describe it"}, + }, + }, + "referee": {"_t": "NaiveReferee", "totalVotes": 10}, + "featureFlags": [], + "campaignFeatureFlags": [], + "createdAt": "2026-10-23T12:00:00Z", + "createdById": "00000000-0000-0000-0000-000000000001", + "createdByMail": "a@rapidata.ai", + } + ) + service.order.job_api.job_definition_definition_id_revision_get.return_value = ( + revision + ) + jd = RapidataJobDefinition("jd-1", "Job", service).reload() + + with pytest.raises(LowResponseCountError): + jd.responses_per_datapoint = 5 + + assert not jd.has_unsaved_changes + + +def test_validation_errors_render_one_line_per_message(): + error = RapidataError( + status_code=400, + details={ + "title": "One or more validation errors occurred.", + "errors": {"Referee": ["At least 10 responses.", "Second."], "": "Top."}, + }, + trace_id="abc", + ) + + assert str(error).splitlines()[:5] == [ + "One or more validation errors occurred.", + "Referee: At least 10 responses.", + "Referee: Second.", + "Top.", + "Trace Id: abc", + ] From d416f20bb864f14bfb0b4ace7dc9f4495e6bd1fc Mon Sep 17 00:00:00 2001 From: RapidPoseidon Date: Wed, 28 Oct 2026 13:41:56 +0000 Subject: [PATCH 2/2] docs(job): drop the enabled-org and ranking note from error handling Co-Authored-By: Claude Opus 5.5 Co-Authored-By: luca@rapidata.ai <25279790+LucStr@users.noreply.github.com> --- docs/error_handling.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/error_handling.md b/docs/error_handling.md index c27089d717..1af4f0d533 100644 --- a/docs/error_handling.md +++ b/docs/error_handling.md @@ -208,8 +208,6 @@ Job definitions collect at least 10 responses per datapoint. Asking for fewer, i Requested 5 responses per datapoint (minimum 10). At least 10 responses per datapoint are required. To collect fewer, email info@rapidata.ai with your use case and we will enable it for your organization. ``` -Once your organization is enabled, the same call goes through; a new `RapidataClient` is not needed. Ranking jobs (`responses_per_comparison`) are not affected. - ## Too many open files If uploads fail with `OSError: [Errno 24] Too many open files`, the process has hit its file-descriptor limit — the upload cache, worker pool, and HTTP connections all consume descriptors. When the SDK detects this it appends a hint to the `FailedUploadException` message pointing at the relevant knobs.