From 6cdd17667b16203f6d44c275c17a2cf327f7d621 Mon Sep 17 00:00:00 2001 From: Sahil Sharma Date: Tue, 25 Aug 2026 12:23:31 -0400 Subject: [PATCH 1/2] Add configurable Sentinel integration tile with org-level settings --- client/src/images/RobloxLogo.png | Bin 0 -> 2734 bytes ...IntegrationConfigApiCredentialsSection.tsx | 60 ++- .../integrations/integrationConfigs.ts | 9 + .../integrations/integrationLogos.ts | 5 + server/.env.example | 5 +- .../integrationManifests.ts | 95 +++++ .../helpers/instantiateBuiltInSignals.ts | 5 +- .../SentinelRareClassAffinitySignal.test.ts | 374 ++++++++++++------ .../SentinelRareClassAffinitySignal.ts | 119 +++++- 9 files changed, 533 insertions(+), 139 deletions(-) create mode 100644 client/src/images/RobloxLogo.png diff --git a/client/src/images/RobloxLogo.png b/client/src/images/RobloxLogo.png new file mode 100644 index 0000000000000000000000000000000000000000..1c651ca76d70ccda4bdf479745e9ecab57b56f3d GIT binary patch literal 2734 zcmd^>`#%%9RDp@c7>}H3Ss5$#S?0Blo-J9@nn}RJc75z!QxE{PA1D4H<3#0DM_I z06uQz`MsCL`|l}_U>4uMJplJx_(e9w9RLtKX>N4QE|_Ow(4L7M=B2MJ$*x60)_h2B zgkQh%!wDTDmD}Ki60DluoFXfMaDIYh8xT(G#AOgpo}eMDz%b8kS<`>q2AiW5dK>Hx zkL+($l>9=&zZOl0^g6Ut!=ARbhaGM);&}y?vY!8^QnzwPR-3Bx7L`szu|~c<{WV@6 z1IygZOFJTS+n;?TrGY^v7Sg(tIKiJ$utv~;k} zVVyIL+sl7y7f^MWH`}$ZBpdMMdb$oQPKY$!o(cc*7}bndN496EJMAG_(=G{}0C|mk zg`_}3ch=bq{&9HKv+WMjUyfyu)EnZF4m3~Sx!$5xL|effN*fVEb*XugilMLmY(}=G z^<>q=vqKk$DmDs#(lDR?`LW4iy?13C2BG2AVc$Y`b;y>Pc+bHyZ|?ybCq)BZTXgEu z1EYwZf;)|hB>0$BEPXq0VE}vT@?zP{X*XrzJ)^OvB*nv`QRH;n+is3pB_F>i&OCH~ zXM?NRocp0+Suby&Y@T{AO+Uo1CH$y4IaVARwwLOR?Zkv2f_k3Rs&%BFrg#Tn_e|Gb zXSQk3(g+sr6D@lBOTe6zm_SSVWew|fLS4{oxAM0g7I;qb3MaACny6dzMZsydi)?v# zu%CVrC4mq<2|g30Dp+{;quUrg++G4N>pfcau(#0uiO_fl-lHY8MrDr6&at(znwLoz zu3NC$2-?W*yuW=p5}=jIrC$Wex|8YBtpcIa>*d>SGI z_hXuf?M&!Wr)RkwP(hgtpHYm7*y{cLBm)WiB6xMiK}I3wpV%K1c=QT(P*ZG=v;6AD z&LmsD;J_OjC-=3c-0!EwULY4Fb0lFi%QXoDY%zGUF?$>#k!1vRPz1$@5?OhmNq63?3f;v_8`z5Q_RomF-y(V0Y2;3zmMr&tUfb&qmU77D79JSG z36fauyZ7Qiic^Kw`F0aAP^Sp%R`d;D7Sa}6CGeO&6iK+2 z7_QV1ThU{fWnvn?GSS$G5{COPjeMK%5wF_li(yrkJ?K#>63m35MfYdWRp`5fi;sqM zvGn|@sn&E3;UZDw1c%Ih1*@w|dqaRFG4T0edc%G9`fM`NaI5blf&*+yJ1>RZT^z0q zV!o00HGAFkr+zS9DCuztHYFcwlim9AlWwNg=whjRbGb!I)4Fm@;QCDGdJ9h3wlq>h zW6U}e`nKW2jZ}yu0dJcr>n1DXBYfwW*{ESd#KK_NpU`K)J5WvM{2e;1peh1QdOknt zR2hhk=>Flh<}zGM#M}yCRv;SV`qdu|mH%D3CfYeOz=>x>gNx1*k?Ed2`M1!n)40*t zl47&W@Sirs9P)Iu$SJ(s>LN_%yp-a+donR`#UUFk=UUfUqMmzN4TadYVRRf}KFT0{ro7|R690O)CYr{wC?SCj)DvE2)vmQ%#@mnq^?+>RczA zpv0U6$;?XJP|&!Fo}Ya`W1;I|!ML8JpK6@hd_}#oyMqa%`rFmp##=jxgsx1TNWl{$ zGn(6^Cz0Rn+NjNjKdYF|Z;@fEAOOzQjI5i+3bq6d*dZXN8j>e9~`9jq|-u5BGh}s zqT*s-Vyt_R);H%@5vMWUmTm)>G}AQV6^AL4l{nvZU{H=)JqTuU;p!eO3~8%(_DoVS zCk7$p>&D?it!lXjMs05{oElZ4D0jWJk)l}Q)%s~D$%3BcQ0jhl$l6rGIn!J?46?Vo z8=dsHKn-Vqv+U7u+R(TJ-UEL@fPL9}@!NL3S+@@ymrsO7s*NgIrP7x^Z7rxDV$IygQ>fK5&dDCl0A`G61_mF = { + apiUrl: + 'Sentinel API URL (optional — falls back to the deployment default)', + topK: 'Top K (nearest neighbors to consider)', + minScoreToConsider: 'Minimum score to consider (0–1)', + threadContextWindowMinutes: 'Thread context window (minutes)', + }; + const SENTINEL_NUMERIC_FIELDS = new Set([ + 'topK', + 'minScoreToConsider', + 'threadContextWindowMinutes', + ]); + + const renderSentinelCredential = (pluginCredential: { + __typename: 'PluginIntegrationApiCredential'; + credential: Record; + }) => { + const credential = pluginCredential.credential ?? {}; + return ( +
+ {Object.entries(SENTINEL_FIELD_LABELS).map(([key, label]) => ( +
+
{label}
+ { + const raw = event.target.value; + let next: Record; + if (raw.trim() === '') { + const { [key]: _removed, ...rest } = credential; + next = rest; + } else if (SENTINEL_NUMERIC_FIELDS.has(key)) { + const parsed = Number(raw); + next = Number.isNaN(parsed) + ? credential + : { ...credential, [key]: parsed }; + } else { + next = { ...credential, [key]: raw }; + } + setApiCredential({ + __typename: 'PluginIntegrationApiCredential', + credential: + next as import('../../../graphql/generated').Scalars['JSONObject'], + }); + }} + /> +
+ ))} +
+ ); + }; + const projectKeysInput = () => { switch (apiCredential.__typename) { case 'GoogleContentSafetyApiIntegrationApiCredential': @@ -194,7 +250,9 @@ export default function IntegrationConfigApiCredentialsSection(props: { case 'ZentropiIntegrationApiCredential': return renderZentropiCredential(apiCredential); case 'PluginIntegrationApiCredential': - return renderPluginCredential(apiCredential); + return props.name === 'SENTINEL' + ? renderSentinelCredential(apiCredential) + : renderPluginCredential(apiCredential); default: throw new Error('Integration not implemented yet'); } diff --git a/client/src/webpages/dashboard/integrations/integrationConfigs.ts b/client/src/webpages/dashboard/integrations/integrationConfigs.ts index f9781f79e..94c0ca11b 100644 --- a/client/src/webpages/dashboard/integrations/integrationConfigs.ts +++ b/client/src/webpages/dashboard/integrations/integrationConfigs.ts @@ -8,6 +8,7 @@ import GoogleLogo from '../../../images/GoogleLogo.png'; import GoogleLogoWithBackground from '../../../images/GoogleLogoWithBackground.png'; import OpenAILogo from '../../../images/OpenAILogo.png'; import OpenAILogoWithBackground from '../../../images/OpenAILogoWithBackground.png'; +import RobloxLogo from '../../../images/RobloxLogo.png'; import ZentropiLogo from '../../../images/ZentropiLogo.png'; export type IntegrationConfig = { @@ -36,6 +37,14 @@ export const INTEGRATION_CONFIGS: IntegrationConfig[] = [ url: 'https://openai.com/', requiresInfo: true, }, + { + name: 'SENTINEL', + title: 'Sentinel', + logo: RobloxLogo, + logoWithBackground: RobloxLogo, + url: 'https://github.com/Roblox/sentinel', + requiresInfo: false, + }, { name: 'ZENTROPI', title: 'Zentropi', diff --git a/client/src/webpages/dashboard/integrations/integrationLogos.ts b/client/src/webpages/dashboard/integrations/integrationLogos.ts index df5acd245..f4f5e8637 100644 --- a/client/src/webpages/dashboard/integrations/integrationLogos.ts +++ b/client/src/webpages/dashboard/integrations/integrationLogos.ts @@ -7,6 +7,7 @@ import GoogleLogo from '../../../images/GoogleLogo.png'; import GoogleLogoWithBackground from '../../../images/GoogleLogoWithBackground.png'; import OpenAILogo from '../../../images/OpenAILogo.png'; import OpenAILogoWithBackground from '../../../images/OpenAILogoWithBackground.png'; +import RobloxLogo from '../../../images/RobloxLogo.png'; import ZentropiLogo from '../../../images/ZentropiLogo.png'; export const INTEGRATION_LOGO_FALLBACKS: Partial< @@ -20,6 +21,10 @@ export const INTEGRATION_LOGO_FALLBACKS: Partial< logo: OpenAILogo, logoWithBackground: OpenAILogoWithBackground, }, + SENTINEL: { + logo: RobloxLogo, + logoWithBackground: RobloxLogo, + }, ZENTROPI: { logo: ZentropiLogo, logoWithBackground: ZentropiLogo, diff --git a/server/.env.example b/server/.env.example index c9c6d5cb4..129c51c7a 100644 --- a/server/.env.example +++ b/server/.env.example @@ -68,7 +68,10 @@ CLICKHOUSE_PROTOCOL=http # CLICKHOUSE_MAX_BYTES_BEFORE_EXTERNAL_GROUP_BY=1500000000 # CLICKHOUSE_MAX_BYTES_BEFORE_EXTERNAL_SORT=1500000000 -# Sentinel API URL (local Sentinel service for rare class affinity signal) +# Sentinel API URL — deployment-wide default for the rare class affinity +# signal. Orgs can override this (and other Sentinel settings) per-org from +# the Integrations dashboard; this env var is just the fallback when an org +# hasn't set its own URL. # Start Sentinel locally: docker compose -f docker-compose.yaml -f docker-compose.sentinel.yaml up -d sentinel SENTINEL_API_URL=http://localhost:8000 diff --git a/server/services/integrationRegistry/integrationManifests.ts b/server/services/integrationRegistry/integrationManifests.ts index 84ade98a6..71708cc8a 100644 --- a/server/services/integrationRegistry/integrationManifests.ts +++ b/server/services/integrationRegistry/integrationManifests.ts @@ -311,12 +311,107 @@ const ZENTROPI: IntegrationManifestEntry = { requiresConfig: true, }; +const SENTINEL: IntegrationManifestEntry = { + modelCard: { + modelName: 'Sentinel', + version: 'BYO model/banks', + releaseDate: 'Ongoing', + sections: [ + { + id: 'trainingData', + title: 'Training Data Sources', + fields: [ + { + label: 'Data Sources', + value: + 'Sentinel scores content against sentence-transformer embeddings of example banks that you (the adopter) curate and supply: a "positive" bank of rare/harmful examples and a "negative" bank of common/normal examples. Coop does not ship pretrained banks — result quality depends entirely on the banks you load.', + }, + ], + }, + { + id: 'policyAndTaxonomy', + title: 'Policy & Taxonomy Definitions', + fields: [ + { + label: 'Policies', + value: + 'No fixed taxonomy. Sentinel is a general-purpose contrastive scorer — the "rare class" it detects is defined entirely by whichever positive/negative example banks are loaded (e.g., grooming language, a specific harassment pattern).', + }, + ], + }, + { + id: 'annotationMethodology', + title: 'Annotation Methodology', + fields: [ + { + label: 'Methodology', + value: + 'Adopter-defined. Sentinel computes a contrastive score (the log ratio of similarity to nearby positive- vs. negative-bank examples), then aggregates scores across a conversation using skewness to surface outlier high-scoring messages. See the upstream repo for the scoring algorithm.', + }, + ], + }, + { + id: 'performanceBenchmarks', + title: 'Performance Benchmarks', + fields: [ + { + label: 'Benchmarks', + value: + 'Depends entirely on the banks and sentence-transformer encoder you configure; there are no standardized benchmark numbers for arbitrary bank/model combinations. Evaluate against your own labeled data before enabling this signal in enforcement rules.', + }, + ], + }, + { + id: 'biasAndLimitations', + title: 'Bias Documentation & Known Limits', + fields: [ + { + label: 'Known Limitations', + value: + 'Quality is bounded by the sentence-transformer encoder and the size/diversity of your example banks; small or unrepresentative banks produce noisy scores. Sentinel scores text only (plus limited thread context) — it does not evaluate images or other media.', + }, + ], + }, + { + id: 'implementationGuidance', + title: 'Implementation Guidance', + fields: [ + { + label: 'Deployment', + value: + 'Coop talks to a self-hosted Sentinel HTTP service (see server/sentinel-api/ in this repo, which wraps the upstream Roblox/sentinel library). Configure the URL of the Sentinel deployment this org should use, plus optional scoring/context overrides, below.', + }, + { + label: 'Credentials', + value: + 'No API key required. Leaving fields blank falls back to the deployment default (SENTINEL_API_URL) and Sentinel’s own scoring defaults.', + }, + ], + }, + { + id: 'relevantLinks', + title: 'Relevant Links', + fields: [ + { + label: 'Source', + value: 'https://github.com/Roblox/sentinel', + }, + ], + }, + ], + }, + title: 'Sentinel', + docsUrl: 'https://github.com/Roblox/sentinel', + requiresConfig: true, +}; + /** Built-in integration manifests (id -> entry). Merged with loaded plugins by the integration registry. */ export const BUILT_IN_MANIFESTS: Readonly< Record > = { GOOGLE_CONTENT_SAFETY_API: GOOGLE_CONTENT_SAFETY, OPEN_AI: OPENAI, + SENTINEL, ZENTROPI, }; diff --git a/server/services/signalsService/helpers/instantiateBuiltInSignals.ts b/server/services/signalsService/helpers/instantiateBuiltInSignals.ts index 5c8892717..b13a3c0ce 100644 --- a/server/services/signalsService/helpers/instantiateBuiltInSignals.ts +++ b/server/services/signalsService/helpers/instantiateBuiltInSignals.ts @@ -5,7 +5,6 @@ import type { HmaService } from '../../hmaService/index.js'; import type { ItemInvestigationService } from '../../itemInvestigationService/index.js'; import type { GetPoliciesByIdEventuallyConsistent } from '../../manualReviewToolService/manualReviewToolQueries.js'; import { type FetchHTTP } from '../../networkingService/index.js'; -import { makeSentinelService } from '../../sentinelService/index.js'; import { type UserScore } from '../../userStatisticsService/userStatisticsService.js'; import { type UserStrikeService } from '../../userStrikeService/index.js'; import AggregationSignal from '../signals/aggregation/AggregationSignal.js'; @@ -171,8 +170,10 @@ export function instantiateBuiltInSignals( [SignalType.AGGREGATION]: new AggregationSignal(aggregationsService), [SignalType.SENTINEL_RARE_CLASS_AFFINITY]: new SentinelRareClassAffinitySignal( - makeSentinelService(fetchHTTP, process.env.SENTINEL_API_URL), + credentialGetters.getForIntegrationId('SENTINEL'), + fetchHTTP, itemInvestigationService, + process.env.SENTINEL_API_URL, ), [SignalType.ZENTROPI_LABELER]: new ZentropiLabelerSignal( credentialGetters.ZENTROPI, diff --git a/server/services/signalsService/signals/third_party_signals/sentinel/SentinelRareClassAffinitySignal.test.ts b/server/services/signalsService/signals/third_party_signals/sentinel/SentinelRareClassAffinitySignal.test.ts index 5be8e4f4c..6ed5f568b 100644 --- a/server/services/signalsService/signals/third_party_signals/sentinel/SentinelRareClassAffinitySignal.test.ts +++ b/server/services/signalsService/signals/third_party_signals/sentinel/SentinelRareClassAffinitySignal.test.ts @@ -1,10 +1,8 @@ import { ScalarTypes } from '@roostorg/types'; +import { jsonParse } from '../../../../../utils/encoding.js'; import { type ItemInvestigationService } from '../../../../itemInvestigationService/index.js'; -import { - SentinelServiceError, - type SentinelService, -} from '../../../../sentinelService/sentinelService.js'; +import { type FetchHTTP } from '../../../../networkingService/index.js'; import { Integration } from '../../../types/Integration.js'; import { SignalPricingStructure } from '../../../types/SignalPricingStructure.js'; import { SignalType } from '../../../types/SignalType.js'; @@ -31,23 +29,68 @@ function makeInput( } as unknown as SentinelSignalInput; } -function makeSentinelService( - overrides: Partial = {}, -): SentinelService { +type FetchResponse = { ok: boolean; status: number; body: unknown }; +type EndpointHandlers = { + health?: () => FetchResponse; + banksStatus?: () => FetchResponse; + score?: () => FetchResponse; +}; + +/** + * Builds a fake `FetchHTTP` that answers Sentinel's `/health`, `/banks/status`, + * and `/score` endpoints (all healthy/loaded/scored by default), so tests can + * exercise the signal through `makeSentinelService` the same way it's wired + * in production, rather than mocking a `SentinelService` instance directly. + * + * Request history is read back from Jest's own `mock.calls`, rather than a + * hand-rolled accumulator, so nothing here mutates an array in place. + */ +function makeFetchHTTP(handlers: EndpointHandlers = {}) { + type Req = { url: string; body?: string }; + const mockFetch = jest.fn(async (req: Req) => { + if (req.url.endsWith('/health')) { + return ( + handlers.health?.() ?? { + ok: true, + status: 200, + body: { status: 'ok', banks_loaded: true }, + } + ); + } + if (req.url.endsWith('/banks/status')) { + return ( + handlers.banksStatus?.() ?? { + ok: true, + status: 200, + body: { loaded: true }, + } + ); + } + if (req.url.endsWith('/score')) { + return ( + handlers.score?.() ?? { + ok: true, + status: 200, + body: { + rare_class_affinity_score: 0.5, + observation_scores: { 'test content': 0.5 }, + num_observations: 1, + }, + } + ); + } + throw new Error(`Unexpected URL in test: ${req.url}`); + }); + const requests = () => mockFetch.mock.calls.map(([req]) => req); return { - healthCheck: jest - .fn() - .mockResolvedValue({ status: 'ok', banks_loaded: true }), - getBanksStatus: jest.fn().mockResolvedValue({ loaded: true }), - scoreTexts: jest.fn().mockResolvedValue({ - rare_class_affinity_score: 0.5, - observation_scores: { 'test content': 0.5 }, - num_observations: 1, - }), - scoreSingleText: jest.fn().mockResolvedValue(0.5), - loadBanks: jest.fn().mockResolvedValue(undefined), - unloadBanks: jest.fn().mockResolvedValue(undefined), - ...overrides, + fetchHTTP: mockFetch as unknown as FetchHTTP, + lastScoreRequest: (): unknown => { + const scoreRequest = requests() + .filter((req) => req.url.endsWith('/score') && req.body != null) + .at(-1); + return scoreRequest ? jsonParse(scoreRequest.body as never) : undefined; + }, + requestUrls: () => requests().map((req) => req.url), }; } @@ -82,18 +125,50 @@ function makeItemInvestigationService( }; } -function makeSignal( - sentinelService?: Partial, - iisOverrides?: Partial, -) { +/** + * `undefined` (the default) means the org has enabled Sentinel with no + * overrides — i.e. `getByIntegrationId` returned `{}`, which is distinct + * from the org never having enabled the integration at all (`undefined`). + */ +function makeSignal(options?: { + orgConfig?: Record | undefined; + fetchHTTP?: FetchHTTP; + iisOverrides?: Partial; + defaultApiUrl?: string | undefined; +}) { + // Destructuring defaults trigger on an explicit `undefined` value, not + // just a missing key — and tests need to distinguish "not provided, + // use the default" from "explicitly no org config / no default URL". So + // check key presence instead of using destructuring defaults for these two. + const opts = options ?? {}; + const orgConfig: Record | undefined = + 'orgConfig' in opts ? opts.orgConfig : {}; + const defaultApiUrl: string | undefined = + 'defaultApiUrl' in opts ? opts.defaultApiUrl : 'http://localhost:8000'; + const fetchHTTP = opts.fetchHTTP ?? makeFetchHTTP().fetchHTTP; + return new SentinelRareClassAffinitySignal( - makeSentinelService(sentinelService), + jest.fn().mockResolvedValue(orgConfig), + fetchHTTP, makeItemInvestigationService( - iisOverrides, + opts.iisOverrides, ) as unknown as ItemInvestigationService, + defaultApiUrl, ); } +/** A minimal thread item, as yielded by `getThreadSubmissionsByTime`. */ +function makeThreadItem(text: string) { + return { + latestSubmission: { + data: { text }, + itemType: { kind: 'CONTENT', schema: [], schemaFieldRoles: {} }, + }, + priorSubmissions: undefined, + parents: (async function* () {})(), + }; +} + describe('SentinelRareClassAffinitySignal', () => { describe('signal metadata', () => { it('returns correct id', () => { @@ -147,32 +222,53 @@ describe('SentinelRareClassAffinitySignal', () => { expect(info.disabled).toBe(false); }); + it('returns disabled=true when the org has not enabled Sentinel', async () => { + const signal = makeSignal({ orgConfig: undefined }); + const info = await signal.getDisabledInfo('org-1'); + expect(info.disabled).toBe(true); + expect(info.disabledMessage).toContain( + 'not enabled for this organization', + ); + }); + + it('returns disabled=true when no URL is configured at all', async () => { + const signal = makeSignal({ orgConfig: {}, defaultApiUrl: undefined }); + const info = await signal.getDisabledInfo('org-1'); + expect(info.disabled).toBe(true); + expect(info.disabledMessage).toContain('No Sentinel API URL'); + }); + it('returns disabled=true when health check fails', async () => { - const signal = makeSignal({ - healthCheck: jest - .fn() - .mockRejectedValue(new Error('Connection refused')), + const { fetchHTTP } = makeFetchHTTP({ + health: () => { + throw new Error('Connection refused'); + }, }); + const signal = makeSignal({ fetchHTTP }); const info = await signal.getDisabledInfo('org-1'); expect(info.disabled).toBe(true); expect(info.disabledMessage).toContain('unavailable'); }); it('returns disabled=true when health status is not ok', async () => { - const signal = makeSignal({ - healthCheck: jest - .fn() - .mockResolvedValue({ status: 'error', banks_loaded: false }), + const { fetchHTTP } = makeFetchHTTP({ + health: () => ({ + ok: true, + status: 200, + body: { status: 'error', banks_loaded: false }, + }), }); + const signal = makeSignal({ fetchHTTP }); const info = await signal.getDisabledInfo('org-1'); expect(info.disabled).toBe(true); expect(info.disabledMessage).toContain('not healthy'); }); it('returns disabled=true when banks are not loaded', async () => { - const signal = makeSignal({ - getBanksStatus: jest.fn().mockResolvedValue({ loaded: false }), + const { fetchHTTP } = makeFetchHTTP({ + banksStatus: () => ({ ok: true, status: 200, body: { loaded: false } }), }); + const signal = makeSignal({ fetchHTTP }); const info = await signal.getDisabledInfo('org-1'); expect(info.disabled).toBe(true); expect(info.disabledMessage).toContain('banks are not loaded'); @@ -181,49 +277,81 @@ describe('SentinelRareClassAffinitySignal', () => { describe('run', () => { it('scores the primary text and returns rare_class_affinity_score', async () => { - const scoreTexts = jest.fn().mockResolvedValue({ - rare_class_affinity_score: 0.72, - observation_scores: { 'test content': 0.72 }, - num_observations: 1, + const { fetchHTTP, lastScoreRequest } = makeFetchHTTP({ + score: () => ({ + ok: true, + status: 200, + body: { + rare_class_affinity_score: 0.72, + observation_scores: { 'test content': 0.72 }, + num_observations: 1, + }, + }), }); - const signal = makeSignal({ scoreTexts }); + const signal = makeSignal({ fetchHTTP }); const result = await signal.run(makeInput()); - expect(scoreTexts).toHaveBeenCalledWith( - expect.objectContaining({ - texts: expect.arrayContaining(['test content']), - }), - ); + expect(lastScoreRequest()).toMatchObject({ + texts: expect.arrayContaining(['test content']), + }); expect(result).toMatchObject({ outputType: { scalarType: ScalarTypes.NUMBER }, score: 0.72, }); }); - it('includes thread context texts when threadIdentifier is provided', async () => { - const scoreTexts = jest.fn().mockResolvedValue({ - rare_class_affinity_score: 0.85, - observation_scores: {}, - num_observations: 2, + it('returns an ERROR result when neither org config nor a default URL is set', async () => { + const signal = makeSignal({ orgConfig: {}, defaultApiUrl: undefined }); + const result = await signal.run(makeInput()); + expect(result.type).toBe('ERROR'); + }); + + it("uses the org's configured apiUrl instead of the deployment default", async () => { + const { fetchHTTP, requestUrls } = makeFetchHTTP(); + const signal = makeSignal({ + fetchHTTP, + orgConfig: { apiUrl: 'http://org-sentinel.internal:9000' }, + defaultApiUrl: 'http://localhost:8000', }); - // Async iterable that yields one prior thread item - const threadItem = { - latestSubmission: { - data: { text: 'prior thread message' }, - itemType: { kind: 'CONTENT', schema: [], schemaFieldRoles: {} }, - }, - priorSubmissions: undefined, - parents: (async function* () {})(), - }; + await signal.run(makeInput()); + + expect( + requestUrls().some((url) => + url.startsWith('http://org-sentinel.internal:9000'), + ), + ).toBe(true); + }); + + it('forwards topK and minScoreToConsider overrides to the /score request', async () => { + const { fetchHTTP, lastScoreRequest } = makeFetchHTTP(); + const signal = makeSignal({ + fetchHTTP, + orgConfig: { topK: 3, minScoreToConsider: 0.4 }, + }); + + await signal.run(makeInput()); + + expect(lastScoreRequest()).toMatchObject({ + top_k: 3, + min_score_to_consider: 0.4, + }); + }); + + it('includes thread context texts when threadIdentifier is provided', async () => { + const { fetchHTTP, lastScoreRequest } = makeFetchHTTP(); + const getThreadSubmissionsByTime = jest.fn().mockReturnValue( (async function* () { - yield threadItem; + yield makeThreadItem('prior thread message'); })(), ); - const signal = makeSignal({ scoreTexts }, { getThreadSubmissionsByTime }); + const signal = makeSignal({ + fetchHTTP, + iisOverrides: { getThreadSubmissionsByTime }, + }); await signal.run( makeInput({ @@ -234,46 +362,60 @@ describe('SentinelRareClassAffinitySignal', () => { }), ); - const texts = scoreTexts.mock.calls[0][0].texts; + const texts = (lastScoreRequest() as { texts: string[] }).texts; expect(texts).toContain('test content'); expect(texts).toContain('prior thread message'); }); + it('passes a threadContextWindowMinutes override as an oldestReturnedSubmissionDate bound', async () => { + const { fetchHTTP } = makeFetchHTTP(); + const getThreadSubmissionsByTime = jest.fn().mockReturnValue( + (async function* () { + // no items needed; we're only asserting on the call args + })(), + ); + + const signal = makeSignal({ + fetchHTTP, + orgConfig: { threadContextWindowMinutes: 30 }, + iisOverrides: { getThreadSubmissionsByTime }, + }); + + const before = Date.now(); + await signal.run( + makeInput({ + runtimeArgs: { + threadIdentifier: { id: 'thread-1', typeId: 'content-type-1' }, + }, + }), + ); + + const call = getThreadSubmissionsByTime.mock.calls[0][0]; + expect(call.oldestReturnedSubmissionDate).toBeInstanceOf(Date); + const windowMs = before - call.oldestReturnedSubmissionDate.getTime(); + // Should be ~30 minutes (allow slack for test execution time). + expect(windowMs).toBeGreaterThan(30 * 60_000 - 5_000); + expect(windowMs).toBeLessThan(30 * 60_000 + 5_000); + }); + it('does not double-count the triggering submission when it is echoed back as thread context', async () => { // submitContent.ts writes the current submission to Scylla before // running rules, so getThreadSubmissionsByTime (which is time-bounded, // not identity-bounded) can return the very submission that triggered // this signal run alongside genuine prior messages. - const scoreTexts = jest.fn().mockResolvedValue({ - rare_class_affinity_score: 0.5, - observation_scores: {}, - num_observations: 2, - }); + const { fetchHTTP, lastScoreRequest } = makeFetchHTTP(); - const priorItem = { - latestSubmission: { - data: { text: 'prior thread message' }, - itemType: { kind: 'CONTENT', schema: [], schemaFieldRoles: {} }, - }, - priorSubmissions: undefined, - parents: (async function* () {})(), - }; - const selfItem = { - latestSubmission: { - data: { text: 'test content' }, - itemType: { kind: 'CONTENT', schema: [], schemaFieldRoles: {} }, - }, - priorSubmissions: undefined, - parents: (async function* () {})(), - }; const getThreadSubmissionsByTime = jest.fn().mockReturnValue( (async function* () { - yield selfItem; - yield priorItem; + yield makeThreadItem('test content'); // echoed-back self item + yield makeThreadItem('prior thread message'); })(), ); - const signal = makeSignal({ scoreTexts }, { getThreadSubmissionsByTime }); + const signal = makeSignal({ + fetchHTTP, + iisOverrides: { getThreadSubmissionsByTime }, + }); await signal.run( makeInput({ @@ -284,33 +426,24 @@ describe('SentinelRareClassAffinitySignal', () => { }), ); - const texts = scoreTexts.mock.calls[0][0].texts; + const texts = (lastScoreRequest() as { texts: string[] }).texts; expect(texts).toEqual(['test content', 'prior thread message']); }); it('only drops one occurrence of duplicate text, in case a genuine duplicate message exists', async () => { - const scoreTexts = jest.fn().mockResolvedValue({ - rare_class_affinity_score: 0.5, - observation_scores: {}, - num_observations: 3, - }); + const { fetchHTTP, lastScoreRequest } = makeFetchHTTP(); - const makeItem = (text: string) => ({ - latestSubmission: { - data: { text }, - itemType: { kind: 'CONTENT', schema: [], schemaFieldRoles: {} }, - }, - priorSubmissions: undefined, - parents: (async function* () {})(), - }); const getThreadSubmissionsByTime = jest.fn().mockReturnValue( (async function* () { - yield makeItem('test content'); - yield makeItem('test content'); + yield makeThreadItem('test content'); + yield makeThreadItem('test content'); })(), ); - const signal = makeSignal({ scoreTexts }, { getThreadSubmissionsByTime }); + const signal = makeSignal({ + fetchHTTP, + iisOverrides: { getThreadSubmissionsByTime }, + }); await signal.run( makeInput({ @@ -321,21 +454,20 @@ describe('SentinelRareClassAffinitySignal', () => { }), ); - const texts = scoreTexts.mock.calls[0][0].texts; + const texts = (lastScoreRequest() as { texts: string[] }).texts; expect(texts).toEqual(['test content', 'test content']); }); it('still returns a score when thread fetch fails', async () => { - const scoreTexts = jest.fn().mockResolvedValue({ - rare_class_affinity_score: 0.5, - observation_scores: { 'test content': 0.5 }, - num_observations: 1, - }); + const { fetchHTTP, lastScoreRequest } = makeFetchHTTP(); const getThreadSubmissionsByTime = jest.fn().mockImplementation(() => { throw new Error('Scylla unavailable'); }); - const signal = makeSignal({ scoreTexts }, { getThreadSubmissionsByTime }); + const signal = makeSignal({ + fetchHTTP, + iisOverrides: { getThreadSubmissionsByTime }, + }); const result = await signal.run( makeInput({ @@ -347,26 +479,16 @@ describe('SentinelRareClassAffinitySignal', () => { // Should still score with just the primary text expect(result).toMatchObject({ score: 0.5 }); - expect(scoreTexts).toHaveBeenCalledWith( - expect.objectContaining({ texts: ['test content'] }), - ); + expect(lastScoreRequest()).toMatchObject({ texts: ['test content'] }); }); - it('returns an error result when Sentinel service throws SentinelServiceError', async () => { - const signal = makeSignal({ - scoreTexts: jest - .fn() - .mockRejectedValue(new SentinelServiceError('Banks not loaded', 503)), + it('returns an error result when Sentinel service returns a non-ok response', async () => { + const { fetchHTTP } = makeFetchHTTP({ + score: () => ({ ok: false, status: 503, body: 'Banks not loaded' }), }); + const signal = makeSignal({ fetchHTTP }); const result = await signal.run(makeInput()); expect(result.type).toBe('ERROR'); }); - - it('re-throws non-SentinelServiceError exceptions', async () => { - const signal = makeSignal({ - scoreTexts: jest.fn().mockRejectedValue(new Error('Unexpected error')), - }); - await expect(signal.run(makeInput())).rejects.toThrow('Unexpected error'); - }); }); }); diff --git a/server/services/signalsService/signals/third_party_signals/sentinel/SentinelRareClassAffinitySignal.ts b/server/services/signalsService/signals/third_party_signals/sentinel/SentinelRareClassAffinitySignal.ts index 070879ba8..acb236f58 100644 --- a/server/services/signalsService/signals/third_party_signals/sentinel/SentinelRareClassAffinitySignal.ts +++ b/server/services/signalsService/signals/third_party_signals/sentinel/SentinelRareClassAffinitySignal.ts @@ -3,29 +3,82 @@ import { ScalarTypes } from '@roostorg/types'; import { makeSignalPermanentError } from '../../../../../utils/errors.js'; import { type ItemInvestigationService } from '../../../../itemInvestigationService/index.js'; import { type ItemSubmission } from '../../../../itemProcessingService/index.js'; +import { type FetchHTTP } from '../../../../networkingService/index.js'; import { + makeSentinelService, SentinelServiceError, - type SentinelService, } from '../../../../sentinelService/index.js'; import { SignalPricingStructure } from '../../../types/SignalPricingStructure.js'; import { SignalType } from '../../../types/SignalType.js'; import SignalBase, { type SignalInput } from '../../SignalBase.js'; -const SENTINEL_DOCS_URL = 'https://github.com/UMass-Rescue/Sentinel'; +const SENTINEL_DOCS_URL = 'https://github.com/Roblox/sentinel'; /** - * How many prior thread items to include in Sentinel scoring. - * More context improves recall for pattern detection but adds latency. + * How many prior thread items to include in Sentinel scoring, when the org + * hasn't configured a `threadContextWindowMinutes` override. More context + * improves recall for pattern detection but adds latency. */ const DEFAULT_THREAD_CONTEXT_LIMIT = 10; +/** + * Per-org Sentinel configuration. Stored as opaque JSON via the generic + * `integration_configs` table — SENTINEL is deliberately not a + * `ConfigurableIntegration` (see signalAuthService.ts), since none of these + * fields are secret credentials. Presence of a saved config (even `{}`) + * means the org has enabled this integration from the dashboard; individual + * fields fall back to deployment-wide defaults when left unset. + */ +type SentinelOrgConfig = { + apiUrl?: string; + topK?: number; + minScoreToConsider?: number; + threadContextWindowMinutes?: number; +}; + +/** Cached getter for an org's raw Sentinel config, keyed by orgId. */ +type GetSentinelConfig = ( + orgId: string, +) => Promise | undefined>; + +function parseSentinelConfig( + raw: Record | undefined, +): SentinelOrgConfig | undefined { + if (raw == null) { + return undefined; + } + const config: SentinelOrgConfig = {}; + if (typeof raw.apiUrl === 'string' && raw.apiUrl.trim() !== '') { + config.apiUrl = raw.apiUrl.trim(); + } + if (typeof raw.topK === 'number' && Number.isFinite(raw.topK)) { + config.topK = raw.topK; + } + if ( + typeof raw.minScoreToConsider === 'number' && + Number.isFinite(raw.minScoreToConsider) + ) { + config.minScoreToConsider = raw.minScoreToConsider; + } + if ( + typeof raw.threadContextWindowMinutes === 'number' && + Number.isFinite(raw.threadContextWindowMinutes) + ) { + config.threadContextWindowMinutes = raw.threadContextWindowMinutes; + } + return config; +} + export default class SentinelRareClassAffinitySignal extends SignalBase< ScalarTypes['STRING'], { scalarType: ScalarTypes['NUMBER'] } > { constructor( - private readonly sentinelService: SentinelService, + private readonly getSentinelConfig: GetSentinelConfig, + private readonly fetchHTTP: FetchHTTP, private readonly itemInvestigationService: ItemInvestigationService, + /** Adopter-level fallback (from SENTINEL_API_URL). Org config, when set, overrides it. */ + private readonly defaultApiUrl?: string, ) { super(); } @@ -92,9 +145,28 @@ It compares submitted content against labeled positive (rare/harmful) and negati return true; } - override async getDisabledInfo(_orgId: string) { + override async getDisabledInfo(orgId: string) { + const config = parseSentinelConfig(await this.getSentinelConfig(orgId)); + if (config == null) { + return { + disabled: true as const, + disabledMessage: + 'Sentinel is not enabled for this organization. Add it from the Integrations page to use this signal.', + }; + } + + const apiUrl = config.apiUrl ?? this.defaultApiUrl; + if (apiUrl == null) { + return { + disabled: true as const, + disabledMessage: + 'No Sentinel API URL is configured. Set one on the Integrations page.', + }; + } + try { - const health = await this.sentinelService.healthCheck(); + const sentinelService = makeSentinelService(this.fetchHTTP, apiUrl); + const health = await sentinelService.healthCheck(); if (health.status !== 'ok' && health.status !== 'healthy') { return { disabled: true as const, @@ -103,7 +175,7 @@ It compares submitted content against labeled positive (rare/harmful) and negati }; } - const banksStatus = await this.sentinelService.getBanksStatus(); + const banksStatus = await sentinelService.getBanksStatus(); if (!banksStatus.loaded) { return { disabled: true as const, @@ -132,6 +204,24 @@ It compares submitted content against labeled positive (rare/harmful) and negati >, ) { const { value, orgId, runtimeArgs } = input; + const config = + parseSentinelConfig(await this.getSentinelConfig(orgId)) ?? {}; + const apiUrl = config.apiUrl ?? this.defaultApiUrl; + + if (apiUrl == null) { + // Permanent: without a configured (or default) URL, this signal can + // never run for this org, and retrying yields the same failure. + return { + type: 'ERROR' as const, + score: makeSignalPermanentError('Sentinel is not configured', { + detail: + 'No Sentinel API URL is configured for this organization or as a deployment default (SENTINEL_API_URL).', + shouldErrorSpan: false, + }), + }; + } + + const sentinelService = makeSentinelService(this.fetchHTTP, apiUrl); // The primary text to score is always the signal input value. const primaryText = String(value.value); @@ -147,6 +237,13 @@ It compares submitted content against labeled positive (rare/harmful) and negati orgId, threadId: runtimeArgs.threadIdentifier, limit: DEFAULT_THREAD_CONTEXT_LIMIT, + ...(config.threadContextWindowMinutes != null + ? { + oldestReturnedSubmissionDate: new Date( + Date.now() - config.threadContextWindowMinutes * 60_000, + ), + } + : {}), }); // `submitContent.ts` writes the current submission to the thread's @@ -179,7 +276,11 @@ It compares submitted content against labeled positive (rare/harmful) and negati } try { - const response = await this.sentinelService.scoreTexts({ texts }); + const response = await sentinelService.scoreTexts({ + texts, + top_k: config.topK, + min_score_to_consider: config.minScoreToConsider, + }); return { outputType: { scalarType: ScalarTypes.NUMBER }, score: response.rare_class_affinity_score, From eb953959d3159dc771008ca6b11100664e993396 Mon Sep 17 00:00:00 2001 From: Sahil Sharma Date: Tue, 1 Sep 2026 11:00:25 -0400 Subject: [PATCH 2/2] Fix decimal input being clobbered in Sentinel numeric config fields Input.onChange parsed event.target.value to Number immediately, which turns "0." into 0 and strips the decimal point on every keystroke, making it impossible to type values like 0.1 into fields such as 'Minimum score to consider'. Track the raw text being typed in local state and display that while editing; only fall back to the canonically formatted/parsed value once the field loses focus. --- ...IntegrationConfigApiCredentialsSection.tsx | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/client/src/webpages/dashboard/integrations/IntegrationConfigApiCredentialsSection.tsx b/client/src/webpages/dashboard/integrations/IntegrationConfigApiCredentialsSection.tsx index 573e88c07..7d60122d8 100644 --- a/client/src/webpages/dashboard/integrations/IntegrationConfigApiCredentialsSection.tsx +++ b/client/src/webpages/dashboard/integrations/IntegrationConfigApiCredentialsSection.tsx @@ -1,5 +1,6 @@ import { Button, Input } from 'antd'; import { Plus, Trash2 } from 'lucide-react'; +import { useState } from 'react'; import { GQLGoogleContentSafetyApiIntegrationApiCredential, @@ -202,6 +203,12 @@ export default function IntegrationConfigApiCredentialsSection(props: { 'threadContextWindowMinutes', ]); + // Raw text currently being typed into numeric fields, so e.g. "0." isn't + // clobbered mid-edit by re-rendering with `String(credential[key])`. + const [sentinelDrafts, setSentinelDrafts] = useState>( + {}, + ); + const renderSentinelCredential = (pluginCredential: { __typename: 'PluginIntegrationApiCredential'; credential: Record; @@ -213,9 +220,14 @@ export default function IntegrationConfigApiCredentialsSection(props: {
{label}
{ const raw = event.target.value; + setSentinelDrafts((prev) => ({ ...prev, [key]: raw })); + let next: Record; if (raw.trim() === '') { const { [key]: _removed, ...rest } = credential; @@ -234,6 +246,12 @@ export default function IntegrationConfigApiCredentialsSection(props: { next as import('../../../graphql/generated').Scalars['JSONObject'], }); }} + onBlur={() => { + // Snap back to the canonical (parsed/committed) formatting + // once the user is done typing, e.g. "0.10" -> "0.1". + const { [key]: _removed, ...rest } = sentinelDrafts; + setSentinelDrafts(rest); + }} />
))}