From 594c89ab765bed471d227d8dfb9c483d2e3ea62b Mon Sep 17 00:00:00 2001 From: Ivan Malison Date: Fri, 31 Jul 2026 18:55:19 -0700 Subject: [PATCH] Select the detection audit payload on getTableState Adds identifierToBox to TableStateGQL and selects it, alongside the homography that was already in the schema but never requested, so the photo path can draw a detection back over the image it came from. The live and shot queries keep the layout-only selection: their frame lives inside a video the app never holds, and one of them polls every five seconds. Co-Authored-By: Claude Opus 5 --- src/index.tsx | 200 +++++++++++++++++++++++++++-- src/operations/shot_simulation.gql | 85 ++++++++++-- src/schema.gql | 1 + 3 files changed, 262 insertions(+), 24 deletions(-) diff --git a/src/index.tsx b/src/index.tsx index 31040f5..ee2243d 100644 --- a/src/index.tsx +++ b/src/index.tsx @@ -4078,6 +4078,7 @@ export type SyncAppleSubscriptionResultGql = { export type TableStateGql = { __typename?: "TableStateGQL"; homography?: Maybe; + identifierToBox: Array; identifierToColor: Array>; identifierToPosition: Array>; }; @@ -7480,6 +7481,76 @@ export type FinalizePlayerAssignmentsMutation = { }>; }; +export type TableStateLayoutFieldsFragment = { + __typename?: "TableStateGQL"; + identifierToPosition: Array>; + identifierToColor: Array<{ __typename?: "RGBColorGQL"; hex: string } | null>; +}; + +export type DetectionBoxFieldsFragment = { + __typename?: "BoundingBoxGQL"; + left: number; + top: number; + width: number; + height: number; +}; + +export type DetectionPocketPointFieldsFragment = { + __typename?: "PocketPointsGQL"; + topLeft: { __typename?: "IntPoint2D"; x: number; y: number }; + topSide: { __typename?: "IntPoint2D"; x: number; y: number }; + topRight: { __typename?: "IntPoint2D"; x: number; y: number }; + bottomLeft: { __typename?: "IntPoint2D"; x: number; y: number }; + bottomSide: { __typename?: "IntPoint2D"; x: number; y: number }; + bottomRight: { __typename?: "IntPoint2D"; x: number; y: number }; +}; + +export type TableStateDetectionFieldsFragment = { + __typename?: "TableStateGQL"; + identifierToBox: Array<{ + __typename?: "BoundingBoxGQL"; + left: number; + top: number; + width: number; + height: number; + }>; + homography?: { + __typename?: "HomographyInfoGQL"; + crop: { + __typename?: "BoundingBoxGQL"; + left: number; + top: number; + width: number; + height: number; + }; + pockets: Array<{ + __typename?: "BoundingBoxGQL"; + left: number; + top: number; + width: number; + height: number; + }>; + sourcePoints: { + __typename?: "PocketPointsGQL"; + topLeft: { __typename?: "IntPoint2D"; x: number; y: number }; + topSide: { __typename?: "IntPoint2D"; x: number; y: number }; + topRight: { __typename?: "IntPoint2D"; x: number; y: number }; + bottomLeft: { __typename?: "IntPoint2D"; x: number; y: number }; + bottomSide: { __typename?: "IntPoint2D"; x: number; y: number }; + bottomRight: { __typename?: "IntPoint2D"; x: number; y: number }; + }; + destPoints: { + __typename?: "PocketPointsGQL"; + topLeft: { __typename?: "IntPoint2D"; x: number; y: number }; + topSide: { __typename?: "IntPoint2D"; x: number; y: number }; + topRight: { __typename?: "IntPoint2D"; x: number; y: number }; + bottomLeft: { __typename?: "IntPoint2D"; x: number; y: number }; + bottomSide: { __typename?: "IntPoint2D"; x: number; y: number }; + bottomRight: { __typename?: "IntPoint2D"; x: number; y: number }; + }; + } | null; +}; + export type GetTableStateQueryVariables = Exact<{ b64Image: Scalars["String"]["input"]; tableSize?: InputMaybe; @@ -7495,6 +7566,48 @@ export type GetTableStateQuery = { __typename?: "RGBColorGQL"; hex: string; } | null>; + identifierToBox: Array<{ + __typename?: "BoundingBoxGQL"; + left: number; + top: number; + width: number; + height: number; + }>; + homography?: { + __typename?: "HomographyInfoGQL"; + crop: { + __typename?: "BoundingBoxGQL"; + left: number; + top: number; + width: number; + height: number; + }; + pockets: Array<{ + __typename?: "BoundingBoxGQL"; + left: number; + top: number; + width: number; + height: number; + }>; + sourcePoints: { + __typename?: "PocketPointsGQL"; + topLeft: { __typename?: "IntPoint2D"; x: number; y: number }; + topSide: { __typename?: "IntPoint2D"; x: number; y: number }; + topRight: { __typename?: "IntPoint2D"; x: number; y: number }; + bottomLeft: { __typename?: "IntPoint2D"; x: number; y: number }; + bottomSide: { __typename?: "IntPoint2D"; x: number; y: number }; + bottomRight: { __typename?: "IntPoint2D"; x: number; y: number }; + }; + destPoints: { + __typename?: "PocketPointsGQL"; + topLeft: { __typename?: "IntPoint2D"; x: number; y: number }; + topSide: { __typename?: "IntPoint2D"; x: number; y: number }; + topRight: { __typename?: "IntPoint2D"; x: number; y: number }; + bottomLeft: { __typename?: "IntPoint2D"; x: number; y: number }; + bottomSide: { __typename?: "IntPoint2D"; x: number; y: number }; + bottomRight: { __typename?: "IntPoint2D"; x: number; y: number }; + }; + } | null; }; }; @@ -9980,6 +10093,73 @@ export const PlayerClusterFieldsFragmentDoc = gql` } ${PlayerClusterShotFieldsFragmentDoc} `; +export const TableStateLayoutFieldsFragmentDoc = gql` + fragment TableStateLayoutFields on TableStateGQL { + identifierToPosition + identifierToColor { + hex + } + } +`; +export const DetectionBoxFieldsFragmentDoc = gql` + fragment DetectionBoxFields on BoundingBoxGQL { + left + top + width + height + } +`; +export const DetectionPocketPointFieldsFragmentDoc = gql` + fragment DetectionPocketPointFields on PocketPointsGQL { + topLeft { + x + y + } + topSide { + x + y + } + topRight { + x + y + } + bottomLeft { + x + y + } + bottomSide { + x + y + } + bottomRight { + x + y + } + } +`; +export const TableStateDetectionFieldsFragmentDoc = gql` + fragment TableStateDetectionFields on TableStateGQL { + identifierToBox { + ...DetectionBoxFields + } + homography { + crop { + ...DetectionBoxFields + } + pockets { + ...DetectionBoxFields + } + sourcePoints { + ...DetectionPocketPointFields + } + destPoints { + ...DetectionPocketPointFields + } + } + } + ${DetectionBoxFieldsFragmentDoc} + ${DetectionPocketPointFieldsFragmentDoc} +`; export const ShotClipRangeFragmentDoc = gql` fragment ShotClipRange on ShotGQL { id @@ -15931,12 +16111,12 @@ export const GetTableStateDocument = gql` tableSize: $tableSize useHomography: $useHomography ) { - identifierToPosition - identifierToColor { - hex - } + ...TableStateLayoutFields + ...TableStateDetectionFields } } + ${TableStateLayoutFieldsFragmentDoc} + ${TableStateDetectionFieldsFragmentDoc} `; /** @@ -16012,13 +16192,11 @@ export const GetLiveTableStateDocument = gql` videoId frameIndex tableState { - identifierToPosition - identifierToColor { - hex - } + ...TableStateLayoutFields } } } + ${TableStateLayoutFieldsFragmentDoc} `; /** @@ -16093,13 +16271,11 @@ export const GetShotTableStateDocument = gql` videoId frameIndex tableState { - identifierToPosition - identifierToColor { - hex - } + ...TableStateLayoutFields } } } + ${TableStateLayoutFieldsFragmentDoc} `; /** diff --git a/src/operations/shot_simulation.gql b/src/operations/shot_simulation.gql index e4ccc29..a4604c8 100644 --- a/src/operations/shot_simulation.gql +++ b/src/operations/shot_simulation.gql @@ -1,3 +1,72 @@ +fragment TableStateLayoutFields on TableStateGQL { + identifierToPosition + identifierToColor { + hex + } +} + +fragment DetectionBoxFields on BoundingBoxGQL { + left + top + width + height +} + +fragment DetectionPocketPointFields on PocketPointsGQL { + topLeft { + x + y + } + topSide { + x + y + } + topRight { + x + y + } + bottomLeft { + x + y + } + bottomSide { + x + y + } + bottomRight { + x + y + } +} + +# Everything needed to redraw a detection over the image it came from: +# the detector's boxes plus the homography that turned them into plane +# positions. Boxes, pocket boxes, and source points share the crop's +# coordinate space; the crop locates that space in the full frame. +# +# Only the photo query selects this. The live and shot queries read a +# frame out of a video the app never holds, so there is no image to +# overlay, and getLiveTableState is polled every five seconds. +fragment TableStateDetectionFields on TableStateGQL { + identifierToBox { + ...DetectionBoxFields + } + homography { + crop { + ...DetectionBoxFields + } + pockets { + ...DetectionBoxFields + } + sourcePoints { + ...DetectionPocketPointFields + } + destPoints { + ...DetectionPocketPointFields + } + } +} + query GetTableState( $b64Image: String! $tableSize: Float @@ -8,10 +77,8 @@ query GetTableState( tableSize: $tableSize useHomography: $useHomography ) { - identifierToPosition - identifierToColor { - hex - } + ...TableStateLayoutFields + ...TableStateDetectionFields } } @@ -20,10 +87,7 @@ query GetLiveTableState($videoId: Int!) { videoId frameIndex tableState { - identifierToPosition - identifierToColor { - hex - } + ...TableStateLayoutFields } } } @@ -34,10 +98,7 @@ query GetShotTableState($shotId: Int!) { videoId frameIndex tableState { - identifierToPosition - identifierToColor { - hex - } + ...TableStateLayoutFields } } } diff --git a/src/schema.gql b/src/schema.gql index c1406e3..8c6d6ba 100644 --- a/src/schema.gql +++ b/src/schema.gql @@ -1708,6 +1708,7 @@ type TableStateGQL { identifierToPosition: [[Float!]!]! homography: HomographyInfoGQL identifierToColor: [RGBColorGQL]! + identifierToBox: [BoundingBoxGQL!]! } type TagClassGQL { -- 2.49.1