Select the detection audit payload on getTableState

Adds identifierToBox to TableStateGQL and selects it, alongside the homography that was already in the schema but never requested, so the photo path can draw a detection back over the image it came from. The live and shot queries keep the layout-only selection: their frame lives inside a video the app never holds, and one of them polls every five seconds. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-07-31 18:55:19 -07:00
parent d7b0b21430
commit 594c89ab76
3 changed files with 262 additions and 24 deletions
--- a/src/index.tsx
+++ b/src/index.tsx
@@ -4078,6 +4078,7 @@ export type SyncAppleSubscriptionResultGql = {
 export type TableStateGql = {
  __typename?: "TableStateGQL";
  homography?: Maybe<HomographyInfoGql>;
+  identifierToBox: Array<BoundingBoxGql>;
  identifierToColor: Array<Maybe<RgbColorGql>>;
  identifierToPosition: Array<Array<Scalars["Float"]["output"]>>;
 };
@@ -7480,6 +7481,76 @@ export type FinalizePlayerAssignmentsMutation = {
  }>;
 };

+export type TableStateLayoutFieldsFragment = {
+  __typename?: "TableStateGQL";
+  identifierToPosition: Array<Array<number>>;
+  identifierToColor: Array<{ __typename?: "RGBColorGQL"; hex: string } | null>;
+};
+
+export type DetectionBoxFieldsFragment = {
+  __typename?: "BoundingBoxGQL";
+  left: number;
+  top: number;
+  width: number;
+  height: number;
+};
+
+export type DetectionPocketPointFieldsFragment = {
+  __typename?: "PocketPointsGQL";
+  topLeft: { __typename?: "IntPoint2D"; x: number; y: number };
+  topSide: { __typename?: "IntPoint2D"; x: number; y: number };
+  topRight: { __typename?: "IntPoint2D"; x: number; y: number };
+  bottomLeft: { __typename?: "IntPoint2D"; x: number; y: number };
+  bottomSide: { __typename?: "IntPoint2D"; x: number; y: number };
+  bottomRight: { __typename?: "IntPoint2D"; x: number; y: number };
+};
+
+export type TableStateDetectionFieldsFragment = {
+  __typename?: "TableStateGQL";
+  identifierToBox: Array<{
+    __typename?: "BoundingBoxGQL";
+    left: number;
+    top: number;
+    width: number;
+    height: number;
+  }>;
+  homography?: {
+    __typename?: "HomographyInfoGQL";
+    crop: {
+      __typename?: "BoundingBoxGQL";
+      left: number;
+      top: number;
+      width: number;
+      height: number;
+    };
+    pockets: Array<{
+      __typename?: "BoundingBoxGQL";
+      left: number;
+      top: number;
+      width: number;
+      height: number;
+    }>;
+    sourcePoints: {
+      __typename?: "PocketPointsGQL";
+      topLeft: { __typename?: "IntPoint2D"; x: number; y: number };
+      topSide: { __typename?: "IntPoint2D"; x: number; y: number };
+      topRight: { __typename?: "IntPoint2D"; x: number; y: number };
+      bottomLeft: { __typename?: "IntPoint2D"; x: number; y: number };
+      bottomSide: { __typename?: "IntPoint2D"; x: number; y: number };
+      bottomRight: { __typename?: "IntPoint2D"; x: number; y: number };
+    };
+    destPoints: {
+      __typename?: "PocketPointsGQL";
+      topLeft: { __typename?: "IntPoint2D"; x: number; y: number };
+      topSide: { __typename?: "IntPoint2D"; x: number; y: number };
+      topRight: { __typename?: "IntPoint2D"; x: number; y: number };
+      bottomLeft: { __typename?: "IntPoint2D"; x: number; y: number };
+      bottomSide: { __typename?: "IntPoint2D"; x: number; y: number };
+      bottomRight: { __typename?: "IntPoint2D"; x: number; y: number };
+    };
+  } | null;
+};
+
 export type GetTableStateQueryVariables = Exact<{
  b64Image: Scalars["String"]["input"];
  tableSize?: InputMaybe<Scalars["Float"]["input"]>;
@@ -7495,6 +7566,48 @@ export type GetTableStateQuery = {
      __typename?: "RGBColorGQL";
      hex: string;
    } | null>;
+    identifierToBox: Array<{
+      __typename?: "BoundingBoxGQL";
+      left: number;
+      top: number;
+      width: number;
+      height: number;
+    }>;
+    homography?: {
+      __typename?: "HomographyInfoGQL";
+      crop: {
+        __typename?: "BoundingBoxGQL";
+        left: number;
+        top: number;
+        width: number;
+        height: number;
+      };
+      pockets: Array<{
+        __typename?: "BoundingBoxGQL";
+        left: number;
+        top: number;
+        width: number;
+        height: number;
+      }>;
+      sourcePoints: {
+        __typename?: "PocketPointsGQL";
+        topLeft: { __typename?: "IntPoint2D"; x: number; y: number };
+        topSide: { __typename?: "IntPoint2D"; x: number; y: number };
+        topRight: { __typename?: "IntPoint2D"; x: number; y: number };
+        bottomLeft: { __typename?: "IntPoint2D"; x: number; y: number };
+        bottomSide: { __typename?: "IntPoint2D"; x: number; y: number };
+        bottomRight: { __typename?: "IntPoint2D"; x: number; y: number };
+      };
+      destPoints: {
+        __typename?: "PocketPointsGQL";
+        topLeft: { __typename?: "IntPoint2D"; x: number; y: number };
+        topSide: { __typename?: "IntPoint2D"; x: number; y: number };
+        topRight: { __typename?: "IntPoint2D"; x: number; y: number };
+        bottomLeft: { __typename?: "IntPoint2D"; x: number; y: number };
+        bottomSide: { __typename?: "IntPoint2D"; x: number; y: number };
+        bottomRight: { __typename?: "IntPoint2D"; x: number; y: number };
+      };
+    } | null;
  };
 };

@@ -9980,6 +10093,73 @@ export const PlayerClusterFieldsFragmentDoc = gql`
  }
  ${PlayerClusterShotFieldsFragmentDoc}
 `;
+export const TableStateLayoutFieldsFragmentDoc = gql`
+  fragment TableStateLayoutFields on TableStateGQL {
+    identifierToPosition
+    identifierToColor {
+      hex
+    }
+  }
+`;
+export const DetectionBoxFieldsFragmentDoc = gql`
+  fragment DetectionBoxFields on BoundingBoxGQL {
+    left
+    top
+    width
+    height
+  }
+`;
+export const DetectionPocketPointFieldsFragmentDoc = gql`
+  fragment DetectionPocketPointFields on PocketPointsGQL {
+    topLeft {
+      x
+      y
+    }
+    topSide {
+      x
+      y
+    }
+    topRight {
+      x
+      y
+    }
+    bottomLeft {
+      x
+      y
+    }
+    bottomSide {
+      x
+      y
+    }
+    bottomRight {
+      x
+      y
+    }
+  }
+`;
+export const TableStateDetectionFieldsFragmentDoc = gql`
+  fragment TableStateDetectionFields on TableStateGQL {
+    identifierToBox {
+      ...DetectionBoxFields
+    }
+    homography {
+      crop {
+        ...DetectionBoxFields
+      }
+      pockets {
+        ...DetectionBoxFields
+      }
+      sourcePoints {
+        ...DetectionPocketPointFields
+      }
+      destPoints {
+        ...DetectionPocketPointFields
+      }
+    }
+  }
+  ${DetectionBoxFieldsFragmentDoc}
+  ${DetectionPocketPointFieldsFragmentDoc}
+`;
 export const ShotClipRangeFragmentDoc = gql`
  fragment ShotClipRange on ShotGQL {
    id
@@ -15931,12 +16111,12 @@ export const GetTableStateDocument = gql`
      tableSize: $tableSize
      useHomography: $useHomography
    ) {
-      identifierToPosition
-      identifierToColor {
-        hex
-      }
+      ...TableStateLayoutFields
+      ...TableStateDetectionFields
    }
  }
+  ${TableStateLayoutFieldsFragmentDoc}
+  ${TableStateDetectionFieldsFragmentDoc}
 `;

 /**
@@ -16012,13 +16192,11 @@ export const GetLiveTableStateDocument = gql`
      videoId
      frameIndex
      tableState {
-        identifierToPosition
-        identifierToColor {
-          hex
-        }
+        ...TableStateLayoutFields
      }
    }
  }
+  ${TableStateLayoutFieldsFragmentDoc}
 `;

 /**
@@ -16093,13 +16271,11 @@ export const GetShotTableStateDocument = gql`
      videoId
      frameIndex
      tableState {
-        identifierToPosition
-        identifierToColor {
-          hex
-        }
+        ...TableStateLayoutFields
      }
    }
  }
+  ${TableStateLayoutFieldsFragmentDoc}
 `;

 /**
--- a/src/operations/shot_simulation.gql
+++ b/src/operations/shot_simulation.gql
@@ -1,3 +1,72 @@
+fragment TableStateLayoutFields on TableStateGQL {
+  identifierToPosition
+  identifierToColor {
+    hex
+  }
+}
+
+fragment DetectionBoxFields on BoundingBoxGQL {
+  left
+  top
+  width
+  height
+}
+
+fragment DetectionPocketPointFields on PocketPointsGQL {
+  topLeft {
+    x
+    y
+  }
+  topSide {
+    x
+    y
+  }
+  topRight {
+    x
+    y
+  }
+  bottomLeft {
+    x
+    y
+  }
+  bottomSide {
+    x
+    y
+  }
+  bottomRight {
+    x
+    y
+  }
+}
+
+# Everything needed to redraw a detection over the image it came from:
+# the detector's boxes plus the homography that turned them into plane
+# positions. Boxes, pocket boxes, and source points share the crop's
+# coordinate space; the crop locates that space in the full frame.
+#
+# Only the photo query selects this. The live and shot queries read a
+# frame out of a video the app never holds, so there is no image to
+# overlay, and getLiveTableState is polled every five seconds.
+fragment TableStateDetectionFields on TableStateGQL {
+  identifierToBox {
+    ...DetectionBoxFields
+  }
+  homography {
+    crop {
+      ...DetectionBoxFields
+    }
+    pockets {
+      ...DetectionBoxFields
+    }
+    sourcePoints {
+      ...DetectionPocketPointFields
+    }
+    destPoints {
+      ...DetectionPocketPointFields
+    }
+  }
+}
+
 query GetTableState(
  $b64Image: String!
  $tableSize: Float
@@ -8,10 +77,8 @@ query GetTableState(
    tableSize: $tableSize
    useHomography: $useHomography
  ) {
-    identifierToPosition
-    identifierToColor {
-      hex
-    }
+    ...TableStateLayoutFields
+    ...TableStateDetectionFields
  }
 }

@@ -20,10 +87,7 @@ query GetLiveTableState($videoId: Int!) {
    videoId
    frameIndex
    tableState {
-      identifierToPosition
-      identifierToColor {
-        hex
-      }
+      ...TableStateLayoutFields
    }
  }
 }
@@ -34,10 +98,7 @@ query GetShotTableState($shotId: Int!) {
    videoId
    frameIndex
    tableState {
-      identifierToPosition
-      identifierToColor {
-        hex
-      }
+      ...TableStateLayoutFields
    }
  }
 }
--- a/src/schema.gql
+++ b/src/schema.gql
@@ -1708,6 +1708,7 @@ type TableStateGQL {
  identifierToPosition: [[Float!]!]!
  homography: HomographyInfoGQL
  identifierToColor: [RGBColorGQL]!
+  identifierToBox: [BoundingBoxGQL!]!
 }

 type TagClassGQL {