Merge pull request 'Return the shot's start frame image from getShotTableState' (#321) from ivan/shot-table-state-frame-image into master

Shot Lab was showing the video's single thumbnail next to every shot's
layout. The backend now returns the frame the layout was detected in as
a base64 JPEG (frameImage), selected by the GetShotTableState operation.
Also dedupes ShareLinkGQL/CreateShotShareLinkInput, which appeared twice
in the committed schema after a bad merge.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

src/index.tsx

@@ -3132,6 +3132,7 @@ export type Query = {
   getResolvedTier: ResolvedTierGql;
   getRuns: GetRunsResult;
   getSessionCoach: SessionCoachGql;
+  getSessionCoachGenerationStatus?: Maybe<SessionCoachGenerationStatusGql>;
   getShotAnnotationTypes: Array<ShotAnnotationTypeGql>;
   getShotTableState: ShotTableStateGql;
   getShots: Array<ShotGql>;
@@ -3293,6 +3294,11 @@ export type QueryGetSessionCoachArgs = {
   videoId: Scalars["Int"]["input"];
 };
 
+export type QueryGetSessionCoachGenerationStatusArgs = {
+  generationId: Scalars["ID"]["input"];
+  videoId: Scalars["Int"]["input"];
+};
+
 export type QueryGetShotAnnotationTypesArgs = {
   errorTypes?: InputMaybe<Scalars["Boolean"]["input"]>;
 };
@@ -3772,6 +3778,18 @@ export enum SessionCoachGenerationStateEnum {
   Running = "RUNNING",
 }
 
+export type SessionCoachGenerationStatusGql = {
+  __typename?: "SessionCoachGenerationStatusGQL";
+  completedAt?: Maybe<Scalars["DateTime"]["output"]>;
+  createdAt: Scalars["DateTime"]["output"];
+  enqueueCount: Scalars["Int"]["output"];
+  id: Scalars["ID"]["output"];
+  lastEnqueuedAt?: Maybe<Scalars["DateTime"]["output"]>;
+  startedAt?: Maybe<Scalars["DateTime"]["output"]>;
+  state: SessionCoachGenerationStateEnum;
+  updatedAt: Scalars["DateTime"]["output"];
+};
+
 export enum SessionCoachIneligibilityReasonEnum {
   Multiplayer = "MULTIPLAYER",
 }
@@ -3864,6 +3882,7 @@ export type ShotProjectionGql = {
 
 export type ShotTableStateGql = {
   __typename?: "ShotTableStateGQL";
+  frameImage?: Maybe<Scalars["String"]["output"]>;
   frameIndex: Scalars["Int"]["output"];
   shotId: Scalars["Int"]["output"];
   tableState: TableStateGql;
@@ -4102,6 +4121,7 @@ export type SyncAppleSubscriptionResultGql = {
 export type TableStateGql = {
   __typename?: "TableStateGQL";
   homography?: Maybe<HomographyInfoGql>;
+  identifierToBox: Array<BoundingBoxGql>;
   identifierToColor: Array<Maybe<RgbColorGql>>;
   identifierToPosition: Array<Array<Scalars["Float"]["output"]>>;
 };
@@ -7237,6 +7257,26 @@ export type GetSessionCoachQuery = {
   };
 };
 
+export type GetSessionCoachGenerationStatusQueryVariables = Exact<{
+  videoId: Scalars["Int"]["input"];
+  generationId: Scalars["ID"]["input"];
+}>;
+
+export type GetSessionCoachGenerationStatusQuery = {
+  __typename?: "Query";
+  getSessionCoachGenerationStatus?: {
+    __typename?: "SessionCoachGenerationStatusGQL";
+    id: string;
+    state: SessionCoachGenerationStateEnum;
+    createdAt: any;
+    updatedAt: any;
+    startedAt?: any | null;
+    lastEnqueuedAt?: any | null;
+    enqueueCount: number;
+    completedAt?: any | null;
+  } | null;
+};
+
 export type RequestSessionCoachGenerationMutationVariables = Exact<{
   videoId: Scalars["Int"]["input"];
 }>;
@@ -7534,6 +7574,76 @@ export type FinalizePlayerAssignmentsMutation = {
   }>;
 };
 
+export type TableStateLayoutFieldsFragment = {
+  __typename?: "TableStateGQL";
+  identifierToPosition: Array<Array<number>>;
+  identifierToColor: Array<{ __typename?: "RGBColorGQL"; hex: string } | null>;
+};
+
+export type DetectionBoxFieldsFragment = {
+  __typename?: "BoundingBoxGQL";
+  left: number;
+  top: number;
+  width: number;
+  height: number;
+};
+
+export type DetectionPocketPointFieldsFragment = {
+  __typename?: "PocketPointsGQL";
+  topLeft: { __typename?: "IntPoint2D"; x: number; y: number };
+  topSide: { __typename?: "IntPoint2D"; x: number; y: number };
+  topRight: { __typename?: "IntPoint2D"; x: number; y: number };
+  bottomLeft: { __typename?: "IntPoint2D"; x: number; y: number };
+  bottomSide: { __typename?: "IntPoint2D"; x: number; y: number };
+  bottomRight: { __typename?: "IntPoint2D"; x: number; y: number };
+};
+
+export type TableStateDetectionFieldsFragment = {
+  __typename?: "TableStateGQL";
+  identifierToBox: Array<{
+    __typename?: "BoundingBoxGQL";
+    left: number;
+    top: number;
+    width: number;
+    height: number;
+  }>;
+  homography?: {
+    __typename?: "HomographyInfoGQL";
+    crop: {
+      __typename?: "BoundingBoxGQL";
+      left: number;
+      top: number;
+      width: number;
+      height: number;
+    };
+    pockets: Array<{
+      __typename?: "BoundingBoxGQL";
+      left: number;
+      top: number;
+      width: number;
+      height: number;
+    }>;
+    sourcePoints: {
+      __typename?: "PocketPointsGQL";
+      topLeft: { __typename?: "IntPoint2D"; x: number; y: number };
+      topSide: { __typename?: "IntPoint2D"; x: number; y: number };
+      topRight: { __typename?: "IntPoint2D"; x: number; y: number };
+      bottomLeft: { __typename?: "IntPoint2D"; x: number; y: number };
+      bottomSide: { __typename?: "IntPoint2D"; x: number; y: number };
+      bottomRight: { __typename?: "IntPoint2D"; x: number; y: number };
+    };
+    destPoints: {
+      __typename?: "PocketPointsGQL";
+      topLeft: { __typename?: "IntPoint2D"; x: number; y: number };
+      topSide: { __typename?: "IntPoint2D"; x: number; y: number };
+      topRight: { __typename?: "IntPoint2D"; x: number; y: number };
+      bottomLeft: { __typename?: "IntPoint2D"; x: number; y: number };
+      bottomSide: { __typename?: "IntPoint2D"; x: number; y: number };
+      bottomRight: { __typename?: "IntPoint2D"; x: number; y: number };
+    };
+  } | null;
+};
+
 export type GetTableStateQueryVariables = Exact<{
   b64Image: Scalars["String"]["input"];
   tableSize?: InputMaybe<Scalars["Float"]["input"]>;
@@ -7549,6 +7659,48 @@ export type GetTableStateQuery = {
       __typename?: "RGBColorGQL";
       hex: string;
     } | null>;
+    identifierToBox: Array<{
+      __typename?: "BoundingBoxGQL";
+      left: number;
+      top: number;
+      width: number;
+      height: number;
+    }>;
+    homography?: {
+      __typename?: "HomographyInfoGQL";
+      crop: {
+        __typename?: "BoundingBoxGQL";
+        left: number;
+        top: number;
+        width: number;
+        height: number;
+      };
+      pockets: Array<{
+        __typename?: "BoundingBoxGQL";
+        left: number;
+        top: number;
+        width: number;
+        height: number;
+      }>;
+      sourcePoints: {
+        __typename?: "PocketPointsGQL";
+        topLeft: { __typename?: "IntPoint2D"; x: number; y: number };
+        topSide: { __typename?: "IntPoint2D"; x: number; y: number };
+        topRight: { __typename?: "IntPoint2D"; x: number; y: number };
+        bottomLeft: { __typename?: "IntPoint2D"; x: number; y: number };
+        bottomSide: { __typename?: "IntPoint2D"; x: number; y: number };
+        bottomRight: { __typename?: "IntPoint2D"; x: number; y: number };
+      };
+      destPoints: {
+        __typename?: "PocketPointsGQL";
+        topLeft: { __typename?: "IntPoint2D"; x: number; y: number };
+        topSide: { __typename?: "IntPoint2D"; x: number; y: number };
+        topRight: { __typename?: "IntPoint2D"; x: number; y: number };
+        bottomLeft: { __typename?: "IntPoint2D"; x: number; y: number };
+        bottomSide: { __typename?: "IntPoint2D"; x: number; y: number };
+        bottomRight: { __typename?: "IntPoint2D"; x: number; y: number };
+      };
+    } | null;
   };
 };
 
@@ -7584,6 +7736,7 @@ export type GetShotTableStateQuery = {
     shotId: number;
     videoId: number;
     frameIndex: number;
+    frameImage?: string | null;
     tableState: {
       __typename?: "TableStateGQL";
       identifierToPosition: Array<Array<number>>;
@@ -10041,6 +10194,73 @@ export const PlayerClusterFieldsFragmentDoc = gql`
   }
   ${PlayerClusterShotFieldsFragmentDoc}
 `;
+export const TableStateLayoutFieldsFragmentDoc = gql`
+  fragment TableStateLayoutFields on TableStateGQL {
+    identifierToPosition
+    identifierToColor {
+      hex
+    }
+  }
+`;
+export const DetectionBoxFieldsFragmentDoc = gql`
+  fragment DetectionBoxFields on BoundingBoxGQL {
+    left
+    top
+    width
+    height
+  }
+`;
+export const DetectionPocketPointFieldsFragmentDoc = gql`
+  fragment DetectionPocketPointFields on PocketPointsGQL {
+    topLeft {
+      x
+      y
+    }
+    topSide {
+      x
+      y
+    }
+    topRight {
+      x
+      y
+    }
+    bottomLeft {
+      x
+      y
+    }
+    bottomSide {
+      x
+      y
+    }
+    bottomRight {
+      x
+      y
+    }
+  }
+`;
+export const TableStateDetectionFieldsFragmentDoc = gql`
+  fragment TableStateDetectionFields on TableStateGQL {
+    identifierToBox {
+      ...DetectionBoxFields
+    }
+    homography {
+      crop {
+        ...DetectionBoxFields
+      }
+      pockets {
+        ...DetectionBoxFields
+      }
+      sourcePoints {
+        ...DetectionPocketPointFields
+      }
+      destPoints {
+        ...DetectionPocketPointFields
+      }
+    }
+  }
+  ${DetectionBoxFieldsFragmentDoc}
+  ${DetectionPocketPointFieldsFragmentDoc}
+`;
 export const ShotClipRangeFragmentDoc = gql`
   fragment ShotClipRange on ShotGQL {
     id
@@ -15743,6 +15963,90 @@ export type GetSessionCoachQueryResult = Apollo.QueryResult<
   GetSessionCoachQuery,
   GetSessionCoachQueryVariables
 >;
+export const GetSessionCoachGenerationStatusDocument = gql`
+  query GetSessionCoachGenerationStatus($videoId: Int!, $generationId: ID!) {
+    getSessionCoachGenerationStatus(
+      videoId: $videoId
+      generationId: $generationId
+    ) {
+      id
+      state
+      createdAt
+      updatedAt
+      startedAt
+      lastEnqueuedAt
+      enqueueCount
+      completedAt
+    }
+  }
+`;
+
+/**
+ * __useGetSessionCoachGenerationStatusQuery__
+ *
+ * To run a query within a React component, call `useGetSessionCoachGenerationStatusQuery` and pass it any options that fit your needs.
+ * When your component renders, `useGetSessionCoachGenerationStatusQuery` returns an object from Apollo Client that contains loading, error, and data properties
+ * you can use to render your UI.
+ *
+ * @param baseOptions options that will be passed into the query, supported options are listed on: https://www.apollographql.com/docs/react/api/react-hooks/#options;
+ *
+ * @example
+ * const { data, loading, error } = useGetSessionCoachGenerationStatusQuery({
+ *   variables: {
+ *      videoId: // value for 'videoId'
+ *      generationId: // value for 'generationId'
+ *   },
+ * });
+ */
+export function useGetSessionCoachGenerationStatusQuery(
+  baseOptions: Apollo.QueryHookOptions<
+    GetSessionCoachGenerationStatusQuery,
+    GetSessionCoachGenerationStatusQueryVariables
+  >,
+) {
+  const options = { ...defaultOptions, ...baseOptions };
+  return Apollo.useQuery<
+    GetSessionCoachGenerationStatusQuery,
+    GetSessionCoachGenerationStatusQueryVariables
+  >(GetSessionCoachGenerationStatusDocument, options);
+}
+export function useGetSessionCoachGenerationStatusLazyQuery(
+  baseOptions?: Apollo.LazyQueryHookOptions<
+    GetSessionCoachGenerationStatusQuery,
+    GetSessionCoachGenerationStatusQueryVariables
+  >,
+) {
+  const options = { ...defaultOptions, ...baseOptions };
+  return Apollo.useLazyQuery<
+    GetSessionCoachGenerationStatusQuery,
+    GetSessionCoachGenerationStatusQueryVariables
+  >(GetSessionCoachGenerationStatusDocument, options);
+}
+export function useGetSessionCoachGenerationStatusSuspenseQuery(
+  baseOptions?: Apollo.SuspenseQueryHookOptions<
+    GetSessionCoachGenerationStatusQuery,
+    GetSessionCoachGenerationStatusQueryVariables
+  >,
+) {
+  const options = { ...defaultOptions, ...baseOptions };
+  return Apollo.useSuspenseQuery<
+    GetSessionCoachGenerationStatusQuery,
+    GetSessionCoachGenerationStatusQueryVariables
+  >(GetSessionCoachGenerationStatusDocument, options);
+}
+export type GetSessionCoachGenerationStatusQueryHookResult = ReturnType<
+  typeof useGetSessionCoachGenerationStatusQuery
+>;
+export type GetSessionCoachGenerationStatusLazyQueryHookResult = ReturnType<
+  typeof useGetSessionCoachGenerationStatusLazyQuery
+>;
+export type GetSessionCoachGenerationStatusSuspenseQueryHookResult = ReturnType<
+  typeof useGetSessionCoachGenerationStatusSuspenseQuery
+>;
+export type GetSessionCoachGenerationStatusQueryResult = Apollo.QueryResult<
+  GetSessionCoachGenerationStatusQuery,
+  GetSessionCoachGenerationStatusQueryVariables
+>;
 export const RequestSessionCoachGenerationDocument = gql`
   mutation RequestSessionCoachGeneration($videoId: Int!) {
     requestSessionCoachGeneration(videoId: $videoId) {
@@ -16054,12 +16358,12 @@ export const GetTableStateDocument = gql`
       tableSize: $tableSize
       useHomography: $useHomography
     ) {
-      identifierToPosition
-      identifierToColor {
-        hex
-      }
+      ...TableStateLayoutFields
+      ...TableStateDetectionFields
     }
   }
+  ${TableStateLayoutFieldsFragmentDoc}
+  ${TableStateDetectionFieldsFragmentDoc}
 `;
 
 /**
@@ -16135,13 +16439,11 @@ export const GetLiveTableStateDocument = gql`
       videoId
       frameIndex
       tableState {
-        identifierToPosition
-        identifierToColor {
-          hex
-        }
+        ...TableStateLayoutFields
       }
     }
   }
+  ${TableStateLayoutFieldsFragmentDoc}
 `;
 
 /**
@@ -16215,14 +16517,13 @@ export const GetShotTableStateDocument = gql`
       shotId
       videoId
       frameIndex
+      frameImage
       tableState {
-        identifierToPosition
-        identifierToColor {
-          hex
-        }
+        ...TableStateLayoutFields
       }
     }
   }
+  ${TableStateLayoutFieldsFragmentDoc}
 `;
 
 /**

src/operations/session-coach.gql

@@ -36,6 +36,22 @@ query GetSessionCoach($videoId: Int!) {
   }
 }
 
+query GetSessionCoachGenerationStatus($videoId: Int!, $generationId: ID!) {
+  getSessionCoachGenerationStatus(
+    videoId: $videoId
+    generationId: $generationId
+  ) {
+    id
+    state
+    createdAt
+    updatedAt
+    startedAt
+    lastEnqueuedAt
+    enqueueCount
+    completedAt
+  }
+}
+
 mutation RequestSessionCoachGeneration($videoId: Int!) {
   requestSessionCoachGeneration(videoId: $videoId) {
     state

src/operations/shot_simulation.gql

@@ -1,3 +1,73 @@
+fragment TableStateLayoutFields on TableStateGQL {
+  identifierToPosition
+  identifierToColor {
+    hex
+  }
+}
+
+fragment DetectionBoxFields on BoundingBoxGQL {
+  left
+  top
+  width
+  height
+}
+
+fragment DetectionPocketPointFields on PocketPointsGQL {
+  topLeft {
+    x
+    y
+  }
+  topSide {
+    x
+    y
+  }
+  topRight {
+    x
+    y
+  }
+  bottomLeft {
+    x
+    y
+  }
+  bottomSide {
+    x
+    y
+  }
+  bottomRight {
+    x
+    y
+  }
+}
+
+# Everything needed to redraw a detection over the image it came from:
+# the detector's boxes plus the homography that turned them into plane
+# positions. Boxes, pocket boxes, and source points share the crop's
+# coordinate space; the crop locates that space in the full frame.
+#
+# Only the photo query selects this. The shot query returns its source
+# frame (frameImage) but positions there come from the stored homography,
+# so there are no fresh boxes to draw; the live query returns no image at
+# all and is polled every five seconds.
+fragment TableStateDetectionFields on TableStateGQL {
+  identifierToBox {
+    ...DetectionBoxFields
+  }
+  homography {
+    crop {
+      ...DetectionBoxFields
+    }
+    pockets {
+      ...DetectionBoxFields
+    }
+    sourcePoints {
+      ...DetectionPocketPointFields
+    }
+    destPoints {
+      ...DetectionPocketPointFields
+    }
+  }
+}
+
 query GetTableState(
   $b64Image: String!
   $tableSize: Float
@@ -8,10 +78,8 @@ query GetTableState(
     tableSize: $tableSize
     useHomography: $useHomography
   ) {
-    identifierToPosition
-    identifierToColor {
-      hex
-    }
+    ...TableStateLayoutFields
+    ...TableStateDetectionFields
   }
 }
 
@@ -20,10 +88,7 @@ query GetLiveTableState($videoId: Int!) {
     videoId
     frameIndex
     tableState {
-      identifierToPosition
-      identifierToColor {
-        hex
-      }
+      ...TableStateLayoutFields
     }
   }
 }
@@ -33,11 +98,11 @@ query GetShotTableState($shotId: Int!) {
     shotId
     videoId
     frameIndex
+    # The b64 JPEG frame the layout was detected in, so the client can show
+    # the shot's actual starting frame instead of the video's thumbnail.
+    frameImage
     tableState {
-      identifierToPosition
-      identifierToColor {
-        hex
-      }
+      ...TableStateLayoutFields
     }
   }
 }

src/schema.gql

@@ -250,11 +250,6 @@ input CreatePoolHallCameraInput {
   claimTermsText: String = null
 }
 
-input CreateShotShareLinkInput {
-  shotIds: [Int!]!
-  paddingSeconds: Float = null
-}
-
 input CreatePoolHallInput {
   name: String!
   address: String = null
@@ -1081,6 +1076,10 @@ type Query {
     limit: Int! = 500
     countRespectsLimit: Boolean! = false
   ): GetRunsResult!
+  getSessionCoachGenerationStatus(
+    videoId: Int!
+    generationId: ID!
+  ): SessionCoachGenerationStatusGQL
   getSessionCoach(videoId: Int!): SessionCoachGQL!
   videoPlayerClusters(videoId: Int!): [PlayerClusterGQL!]!
   getShotAnnotationTypes(errorTypes: Boolean = false): [ShotAnnotationTypeGQL!]!
@@ -1473,6 +1472,17 @@ enum SessionCoachGenerationStateEnum {
   FAILED
 }
 
+type SessionCoachGenerationStatusGQL {
+  id: ID!
+  state: SessionCoachGenerationStateEnum!
+  createdAt: DateTime!
+  updatedAt: DateTime!
+  startedAt: DateTime
+  lastEnqueuedAt: DateTime
+  enqueueCount: Int!
+  completedAt: DateTime
+}
+
 enum SessionCoachIneligibilityReasonEnum {
   MULTIPLAYER
 }
@@ -1558,6 +1568,7 @@ type ShotTableStateGQL {
   videoId: Int!
   frameIndex: Int!
   tableState: TableStateGQL!
+  frameImage: String
 }
 
 input ShotsOrderingComponent @oneOf {
@@ -1739,6 +1750,7 @@ type TableStateGQL {
   identifierToPosition: [[Float!]!]!
   homography: HomographyInfoGQL
   identifierToColor: [RGBColorGQL]!
+  identifierToBox: [BoundingBoxGQL!]!
 }
 
 type TagClassGQL {
@@ -1950,11 +1962,6 @@ type UserSubscriptionStatusGQL {
   stripeSubscriptionId: String
 }
 
-type ShareLinkGQL {
-  slug: String!
-  url: String!
-}
-
 type VideoExportJobGQL {
   id: Int!
   videoId: Int!