From 253ffb595633a4b18221339278f73c8416225f56 Mon Sep 17 00:00:00 2001 From: coofzilla <75731066+coofzilla@users.noreply.github.com> Date: Thu, 29 Feb 2024 22:41:04 +0900 Subject: [PATCH] feat(ios): Add ios support for accessing WebVTT Subtitle Content (#3541) * feature: add support to get subtitle content data * refactor: return a string of the subtitles Push the parsing/formatting to the consumer side. * chore: add types for new subtitle feature * chore: run swiftlint and swiftformat * chore: add documentation for new onSubtitleTracks callback * chore: added test uri; basic implementation of feature; hotfix onTextTracks added optional chaining for `return x?.selected` because tracks that don't have a track selected either by default or manually will return undefined and this can cause an error. * feat: rename onSubtitleTracks to onTextTrackDataChanged Renamed the onSubtitleTracks event to onTextTrackDataChanged across the codebase to clearly indicate the callback's purpose: being called when the text track's data changes. This change is reflected in the events documentation, example usage in VideoPlayer.tsx, and the relevant iOS implementation files for consistency and clarity, in line with PR feedback. * chore: omit target property target could be confusing for users so we have removed it. using the delete operator instead of using {target,...eventData} as that would give an eslint error about unused vars. --- docs/pages/component/events.md | 19 ++++++++++++++++++- examples/basic/src/VideoPlayer.tsx | 16 ++++++++++++++-- ios/Video/Features/RCTPlayerObserver.swift | 14 +++++++++++++- ios/Video/RCTVideo.swift | 7 +++++++ ios/Video/RCTVideoManager.m | 1 + src/Video.tsx | 14 ++++++++++++++ src/VideoNativeComponent.ts | 10 +++++++++- src/types/events.ts | 5 +++++ 8 files changed, 81 insertions(+), 5 deletions(-) diff --git a/docs/pages/component/events.md b/docs/pages/component/events.md index f45a4477..aa7aa0c2 100644 --- a/docs/pages/component/events.md +++ b/docs/pages/component/events.md @@ -493,7 +493,24 @@ Example: } ``` -Platforms: Android +### `onTextTrackDataChanged` +Callback function that is called when new subtitle data is available. It provides the actual subtitle content for the current selected text track, if available (mainly WebVTT). + +Payload: + +Property | Type | Description +--- | --- | --- +`subtitleTracks` | `string` | The subtitles text content in a compatible format. + + +Example: +```javascript +{ + subtitleTracks: "This blade has a dark past.", +} +``` + +Platforms: iOS ### `onVideoTracks` Callback function that is called when video tracks change diff --git a/examples/basic/src/VideoPlayer.tsx b/examples/basic/src/VideoPlayer.tsx index c38b68e0..ed9f06f5 100644 --- a/examples/basic/src/VideoPlayer.tsx +++ b/examples/basic/src/VideoPlayer.tsx @@ -33,6 +33,8 @@ import Video, { ResizeMode, SelectedTrack, DRMType, + OnTextTrackDataChangedData, + SelectedTrackType, } from 'react-native-video'; import ToggleControl from './ToggleControl'; import MultiValueControl, { @@ -120,7 +122,12 @@ class VideoPlayer extends Component { }, ]; - srcIosList = []; + srcIosList = [ + { + description: 'sintel with subtitles', + uri: 'https://bitmovin-a.akamaihd.net/content/sintel/hls/playlist.m3u8', + }, + ]; srcAndroidList = [ { @@ -231,7 +238,7 @@ class VideoPlayer extends Component { onTextTracks = (data: OnTextTracksData) => { const selectedTrack = data.textTracks?.find((x: TextTrack) => { - return x.selected; + return x?.selected; }); this.setState({ @@ -248,6 +255,10 @@ class VideoPlayer extends Component { } }; + onTextTrackDataChanged = (data: OnTextTrackDataChangedData) => { + console.log(`Subtitles: ${JSON.stringify(data, null, 2)}`); + }; + onAspectRatio = (data: OnVideoAspectRatioData) => { console.log('onAspectRadio called ' + JSON.stringify(data)); this.setState({ @@ -749,6 +760,7 @@ class VideoPlayer extends Component { onLoad={this.onLoad} onAudioTracks={this.onAudioTracks} onTextTracks={this.onTextTracks} + onTextTrackDataChanged={this.onTextTrackDataChanged} onProgress={this.onProgress} onEnd={this.onEnd} progressUpdateInterval={1000} diff --git a/ios/Video/Features/RCTPlayerObserver.swift b/ios/Video/Features/RCTPlayerObserver.swift index dfdd8320..a11dcfec 100644 --- a/ios/Video/Features/RCTPlayerObserver.swift +++ b/ios/Video/Features/RCTPlayerObserver.swift @@ -26,11 +26,12 @@ protocol RCTPlayerObserverHandler: RCTPlayerObserverHandlerObjc { func handleExternalPlaybackActiveChange(player: AVPlayer, change: NSKeyValueObservedChange) func handleViewControllerOverlayViewFrameChange(overlayView: UIView, change: NSKeyValueObservedChange) func handleTracksChange(playerItem: AVPlayerItem, change: NSKeyValueObservedChange<[AVPlayerItemTrack]>) + func handleLegibleOutput(strings: [NSAttributedString]) } // MARK: - RCTPlayerObserver -class RCTPlayerObserver: NSObject, AVPlayerItemMetadataOutputPushDelegate { +class RCTPlayerObserver: NSObject, AVPlayerItemMetadataOutputPushDelegate, AVPlayerItemLegibleOutputPushDelegate { weak var _handlers: RCTPlayerObserverHandler? var player: AVPlayer? { @@ -57,8 +58,11 @@ class RCTPlayerObserver: NSObject, AVPlayerItemMetadataOutputPushDelegate { // handle timedMetadata let metadataOutput = AVPlayerItemMetadataOutput() + let legibleOutput = AVPlayerItemLegibleOutput() playerItem.add(metadataOutput) + playerItem.add(legibleOutput) metadataOutput.setDelegate(self, queue: .main) + legibleOutput.setDelegate(self, queue: .main) } } @@ -113,6 +117,14 @@ class RCTPlayerObserver: NSObject, AVPlayerItemMetadataOutputPushDelegate { } } + func legibleOutput(_: AVPlayerItemLegibleOutput, + didOutputAttributedStrings strings: [NSAttributedString], + nativeSampleBuffers _: [Any], + forItemTime _: CMTime) { + guard let _handlers else { return } + _handlers.handleLegibleOutput(strings: strings) + } + func addPlayerObservers() { guard let player, let _handlers else { return diff --git a/ios/Video/RCTVideo.swift b/ios/Video/RCTVideo.swift index 49fe1705..fb465b67 100644 --- a/ios/Video/RCTVideo.swift +++ b/ios/Video/RCTVideo.swift @@ -118,6 +118,7 @@ class RCTVideo: UIView, RCTVideoPlayerViewControllerDelegate, RCTPlayerObserverH @objc var onReceiveAdEvent: RCTDirectEventBlock? @objc var onTextTracks: RCTDirectEventBlock? @objc var onAudioTracks: RCTDirectEventBlock? + @objc var onTextTrackDataChanged: RCTDirectEventBlock? @objc func _onPictureInPictureStatusChanged() { @@ -1388,4 +1389,10 @@ class RCTVideo: UIView, RCTVideoPlayerViewControllerDelegate, RCTPlayerObserverH self.onAudioTracks?(["audioTracks": audioTracks]) } } + + func handleLegibleOutput(strings: [NSAttributedString]) { + if let subtitles = strings.first { + self.onTextTrackDataChanged?(["subtitleTracks": subtitles.string]) + } + } } diff --git a/ios/Video/RCTVideoManager.m b/ios/Video/RCTVideoManager.m index f7b5bcb5..e30f6c9c 100644 --- a/ios/Video/RCTVideoManager.m +++ b/ios/Video/RCTVideoManager.m @@ -66,6 +66,7 @@ RCT_EXPORT_VIEW_PROPERTY(onRestoreUserInterfaceForPictureInPictureStop, RCTDirec RCT_EXPORT_VIEW_PROPERTY(onReceiveAdEvent, RCTDirectEventBlock); RCT_EXPORT_VIEW_PROPERTY(onTextTracks, RCTDirectEventBlock); RCT_EXPORT_VIEW_PROPERTY(onAudioTracks, RCTDirectEventBlock); +RCT_EXPORT_VIEW_PROPERTY(onTextTrackDataChanged, RCTDirectEventBlock); RCT_EXTERN_METHOD(save : (NSDictionary*)options reactTag diff --git a/src/Video.tsx b/src/Video.tsx index 41d2aa64..e0abfbbb 100644 --- a/src/Video.tsx +++ b/src/Video.tsx @@ -29,6 +29,7 @@ import type { OnProgressData, OnReceiveAdEventData, OnSeekData, + OnTextTrackDataChangedData, OnTextTracksData, OnTimedMetadataData, OnVideoAspectRatioData, @@ -93,6 +94,7 @@ const Video = forwardRef( onTimedMetadata, onAudioTracks, onTextTracks, + onTextTrackDataChanged, onVideoTracks, onAspectRatio, ...rest @@ -333,6 +335,17 @@ const Video = forwardRef( [onTextTracks], ); + const _onTextTrackDataChanged = useCallback( + ( + e: NativeSyntheticEvent, + ) => { + const {...eventData} = e.nativeEvent; + delete eventData.target; + onTextTrackDataChanged?.(eventData as OnTextTrackDataChangedData); + }, + [onTextTrackDataChanged], + ); + const _onVideoTracks = useCallback( (e: NativeSyntheticEvent) => { onVideoTracks?.(e.nativeEvent); @@ -509,6 +522,7 @@ const Video = forwardRef( onTimedMetadata={_onTimedMetadata} onAudioTracks={_onAudioTracks} onTextTracks={_onTextTracks} + onTextTrackDataChanged={_onTextTrackDataChanged} onVideoTracks={_onVideoTracks} onVideoFullscreenPlayerDidDismiss={onFullscreenPlayerDidDismiss} onVideoFullscreenPlayerDidPresent={onFullscreenPlayerDidPresent} diff --git a/src/VideoNativeComponent.ts b/src/VideoNativeComponent.ts index 96923561..f173a761 100644 --- a/src/VideoNativeComponent.ts +++ b/src/VideoNativeComponent.ts @@ -7,7 +7,12 @@ import {NativeModules, requireNativeComponent} from 'react-native'; import type ResizeMode from './types/ResizeMode'; import type FilterType from './types/FilterType'; import type Orientation from './types/Orientation'; -import type {AdEvent, EnumValues, OnTextTracksTypeData} from './types'; +import type { + AdEvent, + EnumValues, + OnTextTrackDataChangedData, + OnTextTracksTypeData, +} from './types'; // -------- There are types for native component (future codegen) -------- // if you are looking for types for react component, see src/types/video.ts @@ -366,6 +371,9 @@ export interface VideoNativeProps extends ViewProps { onTimedMetadata?: (event: NativeSyntheticEvent) => void; // ios, android onAudioTracks?: (event: NativeSyntheticEvent) => void; // android onTextTracks?: (event: NativeSyntheticEvent) => void; // android + onTextTrackDataChanged?: ( + event: NativeSyntheticEvent, + ) => void; // iOS onVideoTracks?: (event: NativeSyntheticEvent) => void; // android } diff --git a/src/types/events.ts b/src/types/events.ts index bfde7a4b..9982eda5 100644 --- a/src/types/events.ts +++ b/src/types/events.ts @@ -79,6 +79,10 @@ export type OnTextTracksData = Readonly<{ textTracks: ReadonlyArray; }>; +export type OnTextTrackDataChangedData = Readonly<{ + subtitleTracks: string; +}>; + export type OnVideoTracksData = Readonly<{ videoTracks: ReadonlyArray< Readonly<{ @@ -181,6 +185,7 @@ export interface ReactVideoEvents { onTimedMetadata?: (e: OnTimedMetadataData) => void; //Android, iOS onAudioTracks?: (e: OnAudioTracksData) => void; // Android onTextTracks?: (e: OnTextTracksData) => void; //Android + onTextTrackDataChanged?: (e: OnTextTrackDataChangedData) => void; // iOS onVideoTracks?: (e: OnVideoTracksData) => void; //Android onAspectRatio?: (e: OnVideoAspectRatioData) => void; }