Source: lib/text/text_engine.js

  1. /*! @license
  2.  * Shaka Player
  3.  * Copyright 2016 Google LLC
  4.  * SPDX-License-Identifier: Apache-2.0
  5.  */
  7. goog.provide('shaka.text.TextEngine');
  9. goog.require('goog.asserts');
  10. goog.require('shaka.text.Cue');
  11. goog.require('shaka.util.BufferUtils');
  12. goog.require('shaka.util.Functional');
  13. goog.require('shaka.util.IDestroyable');
  14. goog.require('shaka.util.MimeUtils');
  15. goog.requireType('shaka.cea.ICaptionDecoder');
  18. // TODO: revisit this when Closure Compiler supports partially-exported classes.
  19. /**
  20.  * @summary Manages text parsers and cues.
  21.  * @implements {shaka.util.IDestroyable}
  22.  * @export
  23.  */
  24. shaka.text.TextEngine = class {
  25.   /** @param {shaka.extern.TextDisplayer} displayer */
  26.   constructor(displayer) {
  27.     /** @private {?shaka.extern.TextParser} */
  28.     this.parser_ = null;
  30.     /** @private {shaka.extern.TextDisplayer} */
  31.     this.displayer_ = displayer;
  33.     /** @private {number} */
  34.     this.timestampOffset_ = 0;
  36.     /** @private {number} */
  37.     this.appendWindowStart_ = 0;
  39.     /** @private {number} */
  40.     this.appendWindowEnd_ = Infinity;
  42.     /** @private {?number} */
  43.     this.bufferStart_ = null;
  45.     /** @private {?number} */
  46.     this.bufferEnd_ = null;
  48.     /** @private {string} */
  49.     this.selectedClosedCaptionId_ = '';
  51.     /**
  52.      * The closed captions map stores the CEA closed captions by closed captions
  53.      * id and start and end time.
  54.      * It's used as the buffer of closed caption text streams, to show captions
  55.      * when we start displaying captions or switch caption tracks, we need to be
  56.      * able to get the cues for the other language and display them without
  57.      * re-fetching the video segments they were embedded in.
  58.      * Structure of closed caption map:
  59.      * closed caption id -> {start and end time -> cues}
  60.      * @private {!Map.<string, !Map.<string, !Array.<shaka.text.Cue>>>} */
  61.     this.closedCaptionsMap_ = new Map();
  62.   }
  64.   /**
  65.    * @param {string} mimeType
  66.    * @param {!shaka.extern.TextParserPlugin} plugin
  67.    * @export
  68.    */
  69.   static registerParser(mimeType, plugin) {
  70.     shaka.text.TextEngine.parserMap_[mimeType] = plugin;
  71.   }
  73.   /**
  74.    * @param {string} mimeType
  75.    * @export
  76.    */
  77.   static unregisterParser(mimeType) {
  78.     delete shaka.text.TextEngine.parserMap_[mimeType];
  79.   }
  81.   /**
  82.    * @return {?shaka.extern.TextParserPlugin}
  83.    * @export
  84.    */
  85.   static findParser(mimeType) {
  86.     return shaka.text.TextEngine.parserMap_[mimeType];
  87.   }
  89.   /**
  90.    * @param {string} mimeType
  91.    * @return {boolean}
  92.    */
  93.   static isTypeSupported(mimeType) {
  94.     if (shaka.text.TextEngine.parserMap_[mimeType]) {
  95.       // An actual parser is available.
  96.       return true;
  97.     }
  98.     if (mimeType == shaka.util.MimeUtils.CEA608_CLOSED_CAPTION_MIMETYPE ||
  99.         mimeType == shaka.util.MimeUtils.CEA708_CLOSED_CAPTION_MIMETYPE ) {
  100.       // Closed captions.
  101.       return true;
  102.     }
  103.     return false;
  104.   }
  106.   // TODO: revisit this when the compiler supports partially-exported classes.
  107.   /**
  108.    * @override
  109.    * @export
  110.    */
  111.   destroy() {
  112.     this.parser_ = null;
  113.     this.displayer_ = null;
  114.     this.closedCaptionsMap_.clear();
  116.     return Promise.resolve();
  117.   }
  119.   /**
  120.    * @param {!shaka.extern.TextDisplayer} displayer
  121.    */
  122.   setDisplayer(displayer) {
  123.     this.displayer_ = displayer;
  124.   }
  126.   /**
  127.    * Initialize the parser.  This can be called multiple times, but must be
  128.    * called at least once before appendBuffer.
  129.    *
  130.    * @param {string} mimeType
  131.    */
  132.   initParser(mimeType) {
  133.     // No parser for CEA, which is extracted from video and side-loaded
  134.     // into TextEngine and TextDisplayer.
  135.     if (mimeType == shaka.util.MimeUtils.CEA608_CLOSED_CAPTION_MIMETYPE ||
  136.         mimeType == shaka.util.MimeUtils.CEA708_CLOSED_CAPTION_MIMETYPE) {
  137.       this.parser_ = null;
  138.       return;
  139.     }
  141.     const factory = shaka.text.TextEngine.parserMap_[mimeType];
  142.     goog.asserts.assert(
  143.         factory, 'Text type negotiation should have happened already');
  144.     this.parser_ = shaka.util.Functional.callFactory(factory);
  145.   }
  147.   /**
  148.    * @param {BufferSource} buffer
  149.    * @param {?number} startTime relative to the start of the presentation
  150.    * @param {?number} endTime relative to the start of the presentation
  151.    * @return {!Promise}
  152.    */
  153.   async appendBuffer(buffer, startTime, endTime) {
  154.     goog.asserts.assert(
  155.         this.parser_, 'The parser should already be initialized');
  157.     // Start the operation asynchronously to avoid blocking the caller.
  158.     await Promise.resolve();
  160.     // Check that TextEngine hasn't been destroyed.
  161.     if (!this.parser_ || !this.displayer_) {
  162.       return;
  163.     }
  165.     if (startTime == null || endTime == null) {
  166.       this.parser_.parseInit(shaka.util.BufferUtils.toUint8(buffer));
  167.       return;
  168.     }
  170.     /** @type {shaka.extern.TextParser.TimeContext} **/
  171.     const time = {
  172.       periodStart: this.timestampOffset_,
  173.       segmentStart: startTime,
  174.       segmentEnd: endTime,
  175.     };
  177.     // Parse the buffer and add the new cues.
  178.     const allCues = this.parser_.parseMedia(
  179.         shaka.util.BufferUtils.toUint8(buffer), time);
  180.     const cuesToAppend = allCues.filter((cue) => {
  181.       return cue.startTime >= this.appendWindowStart_ &&
  182.           cue.startTime < this.appendWindowEnd_;
  183.     });
  185.     this.displayer_.append(cuesToAppend);
  187.     // NOTE: We update the buffered range from the start and end times
  188.     // passed down from the segment reference, not with the start and end
  189.     // times of the parsed cues.  This is important because some segments
  190.     // may contain no cues, but we must still consider those ranges
  191.     // buffered.
  192.     if (this.bufferStart_ == null) {
  193.       this.bufferStart_ = Math.max(startTime, this.appendWindowStart_);
  194.     } else {
  195.       // We already had something in buffer, and we assume we are extending
  196.       // the range from the end.
  197.       goog.asserts.assert(
  198.           this.bufferEnd_ != null,
  199.           'There should already be a buffered range end.');
  200.       goog.asserts.assert(
  201.           (startTime - this.bufferEnd_) <= 1,
  202.           'There should not be a gap in text references >1s');
  203.     }
  204.     this.bufferEnd_ = Math.min(endTime, this.appendWindowEnd_);
  205.   }
  207.   /**
  208.    * @param {number} startTime relative to the start of the presentation
  209.    * @param {number} endTime relative to the start of the presentation
  210.    * @return {!Promise}
  211.    */
  212.   async remove(startTime, endTime) {
  213.     // Start the operation asynchronously to avoid blocking the caller.
  214.     await Promise.resolve();
  216.     if (this.displayer_ && this.displayer_.remove(startTime, endTime)) {
  217.       if (this.bufferStart_ == null) {
  218.         goog.asserts.assert(
  219.             this.bufferEnd_ == null, 'end must be null if startTime is null');
  220.       } else {
  221.         goog.asserts.assert(
  222.             this.bufferEnd_ != null,
  223.             'end must be non-null if startTime is non-null');
  225.         // Update buffered range.
  226.         if (endTime <= this.bufferStart_ || startTime >= this.bufferEnd_) {
  227.           // No intersection.  Nothing was removed.
  228.         } else if (startTime <= this.bufferStart_ &&
  229.                    endTime >= this.bufferEnd_) {
  230.           // We wiped out everything.
  231.           this.bufferStart_ = this.bufferEnd_ = null;
  232.         } else if (startTime <= this.bufferStart_ &&
  233.                    endTime < this.bufferEnd_) {
  234.           // We removed from the beginning of the range.
  235.           this.bufferStart_ = endTime;
  236.         } else if (startTime > this.bufferStart_ &&
  237.                    endTime >= this.bufferEnd_) {
  238.           // We removed from the end of the range.
  239.           this.bufferEnd_ = startTime;
  240.         } else {
  241.           // We removed from the middle?  StreamingEngine isn't supposed to.
  242.           goog.asserts.assert(
  243.               false, 'removal from the middle is not supported by TextEngine');
  244.         }
  245.       }
  246.     }
  247.   }
  249.   /** @param {number} timestampOffset */
  250.   setTimestampOffset(timestampOffset) {
  251.     this.timestampOffset_ = timestampOffset;
  252.   }
  254.   /**
  255.    * @param {number} appendWindowStart
  256.    * @param {number} appendWindowEnd
  257.    */
  258.   setAppendWindow(appendWindowStart, appendWindowEnd) {
  259.     this.appendWindowStart_ = appendWindowStart;
  260.     this.appendWindowEnd_ = appendWindowEnd;
  261.   }
  263.   /**
  264.    * @return {?number} Time in seconds of the beginning of the buffered range,
  265.    *   or null if nothing is buffered.
  266.    */
  267.   bufferStart() {
  268.     return this.bufferStart_;
  269.   }
  271.   /**
  272.    * @return {?number} Time in seconds of the end of the buffered range,
  273.    *   or null if nothing is buffered.
  274.    */
  275.   bufferEnd() {
  276.     return this.bufferEnd_;
  277.   }
  279.   /**
  280.    * @param {number} t A timestamp
  281.    * @return {boolean}
  282.    */
  283.   isBuffered(t) {
  284.     if (this.bufferStart_ == null || this.bufferEnd_ == null) {
  285.       return false;
  286.     }
  287.     return t >= this.bufferStart_ && t < this.bufferEnd_;
  288.   }
  290.   /**
  291.    * @param {number} t A timestamp
  292.    * @return {number} Number of seconds ahead of 't' we have buffered
  293.    */
  294.   bufferedAheadOf(t) {
  295.     if (this.bufferEnd_ == null || this.bufferEnd_ < t) {
  296.       return 0;
  297.     }
  299.     goog.asserts.assert(
  300.         this.bufferStart_ != null,
  301.         'start should not be null if end is not null');
  303.     return this.bufferEnd_ - Math.max(t, this.bufferStart_);
  304.   }
  306.   /**
  307.    * Set the selected closed captions id.
  308.    * Append the cues stored in the closed captions map until buffer end time.
  309.    * This is to fill the gap between buffered and unbuffered captions, and to
  310.    * avoid duplicates that would be caused by any future video segments parsed
  311.    * for captions.
  312.    *
  313.    * @param {string} id
  314.    * @param {number} bufferEndTime Load any stored cues up to this time.
  315.    */
  316.   setSelectedClosedCaptionId(id, bufferEndTime) {
  317.     this.selectedClosedCaptionId_ = id;
  319.     const captionsMap = this.closedCaptionsMap_.get(id);
  320.     if (captionsMap) {
  321.       for (const startAndEndTime of captionsMap.keys()) {
  322.         /** @type {Array.<!shaka.text.Cue>} */
  323.         const cues = captionsMap.get(startAndEndTime)
  324.             .filter((c) => c.endTime <= bufferEndTime);
  325.         if (cues) {
  326.           this.displayer_.append(cues);
  327.         }
  328.       }
  329.     }
  330.   }
  332.   /**
  333.   * @param {!Array<muxjs.mp4.ClosedCaption>} closedCaptions
  334.   * @return {!Array<!shaka.cea.ICaptionDecoder.ClosedCaption>}
  335.   */
  336.   convertMuxjsCaptionsToShakaCaptions(closedCaptions) {
  337.     const cues = [];
  338.     for (const caption of closedCaptions) {
  339.       const cue = new shaka.text.Cue(
  340.           caption.startTime, caption.endTime, caption.text);
  341.       cues.push({
  342.         stream: caption.stream,
  343.         cue,
  344.       });
  345.     }
  346.     return cues;
  347.   }
  349.   /**
  350.    * @param {!shaka.text.Cue} cue the cue to apply the timestamp to recursively
  351.    * @param {number} videoTimestampOffset the timestamp offset of the video
  352.    * @private
  353.    */
  354.   applyVideoTimestampOffsetRecursive_(cue, videoTimestampOffset) {
  355.     cue.startTime += videoTimestampOffset;
  356.     cue.endTime += videoTimestampOffset;
  357.     for (const nested of cue.nestedCues) {
  358.       this.applyVideoTimestampOffsetRecursive_(nested, videoTimestampOffset);
  359.     }
  360.   }
  362.   /**
  363.    * Store the closed captions in the text engine, and append the cues to the
  364.    * text displayer.  This is a side-channel used for embedded text only.
  365.    *
  366.    * @param {!Array.<!shaka.cea.ICaptionDecoder.ClosedCaption>} closedCaptions
  367.    * @param {?number} startTime relative to the start of the presentation
  368.    * @param {?number} endTime relative to the start of the presentation
  369.    * @param {number} videoTimestampOffset the timestamp offset of the video
  370.    *   stream in which these captions were embedded
  371.    */
  372.   storeAndAppendClosedCaptions(
  373.       closedCaptions, startTime, endTime, videoTimestampOffset) {
  374.     const startAndEndTime = startTime + ' ' + endTime;
  375.     /** @type {!Map.<string, !Map.<string, !Array.<!shaka.text.Cue>>>} */
  376.     const captionsMap = new Map();
  378.     for (const caption of closedCaptions) {
  379.       const id = caption.stream;
  380.       const cue = caption.cue;
  381.       if (!captionsMap.has(id)) {
  382.         captionsMap.set(id, new Map());
  383.       }
  384.       if (!captionsMap.get(id).has(startAndEndTime)) {
  385.         captionsMap.get(id).set(startAndEndTime, []);
  386.       }
  388.       // Adjust CEA captions with respect to the timestamp offset of the video
  389.       // stream in which they were embedded.
  390.       this.applyVideoTimestampOffsetRecursive_(cue, videoTimestampOffset);
  392.       const keepThisCue =
  393.           cue.startTime >= this.appendWindowStart_ &&
  394.           cue.startTime < this.appendWindowEnd_;
  395.       if (!keepThisCue) {
  396.         continue;
  397.       }
  399.       captionsMap.get(id).get(startAndEndTime).push(cue);
  400.       if (id == this.selectedClosedCaptionId_) {
  401.         this.displayer_.append([cue]);
  402.       }
  403.     }
  405.     for (const id of captionsMap.keys()) {
  406.       if (!this.closedCaptionsMap_.has(id)) {
  407.         this.closedCaptionsMap_.set(id, new Map());
  408.       }
  409.       for (const startAndEndTime of captionsMap.get(id).keys()) {
  410.         const cues = captionsMap.get(id).get(startAndEndTime);
  411.         this.closedCaptionsMap_.get(id).set(startAndEndTime, cues);
  412.       }
  413.     }
  415.     if (this.bufferStart_ == null) {
  416.       this.bufferStart_ = Math.max(startTime, this.appendWindowStart_);
  417.     } else {
  418.       this.bufferStart_ = Math.min(
  419.           this.bufferStart_, Math.max(startTime, this.appendWindowStart_));
  420.     }
  422.     this.bufferEnd_ = Math.max(
  423.         this.bufferEnd_, Math.min(endTime, this.appendWindowEnd_));
  424.   }
  426.   /**
  427.    * Get the number of closed caption channels.
  428.    *
  429.    * This function is for TESTING ONLY. DO NOT USE in the library.
  430.    *
  431.    * @return {number}
  432.    */
  433.   getNumberOfClosedCaptionChannels() {
  434.     return this.closedCaptionsMap_.size;
  435.   }
  437.   /**
  438.    * Get the number of closed caption cues for a given channel. If there is
  439.    * no channel for the given channel id, this will return 0.
  440.    *
  441.    * This function is for TESTING ONLY. DO NOT USE in the library.
  442.    *
  443.    * @param {string} channelId
  444.    * @return {number}
  445.    */
  446.   getNumberOfClosedCaptionsInChannel(channelId) {
  447.     const channel = this.closedCaptionsMap_.get(channelId);
  448.     return channel ? channel.size : 0;
  449.   }
  450. };
  452. /** @private {!Object.<string, !shaka.extern.TextParserPlugin>} */
  453. shaka.text.TextEngine.parserMap_ = {};