Home Reference Source

src/demux/transmuxer.ts

  1. import type { HlsEventEmitter } from '../events';
  2. import { Events } from '../events';
  3. import { ErrorTypes, ErrorDetails } from '../errors';
  4. import Decrypter from '../crypt/decrypter';
  5. import AACDemuxer from '../demux/aacdemuxer';
  6. import MP4Demuxer from '../demux/mp4demuxer';
  7. import TSDemuxer, { TypeSupported } from '../demux/tsdemuxer';
  8. import MP3Demuxer from '../demux/mp3demuxer';
  9. import MP4Remuxer from '../remux/mp4-remuxer';
  10. import PassThroughRemuxer from '../remux/passthrough-remuxer';
  11. import { logger } from '../utils/logger';
  12. import type { Demuxer, DemuxerResult, KeyData } from '../types/demuxer';
  13. import type { Remuxer } from '../types/remuxer';
  14. import type { TransmuxerResult, ChunkMetadata } from '../types/transmuxer';
  15. import type { HlsConfig } from '../config';
  16. import type { DecryptData } from '../loader/level-key';
  17. import type { PlaylistLevelType } from '../types/loader';
  18.  
  19. let now;
  20. // performance.now() not available on WebWorker, at least on Safari Desktop
  21. try {
  22. now = self.performance.now.bind(self.performance);
  23. } catch (err) {
  24. logger.debug('Unable to use Performance API on this environment');
  25. now = self.Date.now;
  26. }
  27.  
  28. type MuxConfig =
  29. | { demux: typeof TSDemuxer; remux: typeof MP4Remuxer }
  30. | { demux: typeof MP4Demuxer; remux: typeof PassThroughRemuxer }
  31. | { demux: typeof AACDemuxer; remux: typeof MP4Remuxer }
  32. | { demux: typeof MP3Demuxer; remux: typeof MP4Remuxer };
  33.  
  34. const muxConfig: MuxConfig[] = [
  35. { demux: TSDemuxer, remux: MP4Remuxer },
  36. { demux: MP4Demuxer, remux: PassThroughRemuxer },
  37. { demux: AACDemuxer, remux: MP4Remuxer },
  38. { demux: MP3Demuxer, remux: MP4Remuxer },
  39. ];
  40.  
  41. export default class Transmuxer {
  42. public async: boolean = false;
  43. private observer: HlsEventEmitter;
  44. private typeSupported: TypeSupported;
  45. private config: HlsConfig;
  46. private vendor: string;
  47. private id: PlaylistLevelType;
  48. private demuxer?: Demuxer;
  49. private remuxer?: Remuxer;
  50. private decrypter?: Decrypter;
  51. private probe!: Function;
  52. private decryptionPromise: Promise<TransmuxerResult> | null = null;
  53. private transmuxConfig!: TransmuxConfig;
  54. private currentTransmuxState!: TransmuxState;
  55.  
  56. constructor(
  57. observer: HlsEventEmitter,
  58. typeSupported: TypeSupported,
  59. config: HlsConfig,
  60. vendor: string,
  61. id: PlaylistLevelType
  62. ) {
  63. this.observer = observer;
  64. this.typeSupported = typeSupported;
  65. this.config = config;
  66. this.vendor = vendor;
  67. this.id = id;
  68. }
  69.  
  70. configure(transmuxConfig: TransmuxConfig) {
  71. this.transmuxConfig = transmuxConfig;
  72. if (this.decrypter) {
  73. this.decrypter.reset();
  74. }
  75. }
  76.  
  77. push(
  78. data: ArrayBuffer,
  79. decryptdata: DecryptData | null,
  80. chunkMeta: ChunkMetadata,
  81. state?: TransmuxState
  82. ): TransmuxerResult | Promise<TransmuxerResult> {
  83. const stats = chunkMeta.transmuxing;
  84. stats.executeStart = now();
  85.  
  86. let uintData: Uint8Array = new Uint8Array(data);
  87. const { currentTransmuxState, transmuxConfig } = this;
  88. if (state) {
  89. this.currentTransmuxState = state;
  90. }
  91.  
  92. const {
  93. contiguous,
  94. discontinuity,
  95. trackSwitch,
  96. accurateTimeOffset,
  97. timeOffset,
  98. initSegmentChange,
  99. } = state || currentTransmuxState;
  100. const {
  101. audioCodec,
  102. videoCodec,
  103. defaultInitPts,
  104. duration,
  105. initSegmentData,
  106. } = transmuxConfig;
  107.  
  108. const keyData = getEncryptionType(uintData, decryptdata);
  109. if (keyData && keyData.method === 'AES-128') {
  110. const decrypter = this.getDecrypter();
  111. // Software decryption is synchronous; webCrypto is not
  112. if (decrypter.isSync()) {
  113. // Software decryption is progressive. Progressive decryption may not return a result on each call. Any cached
  114. // data is handled in the flush() call
  115. const decryptedData = decrypter.softwareDecrypt(
  116. uintData,
  117. keyData.key.buffer,
  118. keyData.iv.buffer
  119. );
  120. if (!decryptedData) {
  121. stats.executeEnd = now();
  122. return emptyResult(chunkMeta);
  123. }
  124. uintData = new Uint8Array(decryptedData);
  125. } else {
  126. this.decryptionPromise = decrypter
  127. .webCryptoDecrypt(uintData, keyData.key.buffer, keyData.iv.buffer)
  128. .then((decryptedData): TransmuxerResult => {
  129. // Calling push here is important; if flush() is called while this is still resolving, this ensures that
  130. // the decrypted data has been transmuxed
  131. const result = this.push(
  132. decryptedData,
  133. null,
  134. chunkMeta
  135. ) as TransmuxerResult;
  136. this.decryptionPromise = null;
  137. return result;
  138. });
  139. return this.decryptionPromise!;
  140. }
  141. }
  142.  
  143. const resetMuxers = this.needsProbing(discontinuity, trackSwitch);
  144. if (resetMuxers) {
  145. this.configureTransmuxer(uintData);
  146. }
  147.  
  148. if (discontinuity || trackSwitch || initSegmentChange || resetMuxers) {
  149. this.resetInitSegment(
  150. initSegmentData,
  151. audioCodec,
  152. videoCodec,
  153. duration,
  154. decryptdata
  155. );
  156. }
  157.  
  158. if (discontinuity || initSegmentChange || resetMuxers) {
  159. this.resetInitialTimestamp(defaultInitPts);
  160. }
  161.  
  162. if (!contiguous) {
  163. this.resetContiguity();
  164. }
  165.  
  166. const result = this.transmux(
  167. uintData,
  168. keyData,
  169. timeOffset,
  170. accurateTimeOffset,
  171. chunkMeta
  172. );
  173. const currentState = this.currentTransmuxState;
  174.  
  175. currentState.contiguous = true;
  176. currentState.discontinuity = false;
  177. currentState.trackSwitch = false;
  178.  
  179. stats.executeEnd = now();
  180. return result;
  181. }
  182.  
  183. // Due to data caching, flush calls can produce more than one TransmuxerResult (hence the Array type)
  184. flush(
  185. chunkMeta: ChunkMetadata
  186. ): TransmuxerResult[] | Promise<TransmuxerResult[]> {
  187. const stats = chunkMeta.transmuxing;
  188. stats.executeStart = now();
  189.  
  190. const { decrypter, currentTransmuxState, decryptionPromise } = this;
  191.  
  192. if (decryptionPromise) {
  193. // Upon resolution, the decryption promise calls push() and returns its TransmuxerResult up the stack. Therefore
  194. // only flushing is required for async decryption
  195. return decryptionPromise.then(() => {
  196. return this.flush(chunkMeta);
  197. });
  198. }
  199.  
  200. const transmuxResults: TransmuxerResult[] = [];
  201. const { timeOffset } = currentTransmuxState;
  202. if (decrypter) {
  203. // The decrypter may have data cached, which needs to be demuxed. In this case we'll have two TransmuxResults
  204. // This happens in the case that we receive only 1 push call for a segment (either for non-progressive downloads,
  205. // or for progressive downloads with small segments)
  206. const decryptedData = decrypter.flush();
  207. if (decryptedData) {
  208. // Push always returns a TransmuxerResult if decryptdata is null
  209. transmuxResults.push(
  210. this.push(decryptedData, null, chunkMeta) as TransmuxerResult
  211. );
  212. }
  213. }
  214.  
  215. const { demuxer, remuxer } = this;
  216. if (!demuxer || !remuxer) {
  217. // If probing failed, then Hls.js has been given content its not able to handle
  218. this.observer.emit(Events.ERROR, Events.ERROR, {
  219. type: ErrorTypes.MEDIA_ERROR,
  220. details: ErrorDetails.FRAG_PARSING_ERROR,
  221. fatal: true,
  222. reason: 'no demux matching with content found',
  223. });
  224. stats.executeEnd = now();
  225. return [emptyResult(chunkMeta)];
  226. }
  227.  
  228. const demuxResultOrPromise = demuxer.flush(timeOffset);
  229. if (isPromise(demuxResultOrPromise)) {
  230. // Decrypt final SAMPLE-AES samples
  231. return demuxResultOrPromise.then((demuxResult) => {
  232. this.flushRemux(transmuxResults, demuxResult, chunkMeta);
  233. return transmuxResults;
  234. });
  235. }
  236.  
  237. this.flushRemux(transmuxResults, demuxResultOrPromise, chunkMeta);
  238. return transmuxResults;
  239. }
  240.  
  241. private flushRemux(
  242. transmuxResults: TransmuxerResult[],
  243. demuxResult: DemuxerResult,
  244. chunkMeta: ChunkMetadata
  245. ) {
  246. const { audioTrack, videoTrack, id3Track, textTrack } = demuxResult;
  247. const { accurateTimeOffset, timeOffset } = this.currentTransmuxState;
  248. logger.log(
  249. `[transmuxer.ts]: Flushed fragment ${chunkMeta.sn}${
  250. chunkMeta.part > -1 ? ' p: ' + chunkMeta.part : ''
  251. } of level ${chunkMeta.level}`
  252. );
  253. const remuxResult = this.remuxer!.remux(
  254. audioTrack,
  255. videoTrack,
  256. id3Track,
  257. textTrack,
  258. timeOffset,
  259. accurateTimeOffset,
  260. true,
  261. this.id
  262. );
  263. transmuxResults.push({
  264. remuxResult,
  265. chunkMeta,
  266. });
  267.  
  268. chunkMeta.transmuxing.executeEnd = now();
  269. }
  270.  
  271. resetInitialTimestamp(defaultInitPts: number | undefined) {
  272. const { demuxer, remuxer } = this;
  273. if (!demuxer || !remuxer) {
  274. return;
  275. }
  276. demuxer.resetTimeStamp(defaultInitPts);
  277. remuxer.resetTimeStamp(defaultInitPts);
  278. }
  279.  
  280. resetContiguity() {
  281. const { demuxer, remuxer } = this;
  282. if (!demuxer || !remuxer) {
  283. return;
  284. }
  285. demuxer.resetContiguity();
  286. remuxer.resetNextTimestamp();
  287. }
  288.  
  289. resetInitSegment(
  290. initSegmentData: Uint8Array | undefined,
  291. audioCodec: string | undefined,
  292. videoCodec: string | undefined,
  293. trackDuration: number,
  294. decryptdata: DecryptData | null
  295. ) {
  296. const { demuxer, remuxer } = this;
  297. if (!demuxer || !remuxer) {
  298. return;
  299. }
  300. demuxer.resetInitSegment(
  301. initSegmentData,
  302. audioCodec,
  303. videoCodec,
  304. trackDuration
  305. );
  306. remuxer.resetInitSegment(
  307. initSegmentData,
  308. audioCodec,
  309. videoCodec,
  310. decryptdata
  311. );
  312. }
  313.  
  314. destroy(): void {
  315. if (this.demuxer) {
  316. this.demuxer.destroy();
  317. this.demuxer = undefined;
  318. }
  319. if (this.remuxer) {
  320. this.remuxer.destroy();
  321. this.remuxer = undefined;
  322. }
  323. }
  324.  
  325. private transmux(
  326. data: Uint8Array,
  327. keyData: KeyData | null,
  328. timeOffset: number,
  329. accurateTimeOffset: boolean,
  330. chunkMeta: ChunkMetadata
  331. ): TransmuxerResult | Promise<TransmuxerResult> {
  332. let result: TransmuxerResult | Promise<TransmuxerResult>;
  333. if (keyData && keyData.method === 'SAMPLE-AES') {
  334. result = this.transmuxSampleAes(
  335. data,
  336. keyData,
  337. timeOffset,
  338. accurateTimeOffset,
  339. chunkMeta
  340. );
  341. } else {
  342. result = this.transmuxUnencrypted(
  343. data,
  344. timeOffset,
  345. accurateTimeOffset,
  346. chunkMeta
  347. );
  348. }
  349. return result;
  350. }
  351.  
  352. private transmuxUnencrypted(
  353. data: Uint8Array,
  354. timeOffset: number,
  355. accurateTimeOffset: boolean,
  356. chunkMeta: ChunkMetadata
  357. ): TransmuxerResult {
  358. const { audioTrack, videoTrack, id3Track, textTrack } = (
  359. this.demuxer as Demuxer
  360. ).demux(data, timeOffset, false, !this.config.progressive);
  361. const remuxResult = this.remuxer!.remux(
  362. audioTrack,
  363. videoTrack,
  364. id3Track,
  365. textTrack,
  366. timeOffset,
  367. accurateTimeOffset,
  368. false,
  369. this.id
  370. );
  371. return {
  372. remuxResult,
  373. chunkMeta,
  374. };
  375. }
  376.  
  377. private transmuxSampleAes(
  378. data: Uint8Array,
  379. decryptData: KeyData,
  380. timeOffset: number,
  381. accurateTimeOffset: boolean,
  382. chunkMeta: ChunkMetadata
  383. ): Promise<TransmuxerResult> {
  384. return (this.demuxer as Demuxer)
  385. .demuxSampleAes(data, decryptData, timeOffset)
  386. .then((demuxResult) => {
  387. const remuxResult = this.remuxer!.remux(
  388. demuxResult.audioTrack,
  389. demuxResult.videoTrack,
  390. demuxResult.id3Track,
  391. demuxResult.textTrack,
  392. timeOffset,
  393. accurateTimeOffset,
  394. false,
  395. this.id
  396. );
  397. return {
  398. remuxResult,
  399. chunkMeta,
  400. };
  401. });
  402. }
  403.  
  404. private configureTransmuxer(data: Uint8Array) {
  405. const { config, observer, typeSupported, vendor } = this;
  406. // probe for content type
  407. let mux;
  408. for (let i = 0, len = muxConfig.length; i < len; i++) {
  409. if (muxConfig[i].demux.probe(data)) {
  410. mux = muxConfig[i];
  411. break;
  412. }
  413. }
  414. if (!mux) {
  415. // If probing previous configs fail, use mp4 passthrough
  416. logger.warn(
  417. 'Failed to find demuxer by probing frag, treating as mp4 passthrough'
  418. );
  419. mux = { demux: MP4Demuxer, remux: PassThroughRemuxer };
  420. }
  421. // so let's check that current remuxer and demuxer are still valid
  422. const demuxer = this.demuxer;
  423. const remuxer = this.remuxer;
  424. const Remuxer: MuxConfig['remux'] = mux.remux;
  425. const Demuxer: MuxConfig['demux'] = mux.demux;
  426. if (!remuxer || !(remuxer instanceof Remuxer)) {
  427. this.remuxer = new Remuxer(observer, config, typeSupported, vendor);
  428. }
  429. if (!demuxer || !(demuxer instanceof Demuxer)) {
  430. this.demuxer = new Demuxer(observer, config, typeSupported);
  431. this.probe = Demuxer.probe;
  432. }
  433. }
  434.  
  435. private needsProbing(discontinuity: boolean, trackSwitch: boolean): boolean {
  436. // in case of continuity change, or track switch
  437. // we might switch from content type (AAC container to TS container, or TS to fmp4 for example)
  438. return !this.demuxer || !this.remuxer || discontinuity || trackSwitch;
  439. }
  440.  
  441. private getDecrypter(): Decrypter {
  442. let decrypter = this.decrypter;
  443. if (!decrypter) {
  444. decrypter = this.decrypter = new Decrypter(this.config);
  445. }
  446. return decrypter;
  447. }
  448. }
  449.  
  450. function getEncryptionType(
  451. data: Uint8Array,
  452. decryptData: DecryptData | null
  453. ): KeyData | null {
  454. let encryptionType: KeyData | null = null;
  455. if (
  456. data.byteLength > 0 &&
  457. decryptData != null &&
  458. decryptData.key != null &&
  459. decryptData.iv !== null &&
  460. decryptData.method != null
  461. ) {
  462. encryptionType = decryptData as KeyData;
  463. }
  464. return encryptionType;
  465. }
  466.  
  467. const emptyResult = (chunkMeta): TransmuxerResult => ({
  468. remuxResult: {},
  469. chunkMeta,
  470. });
  471.  
  472. export function isPromise<T>(p: Promise<T> | any): p is Promise<T> {
  473. return 'then' in p && p.then instanceof Function;
  474. }
  475.  
  476. export class TransmuxConfig {
  477. public audioCodec?: string;
  478. public videoCodec?: string;
  479. public initSegmentData?: Uint8Array;
  480. public duration: number;
  481. public defaultInitPts?: number;
  482.  
  483. constructor(
  484. audioCodec: string | undefined,
  485. videoCodec: string | undefined,
  486. initSegmentData: Uint8Array | undefined,
  487. duration: number,
  488. defaultInitPts?: number
  489. ) {
  490. this.audioCodec = audioCodec;
  491. this.videoCodec = videoCodec;
  492. this.initSegmentData = initSegmentData;
  493. this.duration = duration;
  494. this.defaultInitPts = defaultInitPts;
  495. }
  496. }
  497.  
  498. export class TransmuxState {
  499. public discontinuity: boolean;
  500. public contiguous: boolean;
  501. public accurateTimeOffset: boolean;
  502. public trackSwitch: boolean;
  503. public timeOffset: number;
  504. public initSegmentChange: boolean;
  505.  
  506. constructor(
  507. discontinuity: boolean,
  508. contiguous: boolean,
  509. accurateTimeOffset: boolean,
  510. trackSwitch: boolean,
  511. timeOffset: number,
  512. initSegmentChange: boolean
  513. ) {
  514. this.discontinuity = discontinuity;
  515. this.contiguous = contiguous;
  516. this.accurateTimeOffset = accurateTimeOffset;
  517. this.trackSwitch = trackSwitch;
  518. this.timeOffset = timeOffset;
  519. this.initSegmentChange = initSegmentChange;
  520. }
  521. }