From 5170f492b95dc81b75230312411c5fdb0019eed2 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 10 May 2023 09:28:42 +0200 Subject: [PATCH] Try to have more robust live tests --- .../api/transcoding/create-transcoding.ts | 2 +- server/tests/shared/live.ts | 3 +- server/tests/shared/streaming-playlists.ts | 7 +- shared/server-commands/videos/live-command.ts | 2 +- .../videos/streaming-playlists-command.ts | 78 ++++++++++++++----- 5 files changed, 67 insertions(+), 25 deletions(-) diff --git a/server/tests/api/transcoding/create-transcoding.ts b/server/tests/api/transcoding/create-transcoding.ts index 85389a949..5483c8dba 100644 --- a/server/tests/api/transcoding/create-transcoding.ts +++ b/server/tests/api/transcoding/create-transcoding.ts @@ -226,7 +226,7 @@ function runTests (objectStorage: boolean) { const resolutions = hlsPlaylist.files.map(f => f.resolution.id) await checkResolutionsInMasterPlaylist({ server: servers[0], playlistUrl: hlsPlaylist.playlistUrl, resolutions }) - const shaBody = await servers[0].streamingPlaylists.getSegmentSha256({ url: hlsPlaylist.segmentsSha256Url }) + const shaBody = await servers[0].streamingPlaylists.getSegmentSha256({ url: hlsPlaylist.segmentsSha256Url, withRetry: true }) expect(Object.keys(shaBody)).to.have.lengthOf(5) } } diff --git a/server/tests/shared/live.ts b/server/tests/shared/live.ts index 31f92ef19..793503311 100644 --- a/server/tests/shared/live.ts +++ b/server/tests/shared/live.ts @@ -115,7 +115,8 @@ async function testLiveVideoResolutions (options: { baseUrlSegment: baseUrl, videoUUID: video.uuid, segmentName, - hlsPlaylist + hlsPlaylist, + withRetry: objectStorage // With object storage, the request may fail because of inconsistent data in S3 }) if (originServer.internalServerNumber === server.internalServerNumber) { diff --git a/server/tests/shared/streaming-playlists.ts b/server/tests/shared/streaming-playlists.ts index acfb2b408..e4f88bc25 100644 --- a/server/tests/shared/streaming-playlists.ts +++ b/server/tests/shared/streaming-playlists.ts @@ -51,12 +51,13 @@ async function checkLiveSegmentHash (options: { videoUUID: string segmentName: string hlsPlaylist: VideoStreamingPlaylist + withRetry?: boolean }) { - const { server, baseUrlSegment, videoUUID, segmentName, hlsPlaylist } = options + const { server, baseUrlSegment, videoUUID, segmentName, hlsPlaylist, withRetry = false } = options const command = server.streamingPlaylists - const segmentBody = await command.getFragmentedSegment({ url: `${baseUrlSegment}/${videoUUID}/${segmentName}` }) - const shaBody = await command.getSegmentSha256({ url: hlsPlaylist.segmentsSha256Url }) + const segmentBody = await command.getFragmentedSegment({ url: `${baseUrlSegment}/${videoUUID}/${segmentName}`, withRetry }) + const shaBody = await command.getSegmentSha256({ url: hlsPlaylist.segmentsSha256Url, withRetry }) expect(sha256(segmentBody)).to.equal(shaBody[segmentName]) } diff --git a/shared/server-commands/videos/live-command.ts b/shared/server-commands/videos/live-command.ts index dc3c5a86e..2e4bc10b5 100644 --- a/shared/server-commands/videos/live-command.ts +++ b/shared/server-commands/videos/live-command.ts @@ -224,7 +224,7 @@ export class LiveCommand extends AbstractCommand { const video = await server.videos.get({ id: videoUUID }) const hlsPlaylist = video.streamingPlaylists[0] - const shaBody = await server.streamingPlaylists.getSegmentSha256({ url: hlsPlaylist.segmentsSha256Url }) + const shaBody = await server.streamingPlaylists.getSegmentSha256({ url: hlsPlaylist.segmentsSha256Url, withRetry: objectStorage }) if (!shaBody[segmentName]) { throw new Error('Segment SHA does not exist') diff --git a/shared/server-commands/videos/streaming-playlists-command.ts b/shared/server-commands/videos/streaming-playlists-command.ts index 7b92dcc0a..950808b60 100644 --- a/shared/server-commands/videos/streaming-playlists-command.ts +++ b/shared/server-commands/videos/streaming-playlists-command.ts @@ -44,31 +44,71 @@ export class StreamingPlaylistsCommand extends AbstractCommand { } } - getFragmentedSegment (options: OverrideCommandOptions & { + async getFragmentedSegment (options: OverrideCommandOptions & { url: string range?: string - }) { - return unwrapBody(this.getRawRequest({ - ...options, - url: options.url, - range: options.range, - implicitToken: false, - responseType: 'application/octet-stream', - defaultExpectedStatus: HttpStatusCode.OK_200 - })) + withRetry?: boolean // default false + currentRetry?: number + }) { + const { withRetry, currentRetry = 1 } = options + + try { + const result = await unwrapBody(this.getRawRequest({ + ...options, + + url: options.url, + range: options.range, + implicitToken: false, + responseType: 'application/octet-stream', + defaultExpectedStatus: HttpStatusCode.OK_200 + })) + + return result + } catch (err) { + if (!withRetry || currentRetry > 5) throw err + + await wait(100) + + return this.getFragmentedSegment({ + ...options, + + withRetry, + currentRetry: currentRetry + 1 + }) + } } - getSegmentSha256 (options: OverrideCommandOptions & { + async getSegmentSha256 (options: OverrideCommandOptions & { url: string - }) { - return unwrapBodyOrDecodeToJSON<{ [ id: string ]: string }>(this.getRawRequest({ - ...options, - url: options.url, - contentType: 'application/json', - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.OK_200 - })) + withRetry?: boolean // default false + currentRetry?: number + }) { + const { withRetry, currentRetry = 1 } = options + + try { + const result = await unwrapBodyOrDecodeToJSON<{ [ id: string ]: string }>(this.getRawRequest({ + ...options, + + url: options.url, + contentType: 'application/json', + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.OK_200 + })) + + return result + } catch (err) { + if (!withRetry || currentRetry > 5) throw err + + await wait(100) + + return this.getSegmentSha256({ + ...options, + + withRetry, + currentRetry: currentRetry + 1 + }) + } } }