Don't rely on youtube for tests

Use another import URL when possible, and disable import tests when we
want to do a youtube import test

.github/workflows/test.yml

@@ -22,13 +22,15 @@ jobs:
           POSTGRES_HOST_AUTH_METHOD: trust
 
     strategy:
-      fail-fast: true
+      fail-fast: false
       matrix:
         test_suite: [ misc, api-1, api-2, api-3, api-4, cli, lint ]
 
     env:
       PGUSER: peertube
       PGHOST: localhost
+      DISABLE_HTTP_IMPORT_TESTS: true
+      NODE_PENDING_JOB_WAIT: 2000
 
     steps:
       - uses: actions/checkout@v2
@@ -50,7 +52,9 @@ jobs:
       - name: Cache Node.js modules
         uses: actions/cache@v2
         with:
-          path: ~/.cache/yarn
+          path: |
+            **/node_modules
+            ~/fixtures
           key: ${{ runner.OS }}-node-${{ hashFiles('**/yarn.lock') }}
           restore-keys: |
             ${{ runner.OS }}-node-
@@ -69,5 +73,5 @@ jobs:
         uses: actions/upload-artifact@v2
         if: failure()
         with:
-          name: test-storages
+          name: test-storages-${{ matrix.test_suite }}
           path: test*/logs

scripts/ci.sh

@@ -21,7 +21,7 @@ if [ "$1" = "misc" ]; then
         server/tests/plugins/index.ts
 elif [ "$1" = "cli" ]; then
     npm run build:server
-    CC=gcc-4.9 CXX=g++-4.9 npm run setup:cli
+    npm run setup:cli
     mocha --timeout 5000 --exit --require ts-node/register --require tsconfig-paths/register --bail server/tests/cli/index.ts
 elif [ "$1" = "api-1" ]; then
     npm run build:server

server/tests/api/check-params/users.ts

@@ -1,10 +1,10 @@
 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
 
-import { omit } from 'lodash'
 import 'mocha'
+import { expect } from 'chai'
+import { omit } from 'lodash'
 import { join } from 'path'
 import { User, UserRole, VideoImport, VideoImportState } from '../../../../shared'
-
 import {
   addVideoChannel,
   blockUser,
@@ -31,17 +31,16 @@ import {
   uploadVideo,
   userLogin
 } from '../../../../shared/extra-utils'
+import { MockSmtpServer } from '../../../../shared/extra-utils/miscs/email'
 import {
   checkBadCountPagination,
   checkBadSortPagination,
   checkBadStartPagination
 } from '../../../../shared/extra-utils/requests/check-api-params'
-import { getMagnetURI, getMyVideoImports, getYoutubeVideoUrl, importVideo } from '../../../../shared/extra-utils/videos/video-imports'
-import { VideoPrivacy } from '../../../../shared/models/videos'
 import { waitJobs } from '../../../../shared/extra-utils/server/jobs'
-import { expect } from 'chai'
+import { getMagnetURI, getMyVideoImports, getGoodVideoUrl, importVideo } from '../../../../shared/extra-utils/videos/video-imports'
 import { UserAdminFlag } from '../../../../shared/models/users/user-flag.model'
-import { MockSmtpServer } from '../../../../shared/extra-utils/miscs/email'
+import { VideoPrivacy } from '../../../../shared/models/videos'
 
 describe('Test users API validators', function () {
   const path = '/api/v1/users/'
@@ -1047,7 +1046,7 @@ describe('Test users API validators', function () {
         channelId: 1,
         privacy: VideoPrivacy.PUBLIC
       }
-      await importVideo(server.url, server.accessToken, immutableAssign(baseAttributes, { targetUrl: getYoutubeVideoUrl() }))
+      await importVideo(server.url, server.accessToken, immutableAssign(baseAttributes, { targetUrl: getGoodVideoUrl() }))
       await importVideo(server.url, server.accessToken, immutableAssign(baseAttributes, { magnetUri: getMagnetURI() }))
       await importVideo(server.url, server.accessToken, immutableAssign(baseAttributes, { torrentfile: 'video-720p.torrent' as any }))
 

server/tests/api/check-params/video-imports.ts

@@ -1,9 +1,8 @@
 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
 
-import { omit } from 'lodash'
 import 'mocha'
+import { omit } from 'lodash'
 import { join } from 'path'
-import { VideoPrivacy } from '../../../../shared/models/videos/video-privacy.enum'
 import {
   cleanupTests,
   createUser,
@@ -23,7 +22,8 @@ import {
   checkBadSortPagination,
   checkBadStartPagination
 } from '../../../../shared/extra-utils/requests/check-api-params'
-import { getMagnetURI, getYoutubeVideoUrl } from '../../../../shared/extra-utils/videos/video-imports'
+import { getMagnetURI, getGoodVideoUrl } from '../../../../shared/extra-utils/videos/video-imports'
+import { VideoPrivacy } from '../../../../shared/models/videos/video-privacy.enum'
 
 describe('Test video imports API validator', function () {
   const path = '/api/v1/videos/imports'
@@ -76,7 +76,7 @@ describe('Test video imports API validator', function () {
 
     before(function () {
       baseCorrectParams = {
-        targetUrl: getYoutubeVideoUrl(),
+        targetUrl: getGoodVideoUrl(),
         name: 'my super name',
         category: 5,
         licence: 1,
@@ -246,15 +246,13 @@ describe('Test video imports API validator', function () {
     it('Should succeed with the correct parameters', async function () {
       this.timeout(30000)
 
-      {
+      await makePostBodyRequest({
-        await makePostBodyRequest({
+        url: server.url,
-          url: server.url,
+        path,
-          path,
+        token: server.accessToken,
-          token: server.accessToken,
+        fields: baseCorrectParams,
-          fields: baseCorrectParams,
+        statusCodeExpected: 200
-          statusCodeExpected: 200
+      })
-        })
-      }
     })
 
     it('Should forbid to import http videos', async function () {

server/tests/api/notifications/user-notifications.ts

@@ -24,7 +24,7 @@ import {
   prepareNotificationsTest
 } from '../../../../shared/extra-utils/users/user-notifications'
 import { addUserSubscription, removeUserSubscription } from '../../../../shared/extra-utils/users/user-subscriptions'
-import { getBadVideoUrl, getYoutubeVideoUrl, importVideo } from '../../../../shared/extra-utils/videos/video-imports'
+import { getBadVideoUrl, getGoodVideoUrl, importVideo } from '../../../../shared/extra-utils/videos/video-imports'
 import { UserNotification, UserNotificationType } from '../../../../shared/models/users'
 import { VideoPrivacy } from '../../../../shared/models/videos'
 
@@ -213,7 +213,7 @@ describe('Test user notifications', function () {
         name,
         channelId,
         privacy: VideoPrivacy.PUBLIC,
-        targetUrl: getYoutubeVideoUrl()
+        targetUrl: getGoodVideoUrl()
       }
       const res = await importVideo(servers[0].url, servers[0].accessToken, attributes)
       const uuid = res.body.video.uuid
@@ -284,7 +284,7 @@ describe('Test user notifications', function () {
         name,
         channelId,
         privacy: VideoPrivacy.PUBLIC,
-        targetUrl: getYoutubeVideoUrl(),
+        targetUrl: getGoodVideoUrl(),
         waitTranscoding: true
       }
       const res = await importVideo(servers[1].url, servers[1].accessToken, attributes)
@@ -371,13 +371,13 @@ describe('Test user notifications', function () {
         name,
         channelId,
         privacy: VideoPrivacy.PRIVATE,
-        targetUrl: getYoutubeVideoUrl()
+        targetUrl: getGoodVideoUrl()
       }
       const res = await importVideo(servers[0].url, servers[0].accessToken, attributes)
       const uuid = res.body.video.uuid
 
       await waitJobs(servers)
-      await checkMyVideoImportIsFinished(baseParams, name, uuid, getYoutubeVideoUrl(), true, 'presence')
+      await checkMyVideoImportIsFinished(baseParams, name, uuid, getGoodVideoUrl(), true, 'presence')
     })
   })
 

server/tests/api/videos/video-blacklist.ts

@@ -1,8 +1,8 @@
 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
 
+import 'mocha'
 import * as chai from 'chai'
 import { orderBy } from 'lodash'
-import 'mocha'
 import {
   addVideoToBlacklist,
   cleanupTests,
@@ -25,10 +25,10 @@ import {
 } from '../../../../shared/extra-utils/index'
 import { doubleFollow } from '../../../../shared/extra-utils/server/follows'
 import { waitJobs } from '../../../../shared/extra-utils/server/jobs'
-import { VideoBlacklist, VideoBlacklistType } from '../../../../shared/models/videos'
+import { getGoodVideoUrl, getMagnetURI, importVideo } from '../../../../shared/extra-utils/videos/video-imports'
-import { UserAdminFlag } from '../../../../shared/models/users/user-flag.model'
 import { User, UserRole } from '../../../../shared/models/users'
-import { getMagnetURI, getYoutubeVideoUrl, importVideo } from '../../../../shared/extra-utils/videos/video-imports'
+import { UserAdminFlag } from '../../../../shared/models/users/user-flag.model'
+import { VideoBlacklist, VideoBlacklistType } from '../../../../shared/models/videos'
 
 const expect = chai.expect
 
@@ -424,7 +424,7 @@ describe('Test video blacklist', function () {
       this.timeout(15000)
 
       const attributes = {
-        targetUrl: getYoutubeVideoUrl(),
+        targetUrl: getGoodVideoUrl(),
         name: 'URL import',
         channelId: channelOfUserWithoutFlag
       }

server/tests/api/videos/video-imports.ts

@@ -1,8 +1,7 @@
 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
 
-import * as chai from 'chai'
 import 'mocha'
-import { VideoDetails, VideoImport, VideoPrivacy, VideoCaption } from '../../../../shared/models/videos'
+import * as chai from 'chai'
 import {
   cleanupTests,
   doubleFollow,
@@ -11,15 +10,16 @@ import {
   getMyVideos,
   getVideo,
   getVideosList,
-  listVideoCaptions,
-  testCaptionFile,
   immutableAssign,
+  listVideoCaptions,
   ServerInfo,
-  setAccessTokensToServers
+  setAccessTokensToServers,
+  testCaptionFile
 } from '../../../../shared/extra-utils'
+import { areHttpImportTestsDisabled, testImage } from '../../../../shared/extra-utils/miscs/miscs'
 import { waitJobs } from '../../../../shared/extra-utils/server/jobs'
 import { getMagnetURI, getMyVideoImports, getYoutubeVideoUrl, importVideo } from '../../../../shared/extra-utils/videos/video-imports'
-import { testImage } from '../../../../shared/extra-utils/miscs/miscs'
+import { VideoCaption, VideoDetails, VideoImport, VideoPrivacy } from '../../../../shared/models/videos'
 
 const expect = chai.expect
 
@@ -28,6 +28,8 @@ describe('Test video imports', function () {
   let channelIdServer1: number
   let channelIdServer2: number
 
+  if (areHttpImportTestsDisabled()) return
+
   async function checkVideosServer1 (url: string, idHttp: string, idMagnet: string, idTorrent: string) {
     const resHttp = await getVideo(url, idHttp)
     const videoHttp: VideoDetails = resHttp.body

server/tests/plugins/filter-hooks.ts

@@ -24,7 +24,7 @@ import {
   waitJobs
 } from '../../../shared/extra-utils'
 import { cleanupTests, flushAndRunMultipleServers, ServerInfo } from '../../../shared/extra-utils/server/servers'
-import { getMyVideoImports, getYoutubeVideoUrl, importVideo } from '../../../shared/extra-utils/videos/video-imports'
+import { getGoodVideoUrl, getMyVideoImports, importVideo } from '../../../shared/extra-utils/videos/video-imports'
 import { VideoDetails, VideoImport, VideoImportState, VideoPrivacy } from '../../../shared/models/videos'
 import { VideoCommentThreadTree } from '../../../shared/models/videos/video-comment.model'
 
@@ -92,7 +92,7 @@ describe('Test plugin filter hooks', function () {
       name: 'normal title',
       privacy: VideoPrivacy.PUBLIC,
       channelId: servers[0].videoChannel.id,
-      targetUrl: getYoutubeVideoUrl() + 'bad'
+      targetUrl: getGoodVideoUrl() + 'bad'
     }
     await importVideo(servers[0].url, servers[0].accessToken, baseAttributes, 403)
   })
@@ -117,7 +117,7 @@ describe('Test plugin filter hooks', function () {
         name: 'title with bad word',
         privacy: VideoPrivacy.PUBLIC,
         channelId: servers[0].videoChannel.id,
-        targetUrl: getYoutubeVideoUrl()
+        targetUrl: getGoodVideoUrl()
       }
       const res = await importVideo(servers[0].url, servers[0].accessToken, baseAttributes)
       videoImportId = res.body.id
@@ -220,7 +220,7 @@ describe('Test plugin filter hooks', function () {
 
       const attributes = {
         name: 'video please blacklist me',
-        targetUrl: getYoutubeVideoUrl(),
+        targetUrl: getGoodVideoUrl(),
         channelId: servers[0].videoChannel.id
       }
       const res = await importVideo(servers[0].url, servers[0].accessToken, attributes)

shared/extra-utils/miscs/miscs.ts

@@ -64,19 +64,25 @@ async function testImage (url: string, imageName: string, imagePath: string, ext
 }
 
 function buildAbsoluteFixturePath (path: string, customCIPath = false) {
-  if (isAbsolute(path)) {
+  if (isAbsolute(path)) return path
-    return path
-  }
 
   if (customCIPath) {
     if (process.env.GITLAB_CI) return join(root(), 'cached-fixtures', path)
 
-    if (process.env.TRAVIS) return join(process.env.HOME, 'fixtures', path)
+    return join(process.env.HOME, 'fixtures', path)
   }
 
   return join(root(), 'server', 'tests', 'fixtures', path)
 }
 
+function areHttpImportTestsDisabled () {
+  const disabled = process.env.DISABLE_HTTP_IMPORT_TESTS === 'true'
+
+  if (disabled) console.log('Import tests are disabled')
+
+  return disabled
+}
+
 async function generateHighBitrateVideo () {
   const tempFixturePath = buildAbsoluteFixturePath('video_high_bitrate_1080p.mp4', true)
 
@@ -131,6 +137,7 @@ async function generateVideoWithFramerate (fps = 60) {
 export {
   dateIsValid,
   wait,
+  areHttpImportTestsDisabled,
   buildServerDirectory,
   webtorrentAdd,
   immutableAssign,

shared/extra-utils/videos/video-imports.ts

@@ -15,6 +15,10 @@ function getBadVideoUrl () {
   return 'https://download.cpy.re/peertube/bad_video.mp4'
 }
 
+function getGoodVideoUrl () {
+  return 'https://download.cpy.re/peertube/good_video.mp4'
+}
+
 function importVideo (url: string, token: string, attributes: VideoImportCreate & { torrentfile?: string }, statusCodeExpected = 200) {
   const path = '/api/v1/videos/imports'
 
@@ -53,5 +57,6 @@ export {
   getYoutubeVideoUrl,
   importVideo,
   getMagnetURI,
-  getMyVideoImports
+  getMyVideoImports,
+  getGoodVideoUrl
 }