peertube/shared/server-commands/server/server.ts

import { ChildProcess, fork } from 'child_process'
import { copy } from 'fs-extra'
import { join } from 'path'
import { parallelTests, randomInt, root } from '@shared/core-utils'
import { Video, VideoChannel, VideoChannelSync, VideoCreateResult, VideoDetails } from '@shared/models'
import { BulkCommand } from '../bulk'
import { CLICommand } from '../cli'
import { CustomPagesCommand } from '../custom-pages'
import { FeedCommand } from '../feeds'
import { LogsCommand } from '../logs'
import { AbusesCommand } from '../moderation'
import { OverviewsCommand } from '../overviews'
import { RunnerJobsCommand, RunnerRegistrationTokensCommand, RunnersCommand } from '../runners'
import { SearchCommand } from '../search'
import { SocketIOCommand } from '../socket'
import {
  AccountsCommand,
  BlocklistCommand,
  LoginCommand,
  NotificationsCommand,
  RegistrationsCommand,
  SubscriptionsCommand,
  TwoFactorCommand,
  UsersCommand
} from '../users'
import {
  BlacklistCommand,
  CaptionsCommand,
  ChangeOwnershipCommand,
  ChannelsCommand,
  ChannelSyncsCommand,
  HistoryCommand,
  ImportsCommand,
  LiveCommand,
  PlaylistsCommand,
  ServicesCommand,
  StreamingPlaylistsCommand,
  VideosCommand,
  VideoStudioCommand,
  VideoTokenCommand,
  ViewsCommand
} from '../videos'
import { CommentsCommand } from '../videos/comments-command'
import { VideoStatsCommand } from '../videos/video-stats-command'
import { ConfigCommand } from './config-command'
import { ContactFormCommand } from './contact-form-command'
import { DebugCommand } from './debug-command'
import { FollowsCommand } from './follows-command'
import { JobsCommand } from './jobs-command'
import { MetricsCommand } from './metrics-command'
import { PluginsCommand } from './plugins-command'
import { RedundancyCommand } from './redundancy-command'
import { ServersCommand } from './servers-command'
import { StatsCommand } from './stats-command'

export type RunServerOptions = {
  hideLogs?: boolean
  nodeArgs?: string[]
  peertubeArgs?: string[]
  env?: { [ id: string ]: string }
}

export class PeerTubeServer {
  app?: ChildProcess

  url: string
  host?: string
  hostname?: string
  port?: number

  rtmpPort?: number
  rtmpsPort?: number

  parallel?: boolean
  internalServerNumber: number

  serverNumber?: number
  customConfigFile?: string

  store?: {
    client?: {
      id?: string
      secret?: string
    }

    user?: {
      username: string
      password: string
      email?: string
    }

    channel?: VideoChannel
    videoChannelSync?: Partial<VideoChannelSync>

    video?: Video
    videoCreated?: VideoCreateResult
    videoDetails?: VideoDetails

    videos?: { id: number, uuid: string }[]
  }

  accessToken?: string
  refreshToken?: string

  bulk?: BulkCommand
  cli?: CLICommand
  customPage?: CustomPagesCommand
  feed?: FeedCommand
  logs?: LogsCommand
  abuses?: AbusesCommand
  overviews?: OverviewsCommand
  search?: SearchCommand
  contactForm?: ContactFormCommand
  debug?: DebugCommand
  follows?: FollowsCommand
  jobs?: JobsCommand
  metrics?: MetricsCommand
  plugins?: PluginsCommand
  redundancy?: RedundancyCommand
  stats?: StatsCommand
  config?: ConfigCommand
  socketIO?: SocketIOCommand
  accounts?: AccountsCommand
  blocklist?: BlocklistCommand
  subscriptions?: SubscriptionsCommand
  live?: LiveCommand
  services?: ServicesCommand
  blacklist?: BlacklistCommand
  captions?: CaptionsCommand
  changeOwnership?: ChangeOwnershipCommand
  playlists?: PlaylistsCommand
  history?: HistoryCommand
  imports?: ImportsCommand
  channelSyncs?: ChannelSyncsCommand
  streamingPlaylists?: StreamingPlaylistsCommand
  channels?: ChannelsCommand
  comments?: CommentsCommand
  notifications?: NotificationsCommand
  servers?: ServersCommand
  login?: LoginCommand
  users?: UsersCommand
  videoStudio?: VideoStudioCommand
  videos?: VideosCommand
  videoStats?: VideoStatsCommand
  views?: ViewsCommand
  twoFactor?: TwoFactorCommand
  videoToken?: VideoTokenCommand
  registrations?: RegistrationsCommand

  runners?: RunnersCommand
  runnerRegistrationTokens?: RunnerRegistrationTokensCommand
  runnerJobs?: RunnerJobsCommand

  constructor (options: { serverNumber: number } | { url: string }) {
    if ((options as any).url) {
      this.setUrl((options as any).url)
    } else {
      this.setServerNumber((options as any).serverNumber)
    }

    this.store = {
      client: {
        id: null,
        secret: null
      },
      user: {
        username: null,
        password: null
      }
    }

    this.assignCommands()
  }

  setServerNumber (serverNumber: number) {
    this.serverNumber = serverNumber

    this.parallel = parallelTests()

    this.internalServerNumber = this.parallel ? this.randomServer() : this.serverNumber
    this.rtmpPort = this.parallel ? this.randomRTMP() : 1936
    this.rtmpsPort = this.parallel ? this.randomRTMP() : 1937
    this.port = 9000 + this.internalServerNumber

    this.url = `http://127.0.0.1:${this.port}`
    this.host = `127.0.0.1:${this.port}`
    this.hostname = '127.0.0.1'
  }

  setUrl (url: string) {
    const parsed = new URL(url)

    this.url = url
    this.host = parsed.host
    this.hostname = parsed.hostname
    this.port = parseInt(parsed.port)
  }

  getDirectoryPath (directoryName: string) {
    const testDirectory = 'test' + this.internalServerNumber

    return join(root(), testDirectory, directoryName)
  }

  async flushAndRun (configOverride?: object, options: RunServerOptions = {}) {
    await ServersCommand.flushTests(this.internalServerNumber)

    return this.run(configOverride, options)
  }

  async run (configOverrideArg?: any, options: RunServerOptions = {}) {
    // These actions are async so we need to be sure that they have both been done
    const serverRunString = {
      'HTTP server listening': false
    }
    const key = 'Database peertube_test' + this.internalServerNumber + ' is ready'
    serverRunString[key] = false

    const regexps = {
      client_id: 'Client id: (.+)',
      client_secret: 'Client secret: (.+)',
      user_username: 'Username: (.+)',
      user_password: 'User password: (.+)'
    }

    await this.assignCustomConfigFile()

    const configOverride = this.buildConfigOverride()

    if (configOverrideArg !== undefined) {
      Object.assign(configOverride, configOverrideArg)
    }

    // Share the environment
    const env = Object.create(process.env)
    env['NODE_ENV'] = 'test'
    env['NODE_APP_INSTANCE'] = this.internalServerNumber.toString()
    env['NODE_CONFIG'] = JSON.stringify(configOverride)

    if (options.env) {
      Object.assign(env, options.env)
    }

    const execArgv = options.nodeArgs || []
    // FIXME: too slow :/
    // execArgv.push('--enable-source-maps')

    const forkOptions = {
      silent: true,
      env,
      detached: false,
      execArgv
    }

    const peertubeArgs = options.peertubeArgs || []

    return new Promise<void>((res, rej) => {
      const self = this
      let aggregatedLogs = ''

      this.app = fork(join(root(), 'dist', 'server.js'), peertubeArgs, forkOptions)

      const onPeerTubeExit = () => rej(new Error('Process exited:\n' + aggregatedLogs))
      const onParentExit = () => {
        if (!this.app?.pid) return

        try {
          process.kill(self.app.pid)
        } catch { /* empty */ }
      }

      this.app.on('exit', onPeerTubeExit)
      process.on('exit', onParentExit)

      this.app.stdout.on('data', function onStdout (data) {
        let dontContinue = false

        const log: string = data.toString()
        aggregatedLogs += log

        // Capture things if we want to
        for (const key of Object.keys(regexps)) {
          const regexp = regexps[key]
          const matches = log.match(regexp)
          if (matches !== null) {
            if (key === 'client_id') self.store.client.id = matches[1]
            else if (key === 'client_secret') self.store.client.secret = matches[1]
            else if (key === 'user_username') self.store.user.username = matches[1]
            else if (key === 'user_password') self.store.user.password = matches[1]
          }
        }

        // Check if all required sentences are here
        for (const key of Object.keys(serverRunString)) {
          if (log.includes(key)) serverRunString[key] = true
          if (serverRunString[key] === false) dontContinue = true
        }

        // If no, there is maybe one thing not already initialized (client/user credentials generation...)
        if (dontContinue === true) return

        if (options.hideLogs === false) {
          console.log(log)
        } else {
          process.removeListener('exit', onParentExit)
          self.app.stdout.removeListener('data', onStdout)
          self.app.removeListener('exit', onPeerTubeExit)
        }

        res()
      })
    })
  }

  kill () {
    if (!this.app) return Promise.resolve()

    process.kill(this.app.pid)

    this.app = null

    return Promise.resolve()
  }

  private randomServer () {
    const low = 2500
    const high = 10000

    return randomInt(low, high)
  }

  private randomRTMP () {
    const low = 1900
    const high = 2100

    return randomInt(low, high)
  }

  private async assignCustomConfigFile () {
    if (this.internalServerNumber === this.serverNumber) return

    const basePath = join(root(), 'config')

    const tmpConfigFile = join(basePath, `test-${this.internalServerNumber}.yaml`)
    await copy(join(basePath, `test-${this.serverNumber}.yaml`), tmpConfigFile)

    this.customConfigFile = tmpConfigFile
  }

  private buildConfigOverride () {
    if (!this.parallel) return {}

    return {
      listen: {
        port: this.port
      },
      webserver: {
        port: this.port
      },
      database: {
        suffix: '_test' + this.internalServerNumber
      },
      storage: {
        tmp: this.getDirectoryPath('tmp') + '/',
        tmp_persistent: this.getDirectoryPath('tmp-persistent') + '/',
        bin: this.getDirectoryPath('bin') + '/',
        avatars: this.getDirectoryPath('avatars') + '/',
        videos: this.getDirectoryPath('videos') + '/',
        streaming_playlists: this.getDirectoryPath('streaming-playlists') + '/',
        redundancy: this.getDirectoryPath('redundancy') + '/',
        logs: this.getDirectoryPath('logs') + '/',
        previews: this.getDirectoryPath('previews') + '/',
        thumbnails: this.getDirectoryPath('thumbnails') + '/',
        torrents: this.getDirectoryPath('torrents') + '/',
        captions: this.getDirectoryPath('captions') + '/',
        cache: this.getDirectoryPath('cache') + '/',
        plugins: this.getDirectoryPath('plugins') + '/',
        well_known: this.getDirectoryPath('well-known') + '/'
      },
      admin: {
        email: `admin${this.internalServerNumber}@example.com`
      },
      live: {
        rtmp: {
          port: this.rtmpPort
        }
      }
    }
  }

  private assignCommands () {
    this.bulk = new BulkCommand(this)
    this.cli = new CLICommand(this)
    this.customPage = new CustomPagesCommand(this)
    this.feed = new FeedCommand(this)
    this.logs = new LogsCommand(this)
    this.abuses = new AbusesCommand(this)
    this.overviews = new OverviewsCommand(this)
    this.search = new SearchCommand(this)
    this.contactForm = new ContactFormCommand(this)
    this.debug = new DebugCommand(this)
    this.follows = new FollowsCommand(this)
    this.jobs = new JobsCommand(this)
    this.metrics = new MetricsCommand(this)
    this.plugins = new PluginsCommand(this)
    this.redundancy = new RedundancyCommand(this)
    this.stats = new StatsCommand(this)
    this.config = new ConfigCommand(this)
    this.socketIO = new SocketIOCommand(this)
    this.accounts = new AccountsCommand(this)
    this.blocklist = new BlocklistCommand(this)
    this.subscriptions = new SubscriptionsCommand(this)
    this.live = new LiveCommand(this)
    this.services = new ServicesCommand(this)
    this.blacklist = new BlacklistCommand(this)
    this.captions = new CaptionsCommand(this)
    this.changeOwnership = new ChangeOwnershipCommand(this)
    this.playlists = new PlaylistsCommand(this)
    this.history = new HistoryCommand(this)
    this.imports = new ImportsCommand(this)
    this.channelSyncs = new ChannelSyncsCommand(this)
    this.streamingPlaylists = new StreamingPlaylistsCommand(this)
    this.channels = new ChannelsCommand(this)
    this.comments = new CommentsCommand(this)
    this.notifications = new NotificationsCommand(this)
    this.servers = new ServersCommand(this)
    this.login = new LoginCommand(this)
    this.users = new UsersCommand(this)
    this.videos = new VideosCommand(this)
    this.videoStudio = new VideoStudioCommand(this)
    this.videoStats = new VideoStatsCommand(this)
    this.views = new ViewsCommand(this)
    this.twoFactor = new TwoFactorCommand(this)
    this.videoToken = new VideoTokenCommand(this)
    this.registrations = new RegistrationsCommand(this)

    this.runners = new RunnersCommand(this)
    this.runnerRegistrationTokens = new RunnerRegistrationTokensCommand(this)
    this.runnerJobs = new RunnerJobsCommand(this)
  }
}