From 907b8f9347cca117c77a83cbea27140a10e3311e Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 12 Dec 2019 10:54:01 +0100 Subject: [PATCH] Use a migration script to generate HLS torrent files --- scripts/migrations/peertube-2.1.ts | 77 +++++++++++++++++++ .../0450-streaming-playlist-files.ts | 55 ------------- server/models/video/video-file.ts | 2 +- 3 files changed, 78 insertions(+), 56 deletions(-) create mode 100644 scripts/migrations/peertube-2.1.ts diff --git a/scripts/migrations/peertube-2.1.ts b/scripts/migrations/peertube-2.1.ts new file mode 100644 index 000000000..892497a88 --- /dev/null +++ b/scripts/migrations/peertube-2.1.ts @@ -0,0 +1,77 @@ +import { registerTSPaths } from '../../server/helpers/register-ts-paths' +registerTSPaths() + +import { initDatabaseModels, sequelizeTypescript } from '../../server/initializers' +import * as Sequelize from 'sequelize' +import { join } from 'path' +import { HLS_STREAMING_PLAYLIST_DIRECTORY, STATIC_PATHS, WEBSERVER } from '@server/initializers/constants' +import { pathExists, stat, writeFile } from 'fs-extra' +import { createTorrentPromise } from '@server/helpers/webtorrent' +import { CONFIG } from '@server/initializers/config' +import * as parseTorrent from 'parse-torrent' +import { logger } from '@server/helpers/logger' + +run() + .then(() => process.exit(0)) + .catch(err => { + console.error(err) + process.exit(-1) + }) + +async function run () { + logger.info('Creating torrents and updating database for HSL files.') + + await initDatabaseModels(true) + + const query = 'select "videoFile".id as id, "videoFile".resolution as resolution, "video".uuid as uuid from "videoFile" ' + + 'inner join "videoStreamingPlaylist" ON "videoStreamingPlaylist".id = "videoFile"."videoStreamingPlaylistId" ' + + 'inner join video ON video.id = "videoStreamingPlaylist"."videoId" ' + + 'WHERE video.remote IS FALSE' + const options = { + type: Sequelize.QueryTypes.SELECT + } + const res = await sequelizeTypescript.query(query, options) + + for (const row of res) { + const videoFilename = `${row['uuid']}-${row['resolution']}-fragmented.mp4` + const videoFilePath = join(HLS_STREAMING_PLAYLIST_DIRECTORY, row['uuid'], videoFilename) + + logger.info('Processing %s.', videoFilePath) + + if (!await pathExists(videoFilePath)) { + console.warn('Cannot generate torrent of %s: file does not exist.', videoFilePath) + continue + } + + const createTorrentOptions = { + // Keep the extname, it's used by the client to stream the file inside a web browser + name: `video ${row['uuid']}`, + createdBy: 'PeerTube', + announceList: [ + [ WEBSERVER.WS + '://' + WEBSERVER.HOSTNAME + ':' + WEBSERVER.PORT + '/tracker/socket' ], + [ WEBSERVER.URL + '/tracker/announce' ] + ], + urlList: [ WEBSERVER.URL + join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, row['uuid'], videoFilename) ] + } + const torrent = await createTorrentPromise(videoFilePath, createTorrentOptions) + + const torrentName = `${row['uuid']}-${row['resolution']}-hls.torrent` + const filePath = join(CONFIG.STORAGE.TORRENTS_DIR, torrentName) + + await writeFile(filePath, torrent) + + const parsedTorrent = parseTorrent(torrent) + const infoHash = parsedTorrent.infoHash + + const stats = await stat(videoFilePath) + const size = stats.size + + const queryUpdate = 'UPDATE "videoFile" SET "infoHash" = ?, "size" = ? WHERE id = ?' + + const options = { + type: Sequelize.QueryTypes.UPDATE, + replacements: [ infoHash, size, row['id'] ] + } + await sequelizeTypescript.query(queryUpdate, options) + } +} diff --git a/server/initializers/migrations/0450-streaming-playlist-files.ts b/server/initializers/migrations/0450-streaming-playlist-files.ts index 4e177bef8..460dac8be 100644 --- a/server/initializers/migrations/0450-streaming-playlist-files.ts +++ b/server/initializers/migrations/0450-streaming-playlist-files.ts @@ -48,61 +48,6 @@ async function up (utils: { await utils.sequelize.query(query, { transaction: utils.transaction }) } - - { - const query = 'select "videoFile".id as id, "videoFile".resolution as resolution, "video".uuid as uuid from "videoFile" ' + - 'inner join "videoStreamingPlaylist" ON "videoStreamingPlaylist".id = "videoFile"."videoStreamingPlaylistId" ' + - 'inner join video ON video.id = "videoStreamingPlaylist"."videoId" ' + - 'WHERE video.remote IS FALSE' - const options = { - type: Sequelize.QueryTypes.SELECT, - transaction: utils.transaction - } - const res = await utils.sequelize.query(query, options) - - for (const row of res) { - const videoFilename = `${row['uuid']}-${row['resolution']}-fragmented.mp4` - const videoFilePath = join(HLS_STREAMING_PLAYLIST_DIRECTORY, row['uuid'], videoFilename) - - if (!await pathExists(videoFilePath)) { - console.warn('Cannot generate torrent of %s: file does not exist.', videoFilePath) - continue - } - - const createTorrentOptions = { - // Keep the extname, it's used by the client to stream the file inside a web browser - name: `video ${row['uuid']}`, - createdBy: 'PeerTube', - announceList: [ - [ WEBSERVER.WS + '://' + WEBSERVER.HOSTNAME + ':' + WEBSERVER.PORT + '/tracker/socket' ], - [ WEBSERVER.URL + '/tracker/announce' ] - ], - urlList: [ WEBSERVER.URL + join(HLS_STREAMING_PLAYLIST_DIRECTORY, row['uuid'], videoFilename) ] - } - const torrent = await createTorrentPromise(videoFilePath, createTorrentOptions) - - const torrentName = `${row['uuid']}-${row['resolution']}-hls.torrent` - const filePath = join(CONFIG.STORAGE.TORRENTS_DIR, torrentName) - - await writeFile(filePath, torrent) - - const parsedTorrent = parseTorrent(torrent) - const infoHash = parsedTorrent.infoHash - - const stats = await stat(videoFilePath) - const size = stats.size - - const queryUpdate = 'UPDATE "videoFile" SET "infoHash" = ?, "size" = ? WHERE id = ?' - - const options = { - type: Sequelize.QueryTypes.UPDATE, - replacements: [ infoHash, size, row['id'] ], - transaction: utils.transaction - } - await utils.sequelize.query(queryUpdate, options) - - } - } } function down (options) { diff --git a/server/models/video/video-file.ts b/server/models/video/video-file.ts index cacef0106..fa5a6e76d 100644 --- a/server/models/video/video-file.ts +++ b/server/models/video/video-file.ts @@ -26,7 +26,7 @@ import { VideoStreamingPlaylistModel } from './video-streaming-playlist' import { FindOptions, Op, QueryTypes, Transaction } from 'sequelize' import { MIMETYPES } from '../../initializers/constants' import { MVideoFile, MVideoFileStreamingPlaylistVideo, MVideoFileVideo } from '../../typings/models/video/video-file' -import { MStreamingPlaylist, MStreamingPlaylistVideo, MVideo } from '@server/typings/models' +import { MStreamingPlaylistVideo, MVideo } from '@server/typings/models' @Table({ tableName: 'videoFile',