1
0
Fork 0

More robust webtorrent redundancy download

Avoid issues with inconsistencies between magnet infohash and torrent
infohash, blocking webtorrent upload that will timeout
This commit is contained in:
Chocobozzz 2021-09-06 09:29:25 +02:00
parent 70430c2796
commit 02b286f890
No known key found for this signature in database
GPG key ID: 583A612D890159BE
5 changed files with 8 additions and 12 deletions

View file

@ -74,8 +74,6 @@ export class VideoFilters {
} }
setNSFWPolicy (nsfwPolicy: NSFWPolicyType) { setNSFWPolicy (nsfwPolicy: NSFWPolicyType) {
console.log(nsfwPolicy)
this.updateDefaultNSFW(nsfwPolicy) this.updateDefaultNSFW(nsfwPolicy)
} }

View file

@ -21,8 +21,8 @@ import { extractVideo } from './video'
const createTorrentPromise = promisify2<string, any, any>(createTorrent) const createTorrentPromise = promisify2<string, any, any>(createTorrent)
async function downloadWebTorrentVideo (target: { magnetUri: string, torrentName?: string }, timeout: number) { async function downloadWebTorrentVideo (target: { uri: string, torrentName?: string }, timeout: number) {
const id = target.magnetUri || target.torrentName const id = target.uri || target.torrentName
let timer let timer
const path = generateVideoImportTmpPath(id) const path = generateVideoImportTmpPath(id)
@ -35,7 +35,7 @@ async function downloadWebTorrentVideo (target: { magnetUri: string, torrentName
const webtorrent = new WebTorrent() const webtorrent = new WebTorrent()
let file: TorrentFile let file: TorrentFile
const torrentId = target.magnetUri || join(CONFIG.STORAGE.TORRENTS_DIR, target.torrentName) const torrentId = target.uri || join(CONFIG.STORAGE.TORRENTS_DIR, target.torrentName)
const options = { path: directoryPath } const options = { path: directoryPath }
const torrent = webtorrent.add(torrentId, options, torrent => { const torrent = webtorrent.add(torrentId, options, torrent => {

View file

@ -63,7 +63,7 @@ async function processTorrentImport (job: Job, payload: VideoImportTorrentPayloa
} }
const target = { const target = {
torrentName: videoImport.torrentName ? getSecureTorrentName(videoImport.torrentName) : undefined, torrentName: videoImport.torrentName ? getSecureTorrentName(videoImport.torrentName) : undefined,
magnetUri: videoImport.magnetUri uri: videoImport.magnetUri
} }
return processFile(() => downloadWebTorrentVideo(target, VIDEO_IMPORT_TIMEOUT), videoImport, options) return processFile(() => downloadWebTorrentVideo(target, VIDEO_IMPORT_TIMEOUT), videoImport, options)
} }

View file

@ -1,7 +1,6 @@
import { move } from 'fs-extra' import { move } from 'fs-extra'
import { join } from 'path' import { join } from 'path'
import { getServerActor } from '@server/models/application/application' import { getServerActor } from '@server/models/application/application'
import { TrackerModel } from '@server/models/server/tracker'
import { VideoModel } from '@server/models/video/video' import { VideoModel } from '@server/models/video/video'
import { import {
MStreamingPlaylistFiles, MStreamingPlaylistFiles,
@ -15,7 +14,7 @@ import {
} from '@server/types/models' } from '@server/types/models'
import { VideosRedundancyStrategy } from '../../../shared/models/redundancy' import { VideosRedundancyStrategy } from '../../../shared/models/redundancy'
import { logger, loggerTagsFactory } from '../../helpers/logger' import { logger, loggerTagsFactory } from '../../helpers/logger'
import { downloadWebTorrentVideo, generateMagnetUri } from '../../helpers/webtorrent' import { downloadWebTorrentVideo } from '../../helpers/webtorrent'
import { CONFIG } from '../../initializers/config' import { CONFIG } from '../../initializers/config'
import { HLS_REDUNDANCY_DIRECTORY, REDUNDANCY, VIDEO_IMPORT_TIMEOUT } from '../../initializers/constants' import { HLS_REDUNDANCY_DIRECTORY, REDUNDANCY, VIDEO_IMPORT_TIMEOUT } from '../../initializers/constants'
import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy'
@ -232,10 +231,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, strategy, lTags(video.uuid)) logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, strategy, lTags(video.uuid))
const trackerUrls = await TrackerModel.listUrlsByVideoId(video.id) const tmpPath = await downloadWebTorrentVideo({ uri: file.torrentUrl }, VIDEO_IMPORT_TIMEOUT)
const magnetUri = generateMagnetUri(video, file, trackerUrls)
const tmpPath = await downloadWebTorrentVideo({ magnetUri }, VIDEO_IMPORT_TIMEOUT)
const destPath = join(CONFIG.STORAGE.REDUNDANCY_DIR, file.filename) const destPath = join(CONFIG.STORAGE.REDUNDANCY_DIR, file.filename)
await move(tmpPath, destPath, { overwrite: true }) await move(tmpPath, destPath, { overwrite: true })

View file

@ -194,6 +194,7 @@ export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel>
@Column @Column
metadataUrl: string metadataUrl: string
// Could be null for remote files
@AllowNull(true) @AllowNull(true)
@Column @Column
fileUrl: string fileUrl: string
@ -203,6 +204,7 @@ export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel>
@Column @Column
filename: string filename: string
// Could be null for remote files
@AllowNull(true) @AllowNull(true)
@Column @Column
torrentUrl: string torrentUrl: string