2018-09-11 14:27:07 +00:00
|
|
|
import {
|
|
|
|
AllowNull,
|
2018-10-03 14:43:57 +00:00
|
|
|
BeforeDestroy,
|
2018-09-11 14:27:07 +00:00
|
|
|
BelongsTo,
|
|
|
|
Column,
|
|
|
|
CreatedAt,
|
|
|
|
DataType,
|
|
|
|
ForeignKey,
|
|
|
|
Is,
|
|
|
|
Model,
|
|
|
|
Scopes,
|
|
|
|
Table,
|
|
|
|
UpdatedAt
|
|
|
|
} from 'sequelize-typescript'
|
|
|
|
import { ActorModel } from '../activitypub/actor'
|
2020-01-10 09:11:28 +00:00
|
|
|
import { getSort, getVideoSort, parseAggregateResult, throwIfNotValid } from '../utils'
|
2018-09-11 14:27:07 +00:00
|
|
|
import { isActivityPubUrlValid, isUrlValid } from '../../helpers/custom-validators/activitypub/misc'
|
2019-04-11 12:26:41 +00:00
|
|
|
import { CONSTRAINTS_FIELDS, MIMETYPES } from '../../initializers/constants'
|
2018-09-11 14:27:07 +00:00
|
|
|
import { VideoFileModel } from '../video/video-file'
|
|
|
|
import { getServerActor } from '../../helpers/utils'
|
|
|
|
import { VideoModel } from '../video/video'
|
2020-01-10 09:11:28 +00:00
|
|
|
import { VideoRedundancyStrategy, VideoRedundancyStrategyWithManual } from '../../../shared/models/redundancy'
|
2018-09-11 14:27:07 +00:00
|
|
|
import { logger } from '../../helpers/logger'
|
2018-09-25 16:05:54 +00:00
|
|
|
import { CacheFileObject, VideoPrivacy } from '../../../shared'
|
2018-09-11 14:27:07 +00:00
|
|
|
import { VideoChannelModel } from '../video/video-channel'
|
|
|
|
import { ServerModel } from '../server/server'
|
|
|
|
import { sample } from 'lodash'
|
|
|
|
import { isTestInstance } from '../../helpers/core-utils'
|
2018-09-14 09:05:38 +00:00
|
|
|
import * as Bluebird from 'bluebird'
|
2020-01-10 09:11:28 +00:00
|
|
|
import { col, FindOptions, fn, literal, Op, Transaction, WhereOptions } from 'sequelize'
|
2019-01-29 07:37:25 +00:00
|
|
|
import { VideoStreamingPlaylistModel } from '../video/video-streaming-playlist'
|
2019-04-11 09:33:44 +00:00
|
|
|
import { CONFIG } from '../../initializers/config'
|
2020-01-10 09:11:28 +00:00
|
|
|
import { MVideoForRedundancyAPI, MVideoRedundancy, MVideoRedundancyAP, MVideoRedundancyVideo } from '@server/typings/models'
|
|
|
|
import { VideoRedundanciesTarget } from '@shared/models/redundancy/video-redundancies-filters.model'
|
|
|
|
import {
|
|
|
|
FileRedundancyInformation,
|
|
|
|
StreamingPlaylistRedundancyInformation,
|
|
|
|
VideoRedundancy
|
|
|
|
} from '@shared/models/redundancy/video-redundancy.model'
|
2018-09-11 14:27:07 +00:00
|
|
|
|
|
|
|
export enum ScopeNames {
|
|
|
|
WITH_VIDEO = 'WITH_VIDEO'
|
|
|
|
}
|
|
|
|
|
2019-04-23 07:50:57 +00:00
|
|
|
@Scopes(() => ({
|
2020-01-31 15:56:52 +00:00
|
|
|
[ScopeNames.WITH_VIDEO]: {
|
2018-09-11 14:27:07 +00:00
|
|
|
include: [
|
|
|
|
{
|
2019-04-23 07:50:57 +00:00
|
|
|
model: VideoFileModel,
|
2019-01-29 07:37:25 +00:00
|
|
|
required: false,
|
|
|
|
include: [
|
|
|
|
{
|
2019-04-23 07:50:57 +00:00
|
|
|
model: VideoModel,
|
2019-01-29 07:37:25 +00:00
|
|
|
required: true
|
|
|
|
}
|
|
|
|
]
|
|
|
|
},
|
|
|
|
{
|
2019-04-23 07:50:57 +00:00
|
|
|
model: VideoStreamingPlaylistModel,
|
2019-01-29 07:37:25 +00:00
|
|
|
required: false,
|
2018-09-11 14:27:07 +00:00
|
|
|
include: [
|
|
|
|
{
|
2019-04-23 07:50:57 +00:00
|
|
|
model: VideoModel,
|
2018-09-11 14:27:07 +00:00
|
|
|
required: true
|
|
|
|
}
|
|
|
|
]
|
|
|
|
}
|
2019-04-23 07:50:57 +00:00
|
|
|
]
|
2018-09-11 14:27:07 +00:00
|
|
|
}
|
2019-04-23 07:50:57 +00:00
|
|
|
}))
|
2018-09-11 14:27:07 +00:00
|
|
|
|
|
|
|
@Table({
|
|
|
|
tableName: 'videoRedundancy',
|
|
|
|
indexes: [
|
|
|
|
{
|
|
|
|
fields: [ 'videoFileId' ]
|
|
|
|
},
|
|
|
|
{
|
|
|
|
fields: [ 'actorId' ]
|
|
|
|
},
|
|
|
|
{
|
|
|
|
fields: [ 'url' ],
|
|
|
|
unique: true
|
|
|
|
}
|
|
|
|
]
|
|
|
|
})
|
|
|
|
export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
|
|
|
|
|
|
|
|
@CreatedAt
|
|
|
|
createdAt: Date
|
|
|
|
|
|
|
|
@UpdatedAt
|
|
|
|
updatedAt: Date
|
|
|
|
|
2020-01-10 09:11:28 +00:00
|
|
|
@AllowNull(true)
|
2018-09-11 14:27:07 +00:00
|
|
|
@Column
|
|
|
|
expiresOn: Date
|
|
|
|
|
|
|
|
@AllowNull(false)
|
|
|
|
@Is('VideoRedundancyFileUrl', value => throwIfNotValid(value, isUrlValid, 'fileUrl'))
|
|
|
|
@Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS_REDUNDANCY.URL.max))
|
|
|
|
fileUrl: string
|
|
|
|
|
|
|
|
@AllowNull(false)
|
|
|
|
@Is('VideoRedundancyUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'url'))
|
|
|
|
@Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS_REDUNDANCY.URL.max))
|
|
|
|
url: string
|
|
|
|
|
|
|
|
@AllowNull(true)
|
|
|
|
@Column
|
|
|
|
strategy: string // Only used by us
|
|
|
|
|
|
|
|
@ForeignKey(() => VideoFileModel)
|
|
|
|
@Column
|
|
|
|
videoFileId: number
|
|
|
|
|
|
|
|
@BelongsTo(() => VideoFileModel, {
|
|
|
|
foreignKey: {
|
2019-01-29 07:37:25 +00:00
|
|
|
allowNull: true
|
2018-09-11 14:27:07 +00:00
|
|
|
},
|
|
|
|
onDelete: 'cascade'
|
|
|
|
})
|
|
|
|
VideoFile: VideoFileModel
|
|
|
|
|
2019-01-29 07:37:25 +00:00
|
|
|
@ForeignKey(() => VideoStreamingPlaylistModel)
|
|
|
|
@Column
|
|
|
|
videoStreamingPlaylistId: number
|
|
|
|
|
|
|
|
@BelongsTo(() => VideoStreamingPlaylistModel, {
|
|
|
|
foreignKey: {
|
|
|
|
allowNull: true
|
|
|
|
},
|
|
|
|
onDelete: 'cascade'
|
|
|
|
})
|
|
|
|
VideoStreamingPlaylist: VideoStreamingPlaylistModel
|
|
|
|
|
2018-09-11 14:27:07 +00:00
|
|
|
@ForeignKey(() => ActorModel)
|
|
|
|
@Column
|
|
|
|
actorId: number
|
|
|
|
|
|
|
|
@BelongsTo(() => ActorModel, {
|
|
|
|
foreignKey: {
|
|
|
|
allowNull: false
|
|
|
|
},
|
|
|
|
onDelete: 'cascade'
|
|
|
|
})
|
|
|
|
Actor: ActorModel
|
|
|
|
|
2018-10-03 14:43:57 +00:00
|
|
|
@BeforeDestroy
|
|
|
|
static async removeFile (instance: VideoRedundancyModel) {
|
2018-11-16 10:18:13 +00:00
|
|
|
if (!instance.isOwned()) return
|
2018-09-11 14:27:07 +00:00
|
|
|
|
2019-01-29 07:37:25 +00:00
|
|
|
if (instance.videoFileId) {
|
|
|
|
const videoFile = await VideoFileModel.loadWithVideo(instance.videoFileId)
|
2018-09-11 14:27:07 +00:00
|
|
|
|
2019-01-29 07:37:25 +00:00
|
|
|
const logIdentifier = `${videoFile.Video.uuid}-${videoFile.resolution}`
|
|
|
|
logger.info('Removing duplicated video file %s.', logIdentifier)
|
2018-10-03 14:43:57 +00:00
|
|
|
|
2019-01-29 07:37:25 +00:00
|
|
|
videoFile.Video.removeFile(videoFile, true)
|
|
|
|
.catch(err => logger.error('Cannot delete %s files.', logIdentifier, { err }))
|
|
|
|
}
|
|
|
|
|
|
|
|
if (instance.videoStreamingPlaylistId) {
|
|
|
|
const videoStreamingPlaylist = await VideoStreamingPlaylistModel.loadWithVideo(instance.videoStreamingPlaylistId)
|
|
|
|
|
|
|
|
const videoUUID = videoStreamingPlaylist.Video.uuid
|
|
|
|
logger.info('Removing duplicated video streaming playlist %s.', videoUUID)
|
|
|
|
|
2020-01-24 15:48:05 +00:00
|
|
|
videoStreamingPlaylist.Video.removeStreamingPlaylistFiles(videoStreamingPlaylist, true)
|
2020-01-31 15:56:52 +00:00
|
|
|
.catch(err => logger.error('Cannot delete video streaming playlist files of %s.', videoUUID, { err }))
|
2019-01-29 07:37:25 +00:00
|
|
|
}
|
2018-10-03 14:57:40 +00:00
|
|
|
|
|
|
|
return undefined
|
2018-09-11 14:27:07 +00:00
|
|
|
}
|
|
|
|
|
2019-08-15 09:53:26 +00:00
|
|
|
static async loadLocalByFileId (videoFileId: number): Promise<MVideoRedundancyVideo> {
|
2018-10-01 07:41:48 +00:00
|
|
|
const actor = await getServerActor()
|
|
|
|
|
2018-09-11 14:27:07 +00:00
|
|
|
const query = {
|
|
|
|
where: {
|
2018-10-01 07:41:48 +00:00
|
|
|
actorId: actor.id,
|
2018-09-11 14:27:07 +00:00
|
|
|
videoFileId
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
|
|
|
|
}
|
|
|
|
|
2019-08-15 09:53:26 +00:00
|
|
|
static async loadLocalByStreamingPlaylistId (videoStreamingPlaylistId: number): Promise<MVideoRedundancyVideo> {
|
2019-01-29 07:37:25 +00:00
|
|
|
const actor = await getServerActor()
|
|
|
|
|
|
|
|
const query = {
|
|
|
|
where: {
|
|
|
|
actorId: actor.id,
|
|
|
|
videoStreamingPlaylistId
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
|
|
|
|
}
|
|
|
|
|
2020-01-10 09:11:28 +00:00
|
|
|
static loadByIdWithVideo (id: number, transaction?: Transaction): Bluebird<MVideoRedundancyVideo> {
|
|
|
|
const query = {
|
|
|
|
where: { id },
|
|
|
|
transaction
|
|
|
|
}
|
|
|
|
|
|
|
|
return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
|
|
|
|
}
|
|
|
|
|
2019-08-15 09:53:26 +00:00
|
|
|
static loadByUrl (url: string, transaction?: Transaction): Bluebird<MVideoRedundancy> {
|
2018-09-11 14:27:07 +00:00
|
|
|
const query = {
|
|
|
|
where: {
|
|
|
|
url
|
2018-09-24 11:07:33 +00:00
|
|
|
},
|
|
|
|
transaction
|
2018-09-11 14:27:07 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return VideoRedundancyModel.findOne(query)
|
|
|
|
}
|
|
|
|
|
2018-09-28 08:56:13 +00:00
|
|
|
static async isLocalByVideoUUIDExists (uuid: string) {
|
|
|
|
const actor = await getServerActor()
|
|
|
|
|
|
|
|
const query = {
|
|
|
|
raw: true,
|
|
|
|
attributes: [ 'id' ],
|
|
|
|
where: {
|
|
|
|
actorId: actor.id
|
|
|
|
},
|
|
|
|
include: [
|
|
|
|
{
|
2020-01-31 15:56:52 +00:00
|
|
|
attributes: [],
|
2018-09-28 08:56:13 +00:00
|
|
|
model: VideoFileModel,
|
|
|
|
required: true,
|
|
|
|
include: [
|
|
|
|
{
|
2020-01-31 15:56:52 +00:00
|
|
|
attributes: [],
|
2018-09-28 08:56:13 +00:00
|
|
|
model: VideoModel,
|
|
|
|
required: true,
|
|
|
|
where: {
|
|
|
|
uuid
|
|
|
|
}
|
|
|
|
}
|
|
|
|
]
|
|
|
|
}
|
|
|
|
]
|
|
|
|
}
|
|
|
|
|
|
|
|
return VideoRedundancyModel.findOne(query)
|
2020-01-31 15:56:52 +00:00
|
|
|
.then(r => !!r)
|
2018-09-28 08:56:13 +00:00
|
|
|
}
|
|
|
|
|
2018-09-14 09:05:38 +00:00
|
|
|
static async getVideoSample (p: Bluebird<VideoModel[]>) {
|
|
|
|
const rows = await p
|
2019-08-13 08:22:54 +00:00
|
|
|
if (rows.length === 0) return undefined
|
|
|
|
|
2018-09-14 07:57:21 +00:00
|
|
|
const ids = rows.map(r => r.id)
|
|
|
|
const id = sample(ids)
|
|
|
|
|
2019-01-29 07:37:25 +00:00
|
|
|
return VideoModel.loadWithFiles(id, undefined, !isTestInstance())
|
2018-09-14 07:57:21 +00:00
|
|
|
}
|
|
|
|
|
2018-09-11 14:27:07 +00:00
|
|
|
static async findMostViewToDuplicate (randomizedFactor: number) {
|
|
|
|
// On VideoModel!
|
|
|
|
const query = {
|
2018-09-14 07:57:21 +00:00
|
|
|
attributes: [ 'id', 'views' ],
|
2018-09-11 14:27:07 +00:00
|
|
|
limit: randomizedFactor,
|
2018-09-14 07:57:21 +00:00
|
|
|
order: getVideoSort('-views'),
|
2018-09-25 16:05:54 +00:00
|
|
|
where: {
|
|
|
|
privacy: VideoPrivacy.PUBLIC
|
|
|
|
},
|
2018-09-11 14:27:07 +00:00
|
|
|
include: [
|
2018-09-14 07:57:21 +00:00
|
|
|
await VideoRedundancyModel.buildVideoFileForDuplication(),
|
|
|
|
VideoRedundancyModel.buildServerRedundancyInclude()
|
|
|
|
]
|
|
|
|
}
|
|
|
|
|
2018-09-14 09:05:38 +00:00
|
|
|
return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
|
2018-09-14 07:57:21 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static async findTrendingToDuplicate (randomizedFactor: number) {
|
|
|
|
// On VideoModel!
|
|
|
|
const query = {
|
|
|
|
attributes: [ 'id', 'views' ],
|
|
|
|
subQuery: false,
|
|
|
|
group: 'VideoModel.id',
|
|
|
|
limit: randomizedFactor,
|
|
|
|
order: getVideoSort('-trending'),
|
2018-09-25 16:05:54 +00:00
|
|
|
where: {
|
|
|
|
privacy: VideoPrivacy.PUBLIC
|
|
|
|
},
|
2018-09-14 07:57:21 +00:00
|
|
|
include: [
|
|
|
|
await VideoRedundancyModel.buildVideoFileForDuplication(),
|
|
|
|
VideoRedundancyModel.buildServerRedundancyInclude(),
|
|
|
|
|
|
|
|
VideoModel.buildTrendingQuery(CONFIG.TRENDING.VIDEOS.INTERVAL_DAYS)
|
2018-09-11 14:27:07 +00:00
|
|
|
]
|
|
|
|
}
|
|
|
|
|
2018-09-14 09:05:38 +00:00
|
|
|
return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
|
|
|
|
}
|
|
|
|
|
|
|
|
static async findRecentlyAddedToDuplicate (randomizedFactor: number, minViews: number) {
|
|
|
|
// On VideoModel!
|
|
|
|
const query = {
|
|
|
|
attributes: [ 'id', 'publishedAt' ],
|
|
|
|
limit: randomizedFactor,
|
|
|
|
order: getVideoSort('-publishedAt'),
|
|
|
|
where: {
|
2018-09-25 16:05:54 +00:00
|
|
|
privacy: VideoPrivacy.PUBLIC,
|
2018-09-14 09:05:38 +00:00
|
|
|
views: {
|
2020-01-31 15:56:52 +00:00
|
|
|
[Op.gte]: minViews
|
2018-09-14 09:05:38 +00:00
|
|
|
}
|
|
|
|
},
|
|
|
|
include: [
|
|
|
|
await VideoRedundancyModel.buildVideoFileForDuplication(),
|
|
|
|
VideoRedundancyModel.buildServerRedundancyInclude()
|
|
|
|
]
|
|
|
|
}
|
2018-09-11 14:27:07 +00:00
|
|
|
|
2018-09-14 09:05:38 +00:00
|
|
|
return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
|
2018-09-11 14:27:07 +00:00
|
|
|
}
|
|
|
|
|
2019-08-15 09:53:26 +00:00
|
|
|
static async loadOldestLocalExpired (strategy: VideoRedundancyStrategy, expiresAfterMs: number): Promise<MVideoRedundancyVideo> {
|
2018-09-24 11:07:33 +00:00
|
|
|
const expiredDate = new Date()
|
|
|
|
expiredDate.setMilliseconds(expiredDate.getMilliseconds() - expiresAfterMs)
|
|
|
|
|
|
|
|
const actor = await getServerActor()
|
|
|
|
|
|
|
|
const query = {
|
|
|
|
where: {
|
|
|
|
actorId: actor.id,
|
|
|
|
strategy,
|
|
|
|
createdAt: {
|
2020-01-31 15:56:52 +00:00
|
|
|
[Op.lt]: expiredDate
|
2018-09-24 11:07:33 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findOne(query)
|
|
|
|
}
|
|
|
|
|
2018-09-14 09:05:38 +00:00
|
|
|
static async getTotalDuplicated (strategy: VideoRedundancyStrategy) {
|
2018-09-11 14:27:07 +00:00
|
|
|
const actor = await getServerActor()
|
2019-08-12 06:46:46 +00:00
|
|
|
const redundancyInclude = {
|
|
|
|
attributes: [],
|
|
|
|
model: VideoRedundancyModel,
|
|
|
|
required: true,
|
|
|
|
where: {
|
|
|
|
actorId: actor.id,
|
|
|
|
strategy
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
const queryFiles: FindOptions = {
|
|
|
|
include: [ redundancyInclude ]
|
|
|
|
}
|
2018-09-11 14:27:07 +00:00
|
|
|
|
2019-08-12 06:46:46 +00:00
|
|
|
const queryStreamingPlaylists: FindOptions = {
|
2018-09-14 09:05:38 +00:00
|
|
|
include: [
|
|
|
|
{
|
|
|
|
attributes: [],
|
2019-08-12 06:46:46 +00:00
|
|
|
model: VideoModel.unscoped(),
|
2018-09-14 09:05:38 +00:00
|
|
|
required: true,
|
2019-08-12 06:46:46 +00:00
|
|
|
include: [
|
|
|
|
{
|
2019-08-13 08:22:54 +00:00
|
|
|
required: true,
|
2019-08-12 06:46:46 +00:00
|
|
|
attributes: [],
|
|
|
|
model: VideoStreamingPlaylistModel.unscoped(),
|
|
|
|
include: [
|
|
|
|
redundancyInclude
|
|
|
|
]
|
|
|
|
}
|
|
|
|
]
|
2018-09-14 09:05:38 +00:00
|
|
|
}
|
|
|
|
]
|
2018-09-11 14:27:07 +00:00
|
|
|
}
|
|
|
|
|
2019-08-12 06:46:46 +00:00
|
|
|
return Promise.all([
|
|
|
|
VideoFileModel.aggregate('size', 'SUM', queryFiles),
|
|
|
|
VideoFileModel.aggregate('size', 'SUM', queryStreamingPlaylists)
|
|
|
|
]).then(([ r1, r2 ]) => {
|
|
|
|
return parseAggregateResult(r1) + parseAggregateResult(r2)
|
|
|
|
})
|
2018-09-11 14:27:07 +00:00
|
|
|
}
|
|
|
|
|
2018-09-24 11:07:33 +00:00
|
|
|
static async listLocalExpired () {
|
|
|
|
const actor = await getServerActor()
|
|
|
|
|
|
|
|
const query = {
|
|
|
|
where: {
|
|
|
|
actorId: actor.id,
|
|
|
|
expiresOn: {
|
2020-01-31 15:56:52 +00:00
|
|
|
[Op.lt]: new Date()
|
2018-09-24 11:07:33 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findAll(query)
|
|
|
|
}
|
|
|
|
|
|
|
|
static async listRemoteExpired () {
|
|
|
|
const actor = await getServerActor()
|
|
|
|
|
2018-09-11 14:27:07 +00:00
|
|
|
const query = {
|
|
|
|
where: {
|
2018-09-24 11:07:33 +00:00
|
|
|
actorId: {
|
2019-04-23 07:50:57 +00:00
|
|
|
[Op.ne]: actor.id
|
2018-09-24 11:07:33 +00:00
|
|
|
},
|
2018-09-11 14:27:07 +00:00
|
|
|
expiresOn: {
|
2020-01-31 15:56:52 +00:00
|
|
|
[Op.lt]: new Date(),
|
|
|
|
[Op.ne]: null
|
2018-09-11 14:27:07 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-09-24 11:07:33 +00:00
|
|
|
return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findAll(query)
|
2018-09-11 14:27:07 +00:00
|
|
|
}
|
|
|
|
|
2018-09-28 08:07:05 +00:00
|
|
|
static async listLocalOfServer (serverId: number) {
|
|
|
|
const actor = await getServerActor()
|
2019-01-29 07:37:25 +00:00
|
|
|
const buildVideoInclude = () => ({
|
|
|
|
model: VideoModel,
|
|
|
|
required: true,
|
2018-09-28 08:07:05 +00:00
|
|
|
include: [
|
|
|
|
{
|
2019-01-29 07:37:25 +00:00
|
|
|
attributes: [],
|
|
|
|
model: VideoChannelModel.unscoped(),
|
2018-09-28 08:07:05 +00:00
|
|
|
required: true,
|
|
|
|
include: [
|
|
|
|
{
|
2019-01-29 07:37:25 +00:00
|
|
|
attributes: [],
|
|
|
|
model: ActorModel.unscoped(),
|
2018-09-28 08:07:05 +00:00
|
|
|
required: true,
|
2019-01-29 07:37:25 +00:00
|
|
|
where: {
|
|
|
|
serverId
|
|
|
|
}
|
2018-09-28 08:07:05 +00:00
|
|
|
}
|
|
|
|
]
|
|
|
|
}
|
|
|
|
]
|
2019-01-29 07:37:25 +00:00
|
|
|
})
|
|
|
|
|
|
|
|
const query = {
|
|
|
|
where: {
|
|
|
|
actorId: actor.id
|
|
|
|
},
|
|
|
|
include: [
|
|
|
|
{
|
|
|
|
model: VideoFileModel,
|
|
|
|
required: false,
|
|
|
|
include: [ buildVideoInclude() ]
|
|
|
|
},
|
|
|
|
{
|
|
|
|
model: VideoStreamingPlaylistModel,
|
|
|
|
required: false,
|
|
|
|
include: [ buildVideoInclude() ]
|
|
|
|
}
|
|
|
|
]
|
2018-09-28 08:07:05 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return VideoRedundancyModel.findAll(query)
|
|
|
|
}
|
|
|
|
|
2020-01-10 09:11:28 +00:00
|
|
|
static listForApi (options: {
|
2020-01-31 15:56:52 +00:00
|
|
|
start: number
|
|
|
|
count: number
|
|
|
|
sort: string
|
|
|
|
target: VideoRedundanciesTarget
|
2020-01-10 09:11:28 +00:00
|
|
|
strategy?: string
|
|
|
|
}) {
|
|
|
|
const { start, count, sort, target, strategy } = options
|
2020-01-31 15:56:52 +00:00
|
|
|
const redundancyWhere: WhereOptions = {}
|
|
|
|
const videosWhere: WhereOptions = {}
|
2020-01-10 09:11:28 +00:00
|
|
|
let redundancySqlSuffix = ''
|
|
|
|
|
|
|
|
if (target === 'my-videos') {
|
|
|
|
Object.assign(videosWhere, { remote: false })
|
|
|
|
} else if (target === 'remote-videos') {
|
|
|
|
Object.assign(videosWhere, { remote: true })
|
|
|
|
Object.assign(redundancyWhere, { strategy: { [Op.ne]: null } })
|
|
|
|
redundancySqlSuffix = ' AND "videoRedundancy"."strategy" IS NOT NULL'
|
|
|
|
}
|
|
|
|
|
|
|
|
if (strategy) {
|
|
|
|
Object.assign(redundancyWhere, { strategy: strategy })
|
|
|
|
}
|
|
|
|
|
|
|
|
const videoFilterWhere = {
|
|
|
|
[Op.and]: [
|
|
|
|
{
|
2020-01-31 15:56:52 +00:00
|
|
|
[Op.or]: [
|
2020-01-10 09:11:28 +00:00
|
|
|
{
|
|
|
|
id: {
|
2020-01-31 15:56:52 +00:00
|
|
|
[Op.in]: literal(
|
2020-01-10 09:11:28 +00:00
|
|
|
'(' +
|
|
|
|
'SELECT "videoId" FROM "videoFile" ' +
|
|
|
|
'INNER JOIN "videoRedundancy" ON "videoRedundancy"."videoFileId" = "videoFile".id' +
|
|
|
|
redundancySqlSuffix +
|
|
|
|
')'
|
|
|
|
)
|
|
|
|
}
|
|
|
|
},
|
|
|
|
{
|
|
|
|
id: {
|
2020-01-31 15:56:52 +00:00
|
|
|
[Op.in]: literal(
|
2020-01-10 09:11:28 +00:00
|
|
|
'(' +
|
|
|
|
'select "videoId" FROM "videoStreamingPlaylist" ' +
|
|
|
|
'INNER JOIN "videoRedundancy" ON "videoRedundancy"."videoStreamingPlaylistId" = "videoStreamingPlaylist".id' +
|
|
|
|
redundancySqlSuffix +
|
|
|
|
')'
|
|
|
|
)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
]
|
|
|
|
},
|
|
|
|
|
|
|
|
videosWhere
|
|
|
|
]
|
|
|
|
}
|
|
|
|
|
|
|
|
// /!\ On video model /!\
|
|
|
|
const findOptions = {
|
|
|
|
offset: start,
|
|
|
|
limit: count,
|
|
|
|
order: getSort(sort),
|
|
|
|
include: [
|
|
|
|
{
|
|
|
|
required: false,
|
2020-03-10 13:39:40 +00:00
|
|
|
model: VideoFileModel,
|
2020-01-10 09:11:28 +00:00
|
|
|
include: [
|
|
|
|
{
|
|
|
|
model: VideoRedundancyModel.unscoped(),
|
|
|
|
required: false,
|
|
|
|
where: redundancyWhere
|
|
|
|
}
|
|
|
|
]
|
|
|
|
},
|
|
|
|
{
|
|
|
|
required: false,
|
|
|
|
model: VideoStreamingPlaylistModel.unscoped(),
|
|
|
|
include: [
|
|
|
|
{
|
|
|
|
model: VideoRedundancyModel.unscoped(),
|
|
|
|
required: false,
|
|
|
|
where: redundancyWhere
|
|
|
|
},
|
|
|
|
{
|
2020-03-10 13:39:40 +00:00
|
|
|
model: VideoFileModel,
|
2020-01-10 09:11:28 +00:00
|
|
|
required: false
|
|
|
|
}
|
|
|
|
]
|
|
|
|
}
|
|
|
|
],
|
|
|
|
where: videoFilterWhere
|
|
|
|
}
|
|
|
|
|
|
|
|
// /!\ On video model /!\
|
|
|
|
const countOptions = {
|
|
|
|
where: videoFilterWhere
|
|
|
|
}
|
|
|
|
|
|
|
|
return Promise.all([
|
|
|
|
VideoModel.findAll(findOptions),
|
|
|
|
|
|
|
|
VideoModel.count(countOptions)
|
|
|
|
]).then(([ data, total ]) => ({ total, data }))
|
|
|
|
}
|
|
|
|
|
|
|
|
static async getStats (strategy: VideoRedundancyStrategyWithManual) {
|
2018-09-14 12:57:59 +00:00
|
|
|
const actor = await getServerActor()
|
|
|
|
|
2019-04-23 07:50:57 +00:00
|
|
|
const query: FindOptions = {
|
2018-09-14 12:57:59 +00:00
|
|
|
raw: true,
|
|
|
|
attributes: [
|
2019-04-23 07:50:57 +00:00
|
|
|
[ fn('COALESCE', fn('SUM', col('VideoFile.size')), '0'), 'totalUsed' ],
|
|
|
|
[ fn('COUNT', fn('DISTINCT', col('videoId'))), 'totalVideos' ],
|
|
|
|
[ fn('COUNT', col('videoFileId')), 'totalVideoFiles' ]
|
2018-09-14 12:57:59 +00:00
|
|
|
],
|
|
|
|
where: {
|
|
|
|
strategy,
|
|
|
|
actorId: actor.id
|
|
|
|
},
|
|
|
|
include: [
|
|
|
|
{
|
|
|
|
attributes: [],
|
|
|
|
model: VideoFileModel,
|
|
|
|
required: true
|
|
|
|
}
|
|
|
|
]
|
|
|
|
}
|
|
|
|
|
2019-04-23 07:50:57 +00:00
|
|
|
return VideoRedundancyModel.findOne(query)
|
2020-01-31 15:56:52 +00:00
|
|
|
.then((r: any) => ({
|
|
|
|
totalUsed: parseAggregateResult(r.totalUsed),
|
|
|
|
totalVideos: r.totalVideos,
|
|
|
|
totalVideoFiles: r.totalVideoFiles
|
|
|
|
}))
|
2018-09-14 12:57:59 +00:00
|
|
|
}
|
|
|
|
|
2020-01-10 09:11:28 +00:00
|
|
|
static toFormattedJSONStatic (video: MVideoForRedundancyAPI): VideoRedundancy {
|
2020-01-31 15:56:52 +00:00
|
|
|
const filesRedundancies: FileRedundancyInformation[] = []
|
|
|
|
const streamingPlaylistsRedundancies: StreamingPlaylistRedundancyInformation[] = []
|
2020-01-10 09:11:28 +00:00
|
|
|
|
|
|
|
for (const file of video.VideoFiles) {
|
|
|
|
for (const redundancy of file.RedundancyVideos) {
|
|
|
|
filesRedundancies.push({
|
|
|
|
id: redundancy.id,
|
|
|
|
fileUrl: redundancy.fileUrl,
|
|
|
|
strategy: redundancy.strategy,
|
|
|
|
createdAt: redundancy.createdAt,
|
|
|
|
updatedAt: redundancy.updatedAt,
|
|
|
|
expiresOn: redundancy.expiresOn,
|
|
|
|
size: file.size
|
|
|
|
})
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
for (const playlist of video.VideoStreamingPlaylists) {
|
|
|
|
const size = playlist.VideoFiles.reduce((a, b) => a + b.size, 0)
|
|
|
|
|
|
|
|
for (const redundancy of playlist.RedundancyVideos) {
|
|
|
|
streamingPlaylistsRedundancies.push({
|
|
|
|
id: redundancy.id,
|
|
|
|
fileUrl: redundancy.fileUrl,
|
|
|
|
strategy: redundancy.strategy,
|
|
|
|
createdAt: redundancy.createdAt,
|
|
|
|
updatedAt: redundancy.updatedAt,
|
|
|
|
expiresOn: redundancy.expiresOn,
|
|
|
|
size
|
|
|
|
})
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return {
|
|
|
|
id: video.id,
|
|
|
|
name: video.name,
|
|
|
|
url: video.url,
|
|
|
|
uuid: video.uuid,
|
|
|
|
|
|
|
|
redundancies: {
|
|
|
|
files: filesRedundancies,
|
|
|
|
streamingPlaylists: streamingPlaylistsRedundancies
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-01-29 07:37:25 +00:00
|
|
|
getVideo () {
|
|
|
|
if (this.VideoFile) return this.VideoFile.Video
|
|
|
|
|
|
|
|
return this.VideoStreamingPlaylist.Video
|
|
|
|
}
|
|
|
|
|
2018-11-16 10:18:13 +00:00
|
|
|
isOwned () {
|
|
|
|
return !!this.strategy
|
|
|
|
}
|
|
|
|
|
2019-08-21 12:31:57 +00:00
|
|
|
toActivityPubObject (this: MVideoRedundancyAP): CacheFileObject {
|
2019-01-29 07:37:25 +00:00
|
|
|
if (this.VideoStreamingPlaylist) {
|
|
|
|
return {
|
|
|
|
id: this.url,
|
|
|
|
type: 'CacheFile' as 'CacheFile',
|
|
|
|
object: this.VideoStreamingPlaylist.Video.url,
|
2020-01-10 09:11:28 +00:00
|
|
|
expires: this.expiresOn ? this.expiresOn.toISOString() : null,
|
2019-01-29 07:37:25 +00:00
|
|
|
url: {
|
|
|
|
type: 'Link',
|
|
|
|
mediaType: 'application/x-mpegURL',
|
|
|
|
href: this.fileUrl
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-09-11 14:27:07 +00:00
|
|
|
return {
|
|
|
|
id: this.url,
|
|
|
|
type: 'CacheFile' as 'CacheFile',
|
|
|
|
object: this.VideoFile.Video.url,
|
2020-01-10 09:11:28 +00:00
|
|
|
expires: this.expiresOn ? this.expiresOn.toISOString() : null,
|
2018-09-11 14:27:07 +00:00
|
|
|
url: {
|
|
|
|
type: 'Link',
|
2020-01-31 15:56:52 +00:00
|
|
|
mediaType: MIMETYPES.VIDEO.EXT_MIMETYPE[this.VideoFile.extname] as any,
|
2018-09-11 14:27:07 +00:00
|
|
|
href: this.fileUrl,
|
|
|
|
height: this.VideoFile.resolution,
|
|
|
|
size: this.VideoFile.size,
|
|
|
|
fps: this.VideoFile.fps
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-09-14 07:57:21 +00:00
|
|
|
// Don't include video files we already duplicated
|
|
|
|
private static async buildVideoFileForDuplication () {
|
2018-09-11 14:27:07 +00:00
|
|
|
const actor = await getServerActor()
|
|
|
|
|
2019-04-23 07:50:57 +00:00
|
|
|
const notIn = literal(
|
2018-09-11 14:27:07 +00:00
|
|
|
'(' +
|
2020-01-31 15:56:52 +00:00
|
|
|
`SELECT "videoFileId" FROM "videoRedundancy" WHERE "actorId" = ${actor.id} AND "videoFileId" IS NOT NULL` +
|
2018-09-11 14:27:07 +00:00
|
|
|
')'
|
|
|
|
)
|
2018-09-14 07:57:21 +00:00
|
|
|
|
|
|
|
return {
|
|
|
|
attributes: [],
|
2020-03-10 13:39:40 +00:00
|
|
|
model: VideoFileModel,
|
2018-09-14 07:57:21 +00:00
|
|
|
required: true,
|
|
|
|
where: {
|
|
|
|
id: {
|
2020-01-31 15:56:52 +00:00
|
|
|
[Op.notIn]: notIn
|
2018-09-14 07:57:21 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
private static buildServerRedundancyInclude () {
|
|
|
|
return {
|
|
|
|
attributes: [],
|
|
|
|
model: VideoChannelModel.unscoped(),
|
|
|
|
required: true,
|
|
|
|
include: [
|
|
|
|
{
|
|
|
|
attributes: [],
|
|
|
|
model: ActorModel.unscoped(),
|
|
|
|
required: true,
|
|
|
|
include: [
|
|
|
|
{
|
|
|
|
attributes: [],
|
|
|
|
model: ServerModel.unscoped(),
|
|
|
|
required: true,
|
|
|
|
where: {
|
|
|
|
redundancyAllowed: true
|
|
|
|
}
|
|
|
|
}
|
|
|
|
]
|
|
|
|
}
|
|
|
|
]
|
|
|
|
}
|
2018-09-11 14:27:07 +00:00
|
|
|
}
|
|
|
|
}
|