forks
/
peertube
1
0
Fork 0
Code Releases Activity

Move tags in another table

This commit is contained in:
Chocobozzz 2016-12-24 16:59:17 +01:00
parent 3897209f46
commit 7920c273a2
13 changed files with 333 additions and 138 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

95
server/controllers/api/remote.js
View File

@ -50,28 +50,35 @@ function remoteVideos (req, res, next) {
return res.type('json').status(204).end()
}
function addRemoteVideo (videoToCreateData, fromHost, callback) {
function addRemoteVideo (videoToCreateData, fromHost, finalCallback) {
logger.debug('Adding remote video "%s".', videoToCreateData.name)
waterfall([
function findOrCreatePod (callback) {
function startTransaction (callback) {
db.sequelize.transaction().asCallback(function (err, t) {
return callback(err, t)
})
},
function findOrCreatePod (t, callback) {
const query = {
where: {
host: fromHost
},
defaults: {
host: fromHost
}
},
transaction: t
}
db.Pod.findOrCreate(query).asCallback(function (err, result) {
// [ instance, wasCreated ]
return callback(err, result[0])
return callback(err, t, result[0])
})
},
function findOrCreateAuthor (pod, callback) {
function findOrCreateAuthor (t, pod, callback) {
const username = videoToCreateData.author
const query = {
@ -82,16 +89,45 @@ function addRemoteVideo (videoToCreateData, fromHost, callback) {
defaults: {
name: username,
podId: pod.id
}
},
transaction: t
}
db.Author.findOrCreate(query).asCallback(function (err, result) {
// [ instance, wasCreated ]
return callback(err, result[0])
return callback(err, t, result[0])
})
},
function createVideoObject (author, callback) {
function findOrCreateTags (t, author, callback) {
const tags = videoToCreateData.tags
const tagInstances = []
each(tags, function (tag, callbackEach) {
const query = {
where: {
name: tag
},
defaults: {
name: tag
},
transaction: t
}
db.Tag.findOrCreate(query).asCallback(function (err, res) {
if (err) return callbackEach(err)
// res = [ tag, isCreated ]
const tag = res[0]
tagInstances.push(tag)
return callbackEach()
})
}, function (err) {
return callback(err, t, author, tagInstances)
})
},
function createVideoObject (t, author, tagInstances, callback) {
const videoData = {
name: videoToCreateData.name,
remoteId: videoToCreateData.remoteId,
@ -99,31 +135,58 @@ function addRemoteVideo (videoToCreateData, fromHost, callback) {
infoHash: videoToCreateData.infoHash,
description: videoToCreateData.description,
authorId: author.id,
duration: videoToCreateData.duration,
tags: videoToCreateData.tags
duration: videoToCreateData.duration
}
const video = db.Video.build(videoData)
return callback(null, video)
return callback(null, t, tagInstances, video)
},
function generateThumbnail (video, callback) {
function generateThumbnail (t, tagInstances, video, callback) {
db.Video.generateThumbnailFromBase64(video, videoToCreateData.thumbnailBase64, function (err) {
if (err) {
logger.error('Cannot generate thumbnail from base 64 data.', { error: err })
return callback(err)
}
video.save().asCallback(callback)
return callback(err, t, tagInstances, video)
})
},
function insertIntoDB (video, callback) {
video.save().asCallback(callback)
function insertVideoIntoDB (t, tagInstances, video, callback) {
const options = {
transaction: t
}
video.save(options).asCallback(function (err, videoCreated) {
return callback(err, t, tagInstances, videoCreated)
})
},
function associateTagsToVideo (t, tagInstances, video, callback) {
const options = { transaction: t }
video.setTags(tagInstances, options).asCallback(function (err) {
return callback(err, t)
})
}
], callback)
], function (err, t) {
if (err) {
logger.error('Cannot insert the remote video.')
// Abort transaction?
if (t) t.rollback()
return finalCallback(err)
}
// Commit transaction
t.commit()
return finalCallback()
})
}
function removeRemoteVideo (videoToRemoveData, fromHost, callback) {

99
server/controllers/api/videos.js
View File

@ -1,5 +1,6 @@
'use strict'
const each = require('async/each')
const express = require('express')
const fs = require('fs')
const multer = require('multer')
@ -87,7 +88,13 @@ function addVideo (req, res, next) {
waterfall([
function findOrCreateAuthor (callback) {
function startTransaction (callback) {
db.sequelize.transaction().asCallback(function (err, t) {
return callback(err, t)
})
},
function findOrCreateAuthor (t, callback) {
const username = res.locals.oauth.token.user.username
const query = {
@ -98,75 +105,125 @@ function addVideo (req, res, next) {
defaults: {
name: username,
podId: null // null because it is OUR pod
}
},
transaction: t
}
db.Author.findOrCreate(query).asCallback(function (err, result) {
// [ instance, wasCreated ]
return callback(err, result[0])
return callback(err, t, result[0])
})
},
function createVideoObject (author, callback) {
function findOrCreateTags (t, author, callback) {
const tags = videoInfos.tags
const tagInstances = []
each(tags, function (tag, callbackEach) {
const query = {
where: {
name: tag
},
defaults: {
name: tag
},
transaction: t
}
db.Tag.findOrCreate(query).asCallback(function (err, res) {
if (err) return callbackEach(err)
// res = [ tag, isCreated ]
const tag = res[0]
tagInstances.push(tag)
return callbackEach()
})
}, function (err) {
return callback(err, t, author, tagInstances)
})
},
function createVideoObject (t, author, tagInstances, callback) {
const videoData = {
name: videoInfos.name,
remoteId: null,
extname: path.extname(videoFile.filename),
description: videoInfos.description,
duration: videoFile.duration,
tags: videoInfos.tags,
authorId: author.id
}
const video = db.Video.build(videoData)
return callback(null, author, video)
return callback(null, t, author, tagInstances, video)
},
// Set the videoname the same as the id
function renameVideoFile (author, video, callback) {
function renameVideoFile (t, author, tagInstances, video, callback) {
const videoDir = constants.CONFIG.STORAGE.VIDEOS_DIR
const source = path.join(videoDir, videoFile.filename)
const destination = path.join(videoDir, video.getVideoFilename())
fs.rename(source, destination, function (err) {
return callback(err, author, video)
return callback(err, t, author, tagInstances, video)
})
},
function insertIntoDB (author, video, callback) {
video.save().asCallback(function (err, videoCreated) {
function insertVideoIntoDB (t, author, tagInstances, video, callback) {
const options = { transaction: t }
// Add tags association
video.save(options).asCallback(function (err, videoCreated) {
if (err) return callback(err)
// Do not forget to add Author informations to the created video
videoCreated.Author = author
return callback(err, videoCreated)
return callback(err, t, tagInstances, videoCreated)
})
},
function sendToFriends (video, callback) {
function associateTagsToVideo (t, tagInstances, video, callback) {
const options = { transaction: t }
video.setTags(tagInstances, options).asCallback(function (err) {
video.Tags = tagInstances
return callback(err, t, video)
})
},
function sendToFriends (t, video, callback) {
video.toRemoteJSON(function (err, remoteVideo) {
if (err) return callback(err)
// Now we'll add the video's meta data to our friends
friends.addVideoToFriends(remoteVideo)
return callback(null)
return callback(null, t)
})
}
], function andFinally (err) {
], function andFinally (err, t) {
if (err) {
logger.error('Cannot insert the video.')
// Abort transaction?
if (t) t.rollback()
return next(err)
}
// Commit transaction
t.commit()
// TODO : include Location of the new video -> 201
return res.type('json').status(204).end()
})
}
function getVideo (req, res, next) {
db.Video.loadAndPopulateAuthorAndPod(req.params.id, function (err, video) {
db.Video.loadAndPopulateAuthorAndPodAndTags(req.params.id, function (err, video) {
if (err) return next(err)
if (!video) {
@ -222,12 +279,14 @@ function removeVideo (req, res, next) {
}
function searchVideos (req, res, next) {
db.Video.searchAndPopulateAuthorAndPod(req.params.value, req.query.field, req.query.start, req.query.count, req.query.sort,
function (err, videosList, videosTotal) {
if (err) return next(err)
db.Video.searchAndPopulateAuthorAndPodAndTags(
req.params.value, req.query.field, req.query.start, req.query.count, req.query.sort,
function (err, videosList, videosTotal) {
if (err) return next(err)
res.json(getFormatedVideos(videosList, videosTotal))
})
res.json(getFormatedVideos(videosList, videosTotal))
}
)
}
// ---------------------------------------------------------------------------

2
server/controllers/client.js
View File

@ -93,7 +93,7 @@ function generateWatchHtmlPage (req, res, next) {
},
video: function (callback) {
db.Video.loadAndPopulateAuthorAndPod(videoId, callback)
db.Video.loadAndPopulateAuthorAndPodAndTags(videoId, callback)
}
}, function (err, results) {
if (err) return next(err)

13
server/initializers/database.js
View File

@ -6,13 +6,24 @@ const Sequelize = require('sequelize')
const constants = require('../initializers/constants')
const logger = require('../helpers/logger')
const utils = require('../helpers/utils')
const database = {}
const sequelize = new Sequelize(constants.CONFIG.DATABASE.DBNAME, 'peertube', 'peertube', {
dialect: 'postgres',
host: constants.CONFIG.DATABASE.HOSTNAME,
port: constants.CONFIG.DATABASE.PORT
port: constants.CONFIG.DATABASE.PORT,
benchmark: utils.isTestInstance(),
logging: function (message, benchmark) {
let newMessage = message
if (benchmark !== undefined) {
newMessage += ' | ' + benchmark + 'ms'
}
logger.debug(newMessage)
}
})
const modelDirectory = path.join(__dirname, '..', 'models')

10
server/lib/friends.js
View File

@ -66,10 +66,12 @@ function makeFriends (hosts, callback) {
function quitFriends (callback) {
// Stop pool requests
db.Request.deactivate()
// Flush pool requests
db.Request.flush()
waterfall([
function flushRequests (callbackAsync) {
db.Request.flush(callbackAsync)
},
function getPodsList (callbackAsync) {
return db.Pod.list(callbackAsync)
},
@ -118,7 +120,7 @@ function removeVideoToFriends (videoParams) {
}
function sendOwnedVideosToPod (podId) {
db.Video.listOwnedAndPopulateAuthor(function (err, videosList) {
db.Video.listOwnedAndPopulateAuthorAndTags(function (err, videosList) {
if (err) {
logger.error('Cannot get the list of videos we own.')
return
@ -226,7 +228,7 @@ function makeRequestsToWinningPods (cert, podsList, callback) {
}
// Add our videos to the request scheduler
sendOwnedVideosToPod(podCreated._id)
sendOwnedVideosToPod(podCreated.id)
return callbackEach()
})

3
server/models/pod.js
View File

@ -19,7 +19,6 @@ module.exports = function (sequelize, DataTypes) {
type: DataTypes.INTEGER,
defaultValue: constants.FRIEND_SCORE.BASE
}
// Check createdAt
},
{
classMethods: {
@ -68,7 +67,7 @@ function associate (models) {
this.belongsToMany(models.Request, {
foreignKey: 'podId',
through: models.RequestToPod,
onDelete: 'CASCADE'
onDelete: 'cascade'
})
}

6
server/models/request.js
View File

@ -79,9 +79,11 @@ function deactivate () {
timer = null
}
function flush () {
function flush (callback) {
removeAll.call(this, function (err) {
if (err) logger.error('Cannot flush the requests.', { error: err })
return callback(err)
})
}
@ -298,7 +300,7 @@ function listWithLimitAndRandom (limit, callback) {
function removeAll (callback) {
// Delete all requests
this.destroy({ truncate: true }).asCallback(callback)
this.truncate({ cascade: true }).asCallback(callback)
}
function removeWithEmptyTo (callback) {

30
server/models/tag.js Normal file
View File

@ -0,0 +1,30 @@
'use strict'
// ---------------------------------------------------------------------------
module.exports = function (sequelize, DataTypes) {
const Tag = sequelize.define('Tag',
{
name: {
type: DataTypes.STRING
}
},
{
classMethods: {
associate
}
}
)
return Tag
}
// ---------------------------------------------------------------------------
function associate (models) {
this.belongsToMany(models.Video, {
foreignKey: 'tagId',
through: models.VideoTag,
onDelete: 'cascade'
})
}

103
server/models/video.js
View File

@ -4,6 +4,7 @@ const createTorrent = require('create-torrent')
const ffmpeg = require('fluent-ffmpeg')
const fs = require('fs')
const magnetUtil = require('magnet-uri')
const map = require('lodash/map')
const parallel = require('async/parallel')
const parseTorrent = require('parse-torrent')
const pathUtils = require('path')
@ -41,9 +42,6 @@ module.exports = function (sequelize, DataTypes) {
},
duration: {
type: DataTypes.INTEGER
},
tags: {
type: DataTypes.ARRAY(DataTypes.STRING)
}
},
{
@ -54,12 +52,12 @@ module.exports = function (sequelize, DataTypes) {
getDurationFromFile,
listForApi,
listByHostAndRemoteId,
listOwnedAndPopulateAuthor,
listOwnedAndPopulateAuthorAndTags,
listOwnedByAuthor,
load,
loadAndPopulateAuthor,
loadAndPopulateAuthorAndPod,
searchAndPopulateAuthorAndPod
loadAndPopulateAuthorAndPodAndTags,
searchAndPopulateAuthorAndPodAndTags
},
instanceMethods: {
generateMagnetUri,
@ -170,6 +168,12 @@ function associate (models) {
},
onDelete: 'cascade'
})
this.belongsToMany(models.Tag, {
foreignKey: 'videoId',
through: models.VideoTag,
onDelete: 'cascade'
})
}
function generateMagnetUri () {
@ -248,7 +252,7 @@ function toFormatedJSON () {
magnetUri: this.generateMagnetUri(),
author: this.Author.name,
duration: this.duration,
tags: this.tags,
tags: map(this.Tags, 'name'),
thumbnailPath: constants.STATIC_PATHS.THUMBNAILS + '/' + this.getThumbnailName(),
createdAt: this.createdAt
}
@ -275,7 +279,7 @@ function toRemoteJSON (callback) {
author: self.Author.name,
duration: self.duration,
thumbnailBase64: new Buffer(thumbnailData).toString('base64'),
tags: self.tags,
tags: map(self.Tags, 'name'),
createdAt: self.createdAt,
extname: self.extname
}
@ -310,12 +314,15 @@ function listForApi (start, count, sort, callback) {
const query = {
offset: start,
limit: count,
distinct: true, // For the count, a video can have many tags
order: [ modelUtils.getSort(sort) ],
include: [
{
model: this.sequelize.models.Author,
include: [ this.sequelize.models.Pod ]
}
include: [ { model: this.sequelize.models.Pod, required: false } ]
},
this.sequelize.models.Tag
]
}
@ -337,6 +344,7 @@ function listByHostAndRemoteId (fromHost, remoteId, callback) {
include: [
{
model: this.sequelize.models.Pod,
required: true,
where: {
host: fromHost
}
@ -349,13 +357,13 @@ function listByHostAndRemoteId (fromHost, remoteId, callback) {
return this.findAll(query).asCallback(callback)
}
function listOwnedAndPopulateAuthor (callback) {
function listOwnedAndPopulateAuthorAndTags (callback) {
// If remoteId is null this is *our* video
const query = {
where: {
remoteId: null
},
include: [ this.sequelize.models.Author ]
include: [ this.sequelize.models.Author, this.sequelize.models.Tag ]
}
return this.findAll(query).asCallback(callback)
@ -391,23 +399,26 @@ function loadAndPopulateAuthor (id, callback) {
return this.findById(id, options).asCallback(callback)
}
function loadAndPopulateAuthorAndPod (id, callback) {
function loadAndPopulateAuthorAndPodAndTags (id, callback) {
const options = {
include: [
{
model: this.sequelize.models.Author,
include: [ this.sequelize.models.Pod ]
}
include: [ { model: this.sequelize.models.Pod, required: false } ]
},
this.sequelize.models.Tag
]
}
return this.findById(id, options).asCallback(callback)
}
function searchAndPopulateAuthorAndPod (value, field, start, count, sort, callback) {
function searchAndPopulateAuthorAndPodAndTags (value, field, start, count, sort, callback) {
const podInclude = {
model: this.sequelize.models.Pod
model: this.sequelize.models.Pod,
required: false
}
const authorInclude = {
model: this.sequelize.models.Author,
include: [
@ -415,55 +426,61 @@ function searchAndPopulateAuthorAndPod (value, field, start, count, sort, callba
]
}
const tagInclude = {
model: this.sequelize.models.Tag
}
const query = {
where: {},
include: [
authorInclude
],
offset: start,
limit: count,
distinct: true, // For the count, a video can have many tags
order: [ modelUtils.getSort(sort) ]
}
// TODO: include our pod for podHost searches (we are not stored in the database)
// Make an exact search with the magnet
if (field === 'magnetUri') {
const infoHash = magnetUtil.decode(value).infoHash
query.where.infoHash = infoHash
} else if (field === 'tags') {
query.where[field] = value
const escapedValue = this.sequelize.escape('%' + value + '%')
query.where = {
id: {
$in: this.sequelize.literal(
'(SELECT "VideoTags"."videoId" FROM "Tags" INNER JOIN "VideoTags" ON "Tags"."id" = "VideoTags"."tagId" WHERE name LIKE ' + escapedValue + ')'
)
}
}
} else if (field === 'host') {
const whereQuery = {
'$Author.Pod.host$': {
// FIXME: Include our pod? (not stored in the database)
podInclude.where = {
host: {
$like: '%' + value + '%'
}
}
podInclude.required = true
} else if (field === 'author') {
authorInclude.where = {
name: {
$like: '%' + value + '%'
}
}
// Include our pod? (not stored in the database)
if (constants.CONFIG.WEBSERVER.HOST.indexOf(value) !== -1) {
query.where = {
$or: [
whereQuery,
{
remoteId: null
}
]
}
} else {
query.where = whereQuery
}
} else if (field === 'author') {
query.where = {
'$Author.name$': {
$like: '%' + value + '%'
}
}
// authorInclude.or = true
} else {
query.where[field] = {
$like: '%' + value + '%'
}
}
query.include = [
authorInclude, tagInclude
]
if (tagInclude.where) {
// query.include.push([ this.sequelize.models.Tag ])
}
return this.findAndCountAll(query).asCallback(function (err, result) {
if (err) return callback(err)

9
server/models/videoTag.js Normal file
View File

@ -0,0 +1,9 @@
'use strict'
// ---------------------------------------------------------------------------
module.exports = function (sequelize, DataTypes) {
const VideoTag = sequelize.define('VideoTag', {}, {})
return VideoTag
}

4
server/tests/api/check-params.js
View File

@ -456,7 +456,7 @@ describe('Test parameters validator', function () {
})
})
it('Should fail without a mongodb id', function (done) {
it('Should fail without a correct uuid', function (done) {
request(server.url)
.get(path + 'coucou')
.set('Accept', 'application/json')
@ -481,7 +481,7 @@ describe('Test parameters validator', function () {
.expect(400, done)
})
it('Should fail without a mongodb id', function (done) {
it('Should fail without a correct uuid', function (done) {
request(server.url)
.delete(path + 'hello')
.set('Authorization', 'Bearer ' + server.accessToken)

15
server/tests/api/requests.js
View File

@ -79,15 +79,16 @@ describe('Test requests stats', function () {
uploadVideo(server, function (err) {
if (err) throw err
getRequestsStats(server, function (err, res) {
if (err) throw err
setTimeout(function () {
getRequestsStats(server, function (err, res) {
if (err) throw err
const body = res.body
expect(body.totalRequests).to.equal(1)
const body = res.body
expect(body.totalRequests).to.equal(1)
// Wait one cycle
setTimeout(done, 10000)
})
done()
})
}, 1000)
})
})

82
server/tests/api/single-pod.js
View File

@ -153,31 +153,32 @@ describe('Test a single pod', function () {
})
})
it('Should search the video by podHost', function (done) {
videosUtils.searchVideo(server.url, '9001', 'host', function (err, res) {
if (err) throw err
// Not implemented yet
// it('Should search the video by podHost', function (done) {
// videosUtils.searchVideo(server.url, '9001', 'host', function (err, res) {
// if (err) throw err
expect(res.body.total).to.equal(1)
expect(res.body.data).to.be.an('array')
expect(res.body.data.length).to.equal(1)
// expect(res.body.total).to.equal(1)
// expect(res.body.data).to.be.an('array')
// expect(res.body.data.length).to.equal(1)
const video = res.body.data[0]
expect(video.name).to.equal('my super name')
expect(video.description).to.equal('my super description')
expect(video.podHost).to.equal('localhost:9001')
expect(video.author).to.equal('root')
expect(video.isLocal).to.be.true
expect(video.tags).to.deep.equal([ 'tag1', 'tag2', 'tag3' ])
expect(miscsUtils.dateIsValid(video.createdAt)).to.be.true
// const video = res.body.data[0]
// expect(video.name).to.equal('my super name')
// expect(video.description).to.equal('my super description')
// expect(video.podHost).to.equal('localhost:9001')
// expect(video.author).to.equal('root')
// expect(video.isLocal).to.be.true
// expect(video.tags).to.deep.equal([ 'tag1', 'tag2', 'tag3' ])
// expect(miscsUtils.dateIsValid(video.createdAt)).to.be.true
videosUtils.testVideoImage(server.url, 'video_short.webm', video.thumbnailPath, function (err, test) {
if (err) throw err
expect(test).to.equal(true)
// videosUtils.testVideoImage(server.url, 'video_short.webm', video.thumbnailPath, function (err, test) {
// if (err) throw err
// expect(test).to.equal(true)
done()
})
})
})
// done()
// })
// })
// })
it('Should search the video by tag', function (done) {
videosUtils.searchVideo(server.url, 'tag1', 'tags', function (err, res) {
@ -230,7 +231,7 @@ describe('Test a single pod', function () {
})
it('Should not find a search by tag', function (done) {
videosUtils.searchVideo(server.url, 'tag', 'tags', function (err, res) {
videosUtils.searchVideo(server.url, 'hello', 'tags', function (err, res) {
if (err) throw err
expect(res.body.total).to.equal(0)
@ -424,29 +425,30 @@ describe('Test a single pod', function () {
})
})
it('Should search all the 9001 port videos', function (done) {
videosUtils.searchVideoWithPagination(server.url, '9001', 'host', 0, 15, function (err, res) {
if (err) throw err
// Not implemented yet
// it('Should search all the 9001 port videos', function (done) {
// videosUtils.searchVideoWithPagination(server.url, '9001', 'host', 0, 15, function (err, res) {
// if (err) throw err
const videos = res.body.data
expect(res.body.total).to.equal(6)
expect(videos.length).to.equal(6)
// const videos = res.body.data
// expect(res.body.total).to.equal(6)
// expect(videos.length).to.equal(6)
done()
})
})
// done()
// })
// })
it('Should search all the localhost videos', function (done) {
videosUtils.searchVideoWithPagination(server.url, 'localhost', 'host', 0, 15, function (err, res) {
if (err) throw err
// it('Should search all the localhost videos', function (done) {
// videosUtils.searchVideoWithPagination(server.url, 'localhost', 'host', 0, 15, function (err, res) {
// if (err) throw err
const videos = res.body.data
expect(res.body.total).to.equal(6)
expect(videos.length).to.equal(6)
// const videos = res.body.data
// expect(res.body.total).to.equal(6)
// expect(videos.length).to.equal(6)
done()
})
})
// done()
// })
// })
it('Should search the good magnetUri video', function (done) {
const video = videosListBase[0]