forks/peertube
1
0
Fork 0
Code Releases Activity

Add tags to logs in AP videos

Browse source
This commit is contained in:
Chocobozzz 2021-06-02 16:49:59 +02:00
parent e872632091
commit 463206948d
No known key found for this signature in database
GPG key ID: 583A612D890159BE
7 changed files with 46 additions and 25 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
server/helpers/logger.ts
View file

@ -151,7 +151,8 @@ const bunyanLogger = {
fatal: bunyanLogFactory('error')
}
function loggerTagsFactory (...defaultTags: string[]) {
type LoggerTagsFn = (...tags: string[]) => { tags: string[] }
function loggerTagsFactory (...defaultTags: string[]): LoggerTagsFn {
return (...tags: string[]) => {
return { tags: defaultTags.concat(tags) }
}
@ -160,6 +161,8 @@ function loggerTagsFactory (...defaultTags: string[]) {
// ---------------------------------------------------------------------------
export {
LoggerTagsFn,
buildLogger,
timestampFormatter,
labelFormatter,

10
server/lib/activitypub/videos/refresh.ts
View file

@ -1,4 +1,4 @@
import { logger } from '@server/helpers/logger'
import { logger, loggerTagsFactory } from '@server/helpers/logger'
import { PeerTubeRequestError } from '@server/helpers/requests'
import { VideoFetchByUrlType } from '@server/helpers/video'
import { ActorFollowScoreCache } from '@server/lib/files-cache'
@ -8,6 +8,8 @@ import { HttpStatusCode } from '@shared/core-utils'
import { fetchRemoteVideo, SyncParam, syncVideoExternalAttributes } from './shared'
import { APVideoUpdater } from './updater'
const lTags = loggerTagsFactory('ap', 'video', 'refresh')
async function refreshVideoIfNeeded (options: {
video: MVideoThumbnail
fetchedType: VideoFetchByUrlType
@ -24,7 +26,7 @@ async function refreshVideoIfNeeded (options: {
const { videoObject } = await fetchRemoteVideo(video.url)
if (videoObject === undefined) {
logger.warn('Cannot refresh remote video %s: invalid body.', video.url)
logger.warn('Cannot refresh remote video %s: invalid body.', video.url, lTags(video.uuid))
await video.setAsRefreshed()
return video
@ -40,14 +42,14 @@ async function refreshVideoIfNeeded (options: {
return video
} catch (err) {
if ((err as PeerTubeRequestError).statusCode === HttpStatusCode.NOT_FOUND_404) {
logger.info('Cannot refresh remote video %s: video does not exist anymore. Deleting it.', video.url)
logger.info('Cannot refresh remote video %s: video does not exist anymore. Deleting it.', video.url, lTags(video.uuid))
// Video does not exist anymore
await video.destroy()
return undefined
}
logger.warn('Cannot refresh video %s.', options.video.url, { err })
logger.warn('Cannot refresh video %s.', options.video.url, { err, ...lTags(video.uuid) })
ActorFollowScoreCache.Instance.addBadServerId(video.VideoChannel.Actor.serverId)

5
server/lib/activitypub/videos/shared/abstract-builder.ts
View file

@ -1,7 +1,7 @@
import { Transaction } from 'sequelize/types'
import { checkUrlsSameHost } from '@server/helpers/activitypub'
import { deleteNonExistingModels } from '@server/helpers/database-utils'
import { logger } from '@server/helpers/logger'
import { logger, LoggerTagsFn } from '@server/helpers/logger'
import { createPlaceholderThumbnail, createVideoMiniatureFromUrl } from '@server/lib/thumbnail'
import { setVideoTags } from '@server/lib/video'
import { VideoCaptionModel } from '@server/models/video/video-caption'
@ -24,6 +24,7 @@ import { getTrackerUrls, setVideoTrackers } from './trackers'
export abstract class APVideoAbstractBuilder {
protected abstract videoObject: VideoObject
protected abstract lTags: LoggerTagsFn
protected async getOrCreateVideoChannelFromVideoObject () {
const channel = this.videoObject.attributedTo.find(a => a.type === 'Group')
@ -42,7 +43,7 @@ export abstract class APVideoAbstractBuilder {
video,
type: ThumbnailType.MINIATURE
}).catch(err => {
logger.warn('Cannot generate thumbnail of %s.', this.videoObject.id, { err })
logger.warn('Cannot generate thumbnail of %s.', this.videoObject.id, { err, ...this.lTags(video.uuid) })
return undefined
})

7
server/lib/activitypub/videos/shared/creator.ts
View file

@ -1,5 +1,5 @@
import { logger } from '@server/helpers/logger'
import { logger, loggerTagsFactory } from '@server/helpers/logger'
import { sequelizeTypescript } from '@server/initializers/database'
import { autoBlacklistVideoIfNeeded } from '@server/lib/video-blacklist'
import { VideoModel } from '@server/models/video/video'
@ -9,13 +9,14 @@ import { APVideoAbstractBuilder } from './abstract-builder'
import { getVideoAttributesFromObject } from './object-to-model-attributes'
export class APVideoCreator extends APVideoAbstractBuilder {
protected lTags = loggerTagsFactory('ap', 'video', 'create')
constructor (protected readonly videoObject: VideoObject) {
super()
}
async create (waitThumbnail = false) {
logger.debug('Adding remote video %s.', this.videoObject.id)
logger.debug('Adding remote video %s.', this.videoObject.id, this.lTags(this.videoObject.uuid))
const channelActor = await this.getOrCreateVideoChannelFromVideoObject()
const channel = channelActor.VideoChannel
@ -56,7 +57,7 @@ export class APVideoCreator extends APVideoAbstractBuilder {
transaction: t
})
logger.info('Remote video with uuid %s inserted.', this.videoObject.uuid)
logger.info('Remote video with uuid %s inserted.', this.videoObject.uuid, this.lTags(videoCreated.uuid))
return { autoBlacklisted, videoCreated }
} catch (err) {

9
server/lib/activitypub/videos/shared/url-to-object.ts
View file

@ -1,16 +1,19 @@
import { checkUrlsSameHost } from '@server/helpers/activitypub'
import { sanitizeAndCheckVideoTorrentObject } from '@server/helpers/custom-validators/activitypub/videos'
import { logger } from '@server/helpers/logger'
import { logger, loggerTagsFactory } from '@server/helpers/logger'
import { doJSONRequest } from '@server/helpers/requests'
import { VideoObject } from '@shared/models'
const lTags = loggerTagsFactory('ap', 'video')
async function fetchRemoteVideo (videoUrl: string): Promise<{ statusCode: number, videoObject: VideoObject }> {
logger.info('Fetching remote video %s.', videoUrl)
logger.info('Fetching remote video %s.', videoUrl, lTags(videoUrl))
const { statusCode, body } = await doJSONRequest<any>(videoUrl, { activityPub: true })
if (sanitizeAndCheckVideoTorrentObject(body) === false || checkUrlsSameHost(body.id, videoUrl) !== true) {
logger.debug('Remote video JSON is not valid.', { body })
logger.debug('Remote video JSON is not valid.', { body, ...lTags(videoUrl) })
return { statusCode, videoObject: undefined }
}

22
server/lib/activitypub/videos/shared/video-sync-attributes.ts
View file

@ -1,4 +1,4 @@
import { logger } from '@server/helpers/logger'
import { logger, loggerTagsFactory } from '@server/helpers/logger'
import { JobQueue } from '@server/lib/job-queue'
import { AccountVideoRateModel } from '@server/models/account/account-video-rate'
import { VideoCommentModel } from '@server/models/video/video-comment'
@ -10,6 +10,8 @@ import { addVideoShares } from '../../share'
import { addVideoComments } from '../../video-comments'
import { createRates } from '../../video-rates'
const lTags = loggerTagsFactory('ap', 'video')
type SyncParam = {
likes: boolean
dislikes: boolean
@ -60,29 +62,33 @@ function syncRates (type: 'like' | 'dislike', video: MVideo, fetchedVideo: Video
const cleaner = crawlStartDate => AccountVideoRateModel.cleanOldRatesOf(video.id, type, crawlStartDate)
return crawlCollectionPage<string>(uri, handler, cleaner)
.catch(err => logger.error('Cannot add rate of video %s.', video.uuid, { err, rootUrl: uri }))
.catch(err => logger.error('Cannot add rate of video %s.', video.uuid, { err, rootUrl: uri, ...lTags(video.uuid) }))
}
function syncShares (video: MVideo, fetchedVideo: VideoObject, isSync: boolean) {
const uri = fetchedVideo.shares
if (!isSync) {
return createJob({ uri: fetchedVideo.shares, videoId: video.id, type: 'video-shares' })
return createJob({ uri, videoId: video.id, type: 'video-shares' })
}
const handler = items => addVideoShares(items, video)
const cleaner = crawlStartDate => VideoShareModel.cleanOldSharesOf(video.id, crawlStartDate)
return crawlCollectionPage<string>(fetchedVideo.shares, handler, cleaner)
.catch(err => logger.error('Cannot add shares of video %s.', video.uuid, { err, rootUrl: fetchedVideo.shares }))
return crawlCollectionPage<string>(uri, handler, cleaner)
.catch(err => logger.error('Cannot add shares of video %s.', video.uuid, { err, rootUrl: uri, ...lTags(video.uuid) }))
}
function syncComments (video: MVideo, fetchedVideo: VideoObject, isSync: boolean) {
const uri = fetchedVideo.comments
if (!isSync) {
return createJob({ uri: fetchedVideo.comments, videoId: video.id, type: 'video-comments' })
return createJob({ uri, videoId: video.id, type: 'video-comments' })
}
const handler = items => addVideoComments(items)
const cleaner = crawlStartDate => VideoCommentModel.cleanOldCommentsOf(video.id, crawlStartDate)
return crawlCollectionPage<string>(fetchedVideo.comments, handler, cleaner)
.catch(err => logger.error('Cannot add comments of video %s.', video.uuid, { err, rootUrl: fetchedVideo.comments }))
return crawlCollectionPage<string>(uri, handler, cleaner)
.catch(err => logger.error('Cannot add comments of video %s.', video.uuid, { err, rootUrl: uri, ...lTags(video.uuid) }))
}

13
server/lib/activitypub/videos/updater.ts
View file

@ -1,6 +1,6 @@
import { Transaction } from 'sequelize/types'
import { resetSequelizeInstance } from '@server/helpers/database-utils'
import { logger } from '@server/helpers/logger'
import { logger, loggerTagsFactory } from '@server/helpers/logger'
import { sequelizeTypescript } from '@server/initializers/database'
import { Notifier } from '@server/lib/notifier'
import { PeerTubeSocket } from '@server/lib/peertube-socket'
@ -19,6 +19,8 @@ export class APVideoUpdater extends APVideoAbstractBuilder {
private readonly oldVideoChannel: MChannelAccountLight
protected lTags = loggerTagsFactory('ap', 'video', 'update')
constructor (
protected readonly videoObject: VideoObject,
private readonly video: MVideoAccountLightBlacklistAllFiles
@ -34,7 +36,10 @@ export class APVideoUpdater extends APVideoAbstractBuilder {
}
async update (overrideTo?: string[]) {
logger.debug('Updating remote video "%s".', this.videoObject.uuid, { videoObject: this.videoObject })
logger.debug(
'Updating remote video "%s".', this.videoObject.uuid,
{ videoObject: this.videoObject, ...this.lTags(this.videoObject.uuid) }
)
try {
const channelActor = await this.getOrCreateVideoChannelFromVideoObject()
@ -77,7 +82,7 @@ export class APVideoUpdater extends APVideoAbstractBuilder {
PeerTubeSocket.Instance.sendVideoViewsUpdate(videoUpdated)
}
logger.info('Remote video with uuid %s updated', this.videoObject.uuid)
logger.info('Remote video with uuid %s updated', this.videoObject.uuid, this.lTags(this.videoObject.uuid))
return videoUpdated
} catch (err) {
@ -153,7 +158,7 @@ export class APVideoUpdater extends APVideoAbstractBuilder {
}
// This is just a debug because we will retry the insert
logger.debug('Cannot update the remote video.', { err })
logger.debug('Cannot update the remote video.', { err, ...this.lTags(this.videoObject.uuid) })
throw err
}
}