From fd4831e5022c2ca084fafa1272e87b7aa5e0e616 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 27 Jun 2024 15:29:26 +0200 Subject: [PATCH] Metadata to know if the caption is auto generated --- .../models/src/activitypub/objects/index.ts | 1 + .../objects/video-caption-object.ts | 5 +++ .../src/activitypub/objects/video-object.ts | 3 +- .../video-export.model.ts | 1 + .../src/videos/caption/video-caption.model.ts | 1 + packages/tests/package.json | 2 +- .../tests/src/api/videos/video-captions.ts | 2 ++ .../src/api/videos/video-transcription.ts | 10 +++--- .../peertube-runner/video-transcription.ts | 4 +-- packages/tests/src/shared/transcription.ts | 3 +- .../core/controllers/api/videos/captions.ts | 7 ++++- server/core/helpers/activity-pub-utils.ts | 1 + server/core/initializers/constants.ts | 2 +- .../migrations/0860-caption-generated.ts | 31 +++++++++++++++++++ .../shared/object-to-model-attributes.ts | 1 + .../exporters/videos-exporter.ts | 1 + .../importers/videos-importer.ts | 8 ++++- server/core/lib/video-captions.ts | 9 ++++-- server/core/lib/video-pre-import.ts | 7 ++++- server/core/models/video/video-caption.ts | 26 ++++++++++------ server/core/models/video/video.ts | 2 +- .../core/types/models/video/video-caption.ts | 3 +- 22 files changed, 101 insertions(+), 29 deletions(-) create mode 100644 packages/models/src/activitypub/objects/video-caption-object.ts create mode 100644 server/core/initializers/migrations/0860-caption-generated.ts diff --git a/packages/models/src/activitypub/objects/index.ts b/packages/models/src/activitypub/objects/index.ts index 8e21f584f..bd8351d30 100644 --- a/packages/models/src/activitypub/objects/index.ts +++ b/packages/models/src/activitypub/objects/index.ts @@ -4,6 +4,7 @@ export * from './cache-file-object.js' export * from './common-objects.js' export * from './playlist-element-object.js' export * from './playlist-object.js' +export * from './video-caption-object.js' export * from './video-chapters-object.js' export * from './video-comment-object.js' export * from './video-object.js' diff --git a/packages/models/src/activitypub/objects/video-caption-object.ts b/packages/models/src/activitypub/objects/video-caption-object.ts new file mode 100644 index 000000000..1ced486cc --- /dev/null +++ b/packages/models/src/activitypub/objects/video-caption-object.ts @@ -0,0 +1,5 @@ +import { ActivityIdentifierObject } from './common-objects.js' + +export interface VideoCaptionObject extends ActivityIdentifierObject { + automaticallyGenerated: boolean +} diff --git a/packages/models/src/activitypub/objects/video-object.ts b/packages/models/src/activitypub/objects/video-object.ts index 63177c239..ab44530ab 100644 --- a/packages/models/src/activitypub/objects/video-object.ts +++ b/packages/models/src/activitypub/objects/video-object.ts @@ -6,6 +6,7 @@ import { ActivityTagObject, ActivityUrlObject } from './common-objects.js' +import { VideoCaptionObject } from './video-caption-object.js' import { VideoChapterObject } from './video-chapters-object.js' export interface VideoObject { @@ -18,7 +19,7 @@ export interface VideoObject { category: ActivityIdentifierObject licence: ActivityIdentifierObject language: ActivityIdentifierObject - subtitleLanguage: ActivityIdentifierObject[] + subtitleLanguage: VideoCaptionObject[] views: number diff --git a/packages/models/src/import-export/peertube-export-format/video-export.model.ts b/packages/models/src/import-export/peertube-export-format/video-export.model.ts index 6e49c6193..14ff282a5 100644 --- a/packages/models/src/import-export/peertube-export-format/video-export.model.ts +++ b/packages/models/src/import-export/peertube-export-format/video-export.model.ts @@ -73,6 +73,7 @@ export interface VideoExportJSON { language: string filename: string fileUrl: string + automaticallyGenerated: boolean }[] chapters: { diff --git a/packages/models/src/videos/caption/video-caption.model.ts b/packages/models/src/videos/caption/video-caption.model.ts index d6d625ff7..b28a6a05d 100644 --- a/packages/models/src/videos/caption/video-caption.model.ts +++ b/packages/models/src/videos/caption/video-caption.model.ts @@ -3,5 +3,6 @@ import { VideoConstant } from '../video-constant.model.js' export interface VideoCaption { language: VideoConstant captionPath: string + automaticallyGenerated: boolean updatedAt: string } diff --git a/packages/tests/package.json b/packages/tests/package.json index 566712bf2..65dd3e7ae 100644 --- a/packages/tests/package.json +++ b/packages/tests/package.json @@ -7,7 +7,7 @@ "scripts": { "build": "tsc", "watch": "tsc -w", - "install-dependencies:transcription": "pip install -r ./requirements.txt ../transcription-devtools/requirements.txt" + "install-dependencies:transcription": "pip install -r ./requirements.txt -r ../transcription-devtools/requirements.txt" }, "dependencies": {} } diff --git a/packages/tests/src/api/videos/video-captions.ts b/packages/tests/src/api/videos/video-captions.ts index 027022549..9913041ad 100644 --- a/packages/tests/src/api/videos/video-captions.ts +++ b/packages/tests/src/api/videos/video-captions.ts @@ -72,12 +72,14 @@ describe('Test video captions', function () { expect(caption1.language.id).to.equal('ar') expect(caption1.language.label).to.equal('Arabic') expect(caption1.captionPath).to.match(new RegExp('^/lazy-static/video-captions/' + uuidRegex + '-ar.vtt$')) + expect(caption1.automaticallyGenerated).to.be.false await testCaptionFile(server.url, caption1.captionPath, 'Subtitle good 1.') const caption2 = body.data[1] expect(caption2.language.id).to.equal('zh') expect(caption2.language.label).to.equal('Chinese') expect(caption2.captionPath).to.match(new RegExp('^/lazy-static/video-captions/' + uuidRegex + '-zh.vtt$')) + expect(caption1.automaticallyGenerated).to.be.false await testCaptionFile(server.url, caption2.captionPath, 'Subtitle good 2.') } }) diff --git a/packages/tests/src/api/videos/video-transcription.ts b/packages/tests/src/api/videos/video-transcription.ts index 6548cd9ef..1e1006e30 100644 --- a/packages/tests/src/api/videos/video-transcription.ts +++ b/packages/tests/src/api/videos/video-transcription.ts @@ -13,7 +13,7 @@ import { waitJobs } from '@peertube/peertube-server-commands' import { FIXTURE_URLS } from '@tests/shared/fixture-urls.js' -import { checkCaption, checkLanguage, checkNoCaption, uploadForTranscription } from '@tests/shared/transcription.js' +import { checkAutoCaption, checkLanguage, checkNoCaption, uploadForTranscription } from '@tests/shared/transcription.js' describe('Test video transcription', function () { let servers: PeerTubeServer[] @@ -48,7 +48,7 @@ describe('Test video transcription', function () { await waitJobs(servers) await checkLanguage(servers, uuid, 'en') - await checkCaption(servers, uuid) + await checkAutoCaption(servers, uuid) }) it('Should run transcription on upload by default', async function () { @@ -57,7 +57,7 @@ describe('Test video transcription', function () { const uuid = await uploadForTranscription(servers[0]) await waitJobs(servers) - await checkCaption(servers, uuid) + await checkAutoCaption(servers, uuid) await checkLanguage(servers, uuid, 'en') }) @@ -73,7 +73,7 @@ describe('Test video transcription', function () { }) await waitJobs(servers) - await checkCaption(servers, video.uuid) + await checkAutoCaption(servers, video.uuid) await checkLanguage(servers, video.uuid, 'en') }) @@ -96,7 +96,7 @@ describe('Test video transcription', function () { await servers[0].live.waitUntilReplacedByReplay({ videoId: video.id }) await waitJobs(servers) - await checkCaption(servers, video.uuid, 'WEBVTT\n\n00:') + await checkAutoCaption(servers, video.uuid, 'WEBVTT\n\n00:') await checkLanguage(servers, video.uuid, 'en') await servers[0].config.enableLive({ allowReplay: false }) diff --git a/packages/tests/src/peertube-runner/video-transcription.ts b/packages/tests/src/peertube-runner/video-transcription.ts index a810524f5..aa6f40c54 100644 --- a/packages/tests/src/peertube-runner/video-transcription.ts +++ b/packages/tests/src/peertube-runner/video-transcription.ts @@ -13,7 +13,7 @@ import { } from '@peertube/peertube-server-commands' import { checkPeerTubeRunnerCacheIsEmpty } from '@tests/shared/directories.js' import { PeerTubeRunnerProcess } from '@tests/shared/peertube-runner-process.js' -import { checkCaption, checkLanguage, checkNoCaption, uploadForTranscription } from '@tests/shared/transcription.js' +import { checkAutoCaption, checkLanguage, checkNoCaption, uploadForTranscription } from '@tests/shared/transcription.js' describe('Test transcription in peertube-runner program', function () { let servers: PeerTubeServer[] = [] @@ -46,7 +46,7 @@ describe('Test transcription in peertube-runner program', function () { const uuid = await uploadForTranscription(servers[0]) await waitJobs(servers, { runnerJobs: true }) - await checkCaption(servers, uuid) + await checkAutoCaption(servers, uuid) await checkLanguage(servers, uuid, 'en') }) diff --git a/packages/tests/src/shared/transcription.ts b/packages/tests/src/shared/transcription.ts index 9deb153f0..0bc040062 100644 --- a/packages/tests/src/shared/transcription.ts +++ b/packages/tests/src/shared/transcription.ts @@ -28,7 +28,7 @@ export function getCustomModelPath (modelName: CustomModelName) { // --------------------------------------------------------------------------- -export async function checkCaption (servers: PeerTubeServer[], uuid: string, captionContains = 'WEBVTT\n\n00:00.000 --> 00:') { +export async function checkAutoCaption (servers: PeerTubeServer[], uuid: string, captionContains = 'WEBVTT\n\n00:00.000 --> 00:') { for (const server of servers) { const body = await server.captions.list({ videoId: uuid }) expect(body.total).to.equal(1) @@ -37,6 +37,7 @@ export async function checkCaption (servers: PeerTubeServer[], uuid: string, cap const caption = body.data[0] expect(caption.language.id).to.equal('en') expect(caption.language.label).to.equal('English') + expect(caption.automaticallyGenerated).to.be.true { await testCaptionFile(server.url, caption.captionPath, captionContains) diff --git a/server/core/controllers/api/videos/captions.ts b/server/core/controllers/api/videos/captions.ts index 7703eed0f..1cd620038 100644 --- a/server/core/controllers/api/videos/captions.ts +++ b/server/core/controllers/api/videos/captions.ts @@ -81,7 +81,12 @@ async function createVideoCaption (req: express.Request, res: express.Response) const captionLanguage = req.params.captionLanguage - const videoCaption = await createLocalCaption({ video, language: captionLanguage, path: videoCaptionPhysicalFile.path }) + const videoCaption = await createLocalCaption({ + video, + language: captionLanguage, + path: videoCaptionPhysicalFile.path, + automaticallyGenerated: false + }) await sequelizeTypescript.transaction(async t => { await federateVideoIfNeeded(video, false, t) diff --git a/server/core/helpers/activity-pub-utils.ts b/server/core/helpers/activity-pub-utils.ts index 525793795..3325280bf 100644 --- a/server/core/helpers/activity-pub-utils.ts +++ b/server/core/helpers/activity-pub-utils.ts @@ -73,6 +73,7 @@ const contextStore: { [ id in ContextType ]: (string | { [ id: string ]: string category: 'sc:category', licence: 'sc:license', subtitleLanguage: 'sc:subtitleLanguage', + automaticallyGenerated: 'pt:automaticallyGenerated', sensitive: 'as:sensitive', language: 'sc:inLanguage', identifier: 'sc:identifier', diff --git a/server/core/initializers/constants.ts b/server/core/initializers/constants.ts index faf944249..1395af040 100644 --- a/server/core/initializers/constants.ts +++ b/server/core/initializers/constants.ts @@ -47,7 +47,7 @@ import { cpus } from 'os' // --------------------------------------------------------------------------- -const LAST_MIGRATION_VERSION = 855 +const LAST_MIGRATION_VERSION = 860 // --------------------------------------------------------------------------- diff --git a/server/core/initializers/migrations/0860-caption-generated.ts b/server/core/initializers/migrations/0860-caption-generated.ts new file mode 100644 index 000000000..d49bc3e9a --- /dev/null +++ b/server/core/initializers/migrations/0860-caption-generated.ts @@ -0,0 +1,31 @@ +import * as Sequelize from 'sequelize' + +async function up (utils: { + transaction: Sequelize.Transaction + queryInterface: Sequelize.QueryInterface + sequelize: Sequelize.Sequelize +}): Promise { + const { transaction } = utils + + { + await utils.queryInterface.addColumn('videoCaption', 'automaticallyGenerated', { + type: Sequelize.BOOLEAN, + defaultValue: false, + allowNull: false + }, { transaction }) + + await utils.queryInterface.changeColumn('videoCaption', 'automaticallyGenerated', { + type: Sequelize.BOOLEAN, + defaultValue: null, + allowNull: false + }, { transaction }) + } +} + +function down (options) { + throw new Error('Not implemented.') +} + +export { + down, up +} diff --git a/server/core/lib/activitypub/videos/shared/object-to-model-attributes.ts b/server/core/lib/activitypub/videos/shared/object-to-model-attributes.ts index 9f6eb8b1b..c6d265792 100644 --- a/server/core/lib/activitypub/videos/shared/object-to-model-attributes.ts +++ b/server/core/lib/activitypub/videos/shared/object-to-model-attributes.ts @@ -155,6 +155,7 @@ export function getCaptionAttributesFromObject (video: MVideoId, videoObject: Vi videoId: video.id, filename: VideoCaptionModel.generateCaptionName(c.identifier), language: c.identifier, + automaticallyGenerated: c.automaticallyGenerated === true, fileUrl: c.url })) } diff --git a/server/core/lib/user-import-export/exporters/videos-exporter.ts b/server/core/lib/user-import-export/exporters/videos-exporter.ts index cfa6dd45a..6b4804031 100644 --- a/server/core/lib/user-import-export/exporters/videos-exporter.ts +++ b/server/core/lib/user-import-export/exporters/videos-exporter.ts @@ -198,6 +198,7 @@ export class VideosExporter extends AbstractUserExporter { updatedAt: c.updatedAt.toISOString(), language: c.language, filename: c.filename, + automaticallyGenerated: c.automaticallyGenerated, fileUrl: c.getFileUrl(video) })) } diff --git a/server/core/lib/user-import-export/importers/videos-importer.ts b/server/core/lib/user-import-export/importers/videos-importer.ts index 8877025b1..474a59362 100644 --- a/server/core/lib/user-import-export/importers/videos-importer.ts +++ b/server/core/lib/user-import-export/importers/videos-importer.ts @@ -97,6 +97,7 @@ export class VideosImporter extends AbstractUserImporter isVideoTagValid(t)) + o.captions = o.captions.filter(c => isVideoCaptionLanguageValid(c.language)) o.chapters = o.chapters.filter(c => isVideoChapterTimecodeValid(c.timecode) && isVideoChapterTitleValid(c.title)) @@ -269,7 +270,12 @@ export class VideosImporter extends AbstractUserImporter { diff --git a/server/core/lib/video-pre-import.ts b/server/core/lib/video-pre-import.ts index fbb94edac..615429087 100644 --- a/server/core/lib/video-pre-import.ts +++ b/server/core/lib/video-pre-import.ts @@ -317,7 +317,12 @@ async function processYoutubeSubtitles (youtubeDL: YoutubeDLWrapper, targetUrl: continue } - await createLocalCaption({ language: subtitle.language, path: subtitle.path, video }) + await createLocalCaption({ + language: subtitle.language, + path: subtitle.path, + video, + automaticallyGenerated: false + }) logger.info('Added %s youtube-dl subtitle', subtitle.path) } diff --git a/server/core/models/video/video-caption.ts b/server/core/models/video/video-caption.ts index df2124b86..2be742e63 100644 --- a/server/core/models/video/video-caption.ts +++ b/server/core/models/video/video-caption.ts @@ -1,3 +1,12 @@ +import { VideoCaption, VideoCaptionObject } from '@peertube/peertube-models' +import { buildUUID } from '@peertube/peertube-node-utils' +import { + MVideo, + MVideoCaption, + MVideoCaptionFormattable, + MVideoCaptionLanguageUrl, + MVideoCaptionVideo +} from '@server/types/models/index.js' import { remove } from 'fs-extra/esm' import { join } from 'path' import { Op, OrderItem, Transaction } from 'sequelize' @@ -13,15 +22,6 @@ import { Table, UpdatedAt } from 'sequelize-typescript' -import { ActivityIdentifierObject, VideoCaption } from '@peertube/peertube-models' -import { - MVideo, - MVideoCaption, - MVideoCaptionFormattable, - MVideoCaptionLanguageUrl, - MVideoCaptionVideo -} from '@server/types/models/index.js' -import { buildUUID } from '@peertube/peertube-node-utils' import { isVideoCaptionLanguageValid } from '../../helpers/custom-validators/video-captions.js' import { logger } from '../../helpers/logger.js' import { CONFIG } from '../../initializers/config.js' @@ -81,6 +81,10 @@ export class VideoCaptionModel extends SequelizeModel { @Column(DataType.STRING(CONSTRAINTS_FIELDS.COMMONS.URL.max)) fileUrl: string + @AllowNull(false) + @Column + automaticallyGenerated: boolean + @ForeignKey(() => VideoModel) @Column videoId: number @@ -228,15 +232,17 @@ export class VideoCaptionModel extends SequelizeModel { id: this.language, label: VideoCaptionModel.getLanguageLabel(this.language) }, + automaticallyGenerated: this.automaticallyGenerated, captionPath: this.getCaptionStaticPath(), updatedAt: this.updatedAt.toISOString() } } - toActivityPubObject (this: MVideoCaptionLanguageUrl, video: MVideo): ActivityIdentifierObject { + toActivityPubObject (this: MVideoCaptionLanguageUrl, video: MVideo): VideoCaptionObject { return { identifier: this.language, name: VideoCaptionModel.getLanguageLabel(this.language), + automaticallyGenerated: this.automaticallyGenerated, url: this.getFileUrl(video) } } diff --git a/server/core/models/video/video.ts b/server/core/models/video/video.ts index ec96df0ed..5ef3874e8 100644 --- a/server/core/models/video/video.ts +++ b/server/core/models/video/video.ts @@ -1886,7 +1886,7 @@ export class VideoModel extends SequelizeModel { if (isArray(videoAP.VideoCaptions)) return videoAP.VideoCaptions return this.$get('VideoCaptions', { - attributes: [ 'filename', 'language', 'fileUrl' ], + attributes: [ 'filename', 'language', 'fileUrl', 'automaticallyGenerated' ], transaction }) as Promise } diff --git a/server/core/types/models/video/video-caption.ts b/server/core/types/models/video/video-caption.ts index 3472742a9..3dca15104 100644 --- a/server/core/types/models/video/video-caption.ts +++ b/server/core/types/models/video/video-caption.ts @@ -12,7 +12,8 @@ export type MVideoCaption = Omit export type MVideoCaptionLanguage = Pick export type MVideoCaptionLanguageUrl = - Pick + Pick export type MVideoCaptionVideo = MVideoCaption &