v6.0.1

-----BEGIN PGP SIGNATURE-----
 
 iQEzBAABCAAdFiEExEqtY4NnkSypPt1XWDphLYkBWb4FAmVnAk4ACgkQWDphLYkB
 Wb7adwgAv6e8rk0IyOVER4rezUKnJhTgKyx/MN+yCnKlATqwakT7DVSJinQHXABV
 RjBztQJVGYYY9Spgv5uhRifmmMvxGevXQcTxicndtBODIaWAlJKXsC6KoWQNmyzh
 sBhqQjyPQcZ7gop8Hu8cygx+W8mwEoU41SgA9CtIJc15ox0TbuX53gLzsxdzENOP
 741W7DIJZzsVw4DzSEZASQExt6CCmyECM9JCS3oIlaSBSq+OoMyj69MGv21QU3/F
 H5AgsGTOL/6d9VYCKLwJcYHuYFdSq6xKBEi53QJ3R2EKxfu51rBxP1563WJfDrpH
 T9WzVskYAlrZxcSl6PkMZPiS+5u47A==
 =SVcY
 -----END PGP SIGNATURE-----
gpgsig -----BEGIN PGP SIGNATURE-----
 
 iQJNBAABCAA3FiEEGCBYi9NGimfngx9dVTwOu+tdXwgFAmV6StIZHGtvdG92YWxl
 eGFyaWFuQGdtYWlsLmNvbQAKCRBVPA67611fCOBDEACGjDwXYqJdeVIFUufPYjSo
 vI0DSh6+pMYc3q4Pnp/4ZX0HlbPAX3V2nCl1JQzkXJ/e4aSSkTP+IOwY3+StlVGw
 dEOVuEjc6AaH3LOlgWzFRGKdLBknTJLgv9AelTYWuBfE4rScxMklsx8xFUfRC6ut
 2Wax29RvSCQ9iN9+4xpRyHjrYT7sGJHCN6BLagaDgm5x0ak8blEK3GPVWJTFARhV
 ACp2tOY6XrDnGeljeGZT6Ue/W+N4w4In6JSCsHzzDEM3X2l+qjtFYcghY8kQyRyg
 bQvUQFs7qR04fc0GCCZW7rDxRzqyqJ5jDQJ/sdyuXOXU6Ox7s1pG4RrV8vfBWBpD
 aQ5a79eqWmPDLUSOczDCvwhnYlkq8Pl7AejggIi2ZrJLxlYg5/0k6RQ6lbA2uwwd
 BjVK+XT1WbE8sty5ULyJzt8a9ec5/5HupjFfbDrqtRZbwA3avOrJT12E99h4MsiD
 l7iaDWQ2jb6lu8uKwe8chRKRad3Y9ptVL6btwLzg/W6XSzqGIy0kgzDPvURzxQGi
 9ZowiFlgqc2rjVY/nwdjaVUAL2wsx7iWKMHjdvGV3F4PZSLR9LoMgVpfsDNcfNo6
 DoY98v5r+AcQm9P76Yt5Z9aeZYHdyFOPljx4/zKBotD14I5wePJ2gogfM+SHeyTL
 j42jwix9VPd6t/SxW6JgPw==
 =CHm4
 -----END PGP SIGNATURE-----

Merge tag 'v6.0.1' into changes

v6.0.1

CHANGELOG.md

@@ -1,5 +1,23 @@
 # Changelog
 
+## v6.0.1
+
+### IMPORTANT NOTES
+
+ * If you upgrade from PeerTube **< v6.0.0**, please follow v6.0.0 IMPORTANT NOTES
+ * We've made some modifications in v6.0.0 IMPORTANT NOTES, so if you upgrade from PeerTube v6.0.0:
+   * Ensure `location = /api/v1/videos/upload-resumable {` has been replaced by `location ~ ^/api/v1/videos/(upload-resumable|([^/]+/source/replace-resumable))$ {` in your nginx configuration
+   * Ensure you updated `storage.web_videos` configuration value to use `web-videos/` directory name
+   * Ensure your directory name on filesystem is the same as `storage.web_videos` configuration
+
+### Bug fixes
+
+ * Fix CPU going to 100% on odd cpu count
+ * Increase storyboard generation job TTL
+ * Add missing `generate-video-storyboard` job type in admin jobs list
+ * Regenerate storyboard after studio job
+
+
 ## v6.0.0
 
 ### IMPORTANT NOTES
@@ -15,12 +33,16 @@ We have many important notes in this release. We know it's a pain for sysadmin,
 
   * Remove WebTorrent support in player:
     * "WebTorrent videos" are renamed to "Web Video". The video format is the same, we just stop to use P2P for these videos
-    * There is not "Auto" quality anymore for Web Videos. The viewer has to explicitly choose the video resolution
+    * There is no "Auto" quality anymore for Web Videos. The viewer has to explicitly choose the video resolution
     * We still use P2P with the HLS player, which is the recommended transcoding format since several versions
     * See https://github.com/Chocobozzz/PeerTube/issues/5465 for more information
 
   * Configuration key that you must update in your `production.yaml` if not automatically done by your upgrade script:
-    * `storage.videos` must be **renamed** to `storage.web_videos`. The value of this configuration doesn't need to be changed: https://github.com/Chocobozzz/PeerTube/blob/develop/config/production.yaml.example#L151
+    * `storage.videos` must be **renamed** to `storage.web_videos`: https://github.com/Chocobozzz/PeerTube/blob/develop/config/production.yaml.example#L151
+    * Configuration value of `storage.web_videos` must have the directory name to be **changed** from `videos/` to `web-videos/`: https://github.com/Chocobozzz/PeerTube/blob/develop/config/production.yaml.example#L151
+    * Directory on filesystem must be **renamed** from `videos/` to `web-videos/` to represent the value of `storage.web_videos`
+      * Classic installation: `sudo -u peertube mv '/var/www/peertube/storage/videos/' '/var/www/peertube/storage/web-videos/'`
+      * Docker installation: `mv '/path-to-docker-installation/docker-volume/data/videos/' '/path-to-docker-installation/docker-volume/data/web-videos/'`
     * `transcoding.webtorrent` must be **renamed** to `transcoding.web_videos`: https://github.com/Chocobozzz/PeerTube/blob/develop/config/production.yaml.example#L522
     * `object_storage.videos` must be **renamed** to `object_storage.web_videos`. The value of `object_storage.web_videos.bucket_name` doesn't need to be changed: https://github.com/Chocobozzz/PeerTube/blob/develop/config/production.yaml.example#L223
     * `storage.storyboards` must be **added**: https://github.com/Chocobozzz/PeerTube/blob/develop/config/production.yaml.example#L157
@@ -35,6 +57,7 @@ We have many important notes in this release. We know it's a pain for sysadmin,
 
   * You must update nginx configuration: https://github.com/Chocobozzz/PeerTube/blob/develop/support/nginx/peertube
     * `location ~ ^/static/(thumbnails|avatars)/ {` block must be removed
+    * `location = /api/v1/videos/upload-resumable {` must be updated to `location ~ ^/api/v1/videos/(upload-resumable|([^/]+/source/replace-resumable))$ {`
     * `location ~ ^(/static/(webseed|streaming-playlists)/private/)|^/download {` must be updated to `location ~ ^(/static/(webseed|web-videos|streaming-playlists)/private/)|^/download {`
     * `location ~ ^/static/(webseed|redundancy|streaming-playlists)/ {` must be updated to `location ~ ^/static/(webseed|web-videos|redundancy|streaming-playlists)/ {`
 
@@ -64,7 +87,7 @@ We have many important notes in this release. We know it's a pain for sysadmin,
 
   * Removed unmaintained `peertube-import-videos` (also aliased as `peertube import-videos` or `peertube import`) script
   * PeerTube remote CLI is much more simpler to install using NPM: https://docs.joinpeertube.org/maintain/tools#remote-peertube-cli
-  * Support moving video files from object storage to filesystem: https://docs.joinpeertube.org/maintain/tools#generate-storyboard
+  * Support moving video files from object storage to filesystem: https://docs.joinpeertube.org/maintain/tools#move-video-files-from-object-storage-to-filesystem
 
 ### Features
 

apps/peertube-runner/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@peertube/peertube-runner",
-  "version": "0.0.6",
+  "version": "0.0.7",
   "type": "module",
   "main": "dist/peertube-runner.js",
   "bin": "dist/peertube-runner.js",

client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "peertube-client",
-  "version": "6.0.0",
+  "version": "6.0.1",
   "private": true,
   "license": "AGPL-3.0",
   "author": {

client/src/app/+admin/system/jobs/jobs.component.ts

@@ -35,10 +35,12 @@ export class JobsComponent extends RestTable implements OnInit {
     'after-video-channel-import',
     'email',
     'federate-video',
+    'generate-video-storyboard',
     'manage-video-torrent',
-    'move-to-object-storage',
     'move-to-file-system',
+    'move-to-object-storage',
     'notify',
+    'transcoding-job-builder',
     'video-channel-import',
     'video-file-import',
     'video-import',

package.json

@@ -1,7 +1,7 @@
 {
   "name": "peertube",
   "description": "PeerTube, an ActivityPub-federated video streaming platform using P2P directly in your web browser.",
-  "version": "6.0.0",
+  "version": "6.0.1",
   "private": true,
   "licence": "AGPL-3.0",
   "engines": {
@@ -110,7 +110,6 @@
     "@peertube/http-signature": "^1.7.0",
     "@smithy/node-http-handler": "^2.1.7",
     "@uploadx/core": "^6.0.0",
-    "async-lru": "^1.1.1",
     "async-mutex": "^0.4.0",
     "bcrypt": "5.1.1",
     "bencode": "^4.0.0",

scripts/release.sh

@@ -133,6 +133,8 @@ find dist/ packages/core-utils/dist/ \
       git push origin master
       git checkout "$branch"
 
+      # Rebuild properly the server, with the declaration files
+      npm run build:server
       # Release types package
       npm run generate-types-package "$version"
       cd packages/types-generator/dist

server/core/helpers/custom-jsonld-signature.ts

@@ -1,6 +1,4 @@
-import AsyncLRU from 'async-lru'
 import jsonld from 'jsonld'
-import { logger } from './logger.js'
 
 const CACHE = {
   'https://w3id.org/security/v1': {
@@ -55,36 +53,19 @@ const CACHE = {
   }
 }
 
-const nodeDocumentLoader = (jsonld as any).documentLoaders.node()
-
-const lru = new AsyncLRU({
-  max: 10,
-  load: (url, cb) => {
-    if (CACHE[url] !== undefined) {
-      logger.debug('Using cache for JSON-LD %s.', url)
-
-      return cb(null, {
-        contextUrl: null,
-        document: CACHE[url],
-        documentUrl: url
-      })
-    }
-
-    nodeDocumentLoader(url)
-      .then(value => cb(null, value))
-      .catch(err => cb(err))
-  }
-});
+const nodeDocumentLoader = (jsonld as any).documentLoaders.node();
 
 /* eslint-disable no-import-assign */
 (jsonld as any).documentLoader = (url) => {
-  return new Promise((res, rej) => {
-    lru.get(url, (err, value) => {
-      if (err) return rej(err)
-
-      return res(value)
+  if (url in CACHE) {
+    return Promise.resolve({
+      contextUrl: null,
+      document: CACHE[url],
+      documentUrl: url
     })
-  })
+  }
+
+  return nodeDocumentLoader(url)
 }
 
 export { jsonld }

server/core/initializers/checker-after-init.ts

@@ -13,6 +13,7 @@ import { OAuthClientModel } from '../models/oauth/oauth-client.js'
 import { UserModel } from '../models/user/user.js'
 import { CONFIG, getLocalConfigFilePath, isEmailEnabled, reloadConfig } from './config.js'
 import { WEBSERVER } from './constants.js'
+import { basename } from 'path'
 
 async function checkActivityPubUrls () {
   const actor = await getServerActor()
@@ -224,6 +225,11 @@ function checkStorageConfig () {
         )
       }
     }
+
+    const webVideosDirname = basename(CONFIG.STORAGE.WEB_VIDEOS_DIR)
+    if (webVideosDirname !== 'web-videos') {
+      logger.warn(`storage.web_videos configuration should have a "web-videos" directory name (current value: "${webVideosDirname}")`)
+    }
   }
 
   if (CONFIG.STORAGE.WEB_VIDEOS_DIR === CONFIG.STORAGE.REDUNDANCY_DIR) {

server/core/initializers/constants.ts

@@ -236,7 +236,7 @@ const JOB_TTL: { [id in JobType]: number } = {
   'activitypub-refresher': 60000 * 10, // 10 minutes
   'video-redundancy': 1000 * 3600 * 3, // 3 hours
   'video-live-ending': 1000 * 60 * 10, // 10 minutes
-  'generate-video-storyboard': 1000 * 60 * 30, // 30 minutes
+  'generate-video-storyboard': 1000 * 3600 * 6, // 6 hours
   'manage-video-torrent': 1000 * 3600 * 3, // 3 hours
   'move-to-object-storage': 1000 * 60 * 60 * 3, // 3 hours
   'move-to-file-system': 1000 * 60 * 60 * 3, // 3 hours
@@ -989,7 +989,7 @@ const WORKER_THREADS = {
   },
   BUILD_DIGEST: {
     CONCURRENCY: 1,
-    MAX_THREADS: Math.min(totalCPUs, 2)
+    MAX_THREADS: 1
   }
 }
 

server/core/lib/video-studio.ts

@@ -3,14 +3,11 @@ import { join } from 'path'
 import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
 import { createTorrentAndSetInfoHashFromPath } from '@server/helpers/webtorrent.js'
 import { CONFIG } from '@server/initializers/config.js'
-import { UserModel } from '@server/models/user/user.js'
 import { MUser, MVideo, MVideoFile, MVideoFullLight, MVideoWithAllFiles } from '@server/types/models/index.js'
 import { getVideoStreamDuration } from '@peertube/peertube-ffmpeg'
 import { VideoStudioEditionPayload, VideoStudioTask, VideoStudioTaskPayload } from '@peertube/peertube-models'
-import { federateVideoIfNeeded } from './activitypub/videos/index.js'
 import { JobQueue } from './job-queue/index.js'
 import { VideoStudioTranscodingJobHandler } from './runners/index.js'
-import { createOptimizeOrMergeAudioJobs } from './transcoding/create-transcoding-job.js'
 import { getTranscodingJobPriority } from './transcoding/transcoding-priority.js'
 import { buildNewFile, removeHLSPlaylist, removeWebVideoFile } from './video-file.js'
 import { VideoPathManager } from './video-path-manager.js'
@@ -108,11 +105,33 @@ export async function onVideoStudioEnded (options: {
   video.duration = await getVideoStreamDuration(outputPath)
   await video.save()
 
-  await federateVideoIfNeeded(video, false, undefined)
+  return JobQueue.Instance.createSequentialJobFlow(
+    {
+      type: 'generate-video-storyboard' as 'generate-video-storyboard',
+      payload: {
+        videoUUID: video.uuid,
+        federate: false
+      }
+    },
 
-  const user = await UserModel.loadByVideoId(video.id)
+    {
+      type: 'federate-video' as 'federate-video',
+      payload: {
+        videoUUID: video.uuid,
+        isNewVideo: false
+      }
+    },
 
-  await createOptimizeOrMergeAudioJobs({ video, videoFile: newFile, isNewVideo: false, user, videoFileAlreadyLocked: false })
+    {
+      type: 'transcoding-job-builder' as 'transcoding-job-builder',
+      payload: {
+        videoUUID: video.uuid,
+        optimizeJob: {
+          isNewVideo: false
+        }
+      }
+    }
+  )
 }
 
 // ---------------------------------------------------------------------------

server/core/lib/worker/parent-process.ts

@@ -17,7 +17,8 @@ export function downloadImageFromWorker (options: Parameters<typeof downloadImag
     downloadImageWorker = new Piscina({
       filename: new URL(join('workers', 'image-downloader.js'), import.meta.url).href,
       concurrentTasksPerWorker: WORKER_THREADS.DOWNLOAD_IMAGE.CONCURRENCY,
-      maxThreads: WORKER_THREADS.DOWNLOAD_IMAGE.MAX_THREADS
+      maxThreads: WORKER_THREADS.DOWNLOAD_IMAGE.MAX_THREADS,
+      minThreads: 1
     })
 
     downloadImageWorker.on('error', err => logger.error('Error in download image worker', { err }))
@@ -35,7 +36,8 @@ export function processImageFromWorker (options: Parameters<typeof processImage>
     processImageWorker = new Piscina({
       filename: new URL(join('workers', 'image-processor.js'), import.meta.url).href,
       concurrentTasksPerWorker: WORKER_THREADS.PROCESS_IMAGE.CONCURRENCY,
-      maxThreads: WORKER_THREADS.PROCESS_IMAGE.MAX_THREADS
+      maxThreads: WORKER_THREADS.PROCESS_IMAGE.MAX_THREADS,
+      minThreads: 1
     })
 
     processImageWorker.on('error', err => logger.error('Error in process image worker', { err }))
@@ -53,7 +55,8 @@ export function getImageSizeFromWorker (options: Parameters<typeof getImageSize>
     getImageSizeWorker = new Piscina({
       filename: new URL(join('workers', 'get-image-size.js'), import.meta.url).href,
       concurrentTasksPerWorker: WORKER_THREADS.GET_IMAGE_SIZE.CONCURRENCY,
-      maxThreads: WORKER_THREADS.GET_IMAGE_SIZE.MAX_THREADS
+      maxThreads: WORKER_THREADS.GET_IMAGE_SIZE.MAX_THREADS,
+      minThreads: 1
     })
 
     getImageSizeWorker.on('error', err => logger.error('Error in get image size worker', { err }))
@@ -72,7 +75,8 @@ export function parallelHTTPBroadcastFromWorker (options: Parameters<typeof http
       filename: new URL(join('workers', 'http-broadcast.js'), import.meta.url).href,
       // Keep it sync with job concurrency so the worker will accept all the requests sent by the parallelized jobs
       concurrentTasksPerWorker: JOB_CONCURRENCY['activitypub-http-broadcast-parallel'],
-      maxThreads: 1
+      maxThreads: 1,
+      minThreads: 1
     })
 
     parallelHTTPBroadcastWorker.on('error', err => logger.error('Error in parallel HTTP broadcast worker', { err }))
@@ -93,7 +97,8 @@ export function sequentialHTTPBroadcastFromWorker (
       filename: new URL(join('workers', 'http-broadcast.js'), import.meta.url).href,
       // Keep it sync with job concurrency so the worker will accept all the requests sent by the parallelized jobs
       concurrentTasksPerWorker: JOB_CONCURRENCY['activitypub-http-broadcast'],
-      maxThreads: 1
+      maxThreads: 1,
+      minThreads: 1
     })
 
     sequentialHTTPBroadcastWorker.on('error', err => logger.error('Error in sequential HTTP broadcast image worker', { err }))
@@ -114,7 +119,8 @@ export function httpUnicastFromWorker (
       filename: new URL(join('workers', 'http-unicast.js'), import.meta.url).href,
       // Keep it sync with job concurrency so the worker will accept all the requests sent by the parallelized jobs
       concurrentTasksPerWorker: JOB_CONCURRENCY['activitypub-http-unicast'],
-      maxThreads: 1
+      maxThreads: 1,
+      minThreads: 1
     })
 
     httpUnicastWorker.on('error', err => logger.error('Error in HTTP unicast worker', { err }))
@@ -134,7 +140,8 @@ export function signJsonLDObjectFromWorker <T> (
     signJsonLDObjectWorker = new Piscina({
       filename: new URL(join('workers', 'sign-json-ld-object.js'), import.meta.url).href,
       concurrentTasksPerWorker: WORKER_THREADS.SIGN_JSON_LD_OBJECT.CONCURRENCY,
-      maxThreads: WORKER_THREADS.SIGN_JSON_LD_OBJECT.MAX_THREADS
+      maxThreads: WORKER_THREADS.SIGN_JSON_LD_OBJECT.MAX_THREADS,
+      minThreads: 1
     })
 
     signJsonLDObjectWorker.on('error', err => logger.error('Error in sign JSONLD object worker', { err }))
@@ -155,7 +162,8 @@ export function buildDigestFromWorker (
       filename: new URL(join('workers', 'build-digest.js'), import.meta.url).href,
       // Keep it sync with job concurrency so the worker will accept all the requests sent by the parallelized jobs
       concurrentTasksPerWorker: WORKER_THREADS.BUILD_DIGEST.CONCURRENCY,
-      maxThreads: WORKER_THREADS.BUILD_DIGEST.MAX_THREADS
+      maxThreads: WORKER_THREADS.BUILD_DIGEST.MAX_THREADS,
+      minThreads: 1
     })
 
     buildDigestWorker.on('error', err => logger.error('Error in build digest worker', { err }))

support/docker/production/config/production.yaml

@@ -47,7 +47,7 @@ storage:
   tmp_persistent: '../data/tmp-persistent/' # As tmp but the directory is not cleaned up between PeerTube restarts
   bin: '../data/bin/'
   avatars: '../data/avatars/'
-  web_videos: '../data/videos/' # Keep legacy "videos" directory name to not break existing PeerTube instances on upgrade
+  web_videos: '../data/web-videos/'
   streaming_playlists: '../data/streaming-playlists'
   redundancy: '../data/redundancy/'
   logs: '../data/logs/'

support/nginx/peertube

@@ -78,7 +78,7 @@ server {
     try_files /dev/null @api;
   }
 
-  location = /api/v1/videos/upload-resumable {
+  location ~ ^/api/v1/videos/(upload-resumable|([^/]+/source/replace-resumable))$ {
     client_max_body_size    0;
     proxy_request_buffering off;
 

yarn.lock

@@ -3199,13 +3199,6 @@ assertion-error@^1.1.0:
   resolved "https://registry.yarnpkg.com/assertion-error/-/assertion-error-1.1.0.tgz#e60b6b0e8f301bd97e5375215bda406c85118c0b"
   integrity sha512-jgsaNduz+ndvGyFt3uSuWqvy4lCnIJiovtouQN5JZHOKCS2QuhEdbcQHFhVksz2N2U9hXJo8odG7ETyWlEeuDw==
 
-async-lru@^1.1.1:
-  version "1.1.3"
-  resolved "https://registry.yarnpkg.com/async-lru/-/async-lru-1.1.3.tgz#38f1a99379d4872d969136858be098ca2550f4c6"
-  integrity sha512-tPoYtjE2HCZqZHgOqFJLbgLP+IzY0V1qldIvH/lYd2opuMtO9OURxaCgNBhy5m4lUs3mz4R5iG+xI2uuqoOlwQ==
-  dependencies:
-    lru "^3.1.0"
-
 async-mutex@^0.4.0:
   version "0.4.0"
   resolved "https://registry.yarnpkg.com/async-mutex/-/async-mutex-0.4.0.tgz#ae8048cd4d04ace94347507504b3cf15e631c25f"