1
0
Fork 0
peertube/scripts/ci.sh

159 lines
5.5 KiB
Bash
Raw Normal View History

2020-07-30 10:25:00 -04:00
#!/bin/bash
2018-03-27 04:35:12 -04:00
set -eu
2017-10-31 12:23:57 -04:00
if [ $# -eq 0 ]; then
echo "Need test suite argument."
exit -1
fi
2020-07-30 08:44:08 -04:00
retries=3
speedFactor="${2:-1}"
2020-07-30 08:44:08 -04:00
runJSTest () {
2020-12-10 10:06:09 -05:00
jobname=$1
shift
2020-07-30 08:44:08 -04:00
jobs=$1
shift
2020-12-10 10:06:09 -05:00
2020-07-30 08:44:08 -04:00
files=$@
echo $files
2020-12-10 10:06:09 -05:00
joblog="$jobname-ci.log"
2020-12-11 04:36:05 -05:00
parallel -j $jobs --retries $retries \
"echo Trying {} >> $joblog; npm run mocha -- --timeout 30000 --no-config -c --exit --bail {}" \
2020-07-30 08:44:08 -04:00
::: $files
2020-12-10 10:06:09 -05:00
2022-07-13 05:34:48 -04:00
cat "$joblog" | sort | uniq -c
2020-12-10 10:06:09 -05:00
rm "$joblog"
2020-07-30 08:44:08 -04:00
}
findTestFiles () {
exception="-not -name index.js -not -name index.ts -not -name *.d.ts"
2021-06-17 04:43:34 -04:00
if [ ! -z ${2+x} ]; then
exception="$exception -not -name $2"
fi
find $1 -type f \( -name "*.js" -o -name "*.ts" \) $exception | xargs echo
2020-07-30 08:44:08 -04:00
}
2021-12-28 04:18:15 -05:00
if [ "$1" = "types-package" ]; then
2021-12-28 04:25:30 -05:00
npm run generate-types-package 0.0.0
# Test on in independent directory
rm -fr /tmp/types-generator
mkdir -p /tmp/types-generator
cp -r packages/types-generator/tests /tmp/types-generator/tests
cp -r packages/types-generator/dist /tmp/types-generator/dist
(cd /tmp/types-generator/dist && npm install)
2023-10-11 09:29:11 -04:00
npm run tsc -- --noEmit --esModuleInterop --moduleResolution node16 --module Node16 /tmp/types-generator/tests/test.ts
rm -r /tmp/types-generator
2021-12-24 07:16:55 -05:00
elif [ "$1" = "client" ]; then
2020-12-11 04:36:05 -05:00
npm run build
npm run build:tests
2020-07-30 08:44:08 -04:00
feedsFiles=$(findTestFiles ./packages/tests/dist/feeds)
clientFiles=$(findTestFiles ./packages/tests/dist/client)
2024-01-18 03:09:23 -05:00
miscFiles="./packages/tests/dist/misc-endpoints.js ./packages/tests/dist/nginx.js"
2022-03-17 05:50:26 -04:00
# Not in their own task, they need an index.html
pluginFiles="./packages/tests/dist/plugins/html-injection.js ./packages/tests/dist/api/server/plugins.js"
MOCHA_PARALLEL=true runJSTest "$1" $((2*$speedFactor)) $feedsFiles $miscFiles $pluginFiles $clientFiles
2020-07-30 08:44:08 -04:00
# Use TS tests directly because we import server files
helperFiles=$(findTestFiles ./packages/tests/src/server-helpers)
libFiles=$(findTestFiles ./packages/tests/src/server-lib)
npm run mocha -- --timeout 30000 -c --exit --bail $libFiles $helperFiles
2021-06-17 04:43:34 -04:00
elif [ "$1" = "cli-plugin" ]; then
2023-03-27 02:43:50 -04:00
# Simulate HTML
mkdir -p "./client/dist/en-US/"
cp "./client/src/index.html" "./client/dist/en-US/index.html"
2017-10-31 12:33:23 -04:00
npm run build:server
npm run build:tests
npm run build:peertube-cli
2020-07-30 08:44:08 -04:00
# html-injection test needs an HTML file
pluginsFiles=$(findTestFiles ./packages/tests/dist/plugins html-injection.js)
cliFiles=$(findTestFiles ./packages/tests/dist/cli)
2020-07-30 08:44:08 -04:00
MOCHA_PARALLEL=true runJSTest "$1" $((2*$speedFactor)) $pluginsFiles
runJSTest "$1" 1 $cliFiles
2018-08-22 10:15:35 -04:00
elif [ "$1" = "api-1" ]; then
npm run build:server
npm run build:tests
2020-07-30 08:44:08 -04:00
checkParamFiles=$(findTestFiles ./packages/tests/dist/api/check-params)
notificationsFiles=$(findTestFiles ./packages/tests/dist/api/notifications)
searchFiles=$(findTestFiles ./packages/tests/dist/api/search)
2020-07-30 08:44:08 -04:00
MOCHA_PARALLEL=true runJSTest "$1" $((3*$speedFactor)) $notificationsFiles $searchFiles $checkParamFiles
2018-08-22 10:15:35 -04:00
elif [ "$1" = "api-2" ]; then
npm run build:server
npm run build:tests
2020-07-30 08:44:08 -04:00
liveFiles=$(findTestFiles ./packages/tests/dist/api/live)
# plugins test needs an HTML file
serverFiles=$(findTestFiles ./packages/tests/dist/api/server plugins.js)
usersFiles=$(findTestFiles ./packages/tests/dist/api/users)
2020-07-30 08:44:08 -04:00
MOCHA_PARALLEL=true runJSTest "$1" $((3*$speedFactor)) $liveFiles $serverFiles $usersFiles
2018-08-22 10:15:35 -04:00
elif [ "$1" = "api-3" ]; then
npm run build:server
npm run build:tests
2020-07-30 08:44:08 -04:00
videosFiles=$(findTestFiles ./packages/tests/dist/api/videos)
viewsFiles=$(findTestFiles ./packages/tests/dist/api/views)
2020-07-30 08:44:08 -04:00
MOCHA_PARALLEL=true runJSTest "$1" $((3*$speedFactor)) $viewsFiles $videosFiles
2018-11-15 03:24:56 -05:00
elif [ "$1" = "api-4" ]; then
2019-05-24 11:42:35 -04:00
npm run build:server
npm run build:tests
2020-07-30 08:44:08 -04:00
moderationFiles=$(findTestFiles ./packages/tests/dist/api/moderation)
redundancyFiles=$(findTestFiles ./packages/tests/dist/api/redundancy)
objectStorageFiles=$(findTestFiles ./packages/tests/dist/api/object-storage)
activitypubFiles=$(findTestFiles ./packages/tests/dist/api/activitypub)
2020-07-30 08:44:08 -04:00
MOCHA_PARALLEL=true runJSTest "$1" $((2*$speedFactor)) $moderationFiles $redundancyFiles $activitypubFiles $objectStorageFiles
elif [ "$1" = "api-5" ]; then
npm run build:server
npm run build:tests
transcodingFiles=$(findTestFiles ./packages/tests/dist/api/transcoding)
runnersFiles=$(findTestFiles ./packages/tests/dist/api/runners)
MOCHA_PARALLEL=true runJSTest "$1" $((2*$speedFactor)) $transcodingFiles $runnersFiles
2020-04-27 04:58:09 -04:00
elif [ "$1" = "external-plugins" ]; then
npm run build:server
npm run build:tests
2023-04-21 09:05:27 -04:00
npm run build:peertube-runner
2020-07-30 08:44:08 -04:00
externalPluginsFiles=$(findTestFiles ./packages/tests/dist/external-plugins)
peertubeRunnerFiles=$(findTestFiles ./packages/tests/dist/peertube-runner)
2020-07-30 08:44:08 -04:00
runJSTest "$1" 1 $externalPluginsFiles
MOCHA_PARALLEL=true runJSTest "$1" $((2*$speedFactor)) $peertubeRunnerFiles
2017-10-31 12:23:57 -04:00
elif [ "$1" = "lint" ]; then
npm run eslint -- --ext .ts "server/**/*.ts" "scripts/**/*.ts" "packages/**/*.ts" "apps/**/*.ts"
2020-01-21 08:28:28 -05:00
npm run swagger-cli -- validate support/doc/api/openapi.yaml
2018-09-21 03:26:02 -04:00
2023-08-18 08:12:32 -04:00
( cd client && npm run lint )
feat(transcription): groundwork chore: fiddling around some more chore: add ctranslate2 and timestamped chore: add performance markers chore: refactor test chore: change worflow name chore: ensure Python3 chore(duration): convert to chai/mocha syntahx chore(transcription): add individual tests for others transcribers chore(transcription): implement formats test of all implementations Also compare result of other implementation to the reference implementation chore(transcription): add more test case with other language and models size and local model chore(test): wip ctranslate 2 adapat chore(transcription): wip transcript file and benchmark chore(test): clean a bit chore(test): clean a bit chore(test): refacto timestamed spec chore(test): update workflow chore(test): fix glob expansion with sh chore(test): extract some hw info chore(test): fix async tests chore(benchmark): add model info feat(transcription): allow use of a local mode in timestamped-whisper feat(transcription): extract run and profiling info in own value object feat(transcription): extract run concept in own class an run more bench chore(transcription): somplify run object only a uuid is now needed and add more benchmark scenario docs(transcription): creates own package readme docs(transcription): add local model usage docs(transcription): update README fix(transcription): use fr video for better comparison chore(transcription): make openai comparison passed docs(timestamped): clea chore(transcription): change transcribers transcribe method signature Introduce whisper builtin model. fix(transcription): activate language detection Forbid transcript creation without a language. Add `languageDetection` flag to an engine and some assertions. Fix an issue in `whisper-ctranslate2` : https://github.com/Softcatala/whisper-ctranslate2/pull/93 chore(transcription): use PeerTube time helpers instead of custom ones Update existing time function to output an integer number of seconds and add a ms human-readable time formatter with hints of tests. chore(transcription): use PeerTube UUID helpers chore(transcription): enable CER evaluation Thanks to this recent fix in Jiwer <3 https://github.com/jitsi/jiwer/issues/873 chore(jiwer): creates JiWer package I'm not very happy with the TranscriptFileEvaluator constructor... suggestions ? chore(JiWer): add usage in README docs(jiwer): update JiWer readme chore(transcription): use FunMOOC video in fixtures chore(transcription): add proper english video fixture chore(transcription): use os tmp directory where relevant chore(transcription): fix jiwer cli test reference.txt chore(transcription): move benchmark out of tests chore(transcription): remove transcription workflow docs(transcription): add benchmark info fix(transcription): use ms precision in other transcribers chore(transcription): simplify most of the tests chore(transcription): remove slashes when building path with join chore(transcription): make fromPath method async chore(transcription): assert path to model is a directory for CTranslate2 transcriber chore(transcription): ctranslate2 assertion chore(transcription): ctranslate2 assertion chore(transcription): add preinstall script for Python dependencies chore(transcription): add download and unzip utils functions chore(transcription): add download and unzip utils functions chore(transcription): download & unzip models fixtures chore(transcription): zip chore(transcription): raise download file test timeout chore(transcription): simplify download file test chore(transcription): add transcriptions test to CI chore(transcription): raise test preconditions timeout chore(transcription): run preinstall scripts before running ci chore(transcription): create dedicated tmp folder for transcriber tests chore(transcription): raise timeout some more chore(transcription): raise timeout some more chore(transcription): raise timeout some more chore(transcription): raise timeout some more chore(transcription): raise timeout some more chore(transcription): raise timeout some more chore(transcription): raise timeout some more chore(transcription): raise timeout some more chore(transcription): use short video for local model test chore(transcription): raise timeout some more chore(transcription): raise timeout some more chore(transcription): raise timeout some more chore(transcription): setup verbosity based on NODE_ENV value
2024-03-29 05:34:45 -04:00
elif [ "$1" = "transcription" ]; then
npm run preinstall --workspace=@peertube/peertube-transcription --workspace=@peertube/peertube-jiwer
npm run build:server
npm run build:tests
transcriptionFiles=$(findTestFiles ./packages/tests/dist/transcription)
jiwerFiles=$(findTestFiles ./packages/tests/dist/jiwer)
MOCHA_PARALLEL=true runJSTest "$1" $((3*$speedFactor)) $transcriptionFiles $jiwerFiles
2017-10-31 12:23:57 -04:00
fi