From e9d1cb6cc796a1ce6775e9fb1060d3b4f3bbcb97 Mon Sep 17 00:00:00 2001 From: Itms Date: Tue, 20 Aug 2024 19:15:56 +0200 Subject: [PATCH] Generate nightly SVN builds for end users, fixes #1819. --- .gitignore | 6 +- .../data/mods/public/gui/credits/credits.js | 5 +- .../dockerfiles/translations.Dockerfile | 20 --- build/jenkins/lint-translations.sh | 35 ---- .../pipelines/docker-translations.Jenkinsfile | 76 -------- .../pipelines/nightly-build.Jenkinsfile | 164 ++++++++++++++++++ source/tools/i18n/checkDiff.py | 18 +- source/tools/i18n/cleanTranslationFiles.py | 2 +- .../tools/i18n/get-nightly-translations.bat | 11 ++ source/tools/i18n/get-nightly-translations.sh | 15 ++ source/tools/i18n/maintenanceTasks.sh | 32 ---- source/tools/i18n/pullTranslations.py | 4 +- source/tools/spirv/get-nightly-shaders.bat | 5 + source/tools/spirv/get-nightly-shaders.sh | 10 ++ 14 files changed, 226 insertions(+), 177 deletions(-) delete mode 100644 build/jenkins/dockerfiles/translations.Dockerfile delete mode 100755 build/jenkins/lint-translations.sh delete mode 100644 build/jenkins/pipelines/docker-translations.Jenkinsfile create mode 100644 build/jenkins/pipelines/nightly-build.Jenkinsfile create mode 100644 source/tools/i18n/get-nightly-translations.bat create mode 100755 source/tools/i18n/get-nightly-translations.sh delete mode 100755 source/tools/i18n/maintenanceTasks.sh create mode 100644 source/tools/spirv/get-nightly-shaders.bat create mode 100644 source/tools/spirv/get-nightly-shaders.sh diff --git a/.gitignore b/.gitignore index ae0f5e476e..9570feb7e5 100644 --- a/.gitignore +++ b/.gitignore @@ -64,7 +64,8 @@ test_*.cpp stub_*.cpp !test_setup.cpp -# Binary files generated in nightly builds +# Translation files +binaries/data/mods/public/gui/credits/texts/translators.json *.po *.pot @@ -86,6 +87,9 @@ stub_*.cpp binaries/data/mods/**/*.bmp binaries/data/mods/**/*.jpg +# Vulkan SPIR-V shaders +binaries/data/mods/*/shaders/spirv + # Windows specific data desktop.ini Thumbs.db diff --git a/binaries/data/mods/public/gui/credits/credits.js b/binaries/data/mods/public/gui/credits/credits.js index b9a52ccecd..e5bae01ff6 100644 --- a/binaries/data/mods/public/gui/credits/credits.js +++ b/binaries/data/mods/public/gui/credits/credits.js @@ -37,7 +37,10 @@ function init() let json = Engine.ReadJSONFile("gui/credits/texts/" + category + ".json"); if (!json || !json.Content) { - error("Could not load credits for " + category + "!"); + if (category == "translators") + warn("Translators credits are not present, pull translations from the nightly build to get them.") + else + error("Could not load credits for " + category + "!"); continue; } translateObjectKeys(json, ["Title", "Subtitle", "LangName"]); diff --git a/build/jenkins/dockerfiles/translations.Dockerfile b/build/jenkins/dockerfiles/translations.Dockerfile deleted file mode 100644 index 90fb85712c..0000000000 --- a/build/jenkins/dockerfiles/translations.Dockerfile +++ /dev/null @@ -1,20 +0,0 @@ -FROM debian:buster - -ARG DEBIAN_FRONTEND=noninteractive -ARG DEBCONF_NOWARNINGS="yes" -RUN useradd -ms /bin/bash --uid 1006 builder -RUN apt-get -qq update && apt-get install -qqy --no-install-recommends \ - curl \ - python3-dev \ - python3-pip \ - git \ - subversion \ - && apt-get clean - -ENV SHELL /bin/bash -RUN pip3 install setuptools wheel -RUN pip3 install lxml babel -RUN curl -o- https://raw.githubusercontent.com/transifex/cli/master/install.sh | bash -RUN install tx /usr/bin/tx -USER builder -COPY --chown=builder transifexrc /home/builder/.transifexrc diff --git a/build/jenkins/lint-translations.sh b/build/jenkins/lint-translations.sh deleted file mode 100755 index a29260181a..0000000000 --- a/build/jenkins/lint-translations.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/sh - -# This script uses the Dennis PO(T) linter to find issues. -# See http://dennis.readthedocs.io/en/latest/index.html for -# installation instructions. - -set +e # Lint everything without failing - -# Move to the root of the repository (this script is in build/jenkins/) -cd "$(dirname $0)"/../../ - -# Configuration for the linter -# Ignore -# - W302: Translated string is identical to source string -parameters='--excluderules W302' - -# Run lint and output to a file that will be posted on Phabricator -echo "Running Dennis..." -{ - echo "Linting templates..." - echo "Engine" - dennis-cmd lint ${parameters} binaries/data/l10n/*.pot - echo "Mod mod" - dennis-cmd lint ${parameters} binaries/data/mods/mod/l10n/*.pot - echo "Public mod" - dennis-cmd lint ${parameters} binaries/data/mods/public/l10n/*.pot - - echo "Linting translations..." - echo "Engine" - dennis-cmd lint ${parameters} binaries/data/l10n/*.po - echo "Mod mod" - dennis-cmd lint ${parameters} binaries/data/mods/mod/l10n/*.po - echo "Public mod" - dennis-cmd lint ${parameters} binaries/data/mods/public/l10n/*.po -} > .phabricator-comment diff --git a/build/jenkins/pipelines/docker-translations.Jenkinsfile b/build/jenkins/pipelines/docker-translations.Jenkinsfile deleted file mode 100644 index 4d73879db8..0000000000 --- a/build/jenkins/pipelines/docker-translations.Jenkinsfile +++ /dev/null @@ -1,76 +0,0 @@ -/* Copyright (C) 2023 Wildfire Games. - * This file is part of 0 A.D. - * - * 0 A.D. is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 2 of the License, or - * (at your option) any later version. - * - * 0 A.D. is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with 0 A.D. If not, see . - */ - -// This pipeline is used to update translations to and from Transifex. - -pipeline { - agent { - node { - label 'LinuxSlave' - customWorkspace '/zpool0/trunk' - } - } - - stages { - stage("Prepare volume") { - steps { - sh "sudo zfs clone zpool0/trunk@latest zpool0/translations" - ws('/zpool0/translations') { - sh "svn revert . -R" - sh "svn st | cut -c 9- | xargs rm -rf" - sh "svn up" - } - } - } - - stage("Update translations") { - steps { - ws('/zpool0/translations') { - withDockerContainer("0ad-translations:latest") { - sh "python3 source/tools/i18n/updateTemplates.py" - withCredentials([string(credentialsId: 'redacted', variable: 'token')]) { - sh 'TX_TOKEN="$token" python3 source/tools/i18n/pullTranslations.py' - } - dir("source/tools/i18n/") { - sh "python3 generateDebugTranslation.py --long" - } - sh "python3 source/tools/i18n/cleanTranslationFiles.py" - sh "python3 source/tools/i18n/checkDiff.py --verbose" - sh "python3 source/tools/i18n/creditTranslators.py" - } - } - } - } - - stage("Commit") { - steps { - ws('/zpool0/translations') { - withCredentials([usernamePassword(credentialsId: 'redacted', passwordVariable: 'SVNPASS', usernameVariable: 'SVNUSER')]) { - sh 'svn relocate --username $SVNUSER --password $SVNPASS --no-auth-cache https://svn.wildfiregames.com/svn/ps/trunk' - sh 'svn commit --username $SVNUSER --password $SVNPASS --no-auth-cache --non-interactive -m "[i18n] Updated POT and PO files."' - } - } - } - } - } - post { - always { - sleep 10 - sh "sudo zfs destroy zpool0/translations" - } - } -} diff --git a/build/jenkins/pipelines/nightly-build.Jenkinsfile b/build/jenkins/pipelines/nightly-build.Jenkinsfile new file mode 100644 index 0000000000..7a3a3c2187 --- /dev/null +++ b/build/jenkins/pipelines/nightly-build.Jenkinsfile @@ -0,0 +1,164 @@ +/* Copyright (C) 2024 Wildfire Games. + * This file is part of 0 A.D. + * + * 0 A.D. is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * 0 A.D. is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with 0 A.D. If not, see . + */ + +// This pipeline is used to generate the nightly builds. + +def visualStudioPath = "\"C:\\Program Files (x86)\\Microsoft Visual Studio\\2017\\Community\\MSBuild\\15.0\\Bin\\MSBuild.exe\"" +def buildOptions = "/p:PlatformToolset=v141_xp /p:XPDeprecationWarning=false /t:pyrogenesis /t:AtlasUI /m:2 /nologo -clp:Warningsonly -clp:ErrorsOnly" + +def gitHash = "" +def buildSPIRV = false + +pipeline { + agent { + node { + label 'WindowsAgent' + customWorkspace 'workspace/nightly-build' + } + } + + parameters { + booleanParam(name: 'NEW_REPO', defaultValue: false, description: 'If a brand new nightly repo is being generated, do not attempt to identify unchanged translations.') + stashedFile(name: 'spirv_rules', description: 'rules.json file for generation of SPIR-V shaders. Needed for a new repo, else the existing rules files will be used. Uploading a new rules file will force-rebuild the shaders.') + } + + stages { + stage("Generate build version") { + steps { + checkout scmGit(branches: [[name: "${GIT_BRANCH}"]], extensions: [localBranch()]) + script { gitHash = bat(script:"@git rev-parse --short HEAD", returnStdout: true ).trim() } + bat "cd build\\build_version && build_version.bat" + } + } + + stage("Pull game assets") { + steps { + bat "git lfs pull" + } + } + + stage("Check for shader changes") { + when { + changeset 'binaries/data/mods/**/shaders/**/*.xml' + } + steps { + script { buildSPIRV = true } + } + } + + stage ("Pre-build") { + steps { + bat "cd libraries && get-windows-libs.bat" + bat "(robocopy C:\\wxwidgets3.0.4\\lib libraries\\win32\\wxwidgets\\lib /MIR /NDL /NJH /NJS /NP /NS /NC) ^& IF %ERRORLEVEL% LEQ 1 exit 0" + bat "(robocopy C:\\wxwidgets3.0.4\\include libraries\\win32\\wxwidgets\\include /MIR /NDL /NJH /NJS /NP /NS /NC) ^& IF %ERRORLEVEL% LEQ 1 exit 0" + bat "cd build\\workspaces && update-workspaces.bat --atlas --without-pch --without-tests" + } + } + + stage ("Build") { + steps { + bat("cd build\\workspaces\\vs2017 && ${visualStudioPath} pyrogenesis.sln /p:Configuration=Release ${buildOptions}") + } + } + + stage("Check-in SPIRV generation rules") { + when { + expression { env.spirv_rules_FILENAME } + } + steps { + unstash 'spirv_rules' + bat "move spirv_rules source\\tools\\spirv\\rules.json" + script { buildSPIRV = true } + } + } + + stage("Mirror to SVN") { + steps { + ws("workspace/nightly-svn") { + svn(url: "https://svn.wildfiregames.com/nightly-build/trunk", changelog: false) + bat "svn revert -R . && svn cleanup" + script { env.NIGHTLY_PATH = env.WORKSPACE } + } + bat """ + (robocopy . %NIGHTLY_PATH% ^ + /XD .git ^ + /XF .gitattributes ^ + /XF .gitignore ^ + /XD %cd%\\binaries\\system ^ + /XD %cd%\\build\\workspaces\\vs2017 ^ + /XD %cd%\\libraries\\source\\.svn ^ + /XD %cd%\\libraries\\win32\\.svn ^ + /XD %cd%\\libraries\\win32\\wxwidgets\\include ^ + /XD %cd%\\libraries\\win32\\wxwidgets\\lib ^ + /XD .svn ^ + /XD %NIGHTLY_PATH%\\binaries\\data\\mods\\mod\\shaders\\spirv ^ + /XD %NIGHTLY_PATH%\\binaries\\data\\mods\\public\\shaders\\spirv ^ + /MIR /NDL /NJH /NJS /NP /NS /NC) ^& IF %ERRORLEVEL% LEQ 1 exit 0 + """ + bat """ + (robocopy binaries\\system ..\\nightly-svn\\binaries\\system ^ + /XF *.exp ^ + /XF *.lib ^ + /MIR /NDL /NJH /NJS /NP /NS /NC) ^& IF %ERRORLEVEL% LEQ 1 exit 0 + """ + } + } + + stage("Recompile SPIR-V shaders") { + when { + expression { buildSPIRV } + } + steps { + ws("workspace/nightly-svn") { + bat "python source/tools/spirv/compile.py -d binaries/data/mods/mod binaries/data/mods/mod source/tools/spirv/rules.json binaries/data/mods/mod" + bat "python source/tools/spirv/compile.py -d binaries/data/mods/mod binaries/data/mods/public source/tools/spirv/rules.json binaries/data/mods/public" + } + } + } + + stage("Update translations") { + steps { + ws("workspace/nightly-svn") { + bat "cd source\\tools\\i18n && python updateTemplates.py" + withCredentials([string(credentialsId: 'TX_TOKEN', variable: 'TX_TOKEN')]) { + bat "cd source\\tools\\i18n && python pullTranslations.py" + } + bat "cd source\\tools\\i18n && python generateDebugTranslation.py --long" + bat "cd source\\tools\\i18n && python cleanTranslationFiles.py" + script { if (!params.NEW_REPO) { + bat "python source\\tools\\i18n\\checkDiff.py --verbose" + }} + bat "cd source\\tools\\i18n && python creditTranslators.py" + } + } + } + + stage("Commit") { + steps { + ws("workspace/nightly-svn") { + bat "svn add --force ." + bat "(for /F \"tokens=* delims=! \" %%A in ('svn status ^| findstr /R \"^!\"') do (svn delete %%A)) || (echo No deleted files found) ^& exit 0" + withCredentials([usernamePassword(credentialsId: 'nightly-autobuild', passwordVariable: 'SVNPASS', usernameVariable: 'SVNUSER')]) { + script { env.GITHASH = gitHash + bat 'svn commit --username %SVNUSER% --password %SVNPASS% --no-auth-cache --non-interactive -m "Nightly build for %GITHASH% (%DATE%)"' + } + } + } + } + } + } +} diff --git a/source/tools/i18n/checkDiff.py b/source/tools/i18n/checkDiff.py index 6f3d5484df..727d68208d 100644 --- a/source/tools/i18n/checkDiff.py +++ b/source/tools/i18n/checkDiff.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2021 Wildfire Games. +# Copyright (C) 2024 Wildfire Games. # This file is part of 0 A.D. # # 0 A.D. is free software: you can redistribute it and/or modify @@ -29,9 +29,9 @@ def get_diff(): diff_process = subprocess.run(["svn", "diff", "binaries"], capture_output=True) if diff_process.returncode != 0: - print(f"Error running svn diff: {diff_process.stderr.decode()}. Exiting.") + print(f"Error running svn diff: {diff_process.stderr.decode('utf-8')}. Exiting.") return - return io.StringIO(diff_process.stdout.decode()) + return io.StringIO(diff_process.stdout.decode('utf-8')) def check_diff(diff : io.StringIO, verbose = False) -> List[str]: """Run through a diff of .po files and check that some of the changes @@ -45,10 +45,10 @@ def check_diff(diff : io.StringIO, verbose = False) -> List[str]: l = diff.readline() while l: if l.startswith("Index: binaries"): - if not l.endswith(".pot\n") and not l.endswith(".po\n"): + if not l.strip().endswith(".pot") and not l.strip().endswith(".po"): curfile = None else: - curfile = l[7:-1] + curfile = l[7:].strip() files.add(curfile) # skip patch header diff.readline() @@ -60,7 +60,7 @@ def check_diff(diff : io.StringIO, verbose = False) -> List[str]: if l[0] != '-' and l[0] != '+': l = diff.readline() continue - if l[1] == '\n' or (l[1] == '#' and l[2] == ":"): + if l[1:].strip() == "" or (l[1] == '#' and l[2] == ':'): l = diff.readline() continue if "# Copyright (C)" in l or "POT-Creation-Date:" in l or "PO-Revision-Date" in l or "Last-Translator" in l: @@ -78,7 +78,7 @@ def check_diff(diff : io.StringIO, verbose = False) -> List[str]: def revert_files(files: List[str], verbose = False): revert_process = subprocess.run(["svn", "revert"] + files, capture_output=True) if revert_process.returncode != 0: - print(f"Warning: Some files could not be reverted. Error: {revert_process.stderr.decode()}") + print(f"Warning: Some files could not be reverted. Error: {revert_process.stderr.decode('utf-8')}") if verbose: for file in files: print(f"Reverted {file}") @@ -96,9 +96,9 @@ def add_untracked(verbose = False): continue # Ignore non PO files. This is important so that the translator credits # correctly be updated, note however the script assumes a pristine SVN otherwise. - if not line.endswith(".po") and not line.endswith(".pot"): - continue file = line[1:].strip() + if not file.endswith(".po") and not file.endswith(".pot"): + continue add_process = subprocess.run(["svn", "add", file, "--parents"], capture_output=True) if add_process.stderr != b'': print(f"Warning: file {file} could not be added.") diff --git a/source/tools/i18n/cleanTranslationFiles.py b/source/tools/i18n/cleanTranslationFiles.py index 63c67f8bd1..7ad3981b2a 100644 --- a/source/tools/i18n/cleanTranslationFiles.py +++ b/source/tools/i18n/cleanTranslationFiles.py @@ -43,7 +43,7 @@ def main(): for file in files: usernames = [] reached = False - for line in fileinput.input(file.replace("\\", "/"), inplace=True): + for line in fileinput.input(file.replace("\\", "/"), inplace=True, encoding="utf-8"): if reached: if line == "# \n": line = "" diff --git a/source/tools/i18n/get-nightly-translations.bat b/source/tools/i18n/get-nightly-translations.bat new file mode 100644 index 0000000000..2eca111398 --- /dev/null +++ b/source/tools/i18n/get-nightly-translations.bat @@ -0,0 +1,11 @@ +rem **Download translations from the latest nightly build** + +rem **This will overwrite any uncommitted changes to messages.json files** + +svn export --force --depth files https://svn.wildfiregames.com/nightly-build/trunk/binaries/data/l10n ..\..\..\binaries\data\l10n + +for %%m in (mod public) do ( + svn export --force --depth files https://svn.wildfiregames.com/nightly-build/trunk/binaries/data/mods/%%m/l10n ..\..\..\binaries\data\mods\%%m\l10n +) + +svn export --force https://svn.wildfiregames.com/nightly-build/trunk/binaries/data/mods/public/gui/credits/texts/translators.json ..\..\..\binaries\data\mods\public\gui\credits\texts\translators.json diff --git a/source/tools/i18n/get-nightly-translations.sh b/source/tools/i18n/get-nightly-translations.sh new file mode 100755 index 0000000000..e3b40305f0 --- /dev/null +++ b/source/tools/i18n/get-nightly-translations.sh @@ -0,0 +1,15 @@ +#!/bin/sh +set -ev + +# Download translations from the latest nightly build +# This will overwrite any uncommitted changes to messages.json files + +cd "$(dirname $0)" + +svn export --force --depth files https://svn.wildfiregames.com/nightly-build/trunk/binaries/data/l10n ../../../binaries/data/l10n + +for m in "mod public"; do + svn export --force --depth files https://svn.wildfiregames.com/nightly-build/trunk/binaries/data/mods/${m}/l10n ../../../binaries/data/mods/${m}/l10n +done + +svn export --force https://svn.wildfiregames.com/nightly-build/trunk/binaries/data/mods/public/gui/credits/texts/translators.json ../../../binaries/data/mods/public/gui/credits/texts/translators.json diff --git a/source/tools/i18n/maintenanceTasks.sh b/source/tools/i18n/maintenanceTasks.sh deleted file mode 100755 index e738bb4477..0000000000 --- a/source/tools/i18n/maintenanceTasks.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/sh - -# Regenerates the POT files, downloads the latest translations from Transifex, -# and prepares the commit of the updated POT and PO files. - -SCRIPT_PATH="`dirname \"$0\"`" - - -# POT Generation ############################################################## - -echo ":: Regenerating the translation templates…" -python3 "${SCRIPT_PATH}/updateTemplates.py" - - -# PO Download ################################################################# - -echo ":: Downloading translations from Transifex…" -python3 "${SCRIPT_PATH}/pullTranslations.py" - - -# Pre-Commit Cleanup ######################################################### - -echo ":: Removing unneeded data from the .po files…" -python3 "${SCRIPT_PATH}/cleanTranslationFiles.py" - -echo ":: Reverting unnecessary changes…" -python3 "${SCRIPT_PATH}/checkDiff.py" - -# Commit ###################################################################### - -echo ":: Done" -echo " Now you can commit your changes to the server." diff --git a/source/tools/i18n/pullTranslations.py b/source/tools/i18n/pullTranslations.py index 49b36f8379..1fe80fec97 100644 --- a/source/tools/i18n/pullTranslations.py +++ b/source/tools/i18n/pullTranslations.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2023 Wildfire Games. +# Copyright (C) 2024 Wildfire Games. # This file is part of 0 A.D. # # 0 A.D. is free software: you can redistribute it and/or modify @@ -29,7 +29,7 @@ def main(): path = os.path.join(root, folder) os.chdir(path) print(f"INFO: Starting to pull translations in {path}...") - subprocess.run(["tx", "pull", "-f"]) + subprocess.run(["tx", "pull", "-a", "-f"]) if __name__ == "__main__": diff --git a/source/tools/spirv/get-nightly-shaders.bat b/source/tools/spirv/get-nightly-shaders.bat new file mode 100644 index 0000000000..d34b91d5f4 --- /dev/null +++ b/source/tools/spirv/get-nightly-shaders.bat @@ -0,0 +1,5 @@ +rem **Download SPIR-V shaders from the latest nightly build** + +for %%m in (mod public) do ( + svn export --force https://svn.wildfiregames.com/nightly-build/trunk/binaries/data/mods/%%m/shaders/spirv ..\..\..\binaries\data\mods\%%m\shaders\spirv +) diff --git a/source/tools/spirv/get-nightly-shaders.sh b/source/tools/spirv/get-nightly-shaders.sh new file mode 100644 index 0000000000..8b5ec3a0bc --- /dev/null +++ b/source/tools/spirv/get-nightly-shaders.sh @@ -0,0 +1,10 @@ +#!/bin/sh +set -ev + +# Download SPIR-V shaders from the latest nightly build + +cd "$(dirname $0)" + +for m in "mod public"; do + svn export --force https://svn.wildfiregames.com/nightly-build/trunk/binaries/data/mods/${m}/shaders/spirv ../../../binaries/data/mods/${m}/shaders/spirv +done