diff --git a/.gitignore b/.gitignore
index ae0f5e476e..9570feb7e5 100644
--- a/.gitignore
+++ b/.gitignore
@@ -64,7 +64,8 @@ test_*.cpp
stub_*.cpp
!test_setup.cpp
-# Binary files generated in nightly builds
+# Translation files
+binaries/data/mods/public/gui/credits/texts/translators.json
*.po
*.pot
@@ -86,6 +87,9 @@ stub_*.cpp
binaries/data/mods/**/*.bmp
binaries/data/mods/**/*.jpg
+# Vulkan SPIR-V shaders
+binaries/data/mods/*/shaders/spirv
+
# Windows specific data
desktop.ini
Thumbs.db
diff --git a/binaries/data/mods/public/gui/credits/credits.js b/binaries/data/mods/public/gui/credits/credits.js
index b9a52ccecd..e5bae01ff6 100644
--- a/binaries/data/mods/public/gui/credits/credits.js
+++ b/binaries/data/mods/public/gui/credits/credits.js
@@ -37,7 +37,10 @@ function init()
let json = Engine.ReadJSONFile("gui/credits/texts/" + category + ".json");
if (!json || !json.Content)
{
- error("Could not load credits for " + category + "!");
+ if (category == "translators")
+ warn("Translators credits are not present, pull translations from the nightly build to get them.")
+ else
+ error("Could not load credits for " + category + "!");
continue;
}
translateObjectKeys(json, ["Title", "Subtitle", "LangName"]);
diff --git a/build/jenkins/dockerfiles/translations.Dockerfile b/build/jenkins/dockerfiles/translations.Dockerfile
deleted file mode 100644
index 90fb85712c..0000000000
--- a/build/jenkins/dockerfiles/translations.Dockerfile
+++ /dev/null
@@ -1,20 +0,0 @@
-FROM debian:buster
-
-ARG DEBIAN_FRONTEND=noninteractive
-ARG DEBCONF_NOWARNINGS="yes"
-RUN useradd -ms /bin/bash --uid 1006 builder
-RUN apt-get -qq update && apt-get install -qqy --no-install-recommends \
- curl \
- python3-dev \
- python3-pip \
- git \
- subversion \
- && apt-get clean
-
-ENV SHELL /bin/bash
-RUN pip3 install setuptools wheel
-RUN pip3 install lxml babel
-RUN curl -o- https://raw.githubusercontent.com/transifex/cli/master/install.sh | bash
-RUN install tx /usr/bin/tx
-USER builder
-COPY --chown=builder transifexrc /home/builder/.transifexrc
diff --git a/build/jenkins/lint-translations.sh b/build/jenkins/lint-translations.sh
deleted file mode 100755
index a29260181a..0000000000
--- a/build/jenkins/lint-translations.sh
+++ /dev/null
@@ -1,35 +0,0 @@
-#!/bin/sh
-
-# This script uses the Dennis PO(T) linter to find issues.
-# See http://dennis.readthedocs.io/en/latest/index.html for
-# installation instructions.
-
-set +e # Lint everything without failing
-
-# Move to the root of the repository (this script is in build/jenkins/)
-cd "$(dirname $0)"/../../
-
-# Configuration for the linter
-# Ignore
-# - W302: Translated string is identical to source string
-parameters='--excluderules W302'
-
-# Run lint and output to a file that will be posted on Phabricator
-echo "Running Dennis..."
-{
- echo "Linting templates..."
- echo "Engine"
- dennis-cmd lint ${parameters} binaries/data/l10n/*.pot
- echo "Mod mod"
- dennis-cmd lint ${parameters} binaries/data/mods/mod/l10n/*.pot
- echo "Public mod"
- dennis-cmd lint ${parameters} binaries/data/mods/public/l10n/*.pot
-
- echo "Linting translations..."
- echo "Engine"
- dennis-cmd lint ${parameters} binaries/data/l10n/*.po
- echo "Mod mod"
- dennis-cmd lint ${parameters} binaries/data/mods/mod/l10n/*.po
- echo "Public mod"
- dennis-cmd lint ${parameters} binaries/data/mods/public/l10n/*.po
-} > .phabricator-comment
diff --git a/build/jenkins/pipelines/docker-translations.Jenkinsfile b/build/jenkins/pipelines/docker-translations.Jenkinsfile
deleted file mode 100644
index 4d73879db8..0000000000
--- a/build/jenkins/pipelines/docker-translations.Jenkinsfile
+++ /dev/null
@@ -1,76 +0,0 @@
-/* Copyright (C) 2023 Wildfire Games.
- * This file is part of 0 A.D.
- *
- * 0 A.D. is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 2 of the License, or
- * (at your option) any later version.
- *
- * 0 A.D. is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with 0 A.D. If not, see .
- */
-
-// This pipeline is used to update translations to and from Transifex.
-
-pipeline {
- agent {
- node {
- label 'LinuxSlave'
- customWorkspace '/zpool0/trunk'
- }
- }
-
- stages {
- stage("Prepare volume") {
- steps {
- sh "sudo zfs clone zpool0/trunk@latest zpool0/translations"
- ws('/zpool0/translations') {
- sh "svn revert . -R"
- sh "svn st | cut -c 9- | xargs rm -rf"
- sh "svn up"
- }
- }
- }
-
- stage("Update translations") {
- steps {
- ws('/zpool0/translations') {
- withDockerContainer("0ad-translations:latest") {
- sh "python3 source/tools/i18n/updateTemplates.py"
- withCredentials([string(credentialsId: 'redacted', variable: 'token')]) {
- sh 'TX_TOKEN="$token" python3 source/tools/i18n/pullTranslations.py'
- }
- dir("source/tools/i18n/") {
- sh "python3 generateDebugTranslation.py --long"
- }
- sh "python3 source/tools/i18n/cleanTranslationFiles.py"
- sh "python3 source/tools/i18n/checkDiff.py --verbose"
- sh "python3 source/tools/i18n/creditTranslators.py"
- }
- }
- }
- }
-
- stage("Commit") {
- steps {
- ws('/zpool0/translations') {
- withCredentials([usernamePassword(credentialsId: 'redacted', passwordVariable: 'SVNPASS', usernameVariable: 'SVNUSER')]) {
- sh 'svn relocate --username $SVNUSER --password $SVNPASS --no-auth-cache https://svn.wildfiregames.com/svn/ps/trunk'
- sh 'svn commit --username $SVNUSER --password $SVNPASS --no-auth-cache --non-interactive -m "[i18n] Updated POT and PO files."'
- }
- }
- }
- }
- }
- post {
- always {
- sleep 10
- sh "sudo zfs destroy zpool0/translations"
- }
- }
-}
diff --git a/build/jenkins/pipelines/nightly-build.Jenkinsfile b/build/jenkins/pipelines/nightly-build.Jenkinsfile
new file mode 100644
index 0000000000..7a3a3c2187
--- /dev/null
+++ b/build/jenkins/pipelines/nightly-build.Jenkinsfile
@@ -0,0 +1,164 @@
+/* Copyright (C) 2024 Wildfire Games.
+ * This file is part of 0 A.D.
+ *
+ * 0 A.D. is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * 0 A.D. is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with 0 A.D. If not, see .
+ */
+
+// This pipeline is used to generate the nightly builds.
+
+def visualStudioPath = "\"C:\\Program Files (x86)\\Microsoft Visual Studio\\2017\\Community\\MSBuild\\15.0\\Bin\\MSBuild.exe\""
+def buildOptions = "/p:PlatformToolset=v141_xp /p:XPDeprecationWarning=false /t:pyrogenesis /t:AtlasUI /m:2 /nologo -clp:Warningsonly -clp:ErrorsOnly"
+
+def gitHash = ""
+def buildSPIRV = false
+
+pipeline {
+ agent {
+ node {
+ label 'WindowsAgent'
+ customWorkspace 'workspace/nightly-build'
+ }
+ }
+
+ parameters {
+ booleanParam(name: 'NEW_REPO', defaultValue: false, description: 'If a brand new nightly repo is being generated, do not attempt to identify unchanged translations.')
+ stashedFile(name: 'spirv_rules', description: 'rules.json file for generation of SPIR-V shaders. Needed for a new repo, else the existing rules files will be used. Uploading a new rules file will force-rebuild the shaders.')
+ }
+
+ stages {
+ stage("Generate build version") {
+ steps {
+ checkout scmGit(branches: [[name: "${GIT_BRANCH}"]], extensions: [localBranch()])
+ script { gitHash = bat(script:"@git rev-parse --short HEAD", returnStdout: true ).trim() }
+ bat "cd build\\build_version && build_version.bat"
+ }
+ }
+
+ stage("Pull game assets") {
+ steps {
+ bat "git lfs pull"
+ }
+ }
+
+ stage("Check for shader changes") {
+ when {
+ changeset 'binaries/data/mods/**/shaders/**/*.xml'
+ }
+ steps {
+ script { buildSPIRV = true }
+ }
+ }
+
+ stage ("Pre-build") {
+ steps {
+ bat "cd libraries && get-windows-libs.bat"
+ bat "(robocopy C:\\wxwidgets3.0.4\\lib libraries\\win32\\wxwidgets\\lib /MIR /NDL /NJH /NJS /NP /NS /NC) ^& IF %ERRORLEVEL% LEQ 1 exit 0"
+ bat "(robocopy C:\\wxwidgets3.0.4\\include libraries\\win32\\wxwidgets\\include /MIR /NDL /NJH /NJS /NP /NS /NC) ^& IF %ERRORLEVEL% LEQ 1 exit 0"
+ bat "cd build\\workspaces && update-workspaces.bat --atlas --without-pch --without-tests"
+ }
+ }
+
+ stage ("Build") {
+ steps {
+ bat("cd build\\workspaces\\vs2017 && ${visualStudioPath} pyrogenesis.sln /p:Configuration=Release ${buildOptions}")
+ }
+ }
+
+ stage("Check-in SPIRV generation rules") {
+ when {
+ expression { env.spirv_rules_FILENAME }
+ }
+ steps {
+ unstash 'spirv_rules'
+ bat "move spirv_rules source\\tools\\spirv\\rules.json"
+ script { buildSPIRV = true }
+ }
+ }
+
+ stage("Mirror to SVN") {
+ steps {
+ ws("workspace/nightly-svn") {
+ svn(url: "https://svn.wildfiregames.com/nightly-build/trunk", changelog: false)
+ bat "svn revert -R . && svn cleanup"
+ script { env.NIGHTLY_PATH = env.WORKSPACE }
+ }
+ bat """
+ (robocopy . %NIGHTLY_PATH% ^
+ /XD .git ^
+ /XF .gitattributes ^
+ /XF .gitignore ^
+ /XD %cd%\\binaries\\system ^
+ /XD %cd%\\build\\workspaces\\vs2017 ^
+ /XD %cd%\\libraries\\source\\.svn ^
+ /XD %cd%\\libraries\\win32\\.svn ^
+ /XD %cd%\\libraries\\win32\\wxwidgets\\include ^
+ /XD %cd%\\libraries\\win32\\wxwidgets\\lib ^
+ /XD .svn ^
+ /XD %NIGHTLY_PATH%\\binaries\\data\\mods\\mod\\shaders\\spirv ^
+ /XD %NIGHTLY_PATH%\\binaries\\data\\mods\\public\\shaders\\spirv ^
+ /MIR /NDL /NJH /NJS /NP /NS /NC) ^& IF %ERRORLEVEL% LEQ 1 exit 0
+ """
+ bat """
+ (robocopy binaries\\system ..\\nightly-svn\\binaries\\system ^
+ /XF *.exp ^
+ /XF *.lib ^
+ /MIR /NDL /NJH /NJS /NP /NS /NC) ^& IF %ERRORLEVEL% LEQ 1 exit 0
+ """
+ }
+ }
+
+ stage("Recompile SPIR-V shaders") {
+ when {
+ expression { buildSPIRV }
+ }
+ steps {
+ ws("workspace/nightly-svn") {
+ bat "python source/tools/spirv/compile.py -d binaries/data/mods/mod binaries/data/mods/mod source/tools/spirv/rules.json binaries/data/mods/mod"
+ bat "python source/tools/spirv/compile.py -d binaries/data/mods/mod binaries/data/mods/public source/tools/spirv/rules.json binaries/data/mods/public"
+ }
+ }
+ }
+
+ stage("Update translations") {
+ steps {
+ ws("workspace/nightly-svn") {
+ bat "cd source\\tools\\i18n && python updateTemplates.py"
+ withCredentials([string(credentialsId: 'TX_TOKEN', variable: 'TX_TOKEN')]) {
+ bat "cd source\\tools\\i18n && python pullTranslations.py"
+ }
+ bat "cd source\\tools\\i18n && python generateDebugTranslation.py --long"
+ bat "cd source\\tools\\i18n && python cleanTranslationFiles.py"
+ script { if (!params.NEW_REPO) {
+ bat "python source\\tools\\i18n\\checkDiff.py --verbose"
+ }}
+ bat "cd source\\tools\\i18n && python creditTranslators.py"
+ }
+ }
+ }
+
+ stage("Commit") {
+ steps {
+ ws("workspace/nightly-svn") {
+ bat "svn add --force ."
+ bat "(for /F \"tokens=* delims=! \" %%A in ('svn status ^| findstr /R \"^!\"') do (svn delete %%A)) || (echo No deleted files found) ^& exit 0"
+ withCredentials([usernamePassword(credentialsId: 'nightly-autobuild', passwordVariable: 'SVNPASS', usernameVariable: 'SVNUSER')]) {
+ script { env.GITHASH = gitHash
+ bat 'svn commit --username %SVNUSER% --password %SVNPASS% --no-auth-cache --non-interactive -m "Nightly build for %GITHASH% (%DATE%)"'
+ }
+ }
+ }
+ }
+ }
+ }
+}
diff --git a/source/tools/i18n/checkDiff.py b/source/tools/i18n/checkDiff.py
index 6f3d5484df..727d68208d 100644
--- a/source/tools/i18n/checkDiff.py
+++ b/source/tools/i18n/checkDiff.py
@@ -1,6 +1,6 @@
#!/usr/bin/env python3
#
-# Copyright (C) 2021 Wildfire Games.
+# Copyright (C) 2024 Wildfire Games.
# This file is part of 0 A.D.
#
# 0 A.D. is free software: you can redistribute it and/or modify
@@ -29,9 +29,9 @@ def get_diff():
diff_process = subprocess.run(["svn", "diff", "binaries"], capture_output=True)
if diff_process.returncode != 0:
- print(f"Error running svn diff: {diff_process.stderr.decode()}. Exiting.")
+ print(f"Error running svn diff: {diff_process.stderr.decode('utf-8')}. Exiting.")
return
- return io.StringIO(diff_process.stdout.decode())
+ return io.StringIO(diff_process.stdout.decode('utf-8'))
def check_diff(diff : io.StringIO, verbose = False) -> List[str]:
"""Run through a diff of .po files and check that some of the changes
@@ -45,10 +45,10 @@ def check_diff(diff : io.StringIO, verbose = False) -> List[str]:
l = diff.readline()
while l:
if l.startswith("Index: binaries"):
- if not l.endswith(".pot\n") and not l.endswith(".po\n"):
+ if not l.strip().endswith(".pot") and not l.strip().endswith(".po"):
curfile = None
else:
- curfile = l[7:-1]
+ curfile = l[7:].strip()
files.add(curfile)
# skip patch header
diff.readline()
@@ -60,7 +60,7 @@ def check_diff(diff : io.StringIO, verbose = False) -> List[str]:
if l[0] != '-' and l[0] != '+':
l = diff.readline()
continue
- if l[1] == '\n' or (l[1] == '#' and l[2] == ":"):
+ if l[1:].strip() == "" or (l[1] == '#' and l[2] == ':'):
l = diff.readline()
continue
if "# Copyright (C)" in l or "POT-Creation-Date:" in l or "PO-Revision-Date" in l or "Last-Translator" in l:
@@ -78,7 +78,7 @@ def check_diff(diff : io.StringIO, verbose = False) -> List[str]:
def revert_files(files: List[str], verbose = False):
revert_process = subprocess.run(["svn", "revert"] + files, capture_output=True)
if revert_process.returncode != 0:
- print(f"Warning: Some files could not be reverted. Error: {revert_process.stderr.decode()}")
+ print(f"Warning: Some files could not be reverted. Error: {revert_process.stderr.decode('utf-8')}")
if verbose:
for file in files:
print(f"Reverted {file}")
@@ -96,9 +96,9 @@ def add_untracked(verbose = False):
continue
# Ignore non PO files. This is important so that the translator credits
# correctly be updated, note however the script assumes a pristine SVN otherwise.
- if not line.endswith(".po") and not line.endswith(".pot"):
- continue
file = line[1:].strip()
+ if not file.endswith(".po") and not file.endswith(".pot"):
+ continue
add_process = subprocess.run(["svn", "add", file, "--parents"], capture_output=True)
if add_process.stderr != b'':
print(f"Warning: file {file} could not be added.")
diff --git a/source/tools/i18n/cleanTranslationFiles.py b/source/tools/i18n/cleanTranslationFiles.py
index 63c67f8bd1..7ad3981b2a 100644
--- a/source/tools/i18n/cleanTranslationFiles.py
+++ b/source/tools/i18n/cleanTranslationFiles.py
@@ -43,7 +43,7 @@ def main():
for file in files:
usernames = []
reached = False
- for line in fileinput.input(file.replace("\\", "/"), inplace=True):
+ for line in fileinput.input(file.replace("\\", "/"), inplace=True, encoding="utf-8"):
if reached:
if line == "# \n":
line = ""
diff --git a/source/tools/i18n/get-nightly-translations.bat b/source/tools/i18n/get-nightly-translations.bat
new file mode 100644
index 0000000000..2eca111398
--- /dev/null
+++ b/source/tools/i18n/get-nightly-translations.bat
@@ -0,0 +1,11 @@
+rem **Download translations from the latest nightly build**
+
+rem **This will overwrite any uncommitted changes to messages.json files**
+
+svn export --force --depth files https://svn.wildfiregames.com/nightly-build/trunk/binaries/data/l10n ..\..\..\binaries\data\l10n
+
+for %%m in (mod public) do (
+ svn export --force --depth files https://svn.wildfiregames.com/nightly-build/trunk/binaries/data/mods/%%m/l10n ..\..\..\binaries\data\mods\%%m\l10n
+)
+
+svn export --force https://svn.wildfiregames.com/nightly-build/trunk/binaries/data/mods/public/gui/credits/texts/translators.json ..\..\..\binaries\data\mods\public\gui\credits\texts\translators.json
diff --git a/source/tools/i18n/get-nightly-translations.sh b/source/tools/i18n/get-nightly-translations.sh
new file mode 100755
index 0000000000..e3b40305f0
--- /dev/null
+++ b/source/tools/i18n/get-nightly-translations.sh
@@ -0,0 +1,15 @@
+#!/bin/sh
+set -ev
+
+# Download translations from the latest nightly build
+# This will overwrite any uncommitted changes to messages.json files
+
+cd "$(dirname $0)"
+
+svn export --force --depth files https://svn.wildfiregames.com/nightly-build/trunk/binaries/data/l10n ../../../binaries/data/l10n
+
+for m in "mod public"; do
+ svn export --force --depth files https://svn.wildfiregames.com/nightly-build/trunk/binaries/data/mods/${m}/l10n ../../../binaries/data/mods/${m}/l10n
+done
+
+svn export --force https://svn.wildfiregames.com/nightly-build/trunk/binaries/data/mods/public/gui/credits/texts/translators.json ../../../binaries/data/mods/public/gui/credits/texts/translators.json
diff --git a/source/tools/i18n/maintenanceTasks.sh b/source/tools/i18n/maintenanceTasks.sh
deleted file mode 100755
index e738bb4477..0000000000
--- a/source/tools/i18n/maintenanceTasks.sh
+++ /dev/null
@@ -1,32 +0,0 @@
-#!/bin/sh
-
-# Regenerates the POT files, downloads the latest translations from Transifex,
-# and prepares the commit of the updated POT and PO files.
-
-SCRIPT_PATH="`dirname \"$0\"`"
-
-
-# POT Generation ##############################################################
-
-echo ":: Regenerating the translation templates…"
-python3 "${SCRIPT_PATH}/updateTemplates.py"
-
-
-# PO Download #################################################################
-
-echo ":: Downloading translations from Transifex…"
-python3 "${SCRIPT_PATH}/pullTranslations.py"
-
-
-# Pre-Commit Cleanup #########################################################
-
-echo ":: Removing unneeded data from the .po files…"
-python3 "${SCRIPT_PATH}/cleanTranslationFiles.py"
-
-echo ":: Reverting unnecessary changes…"
-python3 "${SCRIPT_PATH}/checkDiff.py"
-
-# Commit ######################################################################
-
-echo ":: Done"
-echo " Now you can commit your changes to the server."
diff --git a/source/tools/i18n/pullTranslations.py b/source/tools/i18n/pullTranslations.py
index 49b36f8379..1fe80fec97 100644
--- a/source/tools/i18n/pullTranslations.py
+++ b/source/tools/i18n/pullTranslations.py
@@ -1,6 +1,6 @@
#!/usr/bin/env python3
#
-# Copyright (C) 2023 Wildfire Games.
+# Copyright (C) 2024 Wildfire Games.
# This file is part of 0 A.D.
#
# 0 A.D. is free software: you can redistribute it and/or modify
@@ -29,7 +29,7 @@ def main():
path = os.path.join(root, folder)
os.chdir(path)
print(f"INFO: Starting to pull translations in {path}...")
- subprocess.run(["tx", "pull", "-f"])
+ subprocess.run(["tx", "pull", "-a", "-f"])
if __name__ == "__main__":
diff --git a/source/tools/spirv/get-nightly-shaders.bat b/source/tools/spirv/get-nightly-shaders.bat
new file mode 100644
index 0000000000..d34b91d5f4
--- /dev/null
+++ b/source/tools/spirv/get-nightly-shaders.bat
@@ -0,0 +1,5 @@
+rem **Download SPIR-V shaders from the latest nightly build**
+
+for %%m in (mod public) do (
+ svn export --force https://svn.wildfiregames.com/nightly-build/trunk/binaries/data/mods/%%m/shaders/spirv ..\..\..\binaries\data\mods\%%m\shaders\spirv
+)
diff --git a/source/tools/spirv/get-nightly-shaders.sh b/source/tools/spirv/get-nightly-shaders.sh
new file mode 100644
index 0000000000..8b5ec3a0bc
--- /dev/null
+++ b/source/tools/spirv/get-nightly-shaders.sh
@@ -0,0 +1,10 @@
+#!/bin/sh
+set -ev
+
+# Download SPIR-V shaders from the latest nightly build
+
+cd "$(dirname $0)"
+
+for m in "mod public"; do
+ svn export --force https://svn.wildfiregames.com/nightly-build/trunk/binaries/data/mods/${m}/shaders/spirv ../../../binaries/data/mods/${m}/shaders/spirv
+done