Adds #include directive support to shaders

Comments By: Stan, wraitii
Tested By: Freagarach, Stan
Differential Revision: https://code.wildfiregames.com/D3030
This was SVN commit r24553.

source/graphics/PreprocessorWrapper.cpp

@@ -1,4 +1,4 @@
-/* Copyright (C) 2020 Wildfire Games.
+/* Copyright (C) 2021 Wildfire Games.
  * This file is part of 0 A.D.
  *
  * 0 A.D. is free software: you can redistribute it and/or modify
@@ -22,6 +22,88 @@
 #include "graphics/ShaderDefines.h"
 #include "ps/CLogger.h"
 
+#include <cctype>
+
+namespace
+{
+
+struct MatchIncludeResult
+{
+	bool found;
+	bool error;
+	size_t nextLineStart;
+	size_t pathFirst, pathLast;
+
+	static MatchIncludeResult MakeNotFound(const CStr& source, size_t pos)
+	{
+		while (pos < source.size() && source[pos] != '\n')
+			++pos;
+		return MatchIncludeResult{
+			false, false, pos < source.size() ? pos + 1 : source.size(), 0, 0};
+	}
+
+	static MatchIncludeResult MakeError(
+		const char* message, const CStr& source, const size_t lineStart, const size_t currentPos)
+	{
+		ENSURE(currentPos >= lineStart);
+		size_t lineEnd = currentPos;
+		while (lineEnd < source.size() && source[lineEnd] != '\n' && source[lineEnd] != '\r')
+			++lineEnd;
+		const CStr line = source.substr(lineStart, lineEnd - lineStart);
+		while (lineEnd < source.size() && source[lineEnd] != '\n')
+			++lineEnd;
+		const size_t nextLineStart = lineEnd < source.size() ? lineEnd + 1 : source.size();
+		LOGERROR("Preprocessor error: %s: '%s'\n", message, line.c_str());
+		return MatchIncludeResult{false, true, nextLineStart, 0, 0};
+	}
+};
+
+MatchIncludeResult MatchIncludeUntilEOLorEOS(const CStr& source, const size_t lineStart)
+{
+	// We need to match a line like this:
+	// ^[ \t]*#[ \t]*include[ \t]*"[^"]+".*$
+	//  ^     ^^     ^      ^     ^^    ^
+	//  1     23     4      5     67    8    <- steps
+	const CStr INCLUDE = "include";
+	size_t pos = lineStart;
+	// Matching step #1.
+	while (pos < source.size() && std::isblank(source[pos]))
+		++pos;
+	// Matching step #2.
+	if (pos == source.size() || source[pos] != '#')
+		return MatchIncludeResult::MakeNotFound(source, pos);
+	++pos;
+	// Matching step #3.
+	while (pos < source.size() && std::isblank(source[pos]))
+		++pos;
+	// Matching step #4.
+	if (pos + INCLUDE.size() >= source.size() || source.substr(pos, INCLUDE.size()) != INCLUDE)
+		return MatchIncludeResult::MakeNotFound(source, pos);
+	pos += INCLUDE.size();
+	// Matching step #5.
+	while (pos < source.size() && std::isblank(source[pos]))
+		++pos;
+	// Matching step #6.
+	if (pos == source.size() || source[pos] != '"')
+		return MatchIncludeResult::MakeError("#include should be followed by quote", source, lineStart, pos);
+	++pos;
+	// Matching step #7.
+	const size_t pathFirst = pos;
+	while (pos < source.size() && source[pos] != '"' && source[pos] != '\n')
+		++pos;
+	const size_t pathLast = pos;
+	// Matching step #8.
+	if (pos == source.size() || source[pos] != '"')
+		return MatchIncludeResult::MakeError("#include has invalid quote pair", source, lineStart, pos);
+	if (pathLast - pathFirst <= 1)
+		return MatchIncludeResult::MakeError("#include path shouldn't be empty", source, lineStart, pos);
+	while (pos < source.size() && source[pos] != '\n')
+		++pos;
+	return MatchIncludeResult{true, false, pos < source.size() ? pos + 1 : source.size(), pathFirst, pathLast};
+}
+
+} // anonymous namespace
+
 void CPreprocessorWrapper::PyrogenesisShaderError(int iLine, const char* iError, const Ogre::CPreprocessor::Token* iToken)
 {
 	if (iToken)
@@ -31,6 +113,12 @@ void CPreprocessorWrapper::PyrogenesisShaderError(int iLine, const char* iError,
 }
 
 CPreprocessorWrapper::CPreprocessorWrapper()
+	: CPreprocessorWrapper(IncludeRetrieverCallback{})
+{
+}
+
+CPreprocessorWrapper::CPreprocessorWrapper(const IncludeRetrieverCallback& includeCallback)
+	: m_IncludeCallback(includeCallback)
 {
 	Ogre::CPreprocessor::ErrorHandler = CPreprocessorWrapper::PyrogenesisShaderError;
 }
@@ -78,10 +166,63 @@ bool CPreprocessorWrapper::TestConditional(const CStr& expr)
 
 }
 
+CStr CPreprocessorWrapper::ResolveIncludes(CStr source)
+{
+	const CStr lineDirective = "#line ";
+	for (size_t lineStart = 0, line = 1; lineStart < source.size(); ++line)
+	{
+		MatchIncludeResult match = MatchIncludeUntilEOLorEOS(source, lineStart);
+		if (match.error)
+			return {};
+		else if (!match.found)
+		{
+			if (lineStart + lineDirective.size() < source.size() &&
+			    source.substr(lineStart, lineDirective.size()) == lineDirective)
+			{
+				size_t newLineNumber = 0;
+				size_t pos = lineStart + lineDirective.size();
+				while (pos < match.nextLineStart && std::isdigit(source[pos]))
+				{
+					newLineNumber = newLineNumber * 10 + (source[pos] - '0');
+					++pos;
+				}
+				if (newLineNumber > 0)
+					line = newLineNumber - 1;
+			}
+
+			lineStart = match.nextLineStart;
+			continue;
+		}
+		const CStr path = source.substr(match.pathFirst, match.pathLast - match.pathFirst);
+		auto it = m_IncludeCache.find(path);
+		if (it == m_IncludeCache.end())
+		{
+			CStr includeContent;
+			if (!m_IncludeCallback(path, includeContent))
+			{
+				LOGERROR("Preprocessor error: line %zu: Can't load #include file: '%s'", line, path.c_str());
+				return {};
+			}
+			it = m_IncludeCache.emplace(path, std::move(includeContent)).first;
+		}
+		// We need to insert #line directives to have correct line numbers in errors.
+		source =
+			source.substr(0, lineStart) +
+			lineDirective + "1\n" + it->second + "\n" + lineDirective + CStr::FromUInt(line + 1) + "\n" +
+			source.substr(match.nextLineStart);
+		--line;
+	}
+	return source;
+}
+
 CStr CPreprocessorWrapper::Preprocess(const CStr& input)
 {
+	PROFILE("Preprocess shader source");
+
+	CStr source = ResolveIncludes(input);
+
 	size_t len = 0;
-	char* output = m_Preprocessor.Parse(input.c_str(), input.size(), len);
+	char* output = m_Preprocessor.Parse(source.c_str(), source.size(), len);
 
 	if (!output)
 	{
@@ -92,7 +233,7 @@ CStr CPreprocessorWrapper::Preprocess(const CStr& input)
 	CStr ret(output, len);
 
 	// Free output if it's not inside the source string
-	if (!(output >= input.c_str() && output < input.c_str() + input.size()))
+	if (!(output >= source.c_str() && output < source.c_str() + source.size()))
 		free(output);
 
 	return ret;

source/graphics/PreprocessorWrapper.h

@@ -1,4 +1,4 @@
-/* Copyright (C) 2020 Wildfire Games.
+/* Copyright (C) 2021 Wildfire Games.
  * This file is part of 0 A.D.
  *
  * 0 A.D. is free software: you can redistribute it and/or modify
@@ -21,6 +21,9 @@
 #include "ps/CStr.h"
 #include "third_party/ogre3d_preprocessor/OgreGLSLPreprocessor.h"
 
+#include <functional>
+#include <map>
+
 class CShaderDefines;
 
 /**
@@ -29,7 +32,10 @@ class CShaderDefines;
 class CPreprocessorWrapper
 {
 public:
+	using IncludeRetrieverCallback = std::function<bool(const CStr&, CStr& out)>;
+
 	CPreprocessorWrapper();
+	CPreprocessorWrapper(const IncludeRetrieverCallback& includeCallback);
 
 	void AddDefine(const char* name, const char* value);
 
@@ -37,12 +43,20 @@ public:
 
 	bool TestConditional(const CStr& expr);
 
+	// Find all #include directives in the input and replace them by
+	// by a file content from the directive's argument. Parsing is strict
+	// and simple. The directive will be expanded in comments and multiline
+	// strings.
+	CStr ResolveIncludes(CStr source);
+
 	CStr Preprocess(const CStr& input);
 
 	static void PyrogenesisShaderError(int iLine, const char* iError, const Ogre::CPreprocessor::Token* iToken);
 
 private:
 	Ogre::CPreprocessor m_Preprocessor;
+	IncludeRetrieverCallback m_IncludeCallback;
+	std::unordered_map<CStr, CStr> m_IncludeCache;
 };
 
 #endif // INCLUDED_PREPROCESSORWRAPPER

source/graphics/ShaderManager.cpp

@@ -1,4 +1,4 @@
-/* Copyright (C) 2019 Wildfire Games.
+/* Copyright (C) 2021 Wildfire Games.
  * This file is part of 0 A.D.
  *
  * 0 A.D. is free software: you can redistribute it and/or modify
@@ -269,8 +269,8 @@ bool CShaderManager::NewProgram(const char* name, const CShaderDefines& baseDefi
 	program->Reload();
 
 //	m_HotloadFiles[xmlFilename].insert(program); // TODO: should reload somehow when the XML changes
-	m_HotloadFiles[vertexFile].insert(program);
-	m_HotloadFiles[fragmentFile].insert(program);
+	for (const VfsPath& path : program->GetFileDependencies())
+		AddProgramFileDependency(program, path);
 
 	return true;
 }
@@ -566,17 +566,20 @@ Status CShaderManager::ReloadChangedFile(const VfsPath& path)
 {
 	// Find all shaders using this file
 	HotloadFilesMap::iterator files = m_HotloadFiles.find(path);
-	if (files != m_HotloadFiles.end())
-	{
-		// Reload all shaders using this file
-		for (std::set<std::weak_ptr<CShaderProgram> >::iterator it = files->second.begin(); it != files->second.end(); ++it)
-		{
-			if (std::shared_ptr<CShaderProgram> program = it->lock())
-				program->Reload();
-		}
-	}
+	if (files == m_HotloadFiles.end())
+		return INFO::OK;
+
+	// Reload all shaders using this file
+	for (const std::weak_ptr<CShaderProgram>& ptr : files->second)
+		if (std::shared_ptr<CShaderProgram> program = ptr.lock())
+			program->Reload();
 
 	// TODO: hotloading changes to shader XML files and effect XML files would be nice
 
 	return INFO::OK;
 }
+
+void CShaderManager::AddProgramFileDependency(const CShaderProgramPtr& program, const VfsPath& path)
+{
+	m_HotloadFiles[path].insert(program);
+}

source/graphics/ShaderManager.h

@@ -1,4 +1,4 @@
-/* Copyright (C) 2019 Wildfire Games.
+/* Copyright (C) 2021 Wildfire Games.
  * This file is part of 0 A.D.
  *
  * 0 A.D. is free software: you can redistribute it and/or modify
@@ -125,6 +125,11 @@ private:
 
 	static Status ReloadChangedFileCB(void* param, const VfsPath& path);
 	Status ReloadChangedFile(const VfsPath& path);
+
+	/**
+	 * Associates the file with the program to be reloaded if the file has changed.
+	 */
+	void AddProgramFileDependency(const CShaderProgramPtr& program, const VfsPath& path);
 };
 
 #endif // INCLUDED_SHADERMANAGER

source/graphics/ShaderProgram.cpp

@@ -1,4 +1,4 @@
-/* Copyright (C) 2020 Wildfire Games.
+/* Copyright (C) 2021 Wildfire Games.
  * This file is part of 0 A.D.
  *
  * 0 A.D. is free software: you can redistribute it and/or modify
@@ -236,6 +236,11 @@ public:
 		Uniform(id, v[0]);
 	}
 
+	virtual std::vector<VfsPath> GetFileDependencies() const override
+	{
+		return {m_VertexFile, m_FragmentFile};
+	}
+
 private:
 	VfsPath m_VertexFile;
 	VfsPath m_FragmentFile;
@@ -272,6 +277,7 @@ public:
 		m_Program = 0;
 		m_VertexShader = pglCreateShaderObjectARB(GL_VERTEX_SHADER);
 		m_FragmentShader = pglCreateShaderObjectARB(GL_FRAGMENT_SHADER);
+		m_FileDependencies = {m_VertexFile, m_FragmentFile};
 	}
 
 	~CShaderProgramGLSL()
@@ -431,7 +437,17 @@ public:
 		if (fragmentFile.Load(g_VFS, m_FragmentFile) != PSRETURN_OK)
 			return;
 
-		CPreprocessorWrapper preprocessor;
+		std::vector<VfsPath> newFileDependencies = {m_VertexFile, m_FragmentFile};
+
+		CPreprocessorWrapper preprocessor([&newFileDependencies](const CStr& includePath, CStr& out) -> bool {
+			const VfsPath includeFilePath(L"shaders/glsl/" + wstring_from_utf8(includePath));
+			CVFSFile includeFile;
+			if (includeFile.Load(g_VFS, includeFilePath) != PSRETURN_OK)
+				return false;
+			out = includeFile.GetAsString();
+			newFileDependencies.push_back(includeFilePath);
+			return true;
+		});
 		preprocessor.AddDefines(m_Defines);
 
 #if CONFIG2_GLES
@@ -444,6 +460,8 @@ public:
 		CStr vertexCode = preprocessor.Preprocess(vertexFile.GetAsString());
 		CStr fragmentCode = preprocessor.Preprocess(fragmentFile.GetAsString());
 
+		m_FileDependencies = std::move(newFileDependencies);
+
 #if CONFIG2_GLES
 		// Ugly hack to replace desktop GLSL 1.10/1.20 with GLSL ES 1.00,
 		// and also to set default float precision for fragment shaders
@@ -637,9 +655,15 @@ public:
 		}
 	}
 
+	virtual std::vector<VfsPath> GetFileDependencies() const
+	{
+		return m_FileDependencies;
+	}
+
 private:
 	VfsPath m_VertexFile;
 	VfsPath m_FragmentFile;
+	std::vector<VfsPath> m_FileDependencies;
 	CShaderDefines m_Defines;
 	std::map<CStrIntern, int> m_VertexAttribs;
 

source/graphics/ShaderProgram.h

@@ -1,4 +1,4 @@
-/* Copyright (C) 2020 Wildfire Games.
+/* Copyright (C) 2021 Wildfire Games.
  * This file is part of 0 A.D.
  *
  * 0 A.D. is free software: you can redistribute it and/or modify
@@ -25,6 +25,7 @@
 #include "lib/res/handle.h"
 
 #include <map>
+#include <vector>
 
 struct CColor;
 class CMatrix3D;
@@ -193,6 +194,8 @@ public:
 	 */
 	void AssertPointersBound();
 
+	virtual std::vector<VfsPath> GetFileDependencies() const = 0;
+
 protected:
 	CShaderProgram(int streamflags);
 

source/graphics/ShaderProgramFFP.cpp

@@ -1,4 +1,4 @@
-/* Copyright (C) 2014 Wildfire Games.
+/* Copyright (C) 2021 Wildfire Games.
  * This file is part of 0 A.D.
  *
  * 0 A.D. is free software: you can redistribute it and/or modify
@@ -113,6 +113,11 @@ public:
 	{
 	}
 
+	virtual std::vector<VfsPath> GetFileDependencies() const
+	{
+		return {};
+	}
+
 protected:
 	std::map<CStrIntern, int> m_UniformIndexes;
 

source/third_party/ogre3d_preprocessor/tests/test_Preprocessor.h

@@ -1,4 +1,4 @@
-/* Copyright (C) 2020 Wildfire Games.
+/* Copyright (C) 2021 Wildfire Games.
  * This file is part of 0 A.D.
  *
  * 0 A.D. is free software: you can redistribute it and/or modify
@@ -21,6 +21,9 @@
 #include "ps/CStr.h"
 #include "third_party/ogre3d_preprocessor/OgreGLSLPreprocessor.h"
 
+#include <cctype>
+#include <sstream>
+
 class TestPreprocessor : public CxxTest::TestSuite
 {
 public:
@@ -35,6 +38,16 @@ public:
 		CStr8 loggerOutput;
 	};
 
+	// Replaces consecutive spaces/tabs by single space/tab.
+	CStr CompressWhiteSpaces(const CStr& source)
+	{
+		CStr result;
+		for (char ch : source)
+			if (!std::isblank(ch) || (result.empty() || result.back() != ch))
+				result += ch;
+		return result;
+	}
+
 	PreprocessorResult Parse(const char* in)
 	{
 		PreprocessorResult result;
@@ -52,6 +65,18 @@ public:
 		return result;
 	}
 
+	PreprocessorResult ParseWithIncludes(const char* in, CPreprocessorWrapper::IncludeRetrieverCallback includeCallback)
+	{
+		PreprocessorResult result;
+		TestLogger logger;
+
+		CPreprocessorWrapper preprocessor(includeCallback);
+		result.output = preprocessor.Preprocess(in);
+		result.loggerOutput = logger.GetOutput();
+
+		return result;
+	}
+
 	void test_basic()
 	{
 		PreprocessorResult result = Parse("#define TEST 2\n1+1=TEST\n");
@@ -113,4 +138,159 @@ public:
 		TS_ASSERT_EQUALS(result.output.Trim(PS_TRIM_BOTH), "");
 		TS_ASSERT_STR_CONTAINS(result.loggerOutput, "ERROR: Preprocessor error: line 2: Division by zero");
 	}
+
+	void test_include()
+	{
+		bool includeRetrieved = false;
+		CPreprocessorWrapper::IncludeRetrieverCallback includeCallback = [&includeRetrieved](
+				const CStr& includePath, CStr& out) {
+			TS_ASSERT_EQUALS(includePath, "test.h");
+			out = "42";
+			includeRetrieved = true;
+			return true;
+		};
+
+		PreprocessorResult result = ParseWithIncludes(
+			R"(#include "test.h" // Includes something.)", includeCallback);
+
+		TS_ASSERT(includeRetrieved);
+		TS_ASSERT_EQUALS(result.output.Trim(PS_TRIM_BOTH), "#line 1\n42\n#line 2");
+	}
+
+	void test_include_double()
+	{
+		CPreprocessorWrapper::IncludeRetrieverCallback includeCallback = [](
+				const CStr& includePath, CStr& out) {
+			TS_ASSERT_EQUALS(includePath, "test.h");
+			out = "42";
+			return true;
+		};
+
+		PreprocessorResult result = ParseWithIncludes(R"(
+			#include "test.h"
+			#include "test.h"
+			#include "test.h"
+		)", includeCallback);
+
+		TS_ASSERT_EQUALS(result.output.Trim(PS_TRIM_BOTH), "#line 1\n42\n#line 3\n#line 1\n42\n#line 4\n#line 1\n42\n#line 5");
+	}
+
+	void test_include_double_with_guards()
+	{
+		CPreprocessorWrapper::IncludeRetrieverCallback includeCallback = [](
+				const CStr& includePath, CStr& out) {
+			TS_ASSERT_EQUALS(includePath, "test.h");
+			out = R"(#ifndef INCLUDED_TEST
+				#define INCLUDED_TEST
+				42
+			#endif)";
+			return true;
+		};
+
+		PreprocessorResult result = ParseWithIncludes(R"(
+			#include "test.h"
+			#include "test.h"
+			#include "test.h"
+		)", includeCallback);
+
+		TS_ASSERT_EQUALS(result.output.Trim(PS_TRIM_BOTH), "#line 1\n\n\t\t\t\t\n\t\t\t\t42\n\t\t\t\n#line 3\n#line 1\n\n\n\n\n#line 4\n#line 1\n\n\n\n\n#line 5");
+	}
+
+	void test_include_invalid_argument()
+	{
+		int includeRetrievedCounter = 0;
+		CPreprocessorWrapper::IncludeRetrieverCallback includeCallback = [&includeRetrievedCounter](
+				const CStr& includePath, CStr& out) {
+			out = "42";
+			++includeRetrievedCounter;
+			return true;
+		};
+
+		PreprocessorResult result = ParseWithIncludes(R"(
+			#include <test.h>
+			#include test.h
+		)", includeCallback);
+
+		TS_ASSERT_EQUALS(includeRetrievedCounter, 0);
+	}
+
+	void test_include_invalid_file()
+	{
+		CPreprocessorWrapper::IncludeRetrieverCallback includeCallback = [](
+				const CStr& includePath, CStr& out) {
+			return false;
+		};
+
+		PreprocessorResult result = ParseWithIncludes(R"(
+			#include "missed_file.h"
+		)", includeCallback);
+
+		TS_ASSERT_STR_CONTAINS(result.loggerOutput, "ERROR: Preprocessor error: line 2: Can't load #include file: 'missed_file.h'");
+	}
+
+	void test_include_with_defines()
+	{
+		CPreprocessorWrapper::IncludeRetrieverCallback includeCallback = [](
+				const CStr& includePath, CStr& out) {
+			out = R"(
+				#if defined(A)
+					#define X 41
+				#elif defined(B)
+					#define X 42
+				#else
+					#define X 43
+				#endif
+				#ifdef Y
+					#undef Y
+					#define Y 256
+				#endif
+				vec3 color();
+			)";
+			return true;
+		};
+
+		PreprocessorResult result = ParseWithIncludes(R"(
+			#define Y 128
+			#define B 1
+			#include "test.h"
+			X Y
+		)", includeCallback);
+
+		TS_ASSERT_EQUALS(
+			CompressWhiteSpaces(result.output.Trim(PS_TRIM_BOTH)),
+			"#line 1\n\n\t\n\n\n\t\n\t\n\n\n\t\n\t\n\t\n\t\n\tvec3 color();\n\t\n#line 5\n\t42 256");
+	}
+
+	void test_performance_DISABLED()
+	{
+		CPreprocessorWrapper::IncludeRetrieverCallback includeCallback = [](
+			const CStr& includePath, CStr& out) {
+			const size_t dotPosition = includePath.find('.');
+			TS_ASSERT_DIFFERS(dotPosition, CStr::npos);
+			const int depth = CStr(includePath.substr(0, dotPosition)).ToInt();
+			TS_ASSERT_LESS_THAN_EQUALS(0, depth);
+			if (depth < 4)
+			{
+				std::stringstream nextIncludes;
+				for (int idx = 0; idx < 8; ++idx)
+					nextIncludes << "#include \"" << depth + 1 << ".h\"\n";
+				out = nextIncludes.str();
+			}
+			else
+			{
+				out = R"(
+					42
+				)";
+			}
+			return true;
+		};
+
+		const double start = timer_Time();
+		PreprocessorResult result = ParseWithIncludes(R"(
+			#include "0.h"
+		)", includeCallback);
+		const double finish = timer_Time();
+		printf("Total: %lfs\n", finish - start);
+		TS_ASSERT_EQUALS(result.output.Trim(PS_TRIM_BOTH).size(), 132824u);
+	}
 };