Changes XMB format to store UTF-8 strings and removes pointless UTF-16 conversions, refs #204, #244.

Bumps XMB version to 2.

This was SVN commit r15309.

source/ps/XML/XeroXMB.cpp

@@ -1,4 +1,4 @@
-/* Copyright (C) 2011 Wildfire Games.
+/* Copyright (C) 2014 Wildfire Games.
  * This file is part of 0 A.D.
  *
  * 0 A.D. is free software: you can redistribute it and/or modify
@@ -20,7 +20,6 @@
 #include "Xeromyces.h"
 
 #include "lib/byte_order.h"	// FOURCC_LE
-#include "ps/utf16string.h"
 
 // external linkage (also used by Xeromyces.cpp)
 const char* HeaderMagicStr = "XMB0";
@@ -50,11 +49,11 @@ bool XMBFile::Initialise(const char* FileData)
 	// Build a std::map of all the names->ids
 	u32 ElementNameCount = *(u32*)m_Pointer; m_Pointer += 4;
 	for (i = 0; i < ElementNameCount; ++i)
-		m_ElementNames[ReadZStrA()] = i;
+		m_ElementNames[ReadZStr8()] = i;
 
 	u32 AttributeNameCount = *(u32*)m_Pointer; m_Pointer += 4;
 	for (i = 0; i < AttributeNameCount; ++i)
-		m_AttributeNames[ReadZStrA()] = i;
+		m_AttributeNames[ReadZStr8()] = i;
 #else
 	// Ignore all the names for now, and skip over them
 	// (remembering the position of the first)
@@ -72,7 +71,7 @@ bool XMBFile::Initialise(const char* FileData)
 	return true;	// success
 }
 
-std::string XMBFile::ReadZStrA()
+std::string XMBFile::ReadZStr8()
 {
 	int Length = *(int*)m_Pointer;
 	m_Pointer += 4;
@@ -199,7 +198,7 @@ CStr8 XMBElement::GetText() const
 	if (m_Pointer == NULL || *(int*)(m_Pointer + 20) == 0)
 		return CStr8();
 
-	return CStrW(utf16string((utf16_t*)(m_Pointer + 28))).ToUTF8();
+	return CStr8(m_Pointer + 28);
 }
 
 int XMBElement::GetLineNumber() const
@@ -265,7 +264,7 @@ CStr8 XMBAttributeList::GetNamedItem(const int AttributeName) const
 	for (int i = 0; i < Count; ++i)
 	{
 		if (*(int*)Pos == AttributeName)
-			return CStrW(utf16string((utf16_t*)(Pos+8))).ToUTF8();
+			return CStr8(Pos+8);
 		Pos += 8 + *(int*)(Pos+4); // Skip over the string
 	}
 
@@ -297,5 +296,5 @@ XMBAttribute XMBAttributeList::Item(const int id)
 	m_LastItemID = id;
 	m_LastPointer = Pos;
 
-	return XMBAttribute(*(int*)Pos, CStrW(utf16string( (const utf16_t*)(Pos+8) )).ToUTF8());
+	return XMBAttribute(*(int*)Pos, CStr8(Pos+8));
 }

source/ps/XML/XeroXMB.h

@@ -1,4 +1,4 @@
-/* Copyright (C) 2011 Wildfire Games.
+/* Copyright (C) 2014 Wildfire Games.
  * This file is part of 0 A.D.
  *
  * 0 A.D. is free software: you can redistribute it and/or modify
@@ -28,9 +28,6 @@ but much more efficiency (particularly for loading simple data
 classes that don't need much initialisation).
 
 Main limitations:
- * Only handles UTF16 internally. (It's meant to be a feature, but
-   can be detrimental if it's always being converted back to
-   ASCII.)
  * Can't correctly handle mixed text/elements inside elements -
    "<div> <b> Text </b> </div>" and "<div> Te<b/>xt </div>" are
    considered identical.
@@ -47,10 +44,10 @@ XMB_File {
 	char Header[4]; // because everyone has one; currently "XMB0"
 
 	int ElementNameCount;
-	ZStrA ElementNames[];
+	ZStr8 ElementNames[];
 
 	int AttributeNameCount;
-	ZStrA AttributeNames[];
+	ZStr8 AttributeNames[];
 
 	XMB_Node Root;
 }
@@ -72,30 +69,21 @@ XMB_Node {
 
 XMB_Attribute {
 	int Name;
-	ZStrW Value;
+	ZStr8 Value;
 }
 
-ZStrA {
+ZStr8 {
 	int Length; // in bytes
-	char* Text; // null-terminated ASCII
-}
-
-ZStrW {
-	int Length; // in bytes
-	char16* Text; // null-terminated UTF16
+	char* Text; // null-terminated UTF8
 }
 
 XMB_Text {
 20)	int Length; // 0 if there's no text, else 4+sizeof(Text) in bytes including terminator
 	// If Length != 0:
 24)	int LineNumber; // for e.g. debugging scripts
-28)	char16* Text; // null-terminated UTF16
+28)	char* Text; // null-terminated UTF8
 }
 
-TODO: since the API was changed to return UTF-8 CStrs,
-it'd make much more sense to store UTF-8 on disk too
-(plus it'd save space).
-
 */
 
 #ifndef INCLUDED_XEROXMB
@@ -140,7 +128,7 @@ public:
 	XMBElement GetRoot() const;
 
 	
-	// Returns internal ID for a given ASCII element/attribute string.
+	// Returns internal ID for a given element/attribute string.
 	int GetElementID(const char* Name) const;
 	int GetAttributeID(const char* Name) const;
 
@@ -163,7 +151,7 @@ private:
 	const char* m_AttributePointer;
 #endif
 
-	std::string ReadZStrA();
+	std::string ReadZStr8();
 };
 
 class XMBElement

source/ps/XML/Xeromyces.cpp

@@ -1,4 +1,4 @@
-/* Copyright (C) 2012 Wildfire Games.
+/* Copyright (C) 2014 Wildfire Games.
  * This file is part of 0 A.D.
  *
  * 0 A.D. is free software: you can redistribute it and/or modify
@@ -70,7 +70,7 @@ PSRETURN CXeromyces::Load(const PIVFS& vfs, const VfsPath& filename)
 
 	// Arbitrary version number - change this if we update the code and
 	// need to invalidate old users' caches
-	u32 version = 1;
+	u32 version = 2;
 
 	VfsPath xmbPath;
 	Status ret = cacheLoader.TryLoadingCached(filename, MD5(), version, xmbPath);
@@ -277,11 +277,10 @@ static void OutputElement(const xmlNodePtr node, WriteBuffer& writeBuffer,
 	else
 	{
 		// Write length and line number and null-terminated text
-		utf16string textW = CStr8(text).FromUTF8().utf16();
-		u32 nodeLen = u32(4 + 2*(textW.length()+1));
+		u32 nodeLen = u32(4 + text.length()+1);
 		writeBuffer.Append(&nodeLen, 4);
 		writeBuffer.Append(&linenum, 4);
-		writeBuffer.Append((void*)textW.c_str(), nodeLen-4);
+		writeBuffer.Append((void*)text.c_str(), nodeLen-4);
 	}
 
 	// Output attributes
@@ -290,11 +289,10 @@ static void OutputElement(const xmlNodePtr node, WriteBuffer& writeBuffer,
 		writeBuffer.Append(&attributeIDs[(const char*)attr->name], 4);
 
 		xmlChar* value = xmlNodeGetContent(attr->children);
-		utf16string textW = CStr8((const char*)value).FromUTF8().utf16();
-		xmlFree(value);
-		u32 attrLen = u32(2*(textW.length()+1));
+		u32 attrLen = u32(xmlStrlen(value)+1);
 		writeBuffer.Append(&attrLen, 4);
-		writeBuffer.Append((void*)textW.c_str(), attrLen);
+		writeBuffer.Append((void*)value, attrLen);
+		xmlFree(value);
 	}
 
 	// Go back and fill in the child-element offset