Introduce Virtual Address (#2226)

In the current ongoing effort for sourcemod to fully support 64 bits, we are introducing "virtual address".

# Explanation

Because SourcePawn does not yet support a 64 bits-wide type it's been impossible for any plugins to hold addresses in regular 32-bits wide variable.

A first attempt at solving this issue was made in commit ce1a4dc therein dubbed "PseudoAddress", however this turned out to be an unsatisfactory solution, as any 'high' address if offsetted could turn invalid (or outright be impossible to map).

This leaves us with three alternatives :
- New type
- Convert Address into a handle
- Virtual Address

A new type is the most destructive solution, as it entails breaking every single Address related method. While that solution is still not off the table, we're reserving it as the last attempt should this commit fail.

Converting into a handle type is a good compromise between a brand new type whilst also preserving the Address methods. However, this comes with two issues: the first being that you can no longer offset Address, the second is that we would require authors to free the handle type which will be very confusing. This will likely not be implemented.

# Virtual address

Under a reasonable assumption, we've noted that the average plugin is unlikely to play with more than 4 GB of memory; this shouldn't be too surprising as all valve games were once 32bits and therefore limited to 4GB. Assuming this stays mostly true and a plugin isn't interested with the mapped memory of lesser known modules (like soundlib or matlib), it is fair to assume plugins are unlikely to access more than 4GB of mapped memory. Working with this in mind, we map the memory the plugins are likely to access to our custom virtual address ranges (from 0 to 4Gb, the values of which can fit on 32bits variable). If any memory was missed and plugins were to try an access it later those ranges will be late-mapped to our virtual address ranges until we run out of them.

In order to use virtual addressing, whether on 32 bits or 64 bits. Plugins must now "#include <virtual_address>", as well as use the new SDKCall_VirtualAddress, SDKType_VirtualAddress, LoadAddressFromAddress & StoreAddressToAddress where it's appropriate to.

Author: Kenzzer

.gitmodules

@@ -12,3 +12,6 @@
 [submodule "public/safetyhook"]
 	path = public/safetyhook
 	url = https://github.com/alliedmodders/safetyhook
+[submodule "core/logic/libaddrz"]
+	path = core/logic/libaddrz
+	url = https://github.com/dvander/libaddrz.git

core/logic/AMBuilder

@@ -86,9 +86,15 @@ for cxx in builder.targets:
     'DatabaseConfBuilder.cpp',
     'LumpManager.cpp',
     'smn_entitylump.cpp',
+    'libaddrz/addrz.cpp',
+    'libaddrz/mapping.cpp',
+    'libaddrz/platform.cpp',
+    'libaddrz/proc_maps.cpp',
+    'PseudoAddrManager.cpp',
   ]
-
-  if binary.compiler.target.arch == 'x86_64':
-    binary.sources += ['PseudoAddrManager.cpp']
+  if binary.compiler.target.platform == 'linux':
+    binary.sources += ['libaddrz/platform_linux.cpp']
+  elif binary.compiler.target.platform == 'windows':
+    binary.sources += ['libaddrz/platform_windows.cpp']
 
   SM.binaries += [builder.Add(binary)]

core/logic/PluginSys.cpp

@@ -330,6 +330,13 @@ bool CPlugin::ReadInfo()
 	else
 		m_MaxClientsVar = nullptr;
 
+	if (base->FindPubvarByName("PointerSize", &idx) == SP_ERROR_NONE) {
+		sp_pubvar_t* var = nullptr;
+		if (base->GetPubvarByIndex(idx, &var) == SP_ERROR_NONE && var) {
+			*var->offs = sizeof(void*);
+		}
+	}
+
 	return true;
 }
 

core/logic/PseudoAddrManager.cpp

@@ -28,142 +28,96 @@
  */
 
 #include "PseudoAddrManager.h"
+#include <bridge/include/CoreProvider.h>
 #ifdef PLATFORM_APPLE
 #include <mach/mach.h>
 #include <mach/vm_region.h>
 #endif
 #ifdef PLATFORM_LINUX
 #include <inttypes.h>
 #endif
+#ifdef PLATFORM_WINDOWS
+#include <Psapi.h>
+#endif
 
-PseudoAddressManager::PseudoAddressManager() : m_NumEntries(0)
+PseudoAddressManager::PseudoAddressManager() : m_dictionary(am::IPlatform::GetDefault())
 {
 }
 
-// A pseudo address consists of a table index in the upper 6 bits and an offset in the
-// lower 26 bits. The table consists of memory allocation base addresses.
-void *PseudoAddressManager::FromPseudoAddress(uint32_t paddr)
-{
-#ifdef KE_ARCH_X64
-	uint8_t index = paddr >> PSEUDO_OFFSET_BITS;
-	uint32_t offset = paddr & ((1 << PSEUDO_OFFSET_BITS) - 1);
-
-	if (index >= m_NumEntries)
-		return nullptr;
-
-	return reinterpret_cast<void *>(uintptr_t(m_AllocBases[index]) + offset);
-#else
-	return nullptr;
+void PseudoAddressManager::Initialize() {
+#ifdef PLATFORM_WINDOWS
+	auto process = GetCurrentProcess();
+	auto get_module_details = [process](const char* name, void*& baseAddress, size_t& moduleSize) {
+		if (process == NULL) {
+			return false;
+		}
+		auto hndl = GetModuleHandle(name);
+		if (hndl == NULL) {
+			return false;
+		}
+		MODULEINFO info;
+		if (!GetModuleInformation(process, hndl, &info, sizeof(info))) {
+			return false;
+		}
+		moduleSize = info.SizeOfImage;
+		baseAddress = info.lpBaseOfDll;
+		return true;
+	};
 #endif
-}
-
-uint32_t PseudoAddressManager::ToPseudoAddress(void *addr)
-{
-#ifdef KE_ARCH_X64
-	uint8_t index = 0;
-	uint32_t offset = 0;
-	bool hasEntry = false;
-	void *base = GetAllocationBase(addr);
-
-	if (base) {
-		for (int i = 0; i < m_NumEntries; i++) {
-			if (m_AllocBases[i] == base) {
-				index = i;
-				hasEntry = true;
-				break;
-			}
+#ifdef PLATFORM_LINUX
+	auto get_module_details = [](const char* name, void* baseAddress, size_t& moduleSize) {
+		auto hndl = dlopen(name, RTLD_NOLOAD);
+		if (hndl == NULL) {
+			return false;
 		}
-	} else {
-		return 0;
-	}
+		void* addr = dlsym(hndl, "CreateInterface");
+		dlclose(hndl);
 
-	if (!hasEntry) {
-		index = m_NumEntries;
-		if (m_NumEntries < SM_ARRAYSIZE(m_AllocBases))
-			m_AllocBases[m_NumEntries++] = base;
-		else
-			return 0;	// Table is full
-	}
-
-	ptrdiff_t diff = uintptr_t(addr) - uintptr_t(base);
+		if (!addr) {
+			return false;
+		}
 
-	// Ensure difference fits in 26 bits
-	if (diff > (UINT32_MAX >> PSEUDO_INDEX_BITS))
-		return 0;
+		Dl_info info;
+		if (dladdr(addr, &info) == 0) {
+			return false;
+		}
 
-	return (index << PSEUDO_OFFSET_BITS) | diff;
-#else
-	return 0;
+		baseAddress = info.dli_fbase;
+		// It doesn't matter much if we figure out the module size
+		// libaddrz coalesce maps on linux
+		moduleSize = 0;
+		return true;
+	};
 #endif
+
+	// Early map commonly used modules, it's okay if not all of them are here
+	// Everything else will be caught by "ToPseudoAddress" but you risk running out of ranges by then
+	const char* libs[] = { "engine", "server", "tier0", "vstdlib" };
+
+	char formattedName[64];
+	for (int i = 0; i < sizeof(libs) / sizeof(const char*); i++) {
+		bridge->FormatSourceBinaryName(libs[i], formattedName, sizeof(formattedName));
+		void* base_addr = nullptr;
+		size_t module_size = 0;
+		if (get_module_details(formattedName, base_addr, module_size)) {
+			// Create the mapping (hopefully)
+			m_dictionary.Make32bitAddress(base_addr, module_size);
+		}
+	}
 }
 
-void *PseudoAddressManager::GetAllocationBase(void *ptr)
+void *PseudoAddressManager::FromPseudoAddress(uint32_t paddr)
 {
-#if defined PLATFORM_WINDOWS
-
-	MEMORY_BASIC_INFORMATION info;
-	if (!VirtualQuery(ptr, &info, sizeof(MEMORY_BASIC_INFORMATION)))
+	if (paddr == 0) {
 		return nullptr;
-	return info.AllocationBase;
-
-#elif defined PLATFORM_APPLE
-
-#ifdef KE_ARCH_X86
-	typedef vm_region_info_t mach_vm_region_info_t;
-	typedef vm_region_basic_info_data_t mach_vm_region_basic_info_data_t;
-	const vm_region_flavor_t MACH_VM_REGION_BASIC_INFO = VM_REGION_BASIC_INFO;
-	const mach_msg_type_number_t MACH_VM_REGION_BASIC_INFO_COUNT = VM_REGION_BASIC_INFO_COUNT;
-	#define mach_vm_region vm_region
-#elif defined KE_ARCH_X64
-	typedef vm_region_info_64_t mach_vm_region_info_t ;
-	typedef vm_region_basic_info_data_64_t mach_vm_region_basic_info_data_t;
-	const vm_region_flavor_t MACH_VM_REGION_BASIC_INFO = VM_REGION_BASIC_INFO_64;
-	const mach_msg_type_number_t MACH_VM_REGION_BASIC_INFO_COUNT = VM_REGION_BASIC_INFO_COUNT_64;
-	#define mach_vm_region vm_region_64
-#endif
-	vm_size_t size;
-	vm_address_t vmaddr = reinterpret_cast<vm_address_t>(ptr);
-	mach_vm_region_basic_info_data_t info;
-	memory_object_name_t obj;
-	vm_region_flavor_t flavor = MACH_VM_REGION_BASIC_INFO;
-	mach_msg_type_number_t count = MACH_VM_REGION_BASIC_INFO_COUNT;
-	
-	kern_return_t kr = mach_vm_region(mach_task_self(), &vmaddr, &size, flavor,
-	                                  reinterpret_cast<mach_vm_region_info_t>(&info), 
-	                                  &count, &obj);
-
-	if (kr != KERN_SUCCESS)
-		return nullptr;
-
-	return reinterpret_cast<void *>(vmaddr);
-
-#elif defined PLATFORM_LINUX
-
-	uintptr_t addr = reinterpret_cast<uintptr_t>(ptr);
-
-	// Format:
-	// lower    upper    prot     stuff                 path
-	// 08048000-0804c000 r-xp 00000000 03:03 1010107    /bin/cat
-	FILE *fp = fopen("/proc/self/maps", "r");
-	if (fp) {
-		uintptr_t lower, upper;
-		while (fscanf(fp, "%" PRIxPTR "-%" PRIxPTR, &lower, &upper) != EOF) {
-			if (addr >= lower && addr <= upper) {
-				fclose(fp);
-				return reinterpret_cast<void *>(lower);
-			}
+	}
+	return m_dictionary.RecoverAddress(paddr).value_or(nullptr);
+}
 
-			// Read to end of line
-			int c;
-			while ((c = fgetc(fp)) != '\n') {
-				if (c == EOF)
-					break;
-			}
-			if (c == EOF)
-				break;
-		}
-		fclose(fp);
+uint32_t PseudoAddressManager::ToPseudoAddress(void *addr)
+{
+	if (addr == nullptr) {
+		return 0;
 	}
-	return nullptr;
-#endif
+	return m_dictionary.Make32bitAddress(addr).value_or(0);
 }

core/logic/PseudoAddrManager.h

@@ -31,6 +31,7 @@
 #define _INCLUDE_SOURCEMOD_PSEUDOADDRESSMANAGER_H_
 
 #include "common_logic.h"
+#include "libaddrz/addrz.h"
 
 class PseudoAddressManager
 {
@@ -39,13 +40,9 @@ class PseudoAddressManager
 public:
 	void *FromPseudoAddress(uint32_t paddr);
 	uint32_t ToPseudoAddress(void *addr);
+	void Initialize();
 private:
-	void *GetAllocationBase(void *ptr);
-private:
-	static constexpr uint8_t PSEUDO_OFFSET_BITS = 26;
-	static constexpr uint8_t PSEUDO_INDEX_BITS = sizeof(uint32_t) * 8 - PSEUDO_OFFSET_BITS;
-	void *m_AllocBases[1 << PSEUDO_INDEX_BITS];
-	uint8_t m_NumEntries;
+	am::AddressDict m_dictionary;
 };
 
 #endif // _INCLUDE_SOURCEMOD_PSEUDOADDRESSMANAGER_H_

core/logic/common_logic.cpp

@@ -57,6 +57,7 @@
 #include "RootConsoleMenu.h"
 #include "CellArray.h"
 #include "smn_entitylump.h"
+#include "PseudoAddrManager.h"
 #include <bridge/include/BridgeAPI.h>
 #include <bridge/include/IProviderCallbacks.h>
 
@@ -86,9 +87,7 @@ IScriptManager *scripts = &g_PluginSys;
 IExtensionSys *extsys = &g_Extensions;
 ILogger *logger = &g_Logger;
 CNativeOwner g_CoreNatives;
-#ifdef KE_ARCH_X64
 PseudoAddressManager pseudoAddr;
-#endif
 
 EntityLumpParseResult lastParseResult;
 
@@ -122,20 +121,12 @@ static void RegisterProfiler(IProfilingTool *tool)
 
 static void *FromPseudoAddress(uint32_t paddr)
 {
-#ifdef KE_ARCH_X64
 	return pseudoAddr.FromPseudoAddress(paddr);
-#else
-	return nullptr;
-#endif
 }
 
 static uint32_t ToPseudoAddress(void *addr)
 {
-#ifdef KE_ARCH_X64
 	return pseudoAddr.ToPseudoAddress(addr);
-#else
-	return 0;
-#endif
 }
 
 static void SetEntityLumpWritable(bool writable)
@@ -236,6 +227,7 @@ static void logic_init(CoreProvider* core, sm_logic_t* _logic)
 	g_pSourcePawn2 = *core->spe2;
 	SMGlobalClass::head = core->listeners;
 
+	pseudoAddr.Initialize();
 	g_ShareSys.Initialize();
 	g_pCoreIdent = g_ShareSys.CreateCoreIdentity();