技术博客
首页 所有文章 标签 分类 关于
技术博客

目录

windows-C-申请大量内存

 收录于 客户端 C
   约 1999 字   预计阅读 4 分钟 
https://bing.ee123.net/img/rand?artid=146043865
目录

windows C++ 申请大量内存

在程序的应用开发时候，面对内存密集型操作时，需要大量内存，可能需要远超物理内存空间的内存，该怎么做呢？

假设现在的机器是64位的windows，其用户的虚地址空间是0x000’000000000到0x7FFF’FFFFFFFF，约128 TB。

可以考虑提前申请空间到数据段

#include <iostream>
constexpr size_t K_ = 1024;
constexpr size_t KB_ = K_;
constexpr size_t MB_ = K_ * KB_;
constexpr size_t GB_ = K_ * MB_;
constexpr size_t TB_ = K_ * GB_;
uint8_t gBuff[10 * GB_];
int main()
{
	return 0;
}

这样可以得到一个编译错误

错误	C2148	数组的总大小不得超过 0x7fffffff 字节

可以考虑使用malloc动态申请内存

malloc会申请到一块连续的内存，如果失败，会返回空指针

#include "stdafx.h"
#include <iostream>

constexpr size_t K_ = 1024;
constexpr size_t KB_ = K_;
constexpr size_t MB_ = K_ * KB_;
constexpr size_t GB_ = K_ * MB_;
constexpr size_t TB_ = K_ * GB_;

int main()
{
	void* ptr = malloc(48 * GB_);
	if (!ptr) {
		std::cerr << strerror(errno) << std::endl;
		exit(EXIT_FAILURE);
	}
	return 0;
}

这样的话就只能退一步，分批申请小块内存，一直到申请内存的总量是达到预期。

#include "stdafx.h"
#include <iostream>

constexpr size_t K_ = 1024;
constexpr size_t KB_ = K_;
constexpr size_t MB_ = K_ * KB_;
constexpr size_t GB_ = K_ * MB_;
constexpr size_t TB_ = K_ * GB_;

void* MyMalloc(const size_t expect, size_t& actual)
{
	actual = expect;
	void *ptr = nullptr;
	while (actual && !(ptr = malloc(actual)))
	{
		actual = actual>>1;
	}
	return ptr;
}

int main()
{
	size_t expect = 48 * GB_;
	constexpr int len = 0xffff;
	void* buff[len];
	int i = 0;
	for (i = 0; i < len && expect; ++i)
	{
		size_t actual;
		buff[i] = MyMalloc(expect, actual);
		if (buff[i] && actual)
		{
			expect -= actual;
			std::cout << "malloced (" << actual << ") = " << buff[i] << std::endl;
		}
		else {
			break;
		}
	}
	if (expect) {
		std::cerr << "malloc big memory fail!" << std::endl;
	}
	for (i = 0; i < len && buff[i]; ++i)
	{
		free(buff[i]);
		buff[i] = nullptr;
	}
	return 0;
}

可能会得到一个崩溃

https://i-blog.csdnimg.cn/direct/286196b11c2d4a28a3aaa3ffebac4e0d.png

windows上的malloc可能是使用HeapAlloc或VirtualAlloc实现的，内存不够用的时候发生崩溃。

需要借助VirualAlloc

具体查看

#include <iostream>
#include <windows.h>

constexpr size_t K_ = 1024;
constexpr size_t KB_ = K_;
constexpr size_t MB_ = K_ * KB_;
constexpr size_t GB_ = K_ * MB_;
constexpr size_t TB_ = K_ * GB_;

void* MyAlloc(const size_t expect, size_t& actual)
{
	actual = expect;
	void *ptr = nullptr;
	while (actual && !(ptr = VirtualAlloc(nullptr, actual, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)))
	{
		actual = actual >> 1;
	}
	return ptr;
}

bool MyFree(void* ptr)
{
	return (TRUE == VirtualFree(ptr, 0, MEM_RELEASE));
}

int main()
{
	size_t expect = 48 * GB_;
	constexpr int len = 0xffff;
	void* buff[len];
	int i = 0;
	for (i = 0; i < len && expect; ++i)
	{
		size_t actual;
		buff[i] = MyAlloc(expect, actual);
		if (buff[i] && actual)
		{
			expect -= actual;
			std::cout << "myalloced (" << actual << ") = " << buff[i] << std::endl;
		}
		else {
			break;
		}
	}
	if (expect) {
		std::cerr << "alloc big memory fail!" << std::endl;
	}
	else {
		std::cout << "alloced success" << std::endl;
	}
	for (i = 0; i < len && buff[i]; ++i)
	{
		MyFree(buff[i]);
		buff[i] = nullptr;
	}
	return 0;
}

虚拟内存可能在不使用的时候暂存到硬盘交换区中去，可以使用MEM_RESET与MEM_RESET_UNDO来操作。

#include "stdafx.h"
#include <iostream>
#include <cassert>
#include <windows.h>

constexpr size_t K_ = 1024;
constexpr size_t KB_ = K_;
constexpr size_t MB_ = K_ * KB_;
constexpr size_t GB_ = K_ * MB_;
constexpr size_t TB_ = K_ * GB_;

constexpr size_t len = 0xffff;
uint8_t* buff[len];

size_t MyCommit(void* const basePtr, const size_t expect)
{
	size_t actual = expect;
	while (actual && !VirtualAlloc(basePtr, actual, MEM_COMMIT, PAGE_READWRITE))
	{
		actual >>= 1;
	}
	if (actual) {
		MEMORY_BASIC_INFORMATION memInfo;
		VirtualQuery(basePtr, &memInfo, sizeof(memInfo));
		if (memInfo.BaseAddress == basePtr && memInfo.State == MEM_COMMIT) {
			actual = memInfo.RegionSize;
		}
		else {
			actual = 0;
		}
	}
	return actual;
}


int main()
{
	MEMORY_BASIC_INFORMATION memInfo;
	constexpr size_t expect = 64 * GB_;
	// 无法直接申请到超大块内存，需要先预订内存
	void* ptr = VirtualAlloc(nullptr, expect, MEM_RESERVE, PAGE_NOACCESS);
	VirtualQuery(ptr, &memInfo, sizeof(memInfo));
	printf("基地址: %p, 状态: %x, 大小: %zu 字节\n", memInfo.BaseAddress, memInfo.State, memInfo.RegionSize);
	// 然后申请内存
	const size_t actual = MyCommit(ptr, expect);
	assert(actual);
	VirtualQuery(ptr, &memInfo, sizeof(memInfo));
	printf("基地址: %p, 状态: %x, 大小: %zu 字节\n", memInfo.BaseAddress, memInfo.State, memInfo.RegionSize);
	// 在申请到内存内写入数据
	for (int i = 0; i < 10; ++i) {
		((uint8_t*)ptr)[i] = i + 1;
	}
	// 现在对刚刚的内存不感兴趣了，可以暂存的交换区
	VirtualAlloc(ptr, 0, MEM_RESET, PAGE_NOACCESS);
	VirtualQuery(ptr, &memInfo, sizeof(memInfo));
	printf("基地址: %p, 状态: %x, 大小: %zu 字节\n", memInfo.BaseAddress, memInfo.State, memInfo.RegionSize);
	// 去申请下一块内存
	size_t next_expect = expect - actual;
	void* next_ptr = (void*)((size_t)ptr + actual);
	const size_t next_actual = MyCommit(next_ptr, next_expect);
	VirtualQuery(next_ptr, &memInfo, sizeof(memInfo));
	printf("基地址: %p, 状态: %x, 大小: %zu 字节\n", memInfo.BaseAddress, memInfo.State, memInfo.RegionSize);
	assert(next_actual);
	for (int i = 0; i < 10; ++i) {
		((uint8_t*)next_ptr)[i] = i + 1;
	}
	// 把申请到的next_ptr 暂存到交换区
	VirtualAlloc(next_ptr, 0, MEM_RESET, PAGE_NOACCESS);
	VirtualQuery(next_ptr, &memInfo, sizeof(memInfo));
	printf("基地址: %p, 状态: %x, 大小: %zu 字节\n", memInfo.BaseAddress, memInfo.State, memInfo.RegionSize);
	// 把ptr处的内存从交换区中取出
	VirtualAlloc(ptr, actual, MEM_RESET_UNDO, PAGE_READWRITE);
	VirtualQuery(ptr, &memInfo, sizeof(memInfo));
	printf("基地址: %p, 状态: %x, 大小: %zu 字节\n", memInfo.BaseAddress, memInfo.State, memInfo.RegionSize);
	// 读取数据
	for (int i = 0; i < 10; ++i) {
		std::cout << int(((uint8_t*)ptr)[i]) << std::endl;
	}
	VirtualFree(ptr, 0, MEM_RELEASE);
	VirtualQuery(ptr, &memInfo, sizeof(memInfo));
	printf("基地址: %p, 状态: %x, 大小: %zu 字节\n", memInfo.BaseAddress, memInfo.State, memInfo.RegionSize);
	return EXIT_SUCCESS;
}

如果是32位的应用程序像突破4GB的寻址限制，需要使用到 ， 简称AWE。


#include "stdafx.h"
#include <windows.h>
#include <stdio.h>
#include <tchar.h>

constexpr size_t MEMORY_REQUESTED = 1024 * 1024;

BOOL
LoggedSetLockPagesPrivilege(HANDLE hProcess,
	BOOL bEnable);

size_t GetPageSize() {
	static size_t page_size = 0;
	if (page_size == 0) {
		SYSTEM_INFO sSysInfo;
		GetSystemInfo(&sSysInfo);
		page_size = sSysInfo.dwPageSize;
	}
	return page_size;
}

void  main()
{
	BOOL bResult;                   // generic Boolean value
	ULONG_PTR NumberOfPages;        // number of pages to request
	ULONG_PTR NumberOfPagesInitial; // initial number of pages requested
	ULONG_PTR *aPFNs;               // page info; holds opaque data
	PVOID lpMemReserved;            // AWE window

	int PFNArraySize;               // memory to request for PFN array

	_tprintf(_T("This computer has page size %d.\n"), GetPageSize());

	// Calculate the number of pages of memory to request.

	NumberOfPages = MEMORY_REQUESTED / GetPageSize();
	_tprintf(_T("Requesting %d pages of memory.\n"), NumberOfPages);

	// Calculate the size of the user PFN array.

	PFNArraySize = NumberOfPages * sizeof(ULONG_PTR);

	_tprintf(_T("Requesting a PFN array of %d bytes.\n"), PFNArraySize);

	aPFNs = (ULONG_PTR *)HeapAlloc(GetProcessHeap(), 0, PFNArraySize);

	if (aPFNs == NULL)
	{
		_tprintf(_T("Failed to allocate on heap.\n"));
		return;
	}

	// Enable the privilege.

	if (!LoggedSetLockPagesPrivilege(GetCurrentProcess(), TRUE))
	{
		return;
	}

	// Allocate the physical memory.

	NumberOfPagesInitial = NumberOfPages;
	bResult = AllocateUserPhysicalPages(GetCurrentProcess(),
		&NumberOfPages,
		aPFNs);

	if (bResult != TRUE)
	{
		_tprintf(_T("Cannot allocate physical pages (%u)\n"), GetLastError());
		return;
	}

	if (NumberOfPagesInitial != NumberOfPages)
	{
		_tprintf(_T("Allocated only %p pages.\n"), NumberOfPages);
		return;
	}

	// Reserve the virtual memory.

	lpMemReserved = VirtualAlloc(NULL,
		MEMORY_REQUESTED,
		MEM_RESERVE | MEM_PHYSICAL,
		PAGE_READWRITE);

	if (lpMemReserved == NULL)
	{
		_tprintf(_T("Cannot reserve memory.\n"));
		return;
	}

	// Map the physical memory into the window.

	bResult = MapUserPhysicalPages(lpMemReserved,
		NumberOfPages - 3,
		&aPFNs[3]);

	if (bResult != TRUE)
	{
		_tprintf(_T("MapUserPhysicalPages failed (%u)\n"), GetLastError());
		return;
	}

	// unmap

	bResult = MapUserPhysicalPages(lpMemReserved,
		NumberOfPages - 3,
		NULL);

	if (bResult != TRUE)
	{
		_tprintf(_T("MapUserPhysicalPages failed (%u)\n"), GetLastError());
		return;
	}

	// Free the physical pages.

	bResult = FreeUserPhysicalPages(GetCurrentProcess(),
		&NumberOfPages,
		aPFNs);

	if (bResult != TRUE)
	{
		_tprintf(_T("Cannot free physical pages, error %u.\n"), GetLastError());
		return;
	}

	// Free virtual memory.

	bResult = VirtualFree(lpMemReserved,
		0,
		MEM_RELEASE);

	// Release the aPFNs array.

	bResult = HeapFree(GetProcessHeap(), 0, aPFNs);

	if (bResult != TRUE)
	{
		_tprintf(_T("Call to HeapFree has failed (%u)\n"), GetLastError());
	}

}

/*****************************************************************
LoggedSetLockPagesPrivilege: a function to obtain or
release the privilege of locking physical pages.

Inputs:

HANDLE hProcess: Handle for the process for which the
privilege is needed

BOOL bEnable: Enable (TRUE) or disable?

Return value: TRUE indicates success, FALSE failure.

*****************************************************************/
BOOL
LoggedSetLockPagesPrivilege(HANDLE hProcess,
	BOOL bEnable)
{
	struct {
		DWORD Count;
		LUID_AND_ATTRIBUTES Privilege[1];
	} Info;

	HANDLE Token;
	BOOL Result;

	// Open the token.

	Result = OpenProcessToken(hProcess,
		TOKEN_ADJUST_PRIVILEGES,
		&Token);

	if (Result != TRUE)
	{
		_tprintf(_T("Cannot open process token.\n"));
		return FALSE;
	}

	// Enable or disable?

	Info.Count = 1;
	if (bEnable)
	{
		Info.Privilege[0].Attributes = SE_PRIVILEGE_ENABLED;
	}
	else
	{
		Info.Privilege[0].Attributes = 0;
	}

	// Get the LUID.

	Result = LookupPrivilegeValue(NULL,
		SE_LOCK_MEMORY_NAME,
		&(Info.Privilege[0].Luid));

	if (Result != TRUE)
	{
		_tprintf(_T("Cannot get privilege for %s.\n"), SE_LOCK_MEMORY_NAME);
		return FALSE;
	}

	// Adjust the privilege.

	Result = AdjustTokenPrivileges(Token, FALSE,
		(PTOKEN_PRIVILEGES)&Info,
		0, NULL, NULL);

	// Check the result.

	if (Result != TRUE)
	{
		_tprintf(_T("Cannot adjust token privileges (%u)\n"), GetLastError());
		return FALSE;
	}
	else
	{
		if (GetLastError() != ERROR_SUCCESS)
		{
			_tprintf(_T("Cannot enable the SE_LOCK_MEMORY_NAME privilege; "));
			_tprintf(_T("please check the local policy.\n"));
			return FALSE;
		}
	}

	CloseHandle(Token);

	return TRUE;
}
更新于 2025-03-12
 Windows, C
返回 | 主页