pty.c

Go to the documentation of this file.

 
#include "pty.h"
#include <stdio.h>


INTERNAL void
safe_close_handle(HANDLE *h)
{
    if (h != NULL && *h != NULL) {
        CloseHandle(*h);
        *h = NULL;
    }
}
 
b32
pty_init(struct pty_state_t *state, u16 columns, u16 rows)
{
    ASSERT_NOT_NULL(state);
 
    HANDLE hInputRead   = NULL;
    HANDLE hInputWrite  = NULL;
    HANDLE hOutputRead  = NULL;
    HANDLE hOutputWrite = NULL;
 
    /*
     * Mark pipe handles as inheritable so ConPTY can duplicate them
     * internally. bInheritHandle = TRUE opts these handles into the
     * inheritance system, but inheritance itself is disabled in
     * pty_spawn() via bInheritHandles = FALSE.
     */
    SECURITY_ATTRIBUTES saAttr  = {0};
    saAttr.nLength          = sizeof(SECURITY_ATTRIBUTES);
    saAttr.bInheritHandle       = TRUE;
    saAttr.lpSecurityDescriptor = NULL;
 
    if (!CreatePipe(&hInputRead, &hInputWrite, &saAttr, 0)) {
        (void)fprintf(
            stderr,
            "[PTY] Failed to create input pipe. Error: %lu\n",
            GetLastError());
        return false;
    }
 
    if (!CreatePipe(&hOutputRead, &hOutputWrite, &saAttr, 0)) {
        (void)fprintf(
            stderr,
            "[PTY] Failed to create output pipe. Error: %lu\n",
            GetLastError());
        safe_close_handle(&hInputRead);
        safe_close_handle(&hInputWrite);
        return false;
    }
 
    COORD   terminalSize = {(SHORT)columns, (SHORT)rows};
    HRESULT hr = CreatePseudoConsole(terminalSize, hInputRead, hOutputWrite, 0, &state->hPC);
 
    if (FAILED(hr)) {
        (void)fprintf(stderr, "[PTY] Failed to create ConPTY. HRESULT: 0x%08lX\n", hr);
        safe_close_handle(&hInputRead);
        safe_close_handle(&hInputWrite);
        safe_close_handle(&hOutputRead);
        safe_close_handle(&hOutputWrite);
        return false;
    }
 
    /*
     * Close the ConPTY-owned pipe ends in our process. ConPTY has
     * duplicated these handles internally. Retaining our copies would
     * prevent the output pipe from ever signalling EOF  our ReadFile
     * loop would block forever after the shell exits.
     */
    safe_close_handle(&hInputRead);
    safe_close_handle(&hOutputWrite);
 
    state->hInputWrite = hInputWrite;
    state->hOutputRead = hOutputRead;
 
    return true;
}
 
b32
pty_spawn(struct pty_state_t *state, LPCWSTR command_line)
{
    ASSERT_NOT_NULL(state);
    ASSERT_NOT_NULL((void *)command_line);
 
    /*
     * Phase 1: Query the required size of the process thread attribute list.
     *
     * The attribute list is an opaque, variable-length blob. Its size
     * depends on the number of attributes we want to attach (1 in our case).
     * We perform a dry-run call with a NULL buffer to have Windows fill in
     * the required byte count. The function will return FALSE with
     * GetLastError() == ERROR_INSUFFICIENT_BUFFER  this is expected.
     */
    SIZE_T attr_list_size = 0;
    InitializeProcThreadAttributeList(NULL, 1, 0, &attr_list_size);
 
    /*
     * Phase 2: Allocate and initialize the attribute list.
     *
     * HeapAlloc is used rather than malloc because this memory is
     * handed directly to Win32. Using the process default heap
     * (GetProcessHeap) signals that this allocation is OS-facing.
     */
    LPPROC_THREAD_ATTRIBUTE_LIST attr_list =
        (LPPROC_THREAD_ATTRIBUTE_LIST)HeapAlloc(GetProcessHeap(), 0, attr_list_size);
 
    if (attr_list == NULL) {
        (void)fprintf(stderr, "[PTY] Failed to allocate process thread attribute list.\n");
        return false;
    }
 
    if (!InitializeProcThreadAttributeList(attr_list, 1, 0, &attr_list_size)) {
        (void)fprintf(
            stderr,
            "[PTY] Failed to initialize attribute list. Error: %lu\n",
            GetLastError());
        HeapFree(GetProcessHeap(), 0, attr_list);
        return false;
    }
 
    /*
     * Phase 3: Wire the ConPTY handle into the attribute list.
     *
     * PROC_THREAD_ATTRIBUTE_PSEUDOCONSOLE is the key that tells Windows
     * to attach the new process to our ConPTY session. This is the only
     * supported mechanism  there is no way to attach a ConPTY after the
     * process has already been created.
     */
    if (!UpdateProcThreadAttribute(
            attr_list,
            0,
            PROC_THREAD_ATTRIBUTE_PSEUDOCONSOLE,
            state->hPC,
            sizeof(HPCON),
            NULL,
            NULL)) {
        (void)fprintf(
            stderr,
            "[PTY] Failed to set ConPTY process attribute. Error: %lu\n",
            GetLastError());
        DeleteProcThreadAttributeList(attr_list);
        HeapFree(GetProcessHeap(), 0, attr_list);
        return false;
    }
 
    /*
     * Phase 4: Configure STARTUPINFOEXW.
     *
     * cb MUST be sizeof(STARTUPINFOEXW), not sizeof(STARTUPINFOW).
     * Windows reads cb as a version tag to determine which struct it is
     * looking at. Passing the wrong size causes Windows to silently
     * ignore lpAttributeList and spawn the process with no ConPTY.
     *
     * Zero-initialization clears all other STARTUPINFOW fields (window
     * position, console title, standard handle overrides), preventing
     * garbage values from affecting process creation.
     */
    STARTUPINFOEXW si  = {0};
    si.StartupInfo.cb  = sizeof(STARTUPINFOEXW);
    si.lpAttributeList = attr_list;
 
    /*
     * Phase 5: Spawn the shell process.
     *
     * Key flags and parameters:
     *
     * - bInheritHandles = FALSE: the pipe handles are attached via the
     *   ConPTY attribute list, not via inheritance. Enabling inheritance
     *   would give the child duplicate pipe handles it doesn't know about,
     *   causing subtle pipe-lifetime bugs and potential deadlocks.
     *
     * - EXTENDED_STARTUPINFO_PRESENT: tells CreateProcessW to treat the
     *   startup info pointer as STARTUPINFOEXW and read lpAttributeList.
     *   This flag AND the correct cb size are both required. One without
     *   the other will silently skip the attribute list.
     *
     * - NULL for lpApplicationName: Windows resolves the executable by
     *   searching the system PATH, so bare names like L"cmd.exe" work.
     */
    PROCESS_INFORMATION pi      = {0};
    BOOL            success = CreateProcessW(
               NULL,
               (LPWSTR)command_line,
               NULL,
               NULL,
               FALSE,
               EXTENDED_STARTUPINFO_PRESENT,
               NULL,
               NULL,
               &si.StartupInfo,
               &pi);
 
    /*
     * Phase 6: Tear down the attribute list immediately.
     *
     * DeleteProcThreadAttributeList performs internal kernel-side cleanup.
     * It must be called BEFORE HeapFree skipping it can leak memory
     * inside the kernel. The attribute list is no longer needed once
     * CreateProcessW has returned.
     */
    DeleteProcThreadAttributeList(attr_list);
    HeapFree(GetProcessHeap(), 0, attr_list);
 
    if (!success) {
        (void)fprintf(stderr, "[PTY] CreateProcessW failed. Error: %lu\n", GetLastError());
        return false;
    }
 
    /*
     * Phase 7: Store the process handle; discard the thread handle.
     *
     * We retain hProcess for lifecycle management (waiting for exit,
     * querying exit code, etc.). We have no use for the main thread handle
     * at this stage, so it is closed immediately to avoid a handle leak.
     * Closing it does NOT affect the running thread.
     */
    state->hProcess = pi.hProcess;
    safe_close_handle(&pi.hThread);
 
    return true;
}
 
void
pty_cleanup(struct pty_state_t *state)
{
    if (!state) {
        return;
    }
 
    if (state->hPC) {
        ClosePseudoConsole(state->hPC);
        state->hPC = NULL;
    }
 
    safe_close_handle(&state->hProcess);
    safe_close_handle(&state->hInputWrite);
    safe_close_handle(&state->hOutputRead);
}
 // end pty_module