win32: Abort WaitCommEvent() in sp_close()

[HiFiPhile]

Fix an issue with Cypress CY7C652XX driver where overlapped IO is not released once the port handle is closed, cause following port open resulted in "Access denied" error.

Not reproducible with FTDI or SiLabs drivers.

If a process attempts to change the device handle's event mask by using the SetCommMask function while an overlapped WaitCommEvent operation is in progress, WaitCommEvent returns immediately. The variable pointed to by the lpEvtMask parameter is set to zero.

Stripped down demonstration code

#include <Windows.h>
#include <stdbool.h>
#include <stdio.h>

#define RETURN_FAIL(...) do{printf("%s\r\n", __VA_ARGS__); return false;}while(0)

struct sp_port {
	char *name;
	HANDLE hdl;
	COMMTIMEOUTS timeouts;
	OVERLAPPED write_ovl;
	OVERLAPPED read_ovl;
	OVERLAPPED wait_ovl;
	DWORD events;
	BYTE *write_buf;
	DWORD write_buf_size;
	BOOL writing;
	BOOL wait_running;
};

bool sp_open(struct sp_port *port);
bool sp_close(struct sp_port *port);

bool sp_open(struct sp_port *port)
{
    DWORD desired_access = 0, flags_and_attributes = 0, errors;
    char *escaped_port_name;
    COMSTAT status;
    DCB dcb;

    /* Prefix port name with '\\.\' to work with ports above COM9. */
    escaped_port_name = malloc(strlen(port->name) + 5);
    sprintf(escaped_port_name, "\\\\.\\%s", port->name);

    flags_and_attributes = FILE_ATTRIBUTE_NORMAL | FILE_FLAG_OVERLAPPED;
    desired_access |= GENERIC_READ | GENERIC_WRITE;

    port->hdl = CreateFile(escaped_port_name, desired_access, 0, 0,
             OPEN_EXISTING, flags_and_attributes, 0);

    free(escaped_port_name);

    if (port->hdl == INVALID_HANDLE_VALUE)
        RETURN_FAIL("Port CreateFile() failed");

    /* All timeouts initially disabled. */
    port->timeouts.ReadIntervalTimeout = 0;
    port->timeouts.ReadTotalTimeoutMultiplier = 0;
    port->timeouts.ReadTotalTimeoutConstant = 0;
    port->timeouts.WriteTotalTimeoutMultiplier = 0;
    port->timeouts.WriteTotalTimeoutConstant = 0;

    if (SetCommTimeouts(port->hdl, &port->timeouts) == 0) {
        sp_close(port);
        RETURN_FAIL("SetCommTimeouts() failed");
    }

    /* Prepare OVERLAPPED structures. */
#define INIT_OVERLAPPED(ovl) do { \
    memset(&port->ovl, 0, sizeof(port->ovl)); \
    port->ovl.hEvent = INVALID_HANDLE_VALUE; \
    if ((port->ovl.hEvent = CreateEvent(NULL, TRUE, TRUE, NULL)) \
            == INVALID_HANDLE_VALUE) { \
        sp_close(port); \
        RETURN_FAIL(#ovl "CreateEvent() failed"); \
    } \
} while (0)

    INIT_OVERLAPPED(read_ovl);
    INIT_OVERLAPPED(write_ovl);
    INIT_OVERLAPPED(wait_ovl);

    /* Set event mask for RX and error events. */
    if (SetCommMask(port->hdl, EV_RXCHAR | EV_ERR) == 0) {
        sp_close(port);
        RETURN_FAIL("SetCommMask() failed");
    }

    port->writing = FALSE;
    port->wait_running = FALSE;

    if (!GetCommState(port->hdl, &dcb))
        RETURN_FAIL("GetCommState() failed");

    /* Set sane port settings. */
    dcb.fBinary = TRUE;
    dcb.fDsrSensitivity = FALSE;
    dcb.fErrorChar = FALSE;
    dcb.fNull = FALSE;
    dcb.fAbortOnError = FALSE;

    if (ClearCommError(port->hdl, &errors, &status) == 0)
        RETURN_FAIL("ClearCommError() failed");

    if (!SetCommState(port->hdl, &dcb))
        RETURN_FAIL("SetCommState() failed");

    if (WaitCommEvent(port->hdl, &port->events,
        &port->wait_ovl)) {
    } else if (GetLastError() == ERROR_IO_PENDING) {
        port->wait_running = TRUE;
    } else {
        RETURN_FAIL("WaitCommEvent() failed");
    }

    /* Allocate write buffer for 50ms of data at baud rate. */
    port->write_buf_size = min(dcb.BaudRate / (8 * 20), 1);
    port->write_buf = malloc(port->write_buf_size);

    if (!port->write_buf)
        RETURN_FAIL("Allocating write buffer failed");

    return true;
}

bool sp_close(struct sp_port *port)
{
    /* Returns non-zero upon success, 0 upon failure. */
    if (CloseHandle(port->hdl) == 0)
    RETURN_FAIL("Port CloseHandle() failed");
    port->hdl = INVALID_HANDLE_VALUE;

    /* Close event handles for overlapped structures. */
#define CLOSE_OVERLAPPED(ovl) do { \
    if (port->ovl.hEvent != INVALID_HANDLE_VALUE && \
        CloseHandle(port->ovl.hEvent) == 0) \
        RETURN_FAIL(# ovl "event CloseHandle() failed"); \
} while (0)
    CLOSE_OVERLAPPED(read_ovl);
    CLOSE_OVERLAPPED(write_ovl);
    CLOSE_OVERLAPPED(wait_ovl);


    if (port->write_buf) {
        free(port->write_buf);
        port->write_buf = NULL;
    }

    return true;
}

int main(int argc, char** argv)
{
    struct sp_port port;
    port.name = argv[1];

    sp_open(&port);
    sp_close(&port);

    sp_open(&port);
    sp_close(&port);

    sp_open(&port);
    sp_close(&port);
}