#include <mpi.h>
#include <stdlib.h>
#include <sys/mman.h>

int main(int argc, char *argv[])
{
    MPI_Init(&argc, &argv);

    int my_rank;
    MPI_Comm_rank(MPI_COMM_WORLD, &my_rank);

    if (my_rank == 0)
    {
        // send a 1 MB buffer
        char* buffer = (char*)malloc(1024 * 1024);
        for (size_t i = 0; i < 1024 * 1024; ++i)
            buffer[i] = i;

        MPI_Send(buffer, 1024 * 1024, MPI_BYTE,
                 /* dest */ 1, /* tag */ 0, MPI_COMM_WORLD);

        // async send the first 128 KB
        MPI_Request req;
        MPI_Isend(buffer, 128 * 1024, MPI_BYTE,
                  /* dest */ 1, /* tag */ 0, MPI_COMM_WORLD, &req);

        // mark the second half of the buffer as DONTNEED. this triggers the fatal error message
        madvise(buffer + 512 * 1024, 512 * 1024, MADV_DONTNEED);

        MPI_Wait(&req, MPI_STATUS_IGNORE);
    }
    else if (my_rank == 1)
    {
        // receive a 1 MB buffer
        char* buffer = (char*)malloc(1024 * 1024);
        MPI_Recv(buffer, 1024 * 1024, MPI_BYTE,
                 /* source */ 0, /* tag */ 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE);

        // receive 128 KB
        MPI_Recv(buffer, 128 * 1024, MPI_BYTE,
                 /* source */ 0, /* tag */ 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
    }

    MPI_Finalize();

    return 0;
}