c - 使用 MPI_Gather 接收数据结构的正确方法

Question

我尝试使用 MPI_Gather 发送以下数据结构:

struct set {
    int nbits;
    char  bits[];
};

问题是我无法收集上述结构的所有项目，只能收集第一项。其余项目根本没有意义。

这是一个测试用例:

#include <stdio.h>
#include <stdlib.h>

#include "mpi.h"

#define SIZE 10

struct set {
    int nbits;
    char bits[];
};

int main(int argc, char *argv[]) {
    int np, rank, i;
    struct set *subsets, *single;
    void *buf;

    MPI_Init(&argc,&argv);
    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
    MPI_Comm_size(MPI_COMM_WORLD, &np);

    single = malloc(sizeof(struct set) + SIZE);

    if(rank == 0) {
            subsets = malloc((sizeof(struct set) + SIZE) * np);
    }

    buf = &subsets[0];

    MPI_Datatype set_type, oldtypes[2];
    int blockcounts[2];
    MPI_Aint offsets[2];
    MPI_Aint addr[3];

    MPI_Get_address(single, &addr[0]);
    MPI_Get_address(&single->nbits, &addr[1]);
    MPI_Get_address(&single->bits, &addr[2]);

    offsets[0] = addr[1] - addr[0];
    oldtypes[0] = MPI_INT;
    blockcounts[0] = 1;

    offsets[1] = addr[2] - addr[0];
    oldtypes[1] = MPI_CHAR;
    blockcounts[1] = SIZE;

    MPI_Type_create_struct(2, blockcounts, offsets, oldtypes, &set_type);
    MPI_Type_commit(&set_type);

    single->nbits = 2;

    for(i=0; i<single->nbits; i++)
            single->bits[i] = 'A' + rank;

    MPI_Gather(single, 1, set_type, buf, 1, set_type, 0, MPI_COMM_WORLD);

    if(rank == 0) {
            void *ptr;
            struct set *fs;
            int size;

            MPI_Type_size(set_type, &size);

            ptr = buf;

            for(i=0; i<np; i++) {
                    size_t j;

                    fs = ptr;
                    printf("from rank %d: bits => %p nbits => %d\n", i, fs->bits, fs->nbits);

                    for(j=0; j<2; j++)
                            printf("from rank %d: buf[%d] = %#x\n",
                                    i, j, fs->bits[j]);
                    ptr += size;
            }
    }

    MPI_Type_free(&set_type);

    MPI_Finalize();
}

如有任何帮助，我们将不胜感激。

Answer 1

问题不在于MPI，而在于结构和 MPI 类型的指针算法。

你有

        void *ptr;
        struct set *fs;
        int size;

        MPI_Type_size(set_type, &size);

        ptr = buf;

        for(i=0; i<np; i++) {
                size_t j;

                fs = ptr;
                printf("from rank %d: bits => %p nbits => %d\n", i, fs->bits, fs->nbits);

                for(j=0; j<2; j++)
                        printf("from rank %d: buf[%d] = %#x\n",
                                i, j, fs->bits[j]);
                ptr += size;
        }
}

但是MPI_Type_size实际上给出了类型中的数据量；如果有填充(可能会在这里使字符数组位于单词边界上)，这与 sizeof 不同。如果你想在这里使用 MPI 函数，如果你将那个函数调用切换到 MPI_Type_extent ，它实际上告诉您该类型跨越的整个范围，您的代码为我运行...但仍然存在问题。

如果您看一下 sizeof(struct set)+SIZE 和 MPI_Type_extent() 之间的区别，您会发现它们并不相同；这个:

#define SIZE 10
struct set {
  int nbits
  char nbits[]
}

...

malloc(sizeof(struct set)+SIZE);

不一样

struct set {
  int nbits
  char nbits[SIZE]
}

malloc(sizeof(struct set));

因为padding等，这意味着subsets的大小不对，调用MPI_Gather时内存错误。

你可以通过几种不同的方式解决这个问题，但最简单的(也是最短的行数)是定义结构，其中数组已经确定大小，然后使用数组索引而不是指针算法:

#include <stdio.h>
#include <stdlib.h>

#include "mpi.h"

#define SIZE 10

struct set {
    int nbits;
    char bits[SIZE];
};

int main(int argc, char *argv[]) {
    int np, rank, i;
    struct set *subsets, *single;

    MPI_Init(&argc,&argv);
    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
    MPI_Comm_size(MPI_COMM_WORLD, &np);

    single = malloc(sizeof(struct set));

    if(rank == 0) {
            subsets = malloc(sizeof(struct set) * np);
    }

    MPI_Datatype set_type, oldtypes[2];
    int blockcounts[2];
    MPI_Aint offsets[2];
    MPI_Aint addr[3];

    MPI_Get_address(single, &addr[0]);
    MPI_Get_address(&single->nbits, &addr[1]);
    MPI_Get_address(&single->bits, &addr[2]);

    offsets[0] = addr[1] - addr[0];
    oldtypes[0] = MPI_INT;
    blockcounts[0] = 1;

    offsets[1] = addr[2] - addr[0];
    oldtypes[1] = MPI_CHAR;
    blockcounts[1] = SIZE;

    MPI_Type_create_struct(2, blockcounts, offsets, oldtypes, &set_type);
    MPI_Type_commit(&set_type);

    single->nbits = 2;

    for(i=0; i<single->nbits; i++)
            single->bits[i] = 'A' + rank;

    MPI_Gather(single, 1, set_type, &(subsets[0]), 1, set_type, 0, MPI_COMM_WORLD);

    if(rank == 0) {
            for(i=0; i<np; i++) {
                    struct set *fs = &(subsets[i]);
                    printf("from rank %d: bits => %p nbits => %d\n", i, fs->bits, fs->nbits);

                    for(int j=0; j<2; j++)
                            printf("from rank %d: buf[%d] = %#x\n",
                                    i, j, fs->bits[j]);
            }
    }

    MPI_Type_free(&set_type);

    MPI_Finalize();
}

已更新以添加如果您不能这样做，只需更改缓冲区分配的大小以将数据收集到:

#include <stdio.h>
#include <stdlib.h>
#include "mpi.h"

#define SIZE 10

struct set {
    int nbits;
    char bits[];
};

int main(int argc, char *argv[]) {
    int np, rank, i;
    struct set *single;
    void *buf;
    ptrdiff_t extent;

    MPI_Init(&argc,&argv);
    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
    MPI_Comm_size(MPI_COMM_WORLD, &np);

    single = malloc(sizeof(struct set) + SIZE);

    MPI_Datatype set_type, oldtypes[2];
    int blockcounts[2];
    MPI_Aint offsets[2];
    MPI_Aint addr[3];

    MPI_Get_address(single, &addr[0]);
    MPI_Get_address(&single->nbits, &addr[1]);
    MPI_Get_address(&single->bits, &addr[2]);

    offsets[0] = addr[1] - addr[0];
    oldtypes[0] = MPI_INT;
    blockcounts[0] = 1;

    offsets[1] = addr[2] - addr[0];
    oldtypes[1] = MPI_CHAR;
    blockcounts[1] = SIZE;

    MPI_Type_create_struct(2, blockcounts, offsets, oldtypes, &set_type);
    MPI_Type_commit(&set_type);

    MPI_Type_extent(set_type, &extent);
    buf = malloc((int)extent * np);

    single->nbits = 2;

    for(i=0; i<single->nbits; i++)
            single->bits[i] = 'A' + rank;

    MPI_Gather(single, 1, set_type, buf, 1, set_type, 0, MPI_COMM_WORLD);

    if(rank == 0) {
            struct set *fs = buf;
            for(i=0; i<np; i++) {
                    printf("from rank %d: bits => %p nbits => %d\n", i, fs->bits, fs->nbits);

                    for(int j=0; j<2; j++)
                            printf("from rank %d: buf[%d] = %#x\n",
                                    i, j, fs->bits[j]);

                    fs = (struct set *)((char *)fs + extent);
            }
    }

    MPI_Type_free(&set_type);

    MPI_Finalize();
}