MPI Port

Here is my very preliminary work on the MPI infrastructure. There were a few deviations from the specification however they are all fairly minor. We still definitely need to talk about the linked_buffer implementation. The implementation as was sketched out in the specification doesn't seem to have any easy means of filling the pointers table after create_linked_buffers has done its work.

Sorry about the formatting. The wiki strangely denied each of my attempts at attaching the files.

Low-level

#include <mpi/mpi.h> typedef rank_t int; template<typename T> MPI_Datatype get_mpi_type() { return 0; } MPI_Datatype get_mpi_type<char>() { return MPI_CHAR; } MPI_Datatype get_mpi_type<int>() { return MPI_INT; } MPI_Datatype get_mpi_type<short>() { return MPI_SHORT; } MPI_Datatype get_mpi_type<float>() { return MPI_FLOAT; } MPI_Datatype get_mpi_type<double>() { return MPI_DOUBLE; } static void mpi_error(MPI_Comm *comm, int *stat, ...) { int len; char err_string[MPI_MAX_ERROR_STRING]; printf("MPI error number: %i\n", *stat); MPI_Error_string(*stat, err_string, &len); printf("%s\n", err_string); terminate(*stat); } void init_machine(int argc, char **argv) { int i, flag, *tag_ub; MPI_Comm comm; MPI_Errhandler errhandler; flag = MPI_Init(&argc, &argv); comm = MPI_COMM_WORLD; if (flag) mpi_error(&comm, &flag); flag = MPI_Errhandler_create(mpi_error, &errhandler); if (flag) mpi_error(&comm, &flag); flag = MPI_Errhandler_set(MPI_COMM_WORLD, errhandler); if (flag) mpi_error(&comm, &flag); } void shutdown_machine() { MPI_Finalize(); } int get_num_nodes() { int n; MPI_Comm_size(MPI_COMM_WORLD, &n); return n; } rank_t get_node_rank() { rank_t rank; MPI_Comm_rank(MPI_COMM_WORLD, &rank); return rank_t; } void synchronize() { MPI_Barrier(MPI_COMM_WORLD); } void broadcast(T* buffer, int length) { MPI_Bcast(buffer, length, get_mpi_type<T>(), 0, MPI_COMM_WORLD); } template<typename T> void send(T* buffer, int length, rank_t dest) { MPI_Send(buffer, length, get_mpi_type<T>(), dest, tag, MPI_COMM_WORLD); } template<typename T> void recv(T* buffer, int length, rank_t src) { MPI_Status status; MPI_Recv(buffer, length, get_mpi_type<T>(), src, tag, MPI_COMM_WORLD, &status); } template<typename T> T sum(T value) { T work; MPI_Allreduce(&value, &work, 1, get_mpi_type<T>(), MPI_SUM, MPI_COMM_WORLD); return work; } template<typename T> void sum(T* value, T* out, int length) { MPI_Allreduce(&in, out, get_mpi_type<T>(), length, MPI_SUM,, MPI_COMM_WORLD); }

Intermediate layer

#include <map> #include <pair> typedef local_idx_t int; typedef global_idx_t int; class comm_list { // Represents a source_index -> dest_index pair typedef std::pair<global_idx_t, global_idx_t> comm_entry; // Maps a destination rank to a list of comm_entries std::map<rank_t, std::vector<comm_entry> > send; // Maps a source rank to a list of comm_entries std::map<rank_t, std::vector<comm_entry> > recv; // All of the intra-process comm_entries std::vector<comm_entry> local; public comm_list(rank_t rank, const std::vector<comm_entry> l) { for (std::vector<comm_entry>::iterator i=l.begin(); i != l.end(); i++) { global_idx_t src=i->first, dest=i->second; src_rank = get_site_rank(src); dest_rank = get_site_rank(dest) if (src_rank == rank && dest_rank == rank) local.push_back(*i); else if (src_rank == rank) { if (send.find(dest_rank) == send.end()) send[dest_rank] = vector<comm_entry>(); send[dest_rank].append(*i); } else if (dest_rank == rank) { if (recv.find(src_rank) == recv.end()) recv[src_rank] = vector<comm_entry>(); recv[src_rank].append(*i); } } } }; template<typename T> struct linked_buffer { typedef T slot_type; rank_t peer; const int length; shared_ptr<T> buffer; std::vector<T*> pointers; linked_buffer(rank_t peer, int length) : peer(peer), length(length), buffer(new T[length]), pointers(length) { } void gather() { for (int i=0; i<length; i++) buffers[i] = *pointers[i]; } void scatter() { for (int i=0; i<length; i++) *pointers[i] = buffers[i]; } }; // TODO: Need to think about this a little more struct linked_buffer_info { map< vector<linked_buffer> create_linked_buffers(comm_list comm) { }

--Bgamari 19:48, 26 December 2009 (UTC)