models/hierarchical/hierarchical_model.hpp

#ifndef HIERARCHICAL_MODEL_HPP_
#define HIERARCHICAL_MODEL_HPP_

#include <boost/mpi.hpp>
#include <boost/graph/graphviz.hpp>
#include <boost/graph/adjacency_list.hpp>
#include <boost/serialization/deque.hpp>
#include <boost/archive/text_oarchive.hpp>
#include <boost/archive/text_iarchive.hpp>
#include <deque>
#include <utility>
#include "models/cpm.h"

namespace CPM {
namespace hierarchy {
using namespace std;
using namespace boost;

struct vertex_type_t { typedef vertex_property_tag kind; };
struct vertex_model_data_t { typedef vertex_property_tag kind; };
typedef property<vertex_type_t, int, property<vertex_model_data_t, string  > > vertex_properties_t;
typedef property < vertex_name_t, string > vertex_p;  
typedef adjacency_list < vecS, vecS, directedS, vertex_properties_t > Graph;
typedef graph_traits<Graph>::vertex_descriptor Vertex;
typedef graph_traits<Graph>::edge_descriptor Edge;
typedef graph_traits<Graph>::vertex_iterator Vertex_iterator;
typedef graph_traits<Graph>::adjacency_iterator Adjacency_iterator;

class label_writer {
  public:
    label_writer(Graph _g) : g(_g) {}
    void operator()(ostream& out, const Vertex& v) const {
      out << "[type=" << get(vertex_type_t(), g, v ) << ", model_data=\"" << get(vertex_model_data_t(), g, v) << "\"]" ;
    }
  private:
    Graph g;
};

map<pair <int , int> , Vertex > pair2vertex;
map<string, vector<int> > hostname2procs;
map<int, Vertex> rank2vertex;
};
};

using namespace CPM::hierarchy;

bool get_common_parent(Graph graph, Vertex v1, Vertex v2, Vertex& v3) {
        Adjacency_iterator ai1, ai_end1, ai2, ai_end2;
    tie(ai1, ai_end1) = adjacent_vertices(v1, graph);
    tie(ai2, ai_end2) = adjacent_vertices(v2, graph);
    bool found = false;

    while ((ai1 != ai_end1) && (ai2 != ai_end2)) {
        //*ai1 holds the common parent
        if (*ai1 == *ai2) {
            found = true;
            v3 = *ai1;
            break;
        }
        else {
            tie(ai1, ai_end1) = adjacent_vertices(*ai1, graph);
            tie(ai2, ai_end2) = adjacent_vertices(*ai2, graph);
        }
    }
    return found;
}

template <class p2p_model> class H_Model {
private:
    Graph graph;

public:
    void read_hierarchy_file(istream &is){
        dynamic_properties dp;
        property_map<Graph, vertex_type_t>::type type =
          get(vertex_type_t(), graph);
        dp.property("type", type);
        property_map<Graph, vertex_model_data_t>::type model_data =
          get(vertex_model_data_t(), graph);
        dp.property("model_data",model_data); 
        read_graphviz( is, graph, dp, "type");
    }

    void write_hierarchy_file(ostream &os){}

    void read_model_file(istream &is){}

    //ToDo: this function should not include Hockney information! - the specialization should be done in each model 
    void write_model_file(ostream &os) {
        boost::mpi::communicator world;
        //boost::write_graphviz(os, graph, label_writer(graph));
        os << "#Number of nodes: " << world.size() << "\n";
        int i, index;
        map<pair<int,int>, pair<int,int> > pair2pair;
        p2p_model m;
        m.write_header(os);
                
        for(i = 0, index = 0; i < world.size() - 1; i++) {
            int j;
            for(j = i + 1; j < world.size(); j++, index++) {
                Vertex v;
                if (! get_common_parent(graph, rank2vertex[i], rank2vertex[j], v)) {
                    cerr << "could not find common parent to: " << i << " and " << j ;
                    exit(-1);
                }
            
                string model_data = get(vertex_model_data_t(), graph, v);
                istringstream is(model_data);   
                boost::archive::text_iarchive ia(is);
                ia >> m;    
                m.write_elem(i, j, os);
            }
        }
        m.write_footer(os);
    }


    Graph& getGraph(){return graph;}
}; 


template <class p2p_model> void H_Model_estimate(MPI_Comm comm, MPIB_msgset msgset, MPIB_precision precision, int parallel, H_Model<p2p_model>* model) {
    deque<pair<int,int> > proc_pairs;
    generate_benchmark_pairs(model, comm, proc_pairs);
    mpi::communicator world;//ToDo: pass comm argument
    int rank = world.rank(); 

    for (deque<pair<int,int> >::iterator it = proc_pairs.begin(); it != proc_pairs.end(); it++) {
        mpi::request req;
        p2p_model m;

        if ((rank == it->first) || (rank == it->second)) {
            m.estimate(comm, msgset, precision, it->first, it->second);
        }
        //Non-blocking communication here avoids a potential deadlock when process 0 is both benchmarking and sending the results to process 0 (=itself)
        if (rank == it->first)
            req = world.isend(0/*root*/, 0/*tag*/ , m);

        if (rank == 0) {
            world.recv(it->first, 0, m);
            ostringstream os;
            boost::archive::text_oarchive oa(os);
            oa << m;
            //The following line populates the serialized model data into the right vertex in the graph
            //This function assumes that pair2vertex has built identical pairs to the ones we call in this loop
            Vertex v = pair2vertex[make_pair(it->first, it->second)];
            put(vertex_model_data_t(), model->getGraph(), v, os.str()); 
        }       
        if (rank == it->first)
            req.wait();
    }
}

void set_parent_model_data(Graph& graph , Vertex& v1, Vertex& v2, deque<pair<int, int> >& proc_pairs) {

    Vertex v3;
    bool found = get_common_parent(graph, v1, v2, v3);
    bool empty = false;
    if (found) {
        empty = (get(vertex_model_data_t(), graph, v3) == "");
    }
    else {
        fprintf(stderr, "Could not find common parent of vertices");
        exit(-1);
    }

    if (empty) {
        string hostname1 = get(vertex_model_data_t(), graph, v1); 
        string hostname2 = get(vertex_model_data_t(), graph, v2); 
        
        // Case 1: Multi-core communication on a node.
        // Check if at least two processes per node are running. If so, use the first two in the list
        if (!hostname1.compare(hostname2)) {
            if (hostname2procs[hostname1].size() >=2 ) {
                pair<int,int> proc_pair;
                proc_pair.first = hostname2procs[hostname1][0];
                proc_pair.second = hostname2procs[hostname1][1];
                proc_pairs.push_back(proc_pair);
                pair2vertex[proc_pair] = v3;
            }
        }
        // Case 2: Communication between processes on different nodes.
        // Only work with the lowest ranked process on each node
        else {
            if ((!hostname2procs[hostname1].empty()) && (!hostname2procs[hostname2].empty())) {
                pair<int,int> proc_pair;
                proc_pair.first = hostname2procs[hostname1][0];
                proc_pair.second = hostname2procs[hostname2][0];
                proc_pairs.push_back(proc_pair);
                pair2vertex[proc_pair] = v3;
            }
        }
        // temporary mark to avoid duplication of benchmarks on the same level
        put(vertex_model_data_t(), graph, v3, "X") ; 
    }
}

template <class p2p_model>
void generate_benchmark_pairs(H_Model<p2p_model>* model, MPI_Comm comm,  deque<pair<int, int> >& proc_pairs) {
    int rank, size;
    deque<Vertex> bottom_vertices;
    mpi::communicator world;
    rank = world.rank();
    size = world.size();
    Graph& graph = model->getGraph();
    string proc_name = boost::mpi::environment::processor_name();
    if (rank != 0) {
        world.send(0, rank, proc_name);
    }
    //only master process reads the input file
    else {

        //gather information from every process about which host the process is using to run
        Vertex_iterator ai, ai_end;
        Adjacency_iterator ai2, ai_end2;
        tie(ai,ai_end) = vertices(graph);
        //all leafs are stored in bottom_vertices
        while (ai != ai_end) {
            int vertex_type = get(vertex_type_t(), graph, *ai);
            if (vertex_type == 0) {
                bottom_vertices.push_back(*ai); 
            }
            ai++;
        }
        mpi::status status;
        string all_names[size];
        //master process already knows its name, it receives the rest
        all_names[0] = proc_name;
        for (int i=1; i<size; i++) {
            string tmp;
            world.recv(i, i, tmp);
            all_names[i] = tmp;
        }


        for (int i=0; i<size; i++) {
            for (std::deque<Vertex>::iterator it = bottom_vertices.begin(); it != bottom_vertices.end(); it++) {
                string hostname = get(vertex_model_data_t(), graph, *it);
                if (!all_names[i].compare(hostname))
                    rank2vertex[i] = *it;
            }
        }
        
        //master knows who runs on which node now
        for (int i=0; i<size; i++) {
            hostname2procs[all_names[i]].push_back(i);
        }

        std::deque<Vertex>::iterator it1, it2, it3;
        it3 = bottom_vertices.end();
        
        for (it1 = bottom_vertices.begin(); it1 != it3; it1++) {
            for (it2 = it1; it2 != it3; it2++) {
                set_parent_model_data(graph , *it1, *it2, proc_pairs); 
            }
        }
    }
    boost::mpi::broadcast(world, proc_pairs, 0);
}


#endif /*HIERARCHICAL_MODEL_HPP_*/