Skip to content

Commit

Permalink
Merge pull request #3190 from STEllAR-GROUP/topology_improvements
Browse files Browse the repository at this point in the history 
Topology improvements
  • Loading branch information
@hkaiser
hkaiser committed Feb 25, 2018
2 parents 9cdd6bc + 848dd2b commit d35c5fc
Show file tree
Hide file tree
Showing 2 changed files with 92 additions and 49 deletions.
16 changes: 13 additions & 3 deletions hpx/runtime/threads/topology.hpp
View file
Expand Up @@ -18,6 +18,8 @@
#include <hpx/runtime/threads/cpu_mask.hpp>
#include <hpx/runtime/resource/partitioner_fwd.hpp>
#include <hpx/runtime/threads/thread_data_fwd.hpp>
#include <hpx/util/thread_specific_ptr.hpp>

#include <hpx/util/spinlock.hpp>
#include <hpx/util/static.hpp>

Expand Down Expand Up @@ -67,9 +69,13 @@ namespace hpx { namespace threads
membind_firsttouch = HWLOC_MEMBIND_FIRSTTOUCH,
membind_bind = HWLOC_MEMBIND_BIND,
membind_interleave = HWLOC_MEMBIND_INTERLEAVE,
#if HWLOC_API_VERSION < 0x00020000
membind_replicate = HWLOC_MEMBIND_REPLICATE,
#endif
membind_nexttouch = HWLOC_MEMBIND_NEXTTOUCH,
membind_mixed = HWLOC_MEMBIND_MIXED
membind_mixed = HWLOC_MEMBIND_MIXED,
// special HPX addition
membind_user = HWLOC_MEMBIND_MIXED + 256
};

#include <hpx/config/warnings_prefix.hpp>
Expand Down Expand Up @@ -262,12 +268,12 @@ namespace hpx { namespace threads
hpx_hwloc_membind_policy policy, int flags) const;

threads::mask_type get_area_membind_nodeset(
const void *addr, std::size_t len, void *nodeset) const;
const void *addr, std::size_t len) const;

bool set_area_membind_nodeset(
const void *addr, std::size_t len, void *nodeset) const;

int get_numa_domain(const void *addr, void *nodeset) const;
int get_numa_domain(const void *addr) const;

/// Free memory that was previously allocated by allocate
void deallocate(void* addr, std::size_t len) const;
Expand Down Expand Up @@ -385,6 +391,10 @@ namespace hpx { namespace threads
std::vector<mask_type> numa_node_affinity_masks_;
std::vector<mask_type> core_affinity_masks_;
std::vector<mask_type> thread_affinity_masks_;

struct tls_tag {};
static util::thread_specific_ptr<hpx_hwloc_bitmap_wrapper, tls_tag>
bitmap_storage_;
};

#include <hpx/config/warnings_suffix.hpp>
Expand Down
125 changes: 79 additions & 46 deletions src/runtime/threads/topology.cpp
View file
Expand Up @@ -31,6 +31,8 @@
#include <vector>
#include <memory>

#include <errno.h>

#include <hwloc.h>

#if HWLOC_API_VERSION < 0x00010b00
Expand Down Expand Up @@ -58,58 +60,58 @@ namespace hpx { namespace threads { namespace detail
return top.get_number_of_pus();
}

void write_to_log(char const* valuename, std::size_t value)
{
void write_to_log(char const* valuename, std::size_t value)
{
LTM_(debug) << "topology: "
<< valuename << ": " << value; //-V128
}
<< valuename << ": " << value; //-V128
}

void write_to_log_mask(char const* valuename, mask_cref_type value)
{
void write_to_log_mask(char const* valuename, mask_cref_type value)
{
LTM_(debug) << "topology: " << valuename
<< ": " HPX_CPU_MASK_PREFIX
<< std::hex << value;
}
<< ": " HPX_CPU_MASK_PREFIX
<< std::hex << value;
}

void write_to_log(char const* valuename,
std::vector<std::size_t> const& values)
{
void write_to_log(char const* valuename,
std::vector<std::size_t> const& values)
{
LTM_(debug) << "topology: "
<< valuename << "s, size: " //-V128
<< values.size();
<< valuename << "s, size: " //-V128
<< values.size();

std::size_t i = 0;
for (std::size_t value : values)
{
std::size_t i = 0;
for (std::size_t value : values)
{
LTM_(debug) << "topology: " << valuename //-V128
<< "(" << i++ << "): " << value;
<< "(" << i++ << "): " << value;
}
}
}

void write_to_log_mask(char const* valuename,
std::vector<mask_type> const& values)
{
void write_to_log_mask(char const* valuename,
std::vector<mask_type> const& values)
{
LTM_(debug) << "topology: "
<< valuename << "s, size: " //-V128
<< values.size();
<< valuename << "s, size: " //-V128
<< values.size();

std::size_t i = 0;
for (mask_cref_type value : values)
{
std::size_t i = 0;
for (mask_cref_type value : values)
{
LTM_(debug) << "topology: " << valuename //-V128
<< "(" << i++ << "): " HPX_CPU_MASK_PREFIX
<< std::hex << value;
<< "(" << i++ << "): " HPX_CPU_MASK_PREFIX
<< std::hex << value;
}
}
}

std::size_t get_index(hwloc_obj_t obj)
{
// on Windows logical_index is always -1
if (obj->logical_index == ~0x0u)
return static_cast<std::size_t>(obj->os_index);
std::size_t get_index(hwloc_obj_t obj)
{
// on Windows logical_index is always -1
if (obj->logical_index == ~0x0u)
return static_cast<std::size_t>(obj->os_index);

return static_cast<std::size_t>(obj->logical_index);
}
return static_cast<std::size_t>(obj->logical_index);
}
}}}

namespace hpx { namespace threads
Expand Down Expand Up @@ -1223,33 +1225,64 @@ namespace hpx { namespace threads
hwloc_nodeset_t ns = reinterpret_cast<hwloc_nodeset_t>(nodeset);
int ret = hwloc_set_area_membind_nodeset(topo, addr, len, ns, policy, 0);
if (ret<0) {
std::string msg = std::strerror(errno);
if (errno == ENOSYS) msg = "the action is not supported";
if (errno == EXDEV) msg = "the binding cannot be enforced";
HPX_THROW_EXCEPTION(kernel_error
, "hpx::threads::topology::set_area_membind_nodeset"
, "hwloc_set_area_membind_nodeset failed");
, "hwloc_set_area_membind_nodeset failed : " + msg);
return false;
}
return true;
}

util::thread_specific_ptr<hpx_hwloc_bitmap_wrapper, topology::tls_tag>
topology::bitmap_storage_;

threads::mask_type topology::get_area_membind_nodeset(
const void *addr, std::size_t len, void *nodeset) const
const void *addr, std::size_t len) const
{
hpx_hwloc_bitmap_wrapper *nodeset = topology::bitmap_storage_.get();
if (nullptr == nodeset)
{
hwloc_bitmap_t nodeset_ = hwloc_bitmap_alloc();
topology::bitmap_storage_.reset(new hpx_hwloc_bitmap_wrapper(nodeset_));
nodeset = topology::bitmap_storage_.get();
}
//
hwloc_membind_policy_t policy;
hwloc_nodeset_t ns = reinterpret_cast<hwloc_nodeset_t>(nodeset);
hwloc_get_area_membind_nodeset(topo, addr, len, ns, &policy, 0);
hwloc_nodeset_t ns = reinterpret_cast<hwloc_nodeset_t>(nodeset->get_bmp());

if (hwloc_get_area_membind_nodeset(topo, addr, len, ns, &policy, 0)==-1) {
HPX_THROW_EXCEPTION(kernel_error
, "hpx::threads::topology::get_area_membind_nodeset"
, "hwloc_get_area_membind_nodeset failed");
return -1;
std::cout << "error in " ;
}
return bitmap_to_mask(ns, HWLOC_OBJ_NUMANODE);
}

int topology::get_numa_domain(const void *addr, void *nodeset) const
int topology::get_numa_domain(const void *addr) const
{
#if HWLOC_API_VERSION >= 0x00010b03
hwloc_nodeset_t ns = reinterpret_cast<hwloc_nodeset_t>(nodeset);
#if HWLOC_API_VERSION >= 0x00010b00
hpx_hwloc_bitmap_wrapper *nodeset = topology::bitmap_storage_.get();
if (nullptr == nodeset)
{
hwloc_bitmap_t nodeset_ = hwloc_bitmap_alloc();
topology::bitmap_storage_.reset(new hpx_hwloc_bitmap_wrapper(nodeset_));
nodeset = topology::bitmap_storage_.get();
}
//
hwloc_nodeset_t ns = reinterpret_cast<hwloc_nodeset_t>(nodeset->get_bmp());

int ret = hwloc_get_area_memlocation(topo, addr, 1, ns,
HWLOC_MEMBIND_BYNODESET);
if (ret<0) {
std::string msg(strerror(errno));
HPX_THROW_EXCEPTION(kernel_error
, "hpx::threads::topology::set_area_membind_nodeset"
, "hwloc_set_area_membind_nodeset failed");
, "hpx::threads::topology::get_numa_domain"
, "hwloc_get_area_memlocation failed " + msg);
return -1;
}
threads::mask_type mask = bitmap_to_mask(ns, HWLOC_OBJ_NUMANODE);
Expand Down

0 comments on commit d35c5fc

Please sign in to comment.