Fixing line lengths

STEllAR-GROUP · May 29, 2017 · b870220 · b870220
1 parent 28d62a0
commit b870220
Show file tree

Hide file tree

Showing 16 changed files with 259 additions and 216 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -10,10 +10,10 @@
 # file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
 
 # We require at least CMake V3.0.2
-cmake_minimum_required(VERSION 3.0.2 FATAL_ERROR)
+cmake_minimum_required(VERSION 2.8.12 FATAL_ERROR)
 
 # explicitly set certain policies
-cmake_policy(VERSION 3.0.2)
+cmake_policy(VERSION 2.8.12)
 macro(hpx_set_cmake_policy policy value)
   if(POLICY ${policy})
     cmake_policy(SET ${policy} ${value})
@@ -1005,15 +1005,15 @@ if(NOT MSVC)
         add_definitions(-DBOOST_NO_CXX14_CONSTEXPR)
       endif()
     else()
-      # ... otherwise try -std=c++1y
+      # ... otherwise try -std=c++17
 
       if(HPX_WITH_CXX1Y OR NOT (("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Intel")
                                 AND (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 17)))
-        check_cxx_compiler_flag(-std=c++1y HPX_WITH_CXX1y)
+        check_cxx_compiler_flag(-std=c++17 HPX_WITH_CXX1y)
       endif()
 
       if(HPX_WITH_CXX1Y)
-        set(CXX_FLAG -std=c++1y)
+        set(CXX_FLAG -std=c++17)
       else()
         # ... otherwise try -std=c++11
         check_cxx_compiler_flag(-std=c++11 HPX_WITH_CXX11)

diff --git a/cmake/toolchains/Cray.cmake b/cmake/toolchains/Cray.cmake
@@ -61,8 +61,6 @@ set(HPX_WITH_PARCELPORT_MPI_MULTITHREADED OFF CACHE BOOL "")
 set(HPX_WITH_PARCELPORT_LIBFABRIC ON CACHE BOOL "")
 set(HPX_PARCELPORT_LIBFABRIC_PROVIDER "gni" CACHE STRING
   "See libfabric docs for details, gni,verbs,psm2 etc etc")
-set(HPX_PARCELPORT_LIBFABRIC_THROTTLE_SENDS "256" CACHE STRING
-  "Max number of messages in flight at once")
 set(HPX_PARCELPORT_LIBFABRIC_WITH_DEV_MODE OFF CACHE BOOL
   "Custom libfabric logging flag")
 set(HPX_PARCELPORT_LIBFABRIC_WITH_LOGGING  OFF CACHE BOOL

diff --git a/cmake/toolchains/CrayKNL.cmake b/cmake/toolchains/CrayKNL.cmake
@@ -12,6 +12,7 @@
 if(HPX_WITH_STATIC_LINKING)
   set_property(GLOBAL PROPERTY TARGET_SUPPORTS_SHARED_LIBS FALSE)
 else()
+  set_property(GLOBAL PROPERTY TARGET_SUPPORTS_SHARED_LIBS TRUE)
 endif()
 
 # Set the Cray Compiler Wrapper
@@ -59,8 +60,6 @@ set(HPX_WITH_PARCELPORT_MPI_MULTITHREADED OFF CACHE BOOL "")
 set(HPX_WITH_PARCELPORT_LIBFABRIC ON CACHE BOOL "")
 set(HPX_PARCELPORT_LIBFABRIC_PROVIDER "gni" CACHE STRING
   "See libfabric docs for details, gni,verbs,psm2 etc etc")
-set(HPX_PARCELPORT_LIBFABRIC_THROTTLE_SENDS "256" CACHE STRING
-  "Max number of messages in flight at once")
 set(HPX_PARCELPORT_LIBFABRIC_WITH_DEV_MODE OFF CACHE BOOL
   "Custom libfabric logging flag")
 set(HPX_PARCELPORT_LIBFABRIC_WITH_LOGGING  OFF CACHE BOOL

diff --git a/cmake/toolchains/CrayKNLStatic.cmake b/cmake/toolchains/CrayKNLStatic.cmake
@@ -1,5 +1,5 @@
 # Copyright (c) 2014-2017 Thomas Heller
-# Copyright (c) 2017      Bryce Adelstein Lelbach 
+# Copyright (c) 2017      Bryce Adelstein Lelbach
 #
 # Distributed under the Boost Software License, Version 1.0. (See accompanying
 # file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
@@ -44,8 +44,6 @@ set(HPX_WITH_PARCELPORT_MPI_MULTITHREADED ON CACHE BOOL "")
 set(HPX_WITH_PARCELPORT_LIBFABRIC ON CACHE BOOL "")
 set(HPX_PARCELPORT_LIBFABRIC_PROVIDER "gni" CACHE STRING
   "See libfabric docs for details, gni,verbs,psm2 etc etc")
-set(HPX_PARCELPORT_LIBFABRIC_THROTTLE_SENDS "256" CACHE STRING
-  "Max number of messages in flight at once")
 set(HPX_PARCELPORT_LIBFABRIC_WITH_DEV_MODE OFF CACHE BOOL
   "Custom libfabric logging flag")
 set(HPX_PARCELPORT_LIBFABRIC_WITH_LOGGING  OFF CACHE BOOL

diff --git a/cmake/toolchains/CrayStatic.cmake b/cmake/toolchains/CrayStatic.cmake
@@ -50,8 +50,6 @@ set(HPX_WITH_PARCELPORT_MPI_MULTITHREADED ON CACHE BOOL "")
 set(HPX_WITH_PARCELPORT_LIBFABRIC ON CACHE BOOL "")
 set(HPX_PARCELPORT_LIBFABRIC_PROVIDER "gni" CACHE STRING
   "See libfabric docs for details, gni,verbs,psm2 etc etc")
-set(HPX_PARCELPORT_LIBFABRIC_THROTTLE_SENDS "256" CACHE STRING
-  "Max number of messages in flight at once")
 set(HPX_PARCELPORT_LIBFABRIC_WITH_DEV_MODE OFF CACHE BOOL
   "Custom libfabric logging flag")
 set(HPX_PARCELPORT_LIBFABRIC_WITH_LOGGING  OFF CACHE BOOL

diff --git a/hpx/lcos/detail/future_data.hpp b/hpx/lcos/detail/future_data.hpp
@@ -211,6 +211,13 @@ namespace detail
           , f2_(std::move(other.f2_))
         {}
 
+        compose_cb_impl& operator=(compose_cb_impl&& other)
+        {
+            f1_ = std::move(other.f1_);
+            f2_ = std::move(other.f2_);
+            return *this;
+        }
+
         void operator()()
         {
             bool recurse_asynchronously = hpx::threads::get_self_ptr() == nullptr;

diff --git a/hpx/runtime/applier/apply.hpp b/hpx/runtime/applier/apply.hpp
@@ -403,7 +403,8 @@ namespace hpx
                     action_type_(), priority
                 );
 
-            parcelset::detail::parcel_await(std::move(p), parcelset::write_handler_type(), 0,
+            parcelset::detail::parcel_await(std::move(p),
+                parcelset::write_handler_type(), 0,
                 [](parcelset::parcel&& p, parcelset::write_handler_type&&)
                 {
                     hpx::get_runtime().get_parcel_handler()

diff --git a/hpx/runtime/parcelset/detail/parcel_await.hpp b/hpx/runtime/parcelset/detail/parcel_await.hpp
@@ -22,7 +22,8 @@ namespace hpx { namespace parcelset { namespace detail {
     template <typename Parcel, typename Handler, typename Derived>
     struct parcel_await_base : std::enable_shared_from_this<Derived>
     {
-        typedef hpx::util::unique_function_nonser<void(Parcel&&, Handler&&)> put_parcel_type;
+        typedef hpx::util::unique_function_nonser<void(Parcel&&, Handler&&)>
+            put_parcel_type;
 
         parcel_await_base(Parcel&& parcel, Handler&& handler, int archive_flags,
             put_parcel_type pp)
@@ -86,9 +87,11 @@ namespace hpx { namespace parcelset { namespace detail {
     };
 
     struct parcels_await
-      : parcel_await_base<std::vector<parcel>, std::vector<write_handler_type>, parcels_await>
+      : parcel_await_base<std::vector<parcel>, std::vector<write_handler_type>,
+            parcels_await>
     {
-        typedef parcel_await_base<std::vector<parcel>, std::vector<write_handler_type>, parcels_await>
+        typedef parcel_await_base<std::vector<parcel>,
+            std::vector<write_handler_type>, parcels_await>
             base_type;
 
         parcels_await(std::vector<parcel>&& p, std::vector<write_handler_type>&& f,

diff --git a/hpx/runtime/parcelset/parcelhandler.hpp b/hpx/runtime/parcelset/parcelhandler.hpp
@@ -230,7 +230,7 @@ namespace hpx { namespace parcelset
         ///                 where \a err is the status code of the operation and
         ///                       \a size is the number of successfully
         ///                              transferred bytes.
-        void put_parcels(std::vector<parcel> p, std::vector<write_handler_type> f);
+//         void put_parcels(std::vector<parcel> p, std::vector<write_handler_type> f);
 
         /// This put_parcel() function overload is asynchronous, but no
         /// callback is provided by the user.
@@ -241,15 +241,15 @@ namespace hpx { namespace parcelset
         ///                 parcel \a p will be modified in place, as it will
         ///                 get set the resolved destination address and parcel
         ///                 id (if not already set).
-        void put_parcels(std::vector<parcel> parcels)
-        {
-            using util::placeholders::_1;
-            using util::placeholders::_2;
-            std::vector<write_handler_type> handlers(parcels.size(),
-                util::bind(&parcelhandler::invoke_write_handler, this, _1, _2));
-
-            put_parcels(std::move(parcels), std::move(handlers));
-        }
+//         void put_parcels(std::vector<parcel> parcels)
+//         {
+//             using util::placeholders::_1;
+//             using util::placeholders::_2;
+//             std::vector<write_handler_type> handlers(parcels.size(),
+//                 util::bind(&parcelhandler::invoke_write_handler, this, _1, _2));
+//
+//             put_parcels(std::move(parcels), std::move(handlers));
+//         }
 
         double get_current_time() const
         {

diff --git a/hpx/runtime/parcelset/parcelport_impl.hpp b/hpx/runtime/parcelset/parcelport_impl.hpp
@@ -254,31 +254,35 @@ namespace hpx { namespace parcelset
                     parcels[i].destination_locality());
             }
 #endif
+            for (std::size_t i = 0; i != parcels.size(); ++i)
+            {
+                put_parcel(dest, std::move(parcels[i]), std::move(handlers[i]));
+            }
             // We create a shared pointer of the parcels_await object since it
             // needs to be kept alive as long as there are futures not ready
             // or GIDs to be split. This is necessary to preserve the identiy
             // of the this pointer.
-            std::make_shared<detail::parcels_await>(std::move(parcels),
-                std::move(handlers), archive_flags_,
-                [this, dest](std::vector<parcel>&& parcels,
-                    std::vector<write_handler_type>&& handlers)
-                {
-                    if (connection_handler_traits<ConnectionHandler>::
-                        send_immediate_parcels::value &&
-                        can_send_immediate_impl<ConnectionHandler>())
-                    {
-                        send_immediate_impl<ConnectionHandler>(
-                            *this, dest, handlers.data(), parcels.data(),
-                            parcels.size());
-                    }
-                    else
-                    {
-                        enqueue_parcels(
-                            dest, std::move(parcels), std::move(handlers));
-
-                        get_connection_and_send_parcels(dest);
-                    }
-                })->apply();
+//             std::make_shared<detail::parcels_await>(std::move(parcels),
+//                 std::move(handlers), archive_flags_,
+//                 [this, dest](std::vector<parcel>&& parcels,
+//                     std::vector<write_handler_type>&& handlers)
+//                 {
+//                     if (connection_handler_traits<ConnectionHandler>::
+//                         send_immediate_parcels::value &&
+//                         can_send_immediate_impl<ConnectionHandler>())
+//                     {
+//                         send_immediate_impl<ConnectionHandler>(
+//                             *this, dest, handlers.data(), parcels.data(),
+//                             parcels.size());
+//                     }
+//                     else
+//                     {
+//                         enqueue_parcels(
+//                             dest, std::move(parcels), std::move(handlers));
+//
+//                         get_connection_and_send_parcels(dest);
+//                     }
+//                 })->apply();
         }
 
         void send_early_parcel(locality const & dest, parcel p)

diff --git a/plugins/parcelport/libfabric/libfabric_region_provider.hpp b/plugins/parcelport/libfabric/libfabric_region_provider.hpp
@@ -18,6 +18,9 @@
 #include <memory>
 #include <utility>
 //
+
+#include <iostream>
+
 namespace hpx {
 namespace parcelset {
 namespace policies {
@@ -31,7 +34,13 @@ namespace libfabric
 
         template <typename... Args>
         static int register_memory(Args &&... args) {
-            return fi_mr_reg(std::forward<Args>(args)...);
+            int res = fi_mr_reg(std::forward<Args>(args)...);
+            if (res == -FI_ENOKEY)
+                std::cerr << "key already in use\n";
+            if (res == -FI_EKEYREJECTED)
+                std::cerr << "key not available\n";
+
+            return res;
         }
 
         static int unregister_memory(provider_region *region) {

diff --git a/plugins/parcelport/libfabric/pinned_memory_vector.hpp b/plugins/parcelport/libfabric/pinned_memory_vector.hpp
@@ -106,9 +106,10 @@ namespace libfabric
         // move copy operator
         vector_type & operator=(vector_type && other)
         {
+            clear();
             m_array_  = other.m_array_;
             m_size_   = other.m_size_;
-            m_cb_     = other.m_cb_;
+            m_cb_     = std::move(other.m_cb_);
             m_alloc_  = other.m_alloc_;
             m_region_ = other.m_region_;
             LOG_DEBUG_MSG("pinned_memory_vector assigned/moved "
@@ -124,6 +125,23 @@ namespace libfabric
             return *this;
         }
 
+        void clear()
+        {
+            if (m_array_ && m_cb_) {
+                LOG_DEBUG_MSG("pinned_memory_vector clear "
+                    << "size " << hexuint32(m_size_)
+                    << "array " << hexpointer(m_array_)
+                    << "region " << hexpointer(m_region_)
+                    << "alloc " << hexpointer(m_alloc_));
+                m_cb_();
+            }
+            m_size_   = 0;
+            m_array_  = 0;
+            m_cb_     = nullptr;
+            m_alloc_  = nullptr;
+            m_region_ = nullptr;
+        }
+
         size_type size() const {
             return m_size_;
         }
@@ -203,9 +221,14 @@ namespace libfabric
                 << "alloc " << hexpointer(m_alloc_)
                 << "reserving from " << m_size_ << " to " << s);
             if (m_array_ || m_region_) {
-                throw std::runtime_error(
-                    "pinned_memory_vector should never be resized once an "
-                    "allocation has been assigned");
+                if (m_region_ && s > m_region_->get_size())
+                {
+                    LOG_ERROR_MSG(
+                        "Reserve from " << m_region_->get_size() << " to " << s);
+                    throw std::runtime_error(
+                        "pinned_memory_vector should never be resized once an "
+                        "allocation has been assigned");
+                }
             }
             m_region_ = m_alloc_->allocate_region(s);
             m_array_ = static_cast<T*>(m_region_->get_address());

diff --git a/plugins/parcelport/libfabric/sender.cpp b/plugins/parcelport/libfabric/sender.cpp
@@ -234,7 +234,6 @@ namespace libfabric
         ++sends_deleted_;
 
         error_code ec;
-        handler_(ec);
 
         // cleanup header and message region
         memory_pool_->deallocate(message_region_);
@@ -252,7 +251,9 @@ namespace libfabric
         rma_regions_.clear();
         buffer_.data_point_.time_ =
             util::high_resolution_clock::now() - buffer_.data_point_.time_;
+        handler_(ec);
         parcelport_->add_sent_data(buffer_.data_point_);
+        buffer_.clear();
         postprocess_handler_(this);
     }
 

diff --git a/src/runtime/parcelset/detail/parcel_await.cpp b/src/runtime/parcelset/detail/parcel_await.cpp
@@ -14,8 +14,8 @@
 
 namespace hpx { namespace parcelset { namespace detail {
 
-    parcel_await::parcel_await(parcel&& p, write_handler_type&& f, int archive_flags,
-        parcel_await::put_parcel_type pp)
+    parcel_await::parcel_await(parcel&& p, write_handler_type&& f,
+        int archive_flags, parcel_await::put_parcel_type pp)
     : base_type(std::move(p), std::move(f), archive_flags, std::move(pp))
     {
     }
@@ -28,8 +28,9 @@ namespace hpx { namespace parcelset { namespace detail {
         }
     }
 
-    parcels_await::parcels_await(std::vector<parcel>&& p, std::vector<write_handler_type>&& f,
-        int archive_flags, parcels_await::put_parcel_type pp)
+    parcels_await::parcels_await(std::vector<parcel>&& p,
+            std::vector<write_handler_type>&& f, int archive_flags,
+            parcels_await::put_parcel_type pp)
       : base_type(std::move(p), std::move(f), archive_flags, std::move(pp)),
         idx_(0)
     {