settings.hpp (cd61077)

/*******************************<SCHWARZ LIB LICENSE>***********************
Copyright (c) 2019, the SCHWARZ LIB authors
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:

1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.

2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.

3. Neither the name of the copyright holder nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
******************************<SCHWARZ LIB LICENSE>*************************/


#ifndef settings_hpp
#define settings_hpp


#include <map>
#include <memory>
#include <string>
#include <tuple>
#include <vector>


#include <mpi.h>
#include <ginkgo/ginkgo.hpp>

#include <device_guard.hpp>
#include <exception_helpers.hpp>
#include <gather.hpp>
#include <mpi_datatype.hpp>
#include <scatter.hpp>


#if SCHW_HAVE_METIS
#include <metis.h>
#define metis_indextype idx_t
#else
#define metis_indextype gko::int32
#endif


#define MINIMAL_OVERLAP 2


namespace schwz {


struct Settings {
    std::string executor_string;

    std::shared_ptr<gko::Executor> executor = gko::ReferenceExecutor::create();

    std::shared_ptr<device_guard> cuda_device_guard;

    enum partition_settings {
        partition_regular = 0x0,
        partition_regular2d = 0x4,
        partition_metis = 0x1,
        partition_zoltan = 0x2,
        partition_custom = 0x3
    };
    partition_settings partition = partition_settings::partition_regular;

    gko::int32 overlap = MINIMAL_OVERLAP;

    std::string matrix_filename = "null";

    bool explicit_laplacian = true;

    bool use_mixed_precision = false;

    bool enable_random_rhs = false;

    bool print_matrices = false;

    bool debug_print = false;

    enum local_solver_settings {
        direct_solver_cholmod = 0x0,
        direct_solver_umfpack = 0x5,
        direct_solver_ginkgo = 0x1,
        iterative_solver_ginkgo = 0x2,
        iterative_solver_dealii = 0x3,
        solver_custom = 0x4
    };
    local_solver_settings local_solver =
        local_solver_settings::iterative_solver_ginkgo;

    bool non_symmetric_matrix = false;

    unsigned int restart_iter = 1u;

    int reset_local_crit_iter = -1;

    bool naturally_ordered_factor = false;

    std::string metis_objtype;

    bool use_precond = false;

    bool write_debug_out = false;

    bool write_iters_and_residuals = false;

    bool enable_logging = false;

    bool write_perm_data = false;

    int shifted_iter = 1;

    struct comm_settings {
        bool enable_onesided = false;

        bool enable_overlap = false;

        bool enable_put = false;

        bool enable_get = true;

        bool stage_through_host = false;

        bool enable_one_by_one = false;

        bool enable_flush_local = false;

        bool enable_flush_all = true;

        bool enable_lock_local = false;

        bool enable_lock_all = true;
    };
    comm_settings comm_settings;

    struct convergence_settings {
        bool put_all_local_residual_norms = true;
        bool enable_global_simple_tree = false;
        bool enable_decentralized_leader_election = false;
        bool enable_global_check = true;
        bool enable_accumulate = false;

        bool enable_global_check_iter_offset = false;

        enum local_convergence_crit {
            residual_based = 0x0,
            solution_based = 0x1
        };

        local_convergence_crit convergence_crit =
            local_convergence_crit::solution_based;
    };
    convergence_settings convergence_settings;

    std::string factorization = "cholmod";

    std::string reorder;

    Settings(std::string executor_string = "reference")
        : executor_string(executor_string)
    {}
};


template <typename ValueType, typename IndexType>
struct Metadata {
    MPI_Comm mpi_communicator;

    gko::size_type global_size = 0;

    gko::size_type oned_laplacian_size = 0;

    gko::size_type local_size = 0;

    gko::size_type local_size_x = 0;

    gko::size_type local_size_o = 0;

    gko::size_type overlap_size = 0;

    gko::size_type num_subdomains = 1;

    int my_rank;

    int my_local_rank;

    int local_num_procs;

    int comm_size;

    int num_threads;

    IndexType iter_count;

    ValueType tolerance;

    ValueType local_solver_tolerance;

    IndexType max_iters;

    IndexType local_max_iters;

    IndexType updated_max_iters;

    std::string local_precond;

    unsigned int precond_max_block_size;

    ValueType current_residual_norm = -1.0;

    ValueType min_residual_norm = -1.0;

    std::vector<std::tuple<int, int, int, std::string, std::vector<ValueType>>>
        time_struct;

    std::vector<std::tuple<int, std::vector<std::tuple<int, int>>,
                           std::vector<std::tuple<int, int>>, int, int>>
        comm_data_struct;

    struct post_process_data {
        std::vector<std::vector<ValueType>> global_residual_vector_out;
        std::vector<ValueType> local_residual_vector_out;
        std::vector<ValueType> local_converged_iter_count;
        std::vector<ValueType> local_converged_resnorm;
        std::vector<ValueType> local_timestamp;
    };
    post_process_data post_process_data;

    double init_mpi_wtime = 0.0;
    std::shared_ptr<gko::Array<IndexType>> global_to_local;

    std::shared_ptr<gko::Array<IndexType>> local_to_global;

    gko::Array<IndexType> overlap_row;

    std::shared_ptr<gko::Array<IndexType>> first_row;

    std::shared_ptr<gko::Array<IndexType>> permutation;

    std::shared_ptr<gko::Array<IndexType>> i_permutation;
};


#define MEASURE_ELAPSED_FUNC_TIME(_func, _id, _rank, _name, _iter)     \
    {                                                                  \
        auto start_time = std::chrono::steady_clock::now();            \
        _func;                                                         \
        auto elapsed_time = std::chrono::duration<ValueType>(          \
            std::chrono::steady_clock::now() - start_time);            \
        if (_iter == 0) {                                              \
            std::vector<ValueType> temp_vec(1, elapsed_time.count());  \
            metadata.time_struct.push_back(                            \
                std::make_tuple(_id, _rank, _iter, #_name, temp_vec)); \
        } else {                                                       \
            std::get<2>(metadata.time_struct[_id]) = _iter;            \
            (std::get<4>(metadata.time_struct[_id]))                   \
                .push_back(elapsed_time.count());                      \
        }                                                              \
    }


#define INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(_macro) \
    template _macro(float, gko::int32);                   \
    template _macro(double, gko::int32);                  \
    template _macro(float, gko::int64);                   \
    template _macro(double, gko::int64);


#define INSTANTIATE_FOR_EACH_VALUE_MIXEDVALUE_AND_INDEX_TYPE(_macro) \
    template _macro(double, gko::int32, float);                      \
    template _macro(double, gko::int32, double);                     \
    template _macro(double, gko::int64, float);                      \
    template _macro(double, gko::int64, double);

// #define INSTANTIATE_FOR_EACH_VALUE_MIXEDVALUE_AND_INDEX_TYPE(_macro)  \
//   template _macro(float, gko::int32, float);                          \
//   template _macro(double, gko::int32, float);                         \
//   template _macro(double, gko::int32, double);                        \
//   template _macro(float, gko::int64, float);                          \
//   template _macro(double, gko::int64, float);                         \
//   template _macro(double, gko::int64, double);


// explicit instantiations for schwz
#define DECLARE_METADATA(ValueType, IndexType) \
    struct Metadata<ValueType, IndexType>
INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(DECLARE_METADATA);
#undef DECLARE_METADATA


}  // namespace schwz


#endif  // settings.hpp