dnnl.hpp

Go to the documentation of this file.

/*******************************************************************************
* Copyright 2016-2020 Intel Corporation
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
*     http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*******************************************************************************/


#ifndef DNNL_HPP
#define DNNL_HPP

#include "dnnl_config.h"

#include <algorithm>
#include <cstdlib>
#include <iterator>
#include <memory>
#include <string>
#include <vector>
#include <unordered_map>

#include "dnnl.h"

#if DNNL_CPU_THREADING_RUNTIME == DNNL_RUNTIME_THREADPOOL
#include "dnnl_threadpool_iface.hpp"
#endif

#if DNNL_GPU_RUNTIME == DNNL_RUNTIME_OCL
#include <CL/cl.h>
#endif

// __cpp_exceptions is referred from
// https://gcc.gnu.org/onlinedocs/libstdc++/manual/using_exceptions.html
// gcc < 5 does not define __cpp_exceptions but __EXCEPTIONS,
// Microsoft C++ Compiler does not provide an option to disable exceptions
#ifndef DNNL_ENABLE_EXCEPTIONS
#if __cpp_exceptions || __EXCEPTIONS \
        || (defined(_MSC_VER) && !defined(__clang__))
#define DNNL_ENABLE_EXCEPTIONS 1
#else
#define DNNL_ENABLE_EXCEPTIONS 0
#endif
#endif

#if defined(__GNUC__) || defined(__clang__)
#define DNNL_TRAP() __builtin_trap()
#elif defined(__INTEL_COMPILER) || defined(_MSC_VER)
#define DNNL_TRAP() __debugbreak()
#else
#error "unknown compiler"
#endif

#if DNNL_ENABLE_EXCEPTIONS
#define DNNL_THROW_ERROR(status, msg) throw error(status, msg)
#else
#include <cstdio>
#define DNNL_THROW_ERROR(status, msg) \
    do { \
        fputs(msg, stderr); \
        DNNL_TRAP(); \
    } while (0)
#endif


namespace dnnl {


struct error : public std::exception {
    dnnl_status_t status;
    const char *message;

    error(dnnl_status_t status, const char *message)
        : status(status), message(message) {}

    const char *what() const noexcept override { return message; }

    static void wrap_c_api(dnnl_status_t status, const char *message) {
        if (status != dnnl_success) DNNL_THROW_ERROR(status, message);
    }
};

template <typename T>
void validate_container_size(const T &v, const char *error_message,
        int min_size = 1, int max_size = -1) {
    const int size = (int)v.size();
    if (size < min_size || (max_size >= 0 && size > max_size))
        DNNL_THROW_ERROR(dnnl_invalid_arguments, error_message);
}

template <typename T>
struct handle_traits {};

template <typename T, typename traits = handle_traits<T>>
struct handle {
private:
    static dnnl_status_t dummy_destructor(T) { return dnnl_success; }
    std::shared_ptr<typename std::remove_pointer<T>::type> data_ {0};

protected:
    bool operator==(const T other) const { return other == data_.get(); }
    bool operator!=(const T other) const { return !(*this == other); }

public:
    handle() = default;

    handle(const handle<T, traits> &) = default;
    handle<T, traits> &operator=(const handle<T, traits> &) = default;
    handle(handle<T, traits> &&) = default;
    handle<T, traits> &operator=(handle<T, traits> &&) = default;

    explicit handle(T t, bool weak = false) { reset(t, weak); }

    void reset(T t, bool weak = false) {
        data_.reset(t, weak ? &dummy_destructor : traits::destructor);
    }

    T get(bool allow_empty = false) const {
        T result = data_.get();
        if (allow_empty == false && result == nullptr)
            DNNL_THROW_ERROR(
                    dnnl_invalid_arguments, "object is not initialized");
        return result;
    }

    explicit operator T() const { return get(true); }

    explicit operator bool() const { return get(true) != nullptr; }

    bool operator==(const handle<T, traits> &other) const {
        return other.data_.get() == data_.get();
    }

    bool operator!=(const handle &other) const { return !(*this == other); }
};

template <>
struct handle_traits<dnnl_memory_t> {
    static dnnl_status_t destructor(dnnl_memory_t p) {
        return dnnl_memory_destroy(p);
    }
};

template <>
struct handle_traits<dnnl_primitive_desc_t> {
    static dnnl_status_t destructor(dnnl_primitive_desc_t p) {
        return dnnl_primitive_desc_destroy(p);
    }
};

template <>
struct handle_traits<dnnl_primitive_t> {
    static dnnl_status_t destructor(dnnl_primitive_t p) {
        return dnnl_primitive_destroy(p);
    }
};

template <>
struct handle_traits<dnnl_primitive_desc_iterator_t> {
    static dnnl_status_t destructor(dnnl_primitive_desc_iterator_t p) {
        return dnnl_primitive_desc_iterator_destroy(p);
    }
};


struct stream;
struct error;
struct memory;
struct primitive_desc;



struct primitive : public handle<dnnl_primitive_t> {
    friend struct error;
    friend struct stream;

    enum class kind {
        undef = dnnl_undefined_primitive,
        reorder = dnnl_reorder,
        shuffle = dnnl_shuffle,
        concat = dnnl_concat,
        sum = dnnl_sum,
        convolution = dnnl_convolution,
        deconvolution = dnnl_deconvolution,
        eltwise = dnnl_eltwise,
        softmax = dnnl_softmax,
        pooling = dnnl_pooling,
        lrn = dnnl_lrn,
        batch_normalization = dnnl_batch_normalization,
        layer_normalization = dnnl_layer_normalization,
        inner_product = dnnl_inner_product,
        rnn = dnnl_rnn,
        binary = dnnl_binary,
        logsoftmax = dnnl_logsoftmax,
        matmul = dnnl_matmul,
        resampling = dnnl_resampling,
    };

    using handle::handle;

    primitive() = default;

    primitive(const_dnnl_primitive_desc_t c_pd);

    primitive(const primitive_desc &pd);

    inline const_dnnl_primitive_desc_t get_primitive_desc() const;

    inline kind get_kind() const;

    void execute(const stream &stream,
            const std::unordered_map<int, memory> &args) const;
};

inline dnnl_primitive_kind_t convert_to_c(primitive::kind kind) {
    return static_cast<dnnl_primitive_kind_t>(kind);
}

const_dnnl_primitive_desc_t primitive::get_primitive_desc() const {
    const_dnnl_primitive_desc_t pd;
    error::wrap_c_api(dnnl_primitive_get_primitive_desc(get(), &pd),
            "could not get a primitive descriptor from a primitive");
    return pd;
}

dnnl::primitive::kind primitive::get_kind() const {
    const_dnnl_primitive_desc_t pd = get_primitive_desc();
    // TODO (Roma): the code below is only needed because get_primitive_desc
    // returns a C type.
    dnnl_primitive_kind_t kind;
    error::wrap_c_api(dnnl_primitive_desc_query(
                              pd, dnnl_query_primitive_kind, 0, (void *)&kind),
            "could not get a primitive kind from a primitive descriptor");
    return static_cast<dnnl::primitive::kind>(kind);
}



enum class scratchpad_mode {
    library = dnnl_scratchpad_mode_library,
    user = dnnl_scratchpad_mode_user,
};

inline dnnl_scratchpad_mode_t convert_to_c(scratchpad_mode mode) {
    return static_cast<dnnl_scratchpad_mode_t>(mode);
}

enum class prop_kind {
    undef = dnnl_prop_kind_undef,
    forward_training = dnnl_forward_training,
    forward_inference = dnnl_forward_inference,
    forward_scoring = dnnl_forward_scoring,
    forward = dnnl_forward,
    backward = dnnl_backward,
    backward_data = dnnl_backward_data,
    backward_weights = dnnl_backward_weights,
    backward_bias = dnnl_backward_bias
};

inline dnnl_prop_kind_t convert_to_c(prop_kind kind) {
    return static_cast<dnnl_prop_kind_t>(kind);
}

enum class algorithm {
    undef = dnnl_alg_kind_undef,
    convolution_auto = dnnl_convolution_auto,
    convolution_direct = dnnl_convolution_direct,
    convolution_winograd = dnnl_convolution_winograd,
    deconvolution_direct = dnnl_deconvolution_direct,
    deconvolution_winograd = dnnl_deconvolution_winograd,
    eltwise_relu = dnnl_eltwise_relu,
    eltwise_tanh = dnnl_eltwise_tanh,
    eltwise_elu = dnnl_eltwise_elu,
    eltwise_square = dnnl_eltwise_square,
    eltwise_abs = dnnl_eltwise_abs,
    eltwise_sqrt = dnnl_eltwise_sqrt,
    eltwise_swish = dnnl_eltwise_swish,
    eltwise_linear = dnnl_eltwise_linear,
    eltwise_bounded_relu = dnnl_eltwise_bounded_relu,
    eltwise_soft_relu = dnnl_eltwise_soft_relu,
    eltwise_logistic = dnnl_eltwise_logistic,
    eltwise_exp = dnnl_eltwise_exp,
    eltwise_gelu = dnnl_eltwise_gelu,
    eltwise_gelu_tanh = dnnl_eltwise_gelu_tanh,
    eltwise_gelu_erf = dnnl_eltwise_gelu_erf,
    eltwise_log = dnnl_eltwise_log,
    eltwise_clip = dnnl_eltwise_clip,
    eltwise_pow = dnnl_eltwise_pow,
    eltwise_relu_use_dst_for_bwd = dnnl_eltwise_relu_use_dst_for_bwd,
    eltwise_tanh_use_dst_for_bwd = dnnl_eltwise_tanh_use_dst_for_bwd,
    eltwise_elu_use_dst_for_bwd = dnnl_eltwise_elu_use_dst_for_bwd,
    eltwise_sqrt_use_dst_for_bwd = dnnl_eltwise_sqrt_use_dst_for_bwd,
    eltwise_logistic_use_dst_for_bwd = dnnl_eltwise_logistic_use_dst_for_bwd,
    eltwise_exp_use_dst_for_bwd = dnnl_eltwise_exp_use_dst_for_bwd,
    lrn_across_channels = dnnl_lrn_across_channels,
    lrn_within_channel = dnnl_lrn_within_channel,
    pooling_max = dnnl_pooling_max,
    pooling_avg = dnnl_pooling_avg,
    pooling_avg_include_padding = dnnl_pooling_avg_include_padding,
    pooling_avg_exclude_padding = dnnl_pooling_avg_exclude_padding,
    vanilla_rnn = dnnl_vanilla_rnn,
    vanilla_lstm = dnnl_vanilla_lstm,
    vanilla_gru = dnnl_vanilla_gru,
    lbr_gru = dnnl_lbr_gru,
    binary_add = dnnl_binary_add,
    binary_mul = dnnl_binary_mul,
    binary_max = dnnl_binary_max,
    binary_min = dnnl_binary_min,
    resampling_nearest = dnnl_resampling_nearest,
    resampling_linear = dnnl_resampling_linear,
};

inline dnnl_alg_kind_t convert_to_c(algorithm algorithm) {
    return static_cast<dnnl_alg_kind_t>(algorithm);
}



enum class normalization_flags : unsigned {
    none = dnnl_normalization_flags_none,

    use_global_stats = dnnl_use_global_stats,

    use_scale_shift = dnnl_use_scaleshift,

    fuse_norm_relu = dnnl_fuse_norm_relu
};

inline dnnl_normalization_flags_t convert_to_c(normalization_flags flags) {
    return static_cast<dnnl_normalization_flags_t>(flags);
}



enum class rnn_flags : unsigned {
    undef = dnnl_rnn_flags_undef
};

inline dnnl_rnn_flags_t convert_to_c(rnn_flags flags) {
    return static_cast<dnnl_rnn_flags_t>(flags);
}

#define DNNL_DEFINE_BITMASK_OPS(enum_name) \
    inline enum_name operator|(enum_name lhs, enum_name rhs) { \
        return static_cast<enum_name>( \
                static_cast<unsigned>(lhs) | static_cast<unsigned>(rhs)); \
    } \
\
    inline enum_name operator&(enum_name lhs, enum_name rhs) { \
        return static_cast<enum_name>( \
                static_cast<unsigned>(lhs) & static_cast<unsigned>(rhs)); \
    } \
\
    inline enum_name operator^(enum_name lhs, enum_name rhs) { \
        return static_cast<enum_name>( \
                static_cast<unsigned>(lhs) ^ static_cast<unsigned>(rhs)); \
    } \
\
    inline enum_name &operator|=(enum_name &lhs, enum_name rhs) { \
        lhs = static_cast<enum_name>( \
                static_cast<unsigned>(lhs) | static_cast<unsigned>(rhs)); \
        return lhs; \
    } \
\
    inline enum_name &operator&=(enum_name &lhs, enum_name rhs) { \
        lhs = static_cast<enum_name>( \
                static_cast<unsigned>(lhs) & static_cast<unsigned>(rhs)); \
        return lhs; \
    } \
\
    inline enum_name &operator^=(enum_name &lhs, enum_name rhs) { \
        lhs = static_cast<enum_name>( \
                static_cast<unsigned>(lhs) ^ static_cast<unsigned>(rhs)); \
        return lhs; \
    } \
\
    inline enum_name operator~(enum_name rhs) { \
        return static_cast<enum_name>(~static_cast<unsigned>(rhs)); \
    }

DNNL_DEFINE_BITMASK_OPS(normalization_flags)
DNNL_DEFINE_BITMASK_OPS(rnn_flags)

enum class rnn_direction {
    unidirectional_left2right = dnnl_unidirectional_left2right,
    unidirectional_right2left = dnnl_unidirectional_right2left,
    bidirectional_concat = dnnl_bidirectional_concat,
    bidirectional_sum = dnnl_bidirectional_sum,
    unidirectional = dnnl_unidirectional,
};

inline dnnl_rnn_direction_t convert_to_c(rnn_direction dir) {
    return static_cast<dnnl_rnn_direction_t>(dir);
}



enum class query {
    undef = dnnl_query_undef,

    engine = dnnl_query_engine,
    primitive_kind = dnnl_query_primitive_kind,

    num_of_inputs_s32 = dnnl_query_num_of_inputs_s32,
    num_of_outputs_s32 = dnnl_query_num_of_outputs_s32,

    time_estimate_f64 = dnnl_query_time_estimate_f64,
    memory_consumption_s64 = dnnl_query_memory_consumption_s64,

    scratchpad_engine = dnnl_query_scratchpad_engine,

    reorder_src_engine = dnnl_query_reorder_src_engine,
    reorder_dst_engine = dnnl_query_reorder_dst_engine,

    impl_info_str = dnnl_query_impl_info_str,

    prop_kind = dnnl_query_prop_kind,

    op_d = dnnl_query_op_d,
    convolution_d = dnnl_query_convolution_d,
    deconvolution_d = dnnl_query_deconvolution_d,
    shuffle_d = dnnl_query_shuffle_d,
    eltwise_d = dnnl_query_eltwise_d,
    softmax_d = dnnl_query_softmax_d,
    pooling_d = dnnl_query_pooling_d,
    lrn_d = dnnl_query_lrn_d,
    batch_normalization_d = dnnl_query_batch_normalization_d,
    layer_normalization_d = dnnl_query_layer_normalization_d,
    inner_product_d = dnnl_query_inner_product_d,
    rnn_d = dnnl_query_rnn_d,
    binary_d = dnnl_query_binary_d,
    logsoftmax_d = dnnl_query_logsoftmax_d,
    matmul_d = dnnl_query_matmul_d,
    resampling_d = dnnl_query_resampling_d,

    src_md = dnnl_query_src_md,
    diff_src_md = dnnl_query_diff_src_md,
    weights_md = dnnl_query_weights_md,
    diff_weights_md = dnnl_query_diff_weights_md,
    dst_md = dnnl_query_dst_md,
    diff_dst_md = dnnl_query_diff_dst_md,
    workspace_md = dnnl_query_workspace_md,
    scratchpad_md = dnnl_query_scratchpad_md,
    exec_arg_md = dnnl_query_exec_arg_md,
};

inline dnnl_query_t convert_to_c(query query) {
    return static_cast<dnnl_query_t>(query);
}




template <>
struct handle_traits<dnnl_engine_t> {
    static dnnl_status_t destructor(dnnl_engine_t p) {
        return dnnl_engine_destroy(p);
    }
};

struct engine : public handle<dnnl_engine_t> {
    friend struct primitive;
    friend struct reorder;

    enum class kind {
        any = dnnl_any_engine,
        cpu = dnnl_cpu,
        gpu = dnnl_gpu,
    };

    using handle::handle;

    engine() = default;

    static size_t get_count(kind kind) {
        return dnnl_engine_get_count(convert_to_c(kind));
    }

    engine(kind kind, size_t index) {
        dnnl_engine_t engine;
        error::wrap_c_api(
                dnnl_engine_create(&engine, convert_to_c(kind), index),
                "could not create an engine");
        reset(engine);
    }

#if DNNL_GPU_RUNTIME == DNNL_RUNTIME_OCL
    engine(kind kind, cl_device_id device, cl_context context) {
        dnnl_engine_t engine;
        error::wrap_c_api(dnnl_engine_create_ocl(
                                  &engine, convert_to_c(kind), device, context),
                "could not create an engine");
        reset(engine);
    }
#endif

    engine(const handle<dnnl_primitive_desc_t> &pd) {
        dnnl_engine_t c_engine;
        error::wrap_c_api(
                dnnl_primitive_desc_query(pd.get(),
                        dnnl::convert_to_c(dnnl::query::engine), 0, &c_engine),
                "could not get an engine from a primitive_desc");
        reset(c_engine, true);
    }

    kind get_kind() const {
        dnnl_engine_kind_t kind;
        error::wrap_c_api(dnnl_engine_get_kind(get(), &kind),
                "could not get kind of an engine");
        return static_cast<engine::kind>(kind);
    }

#if DNNL_GPU_RUNTIME == DNNL_RUNTIME_OCL
    cl_context get_ocl_context() const {
        cl_context context = nullptr;
        error::wrap_c_api(dnnl_engine_get_ocl_context(get(), &context),
                "could not get an OpenCL context fron an engine");
        return context;
    }

    cl_device_id get_ocl_device() const {
        cl_device_id device = nullptr;
        error::wrap_c_api(dnnl_engine_get_ocl_device(get(), &device),
                "could not get an OpenCL device fron an engine");
        return device;
    }
#endif

    template <typename primitive_desc>
    static engine query(const primitive_desc &pd) {
        return query(pd, dnnl::query::engine);
    }

private:
    static dnnl_engine_kind_t convert_to_c(kind kind) {
        return static_cast<dnnl_engine_kind_t>(kind);
    }

    template <typename primitive_desc>
    static engine query(const primitive_desc &pd, dnnl::query what) {
        dnnl_engine_t c_engine;
        error::wrap_c_api(dnnl_primitive_desc_query(pd.get(),
                                  dnnl::convert_to_c(what), 0, &c_engine),
                "could not get an engine from a primitive_desc");
        return engine(c_engine, true);
    }
};

inline dnnl_engine_kind_t convert_to_c(engine::kind kind) {
    return static_cast<dnnl_engine_kind_t>(kind);
}



template <>
struct handle_traits<dnnl_stream_t> {
    static dnnl_status_t destructor(dnnl_stream_t p) {
        return dnnl_stream_destroy(p);
    }
};
template <>
struct handle_traits<dnnl_stream_attr_t> {
    static dnnl_status_t destructor(dnnl_stream_attr_t p) {
        return dnnl_stream_attr_destroy(p);
    }
};

struct stream_attr : public handle<dnnl_stream_attr_t> {
    using handle::handle;

    stream_attr() = default;

    stream_attr(engine::kind kind) {
        dnnl_stream_attr_t attr;
        error::wrap_c_api(dnnl_stream_attr_create(&attr, convert_to_c(kind)),
                "could not create stream attributes");
        reset(attr);
    }

#if DNNL_CPU_THREADING_RUNTIME == DNNL_RUNTIME_THREADPOOL
    void set_threadpool(threadpool_iface *threadpool) {
        error::wrap_c_api(dnnl_stream_attr_set_threadpool(get(), threadpool),
                "could not set stream threadpool attribute");
    }

    threadpool_iface *get_threadpool() {
        threadpool_iface *tp;
        error::wrap_c_api(dnnl_stream_attr_get_threadpool(get(), (void **)&tp),
                "could not set stream threadpool attribute");
        return tp;
    }
#endif
};

struct stream : public handle<dnnl_stream_t> {
    using handle::handle;

    enum class flags : unsigned {
        default_order = dnnl_stream_default_order,
        in_order = dnnl_stream_default_order,
        out_of_order = dnnl_stream_out_of_order,
        default_flags = dnnl_stream_default_flags,
    };

    stream() = default;

    stream(const engine &engine, flags flags = flags::default_flags,
            const stream_attr &attr = stream_attr()) {
        dnnl_stream_t stream;
        error::wrap_c_api(dnnl_stream_create_v2(&stream, engine.get(),
                                  static_cast<dnnl_stream_flags_t>(flags),
                                  attr.get(true)),
                "could not create a stream");
        reset(stream);
    }

#if DNNL_GPU_RUNTIME == DNNL_RUNTIME_OCL
    stream(const engine &engine, cl_command_queue queue) {
        dnnl_stream_t stream;
        error::wrap_c_api(dnnl_stream_create_ocl(&stream, engine.get(), queue),
                "could not create a stream");
        reset(stream);
    }

    cl_command_queue get_ocl_command_queue() const {
        cl_command_queue queue = nullptr;
        error::wrap_c_api(dnnl_stream_get_ocl_command_queue(get(), &queue),
                "could not get an OpenCL command queue from a stream");
        return queue;
    }
#endif

    stream &wait() {
        error::wrap_c_api(
                dnnl_stream_wait(get()), "could not wait on a stream");
        return *this;
    }
};

DNNL_DEFINE_BITMASK_OPS(stream::flags)



struct memory : public handle<dnnl_memory_t> {
    typedef dnnl_dim_t dim;
    typedef std::vector<dim> dims;

    template <typename T>
    static void validate_dims(const std::vector<T> &v, int min_size = 0) {
        validate_container_size(
                v, "dimensions are invalid", min_size, DNNL_MAX_NDIMS);
    }

    enum class data_type {
        undef = dnnl_data_type_undef,
        f16 = dnnl_f16,
        bf16 = dnnl_bf16,
        f32 = dnnl_f32,
        s32 = dnnl_s32,
        s8 = dnnl_s8,
        u8 = dnnl_u8,
    };

    enum class format_kind {
        undef = dnnl_format_kind_undef,
        any = dnnl_format_kind_any,
        blocked = dnnl_blocked,
        wino = dnnl_format_kind_wino,
        packed = dnnl_format_kind_rnn_packed,
    };

    enum class format_tag {
        undef = dnnl_format_tag_undef,
        any = dnnl_format_tag_any,

        a = dnnl_a,

        ab = dnnl_ab,
        ba = dnnl_ba,

        abc = dnnl_abc,
        acb = dnnl_acb,
        bac = dnnl_bac,
        bca = dnnl_bca,
        cba = dnnl_cba,

        abcd = dnnl_abcd,
        abdc = dnnl_abdc,
        acdb = dnnl_acdb,
        bacd = dnnl_bacd,
        bcda = dnnl_bcda,
        cdba = dnnl_cdba,
        dcab = dnnl_dcab,

        abcde = dnnl_abcde,
        abdec = dnnl_abdec,
        acbde = dnnl_acbde,
        acdeb = dnnl_acdeb,
        bcdea = dnnl_bcdea,
        cdeba = dnnl_cdeba,
        decab = dnnl_decab,
        abcdef = dnnl_abcdef,
        acbdef = dnnl_acbdef,
        defcab = dnnl_defcab,

        x = a,
        nc = ab,
        cn = ba,
        tn = ab,
        nt = ba,
        ncw = abc,
        nwc = acb,
        nchw = abcd,
        nhwc = acdb,
        chwn = bcda,
        ncdhw = abcde,
        ndhwc = acdeb,

        oi = ab,
        io = ba,
        oiw = abc,
        owi = acb,
        wio = cba,
        iwo = bca,
        oihw = abcd,
        hwio = cdba,
        ohwi = acdb,
        ihwo = bcda,
        iohw = bacd,
        oidhw = abcde,
        dhwio = cdeba,
        odhwi = acdeb,
        idhwo = bcdea,

        goiw = abcd,
        wigo = dcab,
        goihw = abcde,
        hwigo = decab,
        giohw = acbde,
        goidhw = abcdef,
        giodhw = acbdef,
        dhwigo = defcab,

        tnc = abc,
        ntc = bac,
        ldnc = abcd,
        ldigo = abcde,
        ldgoi = abdec,
        ldio = abcd,
        ldoi = abdc,
        ldgo = abcd,

        // Opaque blocked formats

        Abc16a = dnnl_Abc16a,
        ABc16a16b = dnnl_ABc16a16b,
        ABc4a4b = dnnl_ABc4a4b,
        aBc16b = dnnl_aBc16b,
        ABc16b16a = dnnl_ABc16b16a,
        Abc4a = dnnl_Abc4a,
        aBc4b = dnnl_aBc4b,
        ABc4b16a4b = dnnl_ABc4b16a4b,
        ABc2b8a4b = dnnl_ABc2b8a4b,
        ABc4b4a = dnnl_ABc4b4a,
        ABc8a16b2a = dnnl_ABc8a16b2a,
        ABc8a8b = dnnl_ABc8a8b,
        aBc8b = dnnl_aBc8b,
        ABc8b16a2b = dnnl_ABc8b16a2b,
        ABc8b8a = dnnl_ABc8b8a,
        Abcd16a = dnnl_Abcd16a,
        ABcd16a16b = dnnl_ABcd16a16b,
        aBcd16b = dnnl_aBcd16b,
        ABcd16b16a = dnnl_ABcd16b16a,
        aBCd16b16c = dnnl_aBCd16b16c,
        aBCd16c16b = dnnl_aBCd16c16b,
        Abcd4a = dnnl_Abcd4a,
        aBcd4b = dnnl_aBcd4b,
        ABcd4b16a4b = dnnl_ABcd4b16a4b,
        ABcd2b8a4b = dnnl_ABcd2b8a4b,
        ABcd4b4a = dnnl_ABcd4b4a,
        ABcd4a4b = dnnl_ABcd4a4b,
        aBCd4c16b4c = dnnl_aBCd4c16b4c,
        aBCd2c8b4c = dnnl_aBCd2c8b4c,
        aBCd4c4b = dnnl_aBCd4c4b,
        aBCd4b4c = dnnl_aBCd4b4c,
        ABcd8a16b2a = dnnl_ABcd8a16b2a,
        ABcd8a8b = dnnl_ABcd8a8b,
        aBcd8b = dnnl_aBcd8b,
        ABcd8b16a2b = dnnl_ABcd8b16a2b,
        aBCd8b16c2b = dnnl_aBCd8b16c2b,
        ABcd8b8a = dnnl_ABcd8b8a,
        aBCd8b8c = dnnl_aBCd8b8c,
        aBCd8c16b2c = dnnl_aBCd8c16b2c,
        aBCd8c8b = dnnl_aBCd8c8b,
        Abcde16a = dnnl_Abcde16a,
        ABcde16a16b = dnnl_ABcde16a16b,
        aBcde16b = dnnl_aBcde16b,
        ABcde16b16a = dnnl_ABcde16b16a,
        aBCde16b16c = dnnl_aBCde16b16c,
        aBCde16c16b = dnnl_aBCde16c16b,
        aBCde2c8b4c = dnnl_aBCde2c8b4c,
        Abcde4a = dnnl_Abcde4a,
        aBcde4b = dnnl_aBcde4b,
        ABcde4b4a = dnnl_ABcde4b4a,
        ABcde4a4b = dnnl_ABcde4a4b,
        aBCde4b4c = dnnl_aBCde4b4c,
        aBCde4c16b4c = dnnl_aBCde4c16b4c,
        aBCde4c4b = dnnl_aBCde4c4b,
        Abcde8a = dnnl_Abcde8a,
        ABcde8a8b = dnnl_ABcde8a8b,
        aBcde8b = dnnl_aBcde8b,
        ABcde8b16a2b = dnnl_ABcde8b16a2b,
        ABcde4b16a4b = dnnl_ABcde4b16a4b,
        ABcde2b8a4b = dnnl_ABcde2b8a4b,
        aBCde8b16c2b = dnnl_aBCde8b16c2b,
        ABcde8b8a = dnnl_ABcde8b8a,
        aBCde8b8c = dnnl_aBCde8b8c,
        ABcd4a8b8a4b = dnnl_ABcd4a8b8a4b,
        ABcd2a8b8a2b = dnnl_ABcd2a8b8a2b,
        aBCde4b8c8b4c = dnnl_aBCde4b8c8b4c,
        aBCde2b8c8b2c = dnnl_aBCde2b8c8b2c,
        aBCde8c16b2c = dnnl_aBCde8c16b2c,
        aBCde8c8b = dnnl_aBCde8c8b,
        aBcdef16b = dnnl_aBcdef16b,
        aBCdef16b16c = dnnl_aBCdef16b16c,
        aBCdef16c16b = dnnl_aBCdef16c16b,
        aBcdef4b = dnnl_aBcdef4b,
        aBCdef4c4b = dnnl_aBCdef4c4b,
        aBCdef4b4c = dnnl_aBCdef4b4c,
        aBCdef8b8c = dnnl_aBCdef8b8c,
        aBCdef8c16b2c = dnnl_aBCdef8c16b2c,
        aBCdef4c16b4c = dnnl_aBCdef4c16b4c,
        aBCdef8c8b = dnnl_aBCdef8c8b,
        aBdc16b = dnnl_aBdc16b,
        aBdc4b = dnnl_aBdc4b,
        aBdc8b = dnnl_aBdc8b,
        aBdec16b = dnnl_aBdec16b,
        aBdec4b = dnnl_aBdec4b,
        aBdec8b = dnnl_aBdec8b,
        aBdefc16b = dnnl_aBdefc16b,
        aCBdef16c16b = dnnl_aCBdef16c16b,
        aCBdef16b16c = dnnl_aCBdef16b16c,
        aBdefc4b = dnnl_aBdefc4b,
        aBdefc8b = dnnl_aBdefc8b,
        Acb16a = dnnl_Acb16a,
        Acb4a = dnnl_Acb4a,
        Acb8a = dnnl_Acb8a,
        aCBd16b16c = dnnl_aCBd16b16c,
        aCBd16c16b = dnnl_aCBd16c16b,
        aCBde16b16c = dnnl_aCBde16b16c,
        aCBde16c16b = dnnl_aCBde16c16b,
        Acdb16a = dnnl_Acdb16a,
        Acdb4a = dnnl_Acdb4a,
        Acdb8a = dnnl_Acdb8a,
        Acdeb16a = dnnl_Acdeb16a,
        Acdeb4a = dnnl_Acdeb4a,
        Acdeb8a = dnnl_Acdeb8a,
        BAc16a16b = dnnl_BAc16a16b,
        BAc16b16a = dnnl_BAc16b16a,
        BAcd16a16b = dnnl_BAcd16a16b,
        BAcd16b16a = dnnl_BAcd16b16a,
        ABcd32a32b = dnnl_ABcd32a32b,
        BAcde16b16a = dnnl_BAcde16b16a,
        BAcde16a16b = dnnl_BAcde16a16b,
        aBdec32b = dnnl_aBdec32b,
        Abcdef16a = dnnl_Abcdef16a,
        Acdb32a = dnnl_Acdb32a,
        aBCd2b4c2b = dnnl_aBCd2b4c2b,
        aBCde2b4c2b = dnnl_aBCde2b4c2b,
        aBCdef2b4c2b = dnnl_aBCdef2b4c2b,
        aBCd2c4b2c = dnnl_aBCd2c4b2c,
        aBCde2c4b2c = dnnl_aBCde2c4b2c,
        aBCdef2c4b2c = dnnl_aBCdef2c4b2c,
        aBCd4b8c2b = dnnl_aBCd4b8c2b,
        aBCde4b8c2b = dnnl_aBCde4b8c2b,
        aBCdef4b8c2b = dnnl_aBCdef4b8c2b,
        aBCd4c8b2c = dnnl_aBCd4c8b2c,
        aBCde4c8b2c = dnnl_aBCde4c8b2c,
        aBCdef4c8b2c = dnnl_aBCdef4c8b2c,

        format_tag_last = dnnl_format_tag_last,

        nCdhw16c = dnnl_nCdhw16c,
        nCdhw4c = dnnl_nCdhw4c,
        nCdhw8c = dnnl_nCdhw8c,
        nChw16c = dnnl_nChw16c,
        nChw4c = dnnl_nChw4c,
        nChw8c = dnnl_nChw8c,
        nCw16c = dnnl_nCw16c,
        nCw4c = dnnl_nCw4c,
        nCw8c = dnnl_nCw8c,
        NCw16n16c = dnnl_NCw16n16c,
        NChw16n16c = dnnl_NChw16n16c,
        NCdhw16n16c = dnnl_NCdhw16n16c,
        NChw32n32c = dnnl_NChw32n32c,
        IOhw16i16o = dnnl_IOhw16i16o,
        Ohwi32o = dnnl_Ohwi32o,
        IOdhw16i16o = dnnl_IOdhw16i16o,
        gIOhw16i16o = dnnl_gIOhw16i16o,
        gOhwi32o = dnnl_gOhwi32o,
        Goidhw16g = dnnl_Goidhw16g,
        IOw16o16i = dnnl_IOw16o16i,
        OIw16i16o = dnnl_OIw16i16o,
        IOw16i16o = dnnl_IOw16i16o,
        gIOw16i16o = dnnl_gIOw16i16o,
        OIw16o16i = dnnl_OIw16o16i,
        Oiw16o = dnnl_Oiw16o,
        OIw4i16o4i = dnnl_OIw4i16o4i,
        OIw2i8o4i = dnnl_OIw2i8o4i,
        OIw4i4o = dnnl_OIw4i4o,
        OIw4o4i = dnnl_OIw4o4i,
        Oiw4o = dnnl_Oiw4o,
        OIw8i16o2i = dnnl_OIw8i16o2i,
        OIw8i8o = dnnl_OIw8i8o,
        OIw8o16i2o = dnnl_OIw8o16i2o,
        OIw8o8i = dnnl_OIw8o8i,
        Owi16o = dnnl_Owi16o,
        OwI16o2i = dnnl_OwI16o2i,
        Owi4o = dnnl_Owi4o,
        Owi8o = dnnl_Owi8o,
        IOhw16o16i = dnnl_IOhw16o16i,
        Ohwi16o = dnnl_Ohwi16o,
        OhwI16o2i = dnnl_OhwI16o2i,
        Ohwi4o = dnnl_Ohwi4o,
        Ohwi8o = dnnl_Ohwi8o,
        OIhw16i16o = dnnl_OIhw16i16o,
        OIhw16o16i = dnnl_OIhw16o16i,
        Oihw16o = dnnl_Oihw16o,
        OIhw4i16o4i = dnnl_OIhw4i16o4i,
        OIhw4i4o = dnnl_OIhw4i4o,
        OIhw4o4i = dnnl_OIhw4o4i,
        Oihw4o = dnnl_Oihw4o,
        OIhw8i16o2i = dnnl_OIhw8i16o2i,
        OIhw8i8o = dnnl_OIhw8i8o,
        OIhw8o16i2o = dnnl_OIhw8o16i2o,
        OIhw8o8i = dnnl_OIhw8o8i,
        OIhw2i8o4i = dnnl_OIhw2i8o4i,
        IOdhw16o16i = dnnl_IOdhw16o16i,
        Odhwi16o = dnnl_Odhwi16o,
        OdhwI16o2i = dnnl_OdhwI16o2i,
        Odhwi4o = dnnl_Odhwi4o,
        Odhwi8o = dnnl_Odhwi8o,
        OIdhw16i16o = dnnl_OIdhw16i16o,
        OIdhw16o16i = dnnl_OIdhw16o16i,
        Oidhw16o = dnnl_Oidhw16o,
        OIdhw4i4o = dnnl_OIdhw4i4o,
        OIdhw4o4i = dnnl_OIdhw4o4i,
        Oidhw4o = dnnl_Oidhw4o,
        OIdhw8i16o2i = dnnl_OIdhw8i16o2i,
        OIdhw4i16o4i = dnnl_OIdhw4i16o4i,
        OIdhw2i8o4i = dnnl_OIdhw2i8o4i,
        OIdhw8i8o = dnnl_OIdhw8i8o,
        OIdhw8o8i = dnnl_OIdhw8o8i,
        gIOw16o16i = dnnl_gIOw16o16i,
        gOIw16i16o = dnnl_gOIw16i16o,
        gOIw16o16i = dnnl_gOIw16o16i,
        gOiw16o = dnnl_gOiw16o,
        gOIw4i16o4i = dnnl_gOIw4i16o4i,
        gOIw2i8o4i = dnnl_gOIw2i8o4i,
        gOIw4i4o = dnnl_gOIw4i4o,
        gOIw4o4i = dnnl_gOIw4o4i,
        gOiw4o = dnnl_gOiw4o,
        gOIw8i16o2i = dnnl_gOIw8i16o2i,
        gOIw8i8o = dnnl_gOIw8i8o,
        gOIw8o16i2o = dnnl_gOIw8o16i2o,
        gOIw8o8i = dnnl_gOIw8o8i,
        gOwi16o = dnnl_gOwi16o,
        gOwI16o2i = dnnl_gOwI16o2i,
        gOwi4o = dnnl_gOwi4o,
        gOwi8o = dnnl_gOwi8o,
        Goiw8g = dnnl_Goiw8g,
        Goiw16g = dnnl_Goiw16g,
        gIOhw16o16i = dnnl_gIOhw16o16i,
        gOhwi16o = dnnl_gOhwi16o,
        gOhwI16o2i = dnnl_gOhwI16o2i,
        gOhwi4o = dnnl_gOhwi4o,
        gOhwi8o = dnnl_gOhwi8o,
        Goihw16g = dnnl_Goihw16g,
        gOIhw16i16o = dnnl_gOIhw16i16o,
        gOIhw16o16i = dnnl_gOIhw16o16i,
        gOihw16o = dnnl_gOihw16o,
        gOIhw4i16o4i = dnnl_gOIhw4i16o4i,
        gOIhw2i8o4i = dnnl_gOIhw2i8o4i,
        gOIhw4i4o = dnnl_gOIhw4i4o,
        gOIhw4o4i = dnnl_gOIhw4o4i,
        gOihw4o = dnnl_gOihw4o,
        Goihw8g = dnnl_Goihw8g,
        gOIhw8i16o2i = dnnl_gOIhw8i16o2i,
        gOIhw8i8o = dnnl_gOIhw8i8o,
        gOIhw8o16i2o = dnnl_gOIhw8o16i2o,
        OIhw4o8i8o4i = dnnl_OIhw4o8i8o4i,
        OIhw2o8i8o2i = dnnl_OIhw2o8i8o2i,
        gOIhw4o8i8o4i = dnnl_gOIhw4o8i8o4i,
        gOIhw2o8i8o2i = dnnl_gOIhw2o8i8o2i,
        gOIhw8o8i = dnnl_gOIhw8o8i,
        gIOdhw16i16o = dnnl_gIOdhw16i16o,
        gIOdhw16o16i = dnnl_gIOdhw16o16i,
        gOdhwi16o = dnnl_gOdhwi16o,
        gOdhwI16o2i = dnnl_gOdhwI16o2i,
        gOdhwi4o = dnnl_gOdhwi4o,
        gOdhwi8o = dnnl_gOdhwi8o,
        gOIdhw16i16o = dnnl_gOIdhw16i16o,
        gOIdhw16o16i = dnnl_gOIdhw16o16i,
        gOidhw16o = dnnl_gOidhw16o,
        gOIdhw4i4o = dnnl_gOIdhw4i4o,
        gOIdhw4o4i = dnnl_gOIdhw4o4i,
        gOidhw4o = dnnl_gOidhw4o,
        gOIdhw8i16o2i = dnnl_gOIdhw8i16o2i,
        gOIdhw4i16o4i = dnnl_gOIdhw4i16o4i,
        gOIdhw2i8o4i = dnnl_gOIdhw2i8o4i,
        gOIdhw8i8o = dnnl_gOIdhw8i8o,
        gOIdhw8o8i = dnnl_gOIdhw8o8i,
        gOIw2i4o2i = dnnl_gOIw2i4o2i,
        gOIhw2i4o2i = dnnl_gOIhw2i4o2i,
        gOIdhw2i4o2i = dnnl_gOIdhw2i4o2i,
        gOIw2o4i2o = dnnl_gOIw2o4i2o,
        gOIhw2o4i2o = dnnl_gOIhw2o4i2o,
        gOIdhw2o4i2o = dnnl_gOIdhw2o4i2o,
        gOIw4i8o2i = dnnl_gOIw4i8o2i,
        gOIhw4i8o2i = dnnl_gOIhw4i8o2i,
        gOIdhw4i8o2i = dnnl_gOIdhw4i8o2i,
        gOIw4o8i2o = dnnl_gOIw4o8i2o,
        gOIhw4o8i2o = dnnl_gOIhw4o8i2o,
        gOIdhw4o8i2o = dnnl_gOIdhw4o8i2o,
    };

    struct desc {
        friend struct memory;
        dnnl_memory_desc_t data;

        desc() : data() {}

        desc(const memory::dims &dims, data_type data_type,
                format_tag format_tag, bool allow_empty = false)
            : data() {
            validate_dims(dims);
            dnnl_status_t status = dnnl_memory_desc_init_by_tag(&data,
                    (int)dims.size(), dims.data(), convert_to_c(data_type),
                    convert_to_c(format_tag));
            if (!allow_empty)
                error::wrap_c_api(status,
                        "could not construct a memory descriptor using a "
                        "format tag");
        }

        desc(const memory::dims &dims, data_type data_type,
                const memory::dims &strides, bool allow_empty = false)
            : data() {
            validate_dims(dims);
            if (!strides.empty()) validate_dims(strides, (int)dims.size());
            dnnl_status_t status = dnnl_memory_desc_init_by_strides(&data,
                    (int)dims.size(), dims.data(), convert_to_c(data_type),
                    strides.empty() ? nullptr : &strides[0]);
            if (!allow_empty)
                error::wrap_c_api(status,
                        "could not construct a memory descriptor using "
                        "strides");
        }

        desc(const dnnl_memory_desc_t &data) : data(data) {}

        //
        desc submemory_desc(const memory::dims &dims,
                const memory::dims &offsets, bool allow_empty = false) const {
            validate_dims(dims, data.ndims);
            validate_dims(offsets, data.ndims);
            dnnl_memory_desc_t sub_md = dnnl_memory_desc_t();
            dnnl_status_t status = dnnl_memory_desc_init_submemory(
                    &sub_md, &data, dims.data(), offsets.data());
            if (!allow_empty)
                error::wrap_c_api(status, "could not construct a sub-memory");
            return desc(sub_md);
        }

        desc reshape(const memory::dims &dims, bool allow_empty = false) const {
            if (data.ndims) validate_dims(dims, 1);
            dnnl_memory_desc_t out_md = dnnl_memory_desc_t();
            dnnl_status_t status = dnnl_memory_desc_reshape(
                    &out_md, &data, (int)dims.size(), dims.data());
            if (!allow_empty)
                error::wrap_c_api(
                        status, "could not reshape a memory descriptor");
            return desc(out_md);
        }

        desc permute_axes(const std::vector<int> &permutation,
                bool allow_empty = false) const {
            validate_dims(permutation, data.ndims);
            dnnl_memory_desc_t out_md = dnnl_memory_desc_t();
            dnnl_status_t status = dnnl_memory_desc_permute_axes(
                    &out_md, &data, permutation.data());
            if (!allow_empty)
                error::wrap_c_api(status,
                        "could not permute axes of a memory descriptor");
            return desc(out_md);
        }

        memory::dims dims() const {
            return memory::dims(data.dims, data.dims + data.ndims);
        }

        memory::data_type data_type() const {
            return static_cast<memory::data_type>(data.data_type);
        }

        size_t get_size() const { return dnnl_memory_desc_get_size(&data); }

        bool is_zero() const { return data.ndims == 0; }

        bool operator==(const desc &other) const {
            return dnnl_memory_desc_equal(&data, &other.data) != 0;
        }

        bool operator!=(const desc &other) const { return !operator==(other); }
    };

    // Default constructor.
    //
    // Constructs an empty memory object, which can be used to indicate absence
    // of a parameter.
    memory() = default;

    memory(const desc &md, const engine &engine, void *handle) {
        dnnl_memory_t result;
        error::wrap_c_api(
                dnnl_memory_create(&result, &md.data, engine.get(), handle),
                "could not create a memory object");
        reset(result);
    }

    memory(const desc &md, const engine &engine)
        : memory(md, engine, DNNL_MEMORY_ALLOCATE) {}

    desc get_desc() const {
        const dnnl_memory_desc_t *cdesc;
        error::wrap_c_api(dnnl_memory_get_memory_desc(get(), &cdesc),
                "could not get a memory descriptor from a memory object");
        return desc(*cdesc);
    }

    engine get_engine() const {
        dnnl_engine_t c_engine;
        error::wrap_c_api(dnnl_memory_get_engine(get(), &c_engine),
                "could not get an engine from a memory object");
        return engine(c_engine, true);
    }

    void *get_data_handle() const {
        void *handle;
        error::wrap_c_api(dnnl_memory_get_data_handle(get(), &handle),
                "could not get a native handle from a memory object");
        return handle;
    }

    void set_data_handle(void *handle, const stream &stream) const {
        error::wrap_c_api(
                dnnl_memory_set_data_handle_v2(get(), handle, stream.get(true)),
                "could not set native handle of a memory object");
    }

    void set_data_handle(void *handle) const {
        error::wrap_c_api(
                dnnl_memory_set_data_handle_v2(get(), handle, nullptr),
                "could not set native handle of a memory object");
    }

    template <typename T = void>
    T *map_data() const {
        void *mapped_ptr;
        error::wrap_c_api(dnnl_memory_map_data(get(), &mapped_ptr),
                "could not map memory object data");
        return static_cast<T *>(mapped_ptr);
    }

    void unmap_data(void *mapped_ptr) const {
        error::wrap_c_api(dnnl_memory_unmap_data(get(), mapped_ptr),
                "could not unmap memory object data");
    }

#if DNNL_GPU_RUNTIME == DNNL_RUNTIME_OCL
    cl_mem get_ocl_mem_object() const {
        cl_mem mem_object;
        error::wrap_c_api(dnnl_memory_get_ocl_mem_object(get(), &mem_object),
                "could not get OpenCL buffer object from a memory object");
        return mem_object;
    }

    void set_ocl_mem_object(cl_mem mem_object) {
        error::wrap_c_api(dnnl_memory_set_ocl_mem_object(get(), mem_object),
                "could not set OpenCL buffer object from a memory object");
    }
#endif

    static dnnl_data_type_t convert_to_c(data_type data_type) {
        return static_cast<dnnl_data_type_t>(data_type);
    }
    static dnnl_format_tag_t convert_to_c(format_tag format) {
        return static_cast<dnnl_format_tag_t>(format);
    }
};

inline bool operator==(dnnl_data_type_t a, memory::data_type b) {
    return a == memory::convert_to_c(b);
}
inline bool operator!=(dnnl_data_type_t a, memory::data_type b) {
    return !(a == b);
}
inline bool operator==(memory::data_type a, dnnl_data_type_t b) {
    return b == a;
}
inline bool operator!=(memory::data_type a, dnnl_data_type_t b) {
    return !(a == b);
}

inline bool operator==(dnnl_format_tag_t a, memory::format_tag b) {
    return a == memory::convert_to_c(b);
}
inline bool operator!=(dnnl_format_tag_t a, memory::format_tag b) {
    return !(a == b);
}
inline bool operator==(memory::format_tag a, dnnl_format_tag_t b) {
    return b == a;
}
inline bool operator!=(memory::format_tag a, dnnl_format_tag_t b) {
    return !(a == b);
}



template <>
struct handle_traits<dnnl_post_ops_t> {
    static dnnl_status_t destructor(dnnl_post_ops_t p) {
        return dnnl_post_ops_destroy(p);
    }
};

struct post_ops : public handle<dnnl_post_ops_t> {
    using handle<dnnl_post_ops_t>::handle;

    post_ops() {
        dnnl_post_ops_t result;
        error::wrap_c_api(
                dnnl_post_ops_create(&result), "could not create post-ops");
        reset(result);
    }

    int len() const { return dnnl_post_ops_len(get()); }

    primitive::kind kind(int index) const {
        error::wrap_c_api(index < len() ? dnnl_success : dnnl_invalid_arguments,
                "post-ops index is out of range");
        return static_cast<primitive::kind>(
                dnnl_post_ops_get_kind(get(), index));
    }

    void append_sum(float scale = 1.) {
        error::wrap_c_api(dnnl_post_ops_append_sum(get(), scale),
                "could not append a sum post-op");
    }

    void get_params_sum(int index, float &scale) const {
        error::wrap_c_api(dnnl_post_ops_get_params_sum(get(), index, &scale),
                "could not get parameters of a sum post-op");
    }

    void append_eltwise(
            float scale, algorithm algorithm, float alpha, float beta) {
        error::wrap_c_api(dnnl_post_ops_append_eltwise(get(), scale,
                                  convert_to_c(algorithm), alpha, beta),
                "could not append an elementwise post-op");
    }

    void get_params_eltwise(int index, float &scale, algorithm &algorithm,
            float &alpha, float &beta) const {
        dnnl_alg_kind_t c_alg;
        error::wrap_c_api(dnnl_post_ops_get_params_eltwise(
                                  get(), index, &scale, &c_alg, &alpha, &beta),
                "could not get parameters of an elementwise post-op");
        algorithm = static_cast<dnnl::algorithm>(c_alg);
    }

    void append_dw_k3s1p1(memory::data_type weights_data_type,
            memory::data_type bias_data_type, memory::data_type dst_data_type,
            int mask, const std::vector<float> &scales) {

        error::wrap_c_api(dnnl_post_ops_append_dw_k3s1p1(get(),
                                  memory::convert_to_c(weights_data_type),
                                  memory::convert_to_c(bias_data_type),
                                  memory::convert_to_c(dst_data_type),
                                  scales.size(), mask, &scales[0]),
                "could not append depthwise post-op");
    }

    void get_params_dw_k3s1p1(int index, memory::data_type &weights_data_type,
            memory::data_type &bias_data_type, memory::data_type &dst_data_type,
            int &mask, std::vector<float> &scales) const {

        dnnl_data_type_t c_weights_data_type;
        dnnl_data_type_t c_bias_data_type;
        dnnl_data_type_t c_dst_data_type;
        dnnl_dim_t count;
        int c_mask;
        const float *c_scales;
        error::wrap_c_api(dnnl_post_ops_get_params_dw_k3s1p1(get(), index,
                                  &c_weights_data_type, &c_bias_data_type,
                                  &c_dst_data_type, &count, &c_mask, &c_scales),
                "could not get parameters of depthwise post-op");

        weights_data_type = static_cast<memory::data_type>(c_weights_data_type);
        bias_data_type = static_cast<memory::data_type>(c_bias_data_type);
        dst_data_type = static_cast<memory::data_type>(c_dst_data_type);
        scales.resize(count);

        mask = c_mask;
        for (dnnl_dim_t c = 0; c < count; ++c)
            scales[c] = c_scales[c];
        return;
    }

    void append_dw_k3s2p1(memory::data_type weights_data_type,
            memory::data_type bias_data_type, memory::data_type dst_data_type,
            int mask, const std::vector<float> &scales) {

        error::wrap_c_api(dnnl_post_ops_append_dw_k3s2p1(get(),
                                  memory::convert_to_c(weights_data_type),
                                  memory::convert_to_c(bias_data_type),
                                  memory::convert_to_c(dst_data_type),
                                  scales.size(), mask, &scales[0]),
                "could not append depthwise post-op");
    }

    void get_params_dw_k3s2p1(int index, memory::data_type &weights_data_type,
            memory::data_type &bias_data_type, memory::data_type &dst_data_type,
            int &mask, std::vector<float> &scales) const {

        dnnl_data_type_t c_weights_data_type;
        dnnl_data_type_t c_bias_data_type;
        dnnl_data_type_t c_dst_data_type;
        dnnl_dim_t count;
        int c_mask;
        const float *c_scales;
        error::wrap_c_api(dnnl_post_ops_get_params_dw_k3s2p1(get(), index,
                                  &c_weights_data_type, &c_bias_data_type,
                                  &c_dst_data_type, &count, &c_mask, &c_scales),
                "could not get parameters of depthwise post-op");

        weights_data_type = static_cast<memory::data_type>(c_weights_data_type);
        bias_data_type = static_cast<memory::data_type>(c_bias_data_type);
        dst_data_type = static_cast<memory::data_type>(c_dst_data_type);
        scales.resize(count);

        mask = c_mask;
        for (dnnl_dim_t c = 0; c < count; ++c)
            scales[c] = c_scales[c];
        return;
    }
};

template <>
struct handle_traits<dnnl_primitive_attr_t> {
    static dnnl_status_t destructor(dnnl_primitive_attr_t p) {
        return dnnl_primitive_attr_destroy(p);
    }
};

struct primitive_attr : public handle<dnnl_primitive_attr_t> {
    using handle<dnnl_primitive_attr_t>::handle;

    primitive_attr() {
        dnnl_primitive_attr_t result;
        error::wrap_c_api(dnnl_primitive_attr_create(&result),
                "could not create primitive attribute");
        reset(result);
    }

    primitive_attr(dnnl_primitive_attr_t attr)
        : handle<dnnl_primitive_attr_t>(attr) {}

    scratchpad_mode get_scratchpad_mode() const {
        dnnl_scratchpad_mode_t result;
        error::wrap_c_api(
                dnnl_primitive_attr_get_scratchpad_mode(get(), &result),
                "could not get scratchpad mode primitive attribute");
        return scratchpad_mode(result);
    }

    void set_scratchpad_mode(scratchpad_mode mode) {
        error::wrap_c_api(dnnl_primitive_attr_set_scratchpad_mode(
                                  get(), dnnl::convert_to_c(mode)),
                "could not set scratchpad mode primitive attribute");
    }

    void get_output_scales(int &mask, std::vector<float> &scales) const {
        dnnl_dim_t count;
        int c_mask;
        const float *c_scales;
        error::wrap_c_api(dnnl_primitive_attr_get_output_scales(
                                  get(), &count, &c_mask, &c_scales),
                "could not get output scales primitive attribute");
        scales.resize(count);

        mask = c_mask;
        for (dnnl_dim_t c = 0; c < count; ++c)
            scales[c] = c_scales[c];
    }

    void set_output_scales(int mask, const std::vector<float> &scales) {
        error::wrap_c_api(
                dnnl_primitive_attr_set_output_scales(
                        get(), (dnnl_dim_t)scales.size(), mask, scales.data()),
                "could not set output scales primitive attribute");
    }

    void get_scales(int arg, int &mask, std::vector<float> &scales) const {
        dnnl_dim_t count;
        int c_mask;
        const float *c_scales;
        error::wrap_c_api(dnnl_primitive_attr_get_scales(
                                  get(), arg, &count, &c_mask, &c_scales),
                "could not get scales primitive attributes");
        scales.resize(count);

        mask = c_mask;
        for (dnnl_dim_t c = 0; c < count; ++c)
            scales[c] = c_scales[c];
    }

    void set_scales(int arg, int mask, const std::vector<float> &scales) {
        error::wrap_c_api(
                dnnl_primitive_attr_set_scales(get(), arg,
                        (dnnl_dim_t)scales.size(), mask, scales.data()),
                "could not set scales primitive attribute");
    }

    void get_zero_points(
            int arg, int &mask, std::vector<int32_t> &zero_points) const {
        dnnl_dim_t count;
        int c_mask;
        const int32_t *c_zero_points;
        error::wrap_c_api(dnnl_primitive_attr_get_zero_points(
                                  get(), arg, &count, &c_mask, &c_zero_points),
                "could not get zero points primitive attribute");
        zero_points.resize(count);

        mask = c_mask;
        for (dnnl_dim_t c = 0; c < count; ++c)
            zero_points[c] = c_zero_points[c];
    }

    void set_zero_points(
            int arg, int mask, const std::vector<int32_t> &zero_points) {
        error::wrap_c_api(dnnl_primitive_attr_set_zero_points(get(), arg,
                                  (dnnl_dim_t)zero_points.size(), mask,
                                  zero_points.data()),
                "could not set zero points primitive attribute");
    }

    const post_ops get_post_ops() const {
        post_ops result;
        const_dnnl_post_ops_t c_result;
        error::wrap_c_api(dnnl_primitive_attr_get_post_ops(get(), &c_result),
                "could not get post-ops primitive attribute");
        result.reset(const_cast<dnnl_post_ops_t>(c_result), true);
        return result;
    }

    void set_post_ops(const post_ops ops) {
        error::wrap_c_api(dnnl_primitive_attr_set_post_ops(get(), ops.get()),
                "could not set post-ops primitive attribute");
    }

    void set_rnn_data_qparams(float scale, float shift) {
        error::wrap_c_api(
                dnnl_primitive_attr_set_rnn_data_qparams(get(), scale, shift),
                "could not get RNN data quantization parameters primitive "
                "attribute");
    }

    void set_rnn_weights_qparams(int mask, const std::vector<float> &scales) {
        error::wrap_c_api(dnnl_primitive_attr_set_rnn_weights_qparams(get(),
                                  (int)scales.size(), mask, scales.data()),
                "could not get RNN weights quantization parameters primitive "
                "attribute");
    }
};



struct primitive_desc_base : public handle<dnnl_primitive_desc_t> {
    using handle<dnnl_primitive_desc_t>::handle;

    primitive_desc_base() = default;

    engine get_engine() const { return engine::query(*this); }

    const char *impl_info_str() const {
        const char *res;
        error::wrap_c_api(dnnl_primitive_desc_query(
                                  get(), dnnl_query_impl_info_str, 0, &res),
                "could not retrieve implementation info string from a "
                "primitive descriptor");
        return res;
    }

    memory::dim query_s64(query what) const {
        memory::dim res;
        dnnl_status_t status = dnnl_primitive_desc_query(
                get(), dnnl::convert_to_c(what), 0, &res);
        return status == dnnl_success ? res : 0;
    }

    memory::desc query_md(query what, int idx = 0) const {
        std::vector<query> valid_q {query::src_md, query::diff_src_md,
                query::weights_md, query::diff_weights_md, query::dst_md,
                query::diff_dst_md, query::workspace_md, query::scratchpad_md,
                query::exec_arg_md};
        if (!std::any_of(valid_q.cbegin(), valid_q.cend(),
                    [=](query q) { return what == q; }))
            DNNL_THROW_ERROR(dnnl_invalid_arguments,
                    "memory descriptor query is invalid");

        const dnnl_memory_desc_t *cdesc = dnnl_primitive_desc_query_md(
                get(), dnnl::convert_to_c(what), idx);
        return cdesc ? memory::desc(*cdesc) : memory::desc();
    }

    memory::desc src_desc(int idx) const {
        return query_md(query::src_md, idx);
    }

    memory::desc dst_desc(int idx) const {
        return query_md(query::dst_md, idx);
    }

    memory::desc weights_desc(int idx) const {
        return query_md(query::weights_md, idx);
    }

    memory::desc diff_src_desc(int idx) const {
        return query_md(query::diff_src_md, idx);
    }

    memory::desc diff_dst_desc(int idx) const {
        return query_md(query::diff_dst_md, idx);
    }

    memory::desc diff_weights_desc(int idx) const {
        return query_md(query::diff_weights_md, idx);
    }

    // Separate versions without the index argument for documentation
    // purposes.

    memory::desc src_desc() const { return src_desc(0); }

    memory::desc dst_desc() const { return dst_desc(0); }

    memory::desc weights_desc() const { return weights_desc(0); }

    memory::desc diff_src_desc() const { return diff_src_desc(0); }

    memory::desc diff_dst_desc() const { return diff_dst_desc(0); }

    memory::desc diff_weights_desc() const { return diff_weights_desc(0); }

    memory::desc workspace_desc() const {
        return query_md(query::workspace_md, 0);
    }

    memory::desc scratchpad_desc() const {
        return query_md(query::scratchpad_md, 0);
    }

    engine scratchpad_engine() const {
        dnnl_engine_t c_engine;
        error::wrap_c_api(dnnl_primitive_desc_query(get(),
                                  dnnl::convert_to_c(query::scratchpad_engine),
                                  0, &c_engine),
                "could not retrieve scratchpad engine from a primitive "
                "descriptor");
        return engine(c_engine, true);
    }

    primitive_attr get_primitive_attr() const {
        const_dnnl_primitive_attr_t const_c_attr;
        error::wrap_c_api(dnnl_primitive_desc_get_attr(get(), &const_c_attr),
                "could not get attributes from a primitive descriptor");
        dnnl_primitive_attr_t c_attr;
        error::wrap_c_api(dnnl_primitive_attr_clone(&c_attr, const_c_attr),
                "could not clone primitive attributes");
        return primitive_attr(c_attr);
    }

    dnnl::primitive::kind get_kind() const {
        dnnl_primitive_kind_t kind;
        error::wrap_c_api(dnnl_primitive_desc_query(get(),
                                  dnnl_query_primitive_kind, 0, (void *)&kind),
                "could not get primitive kind from a primitive descriptor");
        return static_cast<dnnl::primitive::kind>(kind);
    }

protected:
    void reset_with_clone(const_dnnl_primitive_desc_t pd) {
        dnnl_primitive_desc_t new_pd;
        error::wrap_c_api(dnnl_primitive_desc_clone(&new_pd, pd),
                "could not clone a primitive descriptor");
        reset(new_pd);
    }

    primitive_desc_base(
            dnnl_primitive_desc_t pd, dnnl::primitive::kind prim_kind)
        : primitive_desc_base(pd, prim_kind, dnnl::prop_kind::undef) {}

    primitive_desc_base(dnnl_primitive_desc_t pd,
            dnnl::primitive::kind prim_kind, dnnl::prop_kind prop_kind)
        : primitive_desc_base(pd, prim_kind, prop_kind, prop_kind) {}

    primitive_desc_base(dnnl_primitive_desc_t pd,
            dnnl::primitive::kind prim_kind, dnnl::prop_kind prop_kind1,
            dnnl::prop_kind prop_kind2) {
        // It is OK to pass an empty primitive descriptor
        if (pd == nullptr) return;

        dnnl_status_t rc;

        dnnl_primitive_kind_t c_prim_kind = convert_to_c(prim_kind);
        dnnl_prop_kind_t c_prop_kind1 = convert_to_c(prop_kind1);
        dnnl_prop_kind_t c_prop_kind2 = convert_to_c(prop_kind2);

        // Check that primitive kind matches
        dnnl_primitive_kind_t pd_kind;
        rc = dnnl_primitive_desc_query(
                pd, dnnl_query_primitive_kind, 0, (void *)&pd_kind);
        error::wrap_c_api(
                rc, "could not get primitive kind from a primitive descriptor");
        if (pd_kind != c_prim_kind)
            DNNL_THROW_ERROR(dnnl_invalid_arguments,
                    "primitive descriptor operation kind mismatch");

        // Check that propagation kind matches
        dnnl_prop_kind_t pd_prop_kind;
        rc = dnnl_primitive_desc_query(
                pd, dnnl_query_prop_kind, 0, (void *)&pd_prop_kind);

        // Something went wrong
        if (rc != dnnl_success && rc != dnnl_unimplemented)
            DNNL_THROW_ERROR(dnnl_invalid_arguments,
                    "could not get propagation kind from the primitive "
                    "descriptor");

        // Everything is fine
        if ((rc == dnnl_unimplemented && c_prop_kind1 == dnnl_prop_kind_undef)
                || (rc == dnnl_success
                        && (pd_prop_kind == c_prop_kind1
                                || pd_prop_kind == c_prop_kind2))) {
            reset_with_clone(pd);
            return;
        }

        // We could get the propagation kind but there is a mismatch
        DNNL_THROW_ERROR(dnnl_invalid_arguments,
                "primitive descriptor propagation kind mismatch");
    }

    using base = primitive_desc_base;
};



struct reorder : public primitive {
    struct primitive_desc : public primitive_desc_base {
        using primitive_desc_base::primitive_desc_base;

        primitive_desc() = default;

        primitive_desc(const engine &src_engine, const memory::desc &src_md,
                const engine &dst_engine, const memory::desc &dst_md,
                const primitive_attr &attr = primitive_attr()) {
            dnnl_primitive_desc_t result;
            error::wrap_c_api(
                    dnnl_reorder_primitive_desc_create(&result, &src_md.data,
                            src_engine.get(), &dst_md.data, dst_engine.get(),
                            attr.get()),
                    "could not create a primitive descriptor for a reorder "
                    "primitive");
            reset(result);
        }

        primitive_desc(const memory &src, const memory &dst,
                const primitive_attr &attr = primitive_attr()) {
            dnnl_primitive_desc_t result;
            auto src_md = src.get_desc();
            auto dst_md = dst.get_desc();
            error::wrap_c_api(
                    dnnl_reorder_primitive_desc_create(&result, &src_md.data,
                            src.get_engine().get(), &dst_md.data,
                            dst.get_engine().get(), attr.get()),
                    "could not create a primitive descriptor for a reorder "
                    "primitive");
            reset(result);
        }

        primitive_desc(dnnl_primitive_desc_t pd)
            : primitive_desc_base(pd, dnnl::primitive::kind::reorder) {}

        engine get_src_engine() const {
            return engine::query(*this, dnnl::query::reorder_src_engine);
        }

        engine get_dst_engine() const {
            return engine::query(*this, dnnl::query::reorder_dst_engine);
        }

        memory::desc src_desc() const { return base::src_desc(0); }

        memory::desc dst_desc() const { return base::dst_desc(0); }
    };

    reorder() = default;

    reorder(const primitive_desc &pd) : primitive(pd.get()) {}

    reorder(const memory &src, const memory &dst,
            const primitive_attr &attr = primitive_attr())
        : primitive(primitive_desc(src, dst, attr).get()) {}

    using primitive::execute;

    void execute(const stream &stream, memory &src, memory &dst) const {
        primitive::execute(stream, {{DNNL_ARG_FROM, src}, {DNNL_ARG_TO, dst}});
    }
};



inline std::vector<dnnl_memory_desc_t> convert_to_c(
        const std::vector<memory::desc> &mems) {
    std::vector<dnnl_memory_desc_t> c_mems;
    c_mems.reserve(mems.size());
    for (const auto &s : mems)
        c_mems.push_back(s.data);
    return c_mems;
}

struct concat : public primitive {
    struct primitive_desc : public primitive_desc_base {
        using primitive_desc_base::primitive_desc_base;

        primitive_desc() = default;

        primitive_desc(const memory::desc &dst, int concat_dimension,
                const std::vector<memory::desc> &srcs, const engine &engine,
                const primitive_attr &attr = primitive_attr()) {
            auto c_srcs = convert_to_c(srcs);

            dnnl_primitive_desc_t result;
            error::wrap_c_api(
                    dnnl_concat_primitive_desc_create(&result, &dst.data,
                            (int)c_srcs.size(), concat_dimension, c_srcs.data(),
                            attr.get(), engine.get()),
                    "could not create a primitive descriptor for a concat "
                    "primitive");
            reset(result);
        }

        primitive_desc(int concat_dimension,
                const std::vector<memory::desc> &srcs, const engine &engine,
                const primitive_attr &attr = primitive_attr()) {
            auto c_api_srcs = convert_to_c(srcs);

            dnnl_primitive_desc_t result;
            error::wrap_c_api(
                    dnnl_concat_primitive_desc_create(&result, nullptr,
                            (int)c_api_srcs.size(), concat_dimension,
                            c_api_srcs.data(), attr.get(), engine.get()),
                    "could not create a primitive descriptor for a concat "
                    "primitive");
            reset(result);
        }

        primitive_desc(dnnl_primitive_desc_t pd)
            : primitive_desc_base(pd, dnnl::primitive::kind::concat) {}

        memory::desc src_desc(int idx = 0) const { return base::src_desc(idx); }

        memory::desc dst_desc() const { return base::dst_desc(0); }
    };

    concat() = default;

    concat(const primitive_desc &pd) : primitive(pd.get()) {}
};



struct sum : public primitive {
    struct primitive_desc : public primitive_desc_base {
        using primitive_desc_base::primitive_desc_base;

        primitive_desc() = default;

        primitive_desc(const memory::desc &dst,
                const std::vector<float> &scales,
                const std::vector<memory::desc> &srcs, const engine &engine,
                const primitive_attr &attr = primitive_attr()) {
            validate_container_size(scales,
                    "counts of scales and sources are not equal",
                    (int)srcs.size(), (int)srcs.size());

            auto c_api_srcs = convert_to_c(srcs);

            dnnl_primitive_desc_t result;
            error::wrap_c_api(
                    dnnl_sum_primitive_desc_create(&result, &dst.data,
                            (int)c_api_srcs.size(), scales.data(),
                            c_api_srcs.data(), attr.get(), engine.get()),
                    "could not create a primitive descriptor for a sum "
                    "primitive");
            reset(result);
        }

        primitive_desc(const std::vector<float> &scales,
                const std::vector<memory::desc> &srcs, const engine &engine,
                const primitive_attr &attr = primitive_attr()) {
            validate_container_size(scales,
                    "counts of scales and sources are not equal",
                    (int)srcs.size(), (int)srcs.size());

            auto c_api_srcs = convert_to_c(srcs);
            dnnl_primitive_desc_t result;
            error::wrap_c_api(
                    dnnl_sum_primitive_desc_create(&result, nullptr,
                            (int)c_api_srcs.size(), scales.data(),
                            c_api_srcs.data(), attr.get(), engine.get()),
                    "could not create a primitive descriptor for a sum "
                    "primitive");
            reset(result);
        }

        primitive_desc(dnnl_primitive_desc_t pd)
            : primitive_desc_base(pd, dnnl::primitive::kind::sum) {}

        memory::desc src_desc(int idx = 0) const { return base::src_desc(idx); }

        memory::desc dst_desc() const { return base::dst_desc(0); }
    };

    sum() = default;

    sum(const primitive_desc &pd) : primitive(pd.get()) {}
};



struct primitive_desc : public primitive_desc_base {
    using primitive_desc_base::primitive_desc_base;

    primitive_desc() = default;

    primitive_desc(const_dnnl_op_desc_t desc, const primitive_attr *attr,
            const engine &engine, const_dnnl_primitive_desc_t hint_fwd_pd,
            bool allow_empty = false)
        : allow_empty_(allow_empty) {
        dnnl_primitive_desc_iterator_t iterator = nullptr;
        dnnl_status_t status = dnnl_primitive_desc_iterator_create(&iterator,
                desc, attr ? attr->get() : nullptr, engine.get(), hint_fwd_pd);
        if (!allow_empty)
            error::wrap_c_api(
                    status, "could not create a primitive descriptor iterator");
        pd_iterator.reset(iterator);
        fetch_impl();
    }

    bool next_impl() {
        dnnl_status_t status
                = dnnl_primitive_desc_iterator_next(pd_iterator.get());
        if (status == dnnl_iterator_ends) return false;
        error::wrap_c_api(
                status, "could not advance a primitive descriptor iterator");
        fetch_impl();
        return true;
    }

private:
    bool allow_empty_ = false;
    handle<dnnl_primitive_desc_iterator_t> pd_iterator;
    void fetch_impl() {
        dnnl_primitive_desc_t pd = dnnl_primitive_desc_iterator_fetch(
                pd_iterator.get(allow_empty_));
        error::wrap_c_api(pd != nullptr || allow_empty_ ? dnnl_success
                                                        : dnnl_runtime_error,
                "could not fetch a primitive descriptor from a primitive "
                "descriptor iterator");
        reset(pd);
    }
};



struct convolution_forward : public primitive {
    struct desc {
        dnnl_convolution_desc_t data;

        desc(prop_kind prop_kind, algorithm algorithm,
                const memory::desc &src_desc, const memory::desc &weights_desc,
                const memory::desc &bias_desc, const memory::desc &dst_desc,
                const memory::dims &strides, const memory::dims &padding_l,
                const memory::dims &padding_r) {
            memory::validate_dims(strides, src_desc.data.ndims - 2);
            memory::validate_dims(padding_l, src_desc.data.ndims - 2);
            memory::validate_dims(padding_r, src_desc.data.ndims - 2);
            error::wrap_c_api(
                    dnnl_convolution_forward_desc_init(&data,
                            dnnl::convert_to_c(prop_kind),
                            convert_to_c(algorithm), &src_desc.data,
                            &weights_desc.data, &bias_desc.data, &dst_desc.data,
                            &strides[0], &padding_l[0], &padding_r[0]),
                    "could not create a descriptor for a convolution forward "
                    "propagation primitive");
        }

        desc(prop_kind prop_kind, algorithm algorithm,
                const memory::desc &src_desc, const memory::desc &weights_desc,
                const memory::desc &dst_desc, const memory::dims &strides,
                const memory::dims &padding_l, const memory::dims &padding_r) {
            memory::validate_dims(strides, src_desc.data.ndims - 2);
            memory::validate_dims(padding_l, src_desc.data.ndims - 2);
            memory::validate_dims(padding_r, src_desc.data.ndims - 2);
            error::wrap_c_api(
                    dnnl_convolution_forward_desc_init(&data,
                            dnnl::convert_to_c(prop_kind),
                            convert_to_c(algorithm), &src_desc.data,
                            &weights_desc.data, nullptr, &dst_desc.data,
                            &strides[0], &padding_l[0], &padding_r[0]),
                    "could not create a descriptor for a convolution forward "
                    "propagation primitive");
        }

        desc(prop_kind prop_kind, algorithm algorithm,
                const memory::desc &src_desc, const memory::desc &weights_desc,
                const memory::desc &bias_desc, const memory::desc &dst_desc,
                const memory::dims &strides, const memory::dims &dilates,
                const memory::dims &padding_l, const memory::dims &padding_r) {
            memory::validate_dims(strides, src_desc.data.ndims - 2);
            memory::validate_dims(dilates, src_desc.data.ndims - 2);
            memory::validate_dims(padding_l, src_desc.data.ndims - 2);
            memory::validate_dims(padding_r, src_desc.data.ndims - 2);
            error::wrap_c_api(dnnl_dilated_convolution_forward_desc_init(&data,
                                      dnnl::convert_to_c(prop_kind),
                                      convert_to_c(algorithm), &src_desc.data,
                                      &weights_desc.data, &bias_desc.data,
                                      &dst_desc.data, &strides[0], &dilates[0],
                                      &padding_l[0], &padding_r[0]),
                    "could not create a descriptor for a dilated convolution "
                    "forward propagation primitive");
        }

        desc(prop_kind prop_kind, algorithm algorithm,
                const memory::desc &src_desc, const memory::desc &weights_desc,
                const memory::desc &dst_desc, const memory::dims &strides,
                const memory::dims &dilates, const memory::dims &padding_l,
                const memory::dims &padding_r) {
            memory::validate_dims(strides, src_desc.data.ndims - 2);
            memory::validate_dims(dilates, src_desc.data.ndims - 2);
            memory::validate_dims(padding_l, src_desc.data.ndims - 2);
            memory::validate_dims(padding_r, src_desc.data.ndims - 2);
            error::wrap_c_api(dnnl_dilated_convolution_forward_desc_init(&data,
                                      dnnl::convert_to_c(prop_kind),
                                      convert_to_c(algorithm), &src_desc.data,
                                      &weights_desc.data, nullptr,
                                      &dst_desc.data, &strides[0], &dilates[0],
                                      &padding_l[0], &padding_r[0]),
                    "could not create a descriptor for a dilated convolution "
                    "forward propagation primitive");
        }
    };

    struct primitive_desc : public dnnl::primitive_desc {
        primitive_desc() = default;

        primitive_desc(const desc &desc, const engine &engine,
                bool allow_empty = false)
            : dnnl::primitive_desc(
                    &desc.data, nullptr, engine, nullptr, allow_empty) {}

        primitive_desc(const desc &desc, const primitive_attr &attr,
                const engine &engine, bool allow_empty = false)
            : dnnl::primitive_desc(
                    &desc.data, &attr, engine, nullptr, allow_empty) {}

        primitive_desc(dnnl_primitive_desc_t pd)
            : dnnl::primitive_desc(pd, dnnl::primitive::kind::convolution,
                    dnnl::prop_kind::forward_training,
                    dnnl::prop_kind::forward_inference) {}

        memory::desc src_desc() const { return base::src_desc(0); }

        memory::desc weights_desc() const { return base::weights_desc(0); }

        memory::desc dst_desc() const { return base::dst_desc(0); }

        memory::desc bias_desc() const { return base::weights_desc(1); }
    };

    convolution_forward() = default;

    convolution_forward(const primitive_desc &pd) : primitive(pd) {}
};

struct convolution_backward_data : public primitive {

    struct desc {
        dnnl_convolution_desc_t data;

        desc(algorithm algorithm, const memory::desc &diff_src_desc,
                const memory::desc &weights_desc,
                const memory::desc &diff_dst_desc, const memory::dims &strides,
                const memory::dims &padding_l, const memory::dims &padding_r) {
            memory::validate_dims(strides, diff_src_desc.data.ndims - 2);
            memory::validate_dims(padding_l, diff_src_desc.data.ndims - 2);
            memory::validate_dims(padding_r, diff_src_desc.data.ndims - 2);
            error::wrap_c_api(
                    dnnl_convolution_backward_data_desc_init(&data,
                            convert_to_c(algorithm), &diff_src_desc.data,
                            &weights_desc.data, &diff_dst_desc.data,
                            &strides[0], &padding_l[0], &padding_r[0]),
                    "could not create a descriptor for a convolution backward "
                    "propagation primitive");
        }

        desc(algorithm algorithm, const memory::desc &diff_src_desc,
                const memory::desc &weights_desc,
                const memory::desc &diff_dst_desc, const memory::dims &strides,
                const memory::dims &dilates, const memory::dims &padding_l,
                const memory::dims &padding_r) {
            memory::validate_dims(strides, diff_src_desc.data.ndims - 2);
            memory::validate_dims(dilates, diff_src_desc.data.ndims - 2);
            memory::validate_dims(padding_l, diff_src_desc.data.ndims - 2);
            memory::validate_dims(padding_r, diff_src_desc.data.ndims - 2);
            error::wrap_c_api(
                    dnnl_dilated_convolution_backward_data_desc_init(&data,
                            convert_to_c(algorithm), &diff_src_desc.data,
                            &weights_desc.data, &diff_dst_desc.data,
                            &strides[0], &dilates[0], &padding_l[0],
                            &padding_r[0]),
                    "could not create a descriptor for a dilated convolution "
                    "backward propagation primitive");
        }
    };

    struct primitive_desc : public dnnl::primitive_desc {
        primitive_desc() = default;

        primitive_desc(const desc &desc, const engine &engine,
                const convolution_forward::primitive_desc &hint_fwd_pd,
                bool allow_empty = false)
            : dnnl::primitive_desc(&desc.data, nullptr, engine,
                    hint_fwd_pd.get(), allow_empty) {}

        primitive_desc(const desc &desc, const primitive_attr &attr,
                const engine &engine,
                const convolution_forward::primitive_desc &hint_fwd_pd,
                bool allow_empty = false)
            : dnnl::primitive_desc(&desc.data, &attr, engine, hint_fwd_pd.get(),
                    allow_empty) {}

        primitive_desc(dnnl_primitive_desc_t pd)
            : dnnl::primitive_desc(pd, dnnl::primitive::kind::convolution,
                    dnnl::prop_kind::backward_data) {}

        memory::desc diff_src_desc() const { return base::diff_src_desc(0); }

        memory::desc weights_desc() const { return base::weights_desc(0); }

        memory::desc diff_dst_desc() const { return base::diff_dst_desc(0); }
    };

    convolution_backward_data() = default;

    convolution_backward_data(const primitive_desc &pd) : primitive(pd) {}
};

struct convolution_backward_weights : public primitive {
    struct desc {
        dnnl_convolution_desc_t data;

        desc(algorithm algorithm, const memory::desc &src_desc,
                const memory::desc &diff_weights_desc,
                const memory::desc &diff_bias_desc,
                const memory::desc &diff_dst_desc, const memory::dims &strides,
                const memory::dims &padding_l, const memory::dims &padding_r) {
            memory::validate_dims(strides, src_desc.data.ndims - 2);
            memory::validate_dims(padding_l, src_desc.data.ndims - 2);
            memory::validate_dims(padding_r, src_desc.data.ndims - 2);
            error::wrap_c_api(
                    dnnl_convolution_backward_weights_desc_init(&data,
                            convert_to_c(algorithm), &src_desc.data,
                            &diff_weights_desc.data, &diff_bias_desc.data,
                            &diff_dst_desc.data, &strides[0], &padding_l[0],
                            &padding_r[0]),
                    "could not create a descriptor for a convolution weights "
                    "update primitive");
        }

        desc(algorithm algorithm, const memory::desc &src_desc,
                const memory::desc &diff_weights_desc,
                const memory::desc &diff_dst_desc, const memory::dims &strides,
                const memory::dims &padding_l, const memory::dims &padding_r) {
            memory::validate_dims(strides, src_desc.data.ndims - 2);
            memory::validate_dims(padding_l, src_desc.data.ndims - 2);
            memory::validate_dims(padding_r, src_desc.data.ndims - 2);
            error::wrap_c_api(dnnl_convolution_backward_weights_desc_init(&data,
                                      convert_to_c(algorithm), &src_desc.data,
                                      &diff_weights_desc.data, nullptr,
                                      &diff_dst_desc.data, &strides[0],
                                      &padding_l[0], &padding_r[0]),
                    "could not create a descriptor for a convolution weights "
                    "update primitive");
        }

        desc(algorithm algorithm, const memory::desc &src_desc,
                const memory::desc &diff_weights_desc,
                const memory::desc &diff_bias_desc,
                const memory::desc &diff_dst_desc, const memory::dims &strides,
                const memory::dims &dilates, const memory::dims &padding_l,
                const memory::dims &padding_r) {
            memory::validate_dims(strides, src_desc.data.ndims - 2);
            memory::validate_dims(dilates, src_desc.data.ndims - 2);
            memory::validate_dims(padding_l, src_desc.data.ndims - 2);
            memory::validate_dims(padding_r, src_desc.data.ndims - 2);
            error::wrap_c_api(
                    dnnl_dilated_convolution_backward_weights_desc_init(&data,
                            convert_to_c(algorithm), &src_desc.data,
                            &diff_weights_desc.data, &diff_bias_desc.data,
                            &diff_dst_desc.data, &strides[0], &dilates[0],
                            &padding_l[0], &padding_r[0]),
                    "could not create a descriptor for a dilated convolution "
                    "weights gradient primitive");
        }

        desc(algorithm algorithm, const memory::desc &src_desc,
                const memory::desc &diff_weights_desc,
                const memory::desc &diff_dst_desc, const memory::dims &strides,
                const memory::dims &dilates, const memory::dims &padding_l,
                const memory::dims &padding_r) {
            memory::validate_dims(strides, src_desc.data.ndims - 2);
            memory::validate_dims(dilates, src_desc.data.ndims - 2);
            memory::validate_dims(padding_l, src_desc.data.ndims - 2);
            memory::validate_dims(padding_r, src_desc.data.ndims - 2);
            error::wrap_c_api(
                    dnnl_dilated_convolution_backward_weights_desc_init(&data,
                            convert_to_c(algorithm), &src_desc.data,
                            &diff_weights_desc.data, nullptr,
                            &diff_dst_desc.data, &strides[0], &dilates[0],
                            &padding_l[0], &padding_r[0]),
                    "could not create a descriptor for a dilated convolution "
                    "weights gradient primitive");
        }
    };

    struct primitive_desc : public dnnl::primitive_desc {
        primitive_desc() = default;

        primitive_desc(const desc &desc, const engine &engine,
                const convolution_forward::primitive_desc &hint_fwd_pd,
                bool allow_empty = false)
            : dnnl::primitive_desc(&desc.data, nullptr, engine,
                    hint_fwd_pd.get(), allow_empty) {}

        primitive_desc(const desc &desc, const primitive_attr &attr,
                const engine &engine,
                const convolution_forward::primitive_desc &hint_fwd_pd,
                bool allow_empty = false)
            : dnnl::primitive_desc(&desc.data, &attr, engine, hint_fwd_pd.get(),
                    allow_empty) {}

        primitive_desc(dnnl_primitive_desc_t pd)
            : dnnl::primitive_desc(pd, dnnl::primitive::kind::convolution,
                    dnnl::prop_kind::backward_weights) {}

        memory::desc src_desc() const { return base::src_desc(0); }

        memory::desc diff_weights_desc() const {
            return base::diff_weights_desc(0);
        }

        memory::desc diff_dst_desc() const { return base::diff_dst_desc(0); }

        memory::desc diff_bias_desc() const {
            return base::diff_weights_desc(1);
        }
    };

    convolution_backward_weights() = default;

    convolution_backward_weights(const primitive_desc &pd) : primitive(pd) {}
};

//

struct deconvolution_forward : public primitive {
    struct desc {
        dnnl_deconvolution_desc_t data;

        desc(prop_kind prop_kind, algorithm algorithm,
                const memory::desc &src_desc, const memory::desc &weights_desc,
                const memory::desc &bias_desc, const memory::desc &dst_desc,
                const memory::dims &strides, const memory::dims &padding_l,
                const memory::dims &padding_r) {
            memory::validate_dims(strides, src_desc.data.ndims - 2);
            memory::validate_dims(padding_l, src_desc.data.ndims - 2);
            memory::validate_dims(padding_r, src_desc.data.ndims - 2);
            error::wrap_c_api(
                    dnnl_deconvolution_forward_desc_init(&data,
                            dnnl::convert_to_c(prop_kind),
                            convert_to_c(algorithm), &src_desc.data,
                            &weights_desc.data, &bias_desc.data, &dst_desc.data,
                            &strides[0], &padding_l[0], &padding_r[0]),
                    "could not create a descriptor for a deconvolution forward "
                    "propagation primitive");
        }

        desc(prop_kind prop_kind, algorithm algorithm,
                const memory::desc &src_desc, const memory::desc &weights_desc,
                const memory::desc &dst_desc, const memory::dims &strides,
                const memory::dims &padding_l, const memory::dims &padding_r) {
            memory::validate_dims(strides, src_desc.data.ndims - 2);
            memory::validate_dims(padding_l, src_desc.data.ndims - 2);
            memory::validate_dims(padding_r, src_desc.data.ndims - 2);
            error::wrap_c_api(
                    dnnl_deconvolution_forward_desc_init(&data,
                            dnnl::convert_to_c(prop_kind),
                            convert_to_c(algorithm), &src_desc.data,
                            &weights_desc.data, nullptr, &dst_desc.data,
                            &strides[0], &padding_l[0], &padding_r[0]),
                    "could not create a descriptor for a deconvolution forward "
                    "propagation primitive");
        }

        desc(prop_kind prop_kind, algorithm algorithm,
                const memory::desc &src_desc, const memory::desc &weights_desc,
                const memory::desc &bias_desc, const memory::desc &dst_desc,
                const memory::dims &strides, const memory::dims &dilates,
                const memory::dims &padding_l, const memory::dims &padding_r) {
            memory::validate_dims(strides, src_desc.data.ndims - 2);
            memory::validate_dims(dilates, src_desc.data.ndims - 2);
            memory::validate_dims(padding_l, src_desc.data.ndims - 2);
            memory::validate_dims(padding_r, src_desc.data.ndims - 2);
            error::wrap_c_api(dnnl_dilated_deconvolution_forward_desc_init(
                                      &data, dnnl::convert_to_c(prop_kind),
                                      convert_to_c(algorithm), &src_desc.data,
                                      &weights_desc.data, &bias_desc.data,
                                      &dst_desc.data, &strides[0], &dilates[0],
                                      &padding_l[0], &padding_r[0]),
                    "could not create a descriptor for a dilated deconvolution "
                    "forward propagation primitive");
        }

        desc(prop_kind prop_kind, algorithm algorithm,
                const memory::desc &src_desc, const memory::desc &weights_desc,
                const memory::desc &dst_desc, const memory::dims &strides,
                const memory::dims &dilates, const memory::dims &padding_l,
                const memory::dims &padding_r) {
            memory::validate_dims(strides, src_desc.data.ndims - 2);
            memory::validate_dims(dilates, src_desc.data.ndims - 2);
            memory::validate_dims(padding_l, src_desc.data.ndims - 2);
            memory::validate_dims(padding_r, src_desc.data.ndims - 2);
            error::wrap_c_api(dnnl_dilated_deconvolution_forward_desc_init(
                                      &data, dnnl::convert_to_c(prop_kind),
                                      convert_to_c(algorithm), &src_desc.data,
                                      &weights_desc.data, nullptr,
                                      &dst_desc.data, &strides[0], &dilates[0],
                                      &padding_l[0], &padding_r[0]),
                    "could not create a descriptor for a dilated deconvolution "
                    "forward propagation primitive");
        }
    };

    struct primitive_desc : public dnnl::primitive_desc {
        primitive_desc() = default;

        primitive_desc(const desc &desc, const engine &engine,
                bool allow_empty = false)
            : dnnl::primitive_desc(
                    &desc.data, nullptr, engine, nullptr, allow_empty) {}

        primitive_desc(const desc &desc, const primitive_attr &attr,
                const engine &engine, bool allow_empty = false)
            : dnnl::primitive_desc(
                    &desc.data, &attr, engine, nullptr, allow_empty) {}

        primitive_desc(dnnl_primitive_desc_t pd)
            : dnnl::primitive_desc(pd, dnnl::primitive::kind::deconvolution,
                    dnnl::prop_kind::forward_training,
                    dnnl::prop_kind::forward_inference) {}

        memory::desc src_desc() const { return base::src_desc(0); }

        memory::desc weights_desc() const { return base::weights_desc(0); }

        memory::desc dst_desc() const { return base::dst_desc(0); }

        memory::desc bias_desc() const { return base::weights_desc(1); }
    };

    deconvolution_forward() = default;

    deconvolution_forward(const primitive_desc &pd) : primitive(pd) {}
};

struct deconvolution_backward_data : public primitive {
    struct desc {
        dnnl_deconvolution_desc_t data;

        desc(algorithm algorithm, const memory::desc &diff_src_desc,
                const memory::desc &weights_desc,
                const memory::desc &diff_dst_desc, const memory::dims &strides,
                const memory::dims &padding_l, const memory::dims &padding_r) {
            memory::validate_dims(strides, diff_src_desc.data.ndims - 2);
            memory::validate_dims(padding_l, diff_src_desc.data.ndims - 2);
            memory::validate_dims(padding_r, diff_src_desc.data.ndims - 2);
            error::wrap_c_api(
                    dnnl_deconvolution_backward_data_desc_init(&data,
                            convert_to_c(algorithm), &diff_src_desc.data,
                            &weights_desc.data, &diff_dst_desc.data,
                            &strides[0], &padding_l[0], &padding_r[0]),
                    "could not create a descriptor for a deconvolution "
                    "backward propagation primitive");
        }

        desc(algorithm algorithm, const memory::desc &diff_src_desc,
                const memory::desc &weights_desc,
                const memory::desc &diff_dst_desc, const memory::dims &strides,
                const memory::dims &dilates, const memory::dims &padding_l,
                const memory::dims &padding_r) {
            memory::validate_dims(strides, diff_src_desc.data.ndims - 2);
            memory::validate_dims(dilates, diff_src_desc.data.ndims - 2);
            memory::validate_dims(padding_l, diff_src_desc.data.ndims - 2);
            memory::validate_dims(padding_r, diff_src_desc.data.ndims - 2);
            error::wrap_c_api(
                    dnnl_dilated_deconvolution_backward_data_desc_init(&data,
                            convert_to_c(algorithm), &diff_src_desc.data,
                            &weights_desc.data, &diff_dst_desc.data,
                            &strides[0], &dilates[0], &padding_l[0],
                            &padding_r[0]),
                    "could not create a descriptor for a dilated deconvolution "
                    "backward propagation primitive");
        }
    };

    struct primitive_desc : public dnnl::primitive_desc {
        primitive_desc() = default;

        primitive_desc(const desc &desc, const engine &engine,
                const deconvolution_forward::primitive_desc &hint_fwd_pd,
                bool allow_empty = false)
            : dnnl::primitive_desc(&desc.data, nullptr, engine,
                    hint_fwd_pd.get(), allow_empty) {}

        primitive_desc(const desc &desc, const primitive_attr &attr,
                const engine &engine,
                const deconvolution_forward::primitive_desc &hint_fwd_pd,
                bool allow_empty = false)
            : dnnl::primitive_desc(&desc.data, &attr, engine, hint_fwd_pd.get(),
                    allow_empty) {}

        primitive_desc(dnnl_primitive_desc_t pd)
            : dnnl::primitive_desc(pd, dnnl::primitive::kind::deconvolution,
                    dnnl::prop_kind::backward_data) {}

        memory::desc diff_src_desc() const { return base::diff_src_desc(0); }

        memory::desc weights_desc() const { return base::weights_desc(0); }

        memory::desc diff_dst_desc() const { return base::diff_dst_desc(0); }
    };

    deconvolution_backward_data() = default;

    deconvolution_backward_data(const primitive_desc &pd) : primitive(pd) {}
};

struct deconvolution_backward_weights : public primitive {
    struct desc {
        dnnl_deconvolution_desc_t data;

        desc(algorithm algorithm, const memory::desc &src_desc,
                const memory::desc &diff_weights_desc,
                const memory::desc &diff_bias_desc,
                const memory::desc &diff_dst_desc, const memory::dims &strides,
                const memory::dims &padding_l, const memory::dims &padding_r) {
            memory::validate_dims(strides, src_desc.data.ndims - 2);
            memory::validate_dims(padding_l, src_desc.data.ndims - 2);
            memory::validate_dims(padding_r, src_desc.data.ndims - 2);
            error::wrap_c_api(
                    dnnl_deconvolution_backward_weights_desc_init(&data,
                            convert_to_c(algorithm), &src_desc.data,
                            &diff_weights_desc.data, &diff_bias_desc.data,
                            &diff_dst_desc.data, &strides[0], &padding_l[0],
                            &padding_r[0]),
                    "could not create a descriptor for a deconvolution weights "
                    "update primitive");
        }

        desc(algorithm algorithm, const memory::desc &src_desc,
                const memory::desc &diff_weights_desc,
                const memory::desc &diff_dst_desc, const memory::dims &strides,
                const memory::dims &padding_l, const memory::dims &padding_r) {
            memory::validate_dims(strides, src_desc.data.ndims - 2);
            memory::validate_dims(padding_l, src_desc.data.ndims - 2);
            memory::validate_dims(padding_r, src_desc.data.ndims - 2);
            error::wrap_c_api(dnnl_deconvolution_backward_weights_desc_init(
                                      &data, convert_to_c(algorithm),
                                      &src_desc.data, &diff_weights_desc.data,
                                      nullptr, &diff_dst_desc.data, &strides[0],
                                      &padding_l[0], &padding_r[0]),
                    "could not create a descriptor for a deconvolution weights "
                    "update primitive");
        }

        desc(algorithm algorithm, const memory::desc &src_desc,
                const memory::desc &diff_weights_desc,
                const memory::desc &diff_bias_desc,
                const memory::desc &diff_dst_desc, const memory::dims &strides,
                const memory::dims &dilates, const memory::dims &padding_l,
                const memory::dims &padding_r) {
            memory::validate_dims(strides, src_desc.data.ndims - 2);
            memory::validate_dims(dilates, src_desc.data.ndims - 2);
            memory::validate_dims(padding_l, src_desc.data.ndims - 2);
            memory::validate_dims(padding_r, src_desc.data.ndims - 2);
            error::wrap_c_api(
                    dnnl_dilated_deconvolution_backward_weights_desc_init(&data,
                            convert_to_c(algorithm), &src_desc.data,
                            &diff_weights_desc.data, &diff_bias_desc.data,
                            &diff_dst_desc.data, &strides[0], &dilates[0],
                            &padding_l[0], &padding_r[0]),
                    "could not create a descriptor for a dilated deconvolution "
                    "weights gradient primitive");
        }

        desc(algorithm algorithm, const memory::desc &src_desc,
                const memory::desc &diff_weights_desc,
                const memory::desc &diff_dst_desc, const memory::dims &strides,
                const memory::dims &dilates, const memory::dims &padding_l,
                const memory::dims &padding_r) {
            memory::validate_dims(strides, src_desc.data.ndims - 2);
            memory::validate_dims(dilates, src_desc.data.ndims - 2);
            memory::validate_dims(padding_l, src_desc.data.ndims - 2);
            memory::validate_dims(padding_r, src_desc.data.ndims - 2);
            error::wrap_c_api(
                    dnnl_dilated_deconvolution_backward_weights_desc_init(&data,
                            convert_to_c(algorithm), &src_desc.data,
                            &diff_weights_desc.data, nullptr,
                            &diff_dst_desc.data, &strides[0], &dilates[0],
                            &padding_l[0], &padding_r[0]),
                    "could not create a descriptor for a dilated deconvolution "
                    "weights gradient primitive");
        }
    };

    struct primitive_desc : public dnnl::primitive_desc {
        primitive_desc() = default;

        primitive_desc(const desc &desc, const engine &engine,
                const deconvolution_forward::primitive_desc &hint_fwd_pd,
                bool allow_empty = false)
            : dnnl::primitive_desc(&desc.data, nullptr, engine,
                    hint_fwd_pd.get(), allow_empty) {}

        primitive_desc(const desc &desc, const primitive_attr &attr,
                const engine &engine,
                const deconvolution_forward::primitive_desc &hint_fwd_pd,
                bool allow_empty = false)
            : dnnl::primitive_desc(&desc.data, &attr, engine, hint_fwd_pd.get(),
                    allow_empty) {}

        primitive_desc(dnnl_primitive_desc_t pd)
            : dnnl::primitive_desc(pd, dnnl::primitive::kind::deconvolution,
                    dnnl::prop_kind::backward_weights) {}

        memory::desc src_desc() const { return base::src_desc(0); }

        memory::desc diff_weights_desc() const {
            return base::diff_weights_desc(0);
        }

        memory::desc diff_dst_desc() const { return base::diff_dst_desc(0); }

        memory::desc diff_bias_desc() const {
            return base::diff_weights_desc(1);
        }
    };

    deconvolution_backward_weights() = default;

    deconvolution_backward_weights(const primitive_desc &pd) : primitive(pd) {}
};



struct lrn_forward : public primitive {
    struct desc {
        dnnl_lrn_desc_t data;

        desc(prop_kind prop_kind, algorithm algorithm,
                const memory::desc &data_desc, memory::dim local_size,
                float alpha, float beta, float k = 1.f) {
            error::wrap_c_api(dnnl_lrn_forward_desc_init(&data,
                                      dnnl::convert_to_c(prop_kind),
                                      convert_to_c(algorithm), &data_desc.data,
                                      local_size, alpha, beta, k),
                    "could not create a descriptor for a lrn forward "
                    "propagation primitive");
        }
    };

    struct primitive_desc : public dnnl::primitive_desc {
        primitive_desc() = default;

        primitive_desc(const desc &desc, const engine &engine,
                bool allow_empty = false)
            : dnnl::primitive_desc(
                    &desc.data, nullptr, engine, nullptr, allow_empty) {}

        primitive_desc(const desc &desc, const primitive_attr &attr,
                const engine &engine, bool allow_empty = false)
            : dnnl::primitive_desc(
                    &desc.data, &attr, engine, nullptr, allow_empty) {}

        primitive_desc(dnnl_primitive_desc_t pd)
            : dnnl::primitive_desc(pd, dnnl::primitive::kind::lrn,
                    dnnl::prop_kind::forward_training,
                    dnnl::prop_kind::forward_inference) {}

        memory::desc src_desc() const { return base::src_desc(0); }

        memory::desc dst_desc() const { return base::dst_desc(0); }

        memory::desc workspace_desc() const { return base::workspace_desc(); }
    };

    lrn_forward() = default;

    lrn_forward(const primitive_desc &pd) : primitive(pd) {}
};

struct lrn_backward : public primitive {
    struct desc {
        dnnl_lrn_desc_t data;

        desc(algorithm algorithm, const memory::desc &data_desc,
                const memory::desc &diff_data_desc, memory::dim local_size,
                float alpha, float beta, float k = 1.f) {
            error::wrap_c_api(
                    dnnl_lrn_backward_desc_init(&data, convert_to_c(algorithm),
                            &diff_data_desc.data, &data_desc.data, local_size,
                            alpha, beta, k),
                    "could not create a descriptor for a lrn backward "
                    "propagation primitive");
        }
    };

    struct primitive_desc : public dnnl::primitive_desc {
        primitive_desc() = default;

        primitive_desc(const desc &desc, const engine &engine,
                const lrn_forward::primitive_desc &hint_fwd_pd,
                bool allow_empty = false)
            : dnnl::primitive_desc(&desc.data, nullptr, engine,
                    hint_fwd_pd.get(), allow_empty) {}

        primitive_desc(const desc &desc, const primitive_attr &attr,
                const engine &engine,
                const lrn_forward::primitive_desc &hint_fwd_pd,
                bool allow_empty = false)
            : dnnl::primitive_desc(&desc.data, &attr, engine, hint_fwd_pd.get(),
                    allow_empty) {}

        primitive_desc(dnnl_primitive_desc_t pd)
            : dnnl::primitive_desc(pd, dnnl::primitive::kind::lrn,
                    dnnl::prop_kind::backward_data) {}

        memory::desc diff_src_desc() const { return base::diff_src_desc(0); }

        memory::desc diff_dst_desc() const { return base::diff_dst_desc(0); }

        memory::desc workspace_desc() const { return base::workspace_desc(); }
    };

    lrn_backward() = default;

    lrn_backward(const primitive_desc &pd) : primitive(pd) {}
};



struct pooling_forward : public primitive {
    struct desc {
        dnnl_pooling_desc_t data;

        desc(prop_kind prop_kind, algorithm algorithm,
                const memory::desc &src_desc, const memory::desc &dst_desc,
                const memory::dims &strides, const memory::dims &kernel,
                const memory::dims &padding_l, const memory::dims &padding_r) {
            memory::validate_dims(strides, src_desc.data.ndims - 2);
            memory::validate_dims(kernel, src_desc.data.ndims - 2);
            memory::validate_dims(padding_l, src_desc.data.ndims - 2);
            memory::validate_dims(padding_r, src_desc.data.ndims - 2);
            error::wrap_c_api(dnnl_pooling_forward_desc_init(&data,
                                      dnnl::convert_to_c(prop_kind),
                                      convert_to_c(algorithm), &src_desc.data,
                                      &dst_desc.data, &strides[0], &kernel[0],
                                      &padding_l[0], &padding_r[0]),
                    "could not create a descriptor for a pooling forward "
                    "propagation primitive");
        }
    };

    struct primitive_desc : public dnnl::primitive_desc {
        primitive_desc() = default;

        primitive_desc(const desc &desc, const engine &engine,
                bool allow_empty = false)
            : dnnl::primitive_desc(
                    &desc.data, nullptr, engine, nullptr, allow_empty) {}

        primitive_desc(const desc &desc, const primitive_attr &attr,
                const engine &engine, bool allow_empty = false)
            : dnnl::primitive_desc(
                    &desc.data, &attr, engine, nullptr, allow_empty) {}

        primitive_desc(dnnl_primitive_desc_t pd)
            : dnnl::primitive_desc(pd, dnnl::primitive::kind::pooling,
                    dnnl::prop_kind::forward_training,
                    dnnl::prop_kind::forward_inference) {}

        memory::desc src_desc() const { return base::src_desc(0); }

        memory::desc dst_desc() const { return base::dst_desc(0); }

        memory::desc workspace_desc() const { return base::workspace_desc(); }
    };

    pooling_forward() = default;

    pooling_forward(const primitive_desc &pd) : primitive(pd) {}
};

struct pooling_backward : public primitive {
    struct desc {
        dnnl_pooling_desc_t data;

        desc(algorithm algorithm, const memory::desc &diff_src_desc,
                const memory::desc &diff_dst_desc, const memory::dims &strides,
                const memory::dims &kernel, const memory::dims &padding_l,
                const memory::dims &padding_r) {
            memory::validate_dims(strides, diff_src_desc.data.ndims - 2);
            memory::validate_dims(kernel, diff_src_desc.data.ndims - 2);
            memory::validate_dims(padding_l, diff_src_desc.data.ndims - 2);
            memory::validate_dims(padding_r, diff_src_desc.data.ndims - 2);
            error::wrap_c_api(
                    dnnl_pooling_backward_desc_init(&data,
                            convert_to_c(algorithm), &diff_src_desc.data,
                            &diff_dst_desc.data, &strides[0], &kernel[0],
                            &padding_l[0], &padding_r[0]),
                    "could not create a descriptor for a pooling backward "
                    "propagation primitive");
        }
    };

    struct primitive_desc : public dnnl::primitive_desc {
        primitive_desc() = default;

        primitive_desc(const desc &desc, const engine &engine,
                const pooling_forward::primitive_desc &hint_fwd_pd,
                bool allow_empty = false)
            : dnnl::primitive_desc(&desc.data, nullptr, engine,
                    hint_fwd_pd.get(), allow_empty) {}

        primitive_desc(const desc &desc, const primitive_attr &attr,
                const engine &engine,
                const pooling_forward::primitive_desc &hint_fwd_pd,
                bool allow_empty = false)
            : dnnl::primitive_desc(&desc.data, &attr, engine, hint_fwd_pd.get(),
                    allow_empty) {}

        primitive_desc(dnnl_primitive_desc_t pd)
            : dnnl::primitive_desc(pd, dnnl::primitive::kind::pooling,
                    dnnl::prop_kind::backward_data) {}

        memory::desc diff_src_desc() const { return base::diff_src_desc(0); }

        memory::desc diff_dst_desc() const { return base::diff_dst_desc(0); }

        memory::desc workspace_desc() const { return base::workspace_desc(); }
    };

    pooling_backward() = default;

    pooling_backward(const primitive_desc &pd) : primitive(pd) {}
};



struct eltwise_forward : public primitive {
    struct desc {
        dnnl_eltwise_desc_t data;

        desc(prop_kind prop_kind, algorithm algorithm,
                const memory::desc &data_desc, float alpha = 0,
                float beta = 0) {
            error::wrap_c_api(dnnl_eltwise_forward_desc_init(&data,
                                      dnnl::convert_to_c(prop_kind),
                                      dnnl::convert_to_c(algorithm),
                                      &data_desc.data, alpha, beta),
                    "could not create a descriptor for an eltwise forward "
                    "propagation primitive");
        }
    };

    struct primitive_desc : public dnnl::primitive_desc {
        primitive_desc() = default;

        primitive_desc(const desc &desc, const engine &engine,
                bool allow_empty = false)
            : dnnl::primitive_desc(
                    &desc.data, nullptr, engine, nullptr, allow_empty) {}

        primitive_desc(const desc &desc, const primitive_attr &attr,
                const engine &engine, bool allow_empty = false)
            : dnnl::primitive_desc(
                    &desc.data, &attr, engine, nullptr, allow_empty) {}

        primitive_desc(dnnl_primitive_desc_t pd)
            : dnnl::primitive_desc(pd, dnnl::primitive::kind::eltwise,
                    dnnl::prop_kind::forward_training,
                    dnnl::prop_kind::forward_inference) {}

        memory::desc src_desc() const { return base::src_desc(0); }

        memory::desc dst_desc() const { return base::dst_desc(0); }
    };

    eltwise_forward() = default;

    eltwise_forward(const primitive_desc &pd) : primitive(pd) {}
};

struct eltwise_backward : public primitive {
    struct desc {
        dnnl_eltwise_desc_t data;

        desc(algorithm algorithm, const memory::desc &diff_data_desc,
                const memory::desc &data_desc, float alpha = 0,
                float beta = 0) {
            error::wrap_c_api(
                    dnnl_eltwise_backward_desc_init(&data,
                            dnnl::convert_to_c(algorithm), &diff_data_desc.data,
                            &data_desc.data, alpha, beta),
                    "could not create a descriptor for an eltwise backward "
                    "propagation primitive");
        }
    };

    struct primitive_desc : public dnnl::primitive_desc {
        primitive_desc() = default;

        primitive_desc(const desc &desc, const engine &engine,
                const eltwise_forward::primitive_desc &hint_fwd_pd,
                bool allow_empty = false)
            : dnnl::primitive_desc(&desc.data, nullptr, engine,
                    hint_fwd_pd.get(), allow_empty) {}

        primitive_desc(const desc &desc, const primitive_attr &attr,
                const engine &engine,
                const eltwise_forward::primitive_desc &hint_fwd_pd,
                bool allow_empty = false)
            : dnnl::primitive_desc(&desc.data, &attr, engine, hint_fwd_pd.get(),
                    allow_empty) {}

        primitive_desc(dnnl_primitive_desc_t pd)
            : dnnl::primitive_desc(pd, dnnl::primitive::kind::eltwise,
                    dnnl::prop_kind::backward_data) {}

        memory::desc src_desc() const { return base::src_desc(0); }

        memory::desc diff_src_desc() const { return base::diff_src_desc(0); }

        memory::desc diff_dst_desc() const { return base::diff_dst_desc(0); }
    };

    eltwise_backward() = default;

    eltwise_backward(const primitive_desc &pd) : primitive(pd) {}
};



struct softmax_forward : public primitive {
    struct desc {
        dnnl_softmax_desc_t data;

        desc() = default;

        desc(prop_kind prop_kind, const memory::desc &data_desc,
                int softmax_axis) {
            error::wrap_c_api(dnnl_softmax_forward_desc_init(&data,
                                      dnnl::convert_to_c(prop_kind),
                                      &data_desc.data, softmax_axis),
                    "could not create a descriptor for a softmax forward "
                    "propagation primitive");
        }
    };

    struct primitive_desc : public dnnl::primitive_desc {
        primitive_desc() = default;

        primitive_desc(const desc &desc, const engine &engine,
                bool allow_empty = false)
            : dnnl::primitive_desc(
                    &desc.data, nullptr, engine, nullptr, allow_empty) {}

        primitive_desc(const desc &desc, const primitive_attr &attr,
                const engine &engine, bool allow_empty = false)
            : dnnl::primitive_desc(
                    &desc.data, &attr, engine, nullptr, allow_empty) {}

        primitive_desc(dnnl_primitive_desc_t pd)
            : dnnl::primitive_desc(pd, dnnl::primitive::kind::softmax,
                    dnnl::prop_kind::forward_training,
                    dnnl::prop_kind::forward_inference) {}

        memory::desc src_desc() const { return base::src_desc(0); }

        memory::desc dst_desc() const { return base::dst_desc(0); }
    };

    softmax_forward() = default;

    softmax_forward(const primitive_desc &pd) : primitive(pd) {}
};

struct softmax_backward : public primitive {
    struct desc {
        dnnl_softmax_desc_t data;

        desc() = default;

        desc(const memory::desc &diff_data_desc, const memory::desc &data_desc,
                int softmax_axis) {
            error::wrap_c_api(
                    dnnl_softmax_backward_desc_init(&data, &diff_data_desc.data,
                            &data_desc.data, softmax_axis),
                    "could not create a descriptor for a softmax backward "
                    "propagation primitive");
        }
    };

    struct primitive_desc : public dnnl::primitive_desc {
        primitive_desc() = default;

        primitive_desc(const desc &desc, const engine &engine,
                const softmax_forward::primitive_desc &hint_fwd_pd,
                bool allow_empty = false)
            : dnnl::primitive_desc(&desc.data, nullptr, engine,
                    hint_fwd_pd.get(), allow_empty) {}

        primitive_desc(const desc &desc, const primitive_attr &attr,
                const engine &engine,
                const softmax_forward::primitive_desc &hint_fwd_pd,
                bool allow_empty = false)
            : dnnl::primitive_desc(&desc.data, &attr, engine, hint_fwd_pd.get(),
                    allow_empty) {}

        primitive_desc(dnnl_primitive_desc_t pd)
            : dnnl::primitive_desc(pd, dnnl::primitive::kind::softmax,
                    dnnl::prop_kind::backward_data) {}

        memory::desc dst_desc() const { return base::dst_desc(0); }

        memory::desc diff_src_desc() const { return base::diff_src_desc(0); }

        memory::desc diff_dst_desc() const { return base::diff_dst_desc(0); }
    };

    softmax_backward() = default;

    softmax_backward(const primitive_desc &pd) : primitive(pd) {}
};



struct logsoftmax_forward : public primitive {
    struct desc {
        dnnl_logsoftmax_desc_t data;

        desc() = default;

        desc(prop_kind prop_kind, const memory::desc &data_desc,
                int logsoftmax_axis) {
            error::wrap_c_api(dnnl_logsoftmax_forward_desc_init(&data,
                                      dnnl::convert_to_c(prop_kind),
                                      &data_desc.data, logsoftmax_axis),
                    "could not create a descriptor for a logsoftmax forward "
                    "propagation primitive");
        }
    };

    struct primitive_desc : public dnnl::primitive_desc {
        primitive_desc() = default;

        primitive_desc(const desc &desc, const engine &engine,
                bool allow_empty = false)
            : dnnl::primitive_desc(
                    &desc.data, nullptr, engine, nullptr, allow_empty) {}

        primitive_desc(const desc &desc, const primitive_attr &attr,
                const engine &engine, bool allow_empty = false)
            : dnnl::primitive_desc(
                    &desc.data, &attr, engine, nullptr, allow_empty) {}

        primitive_desc(dnnl_primitive_desc_t pd)
            : dnnl::primitive_desc(pd,
                    // Logsoftmax and softmax share the implementation and
                    // currently report the same primitive kind. Hence this
                    // must be softmax and not logsoftmax.
                    dnnl::primitive::kind::softmax,
                    dnnl::prop_kind::forward_training,
                    dnnl::prop_kind::forward_inference) {}

        memory::desc src_desc() const { return base::src_desc(0); }

        memory::desc dst_desc() const { return base::dst_desc(0); }
    };

    logsoftmax_forward() = default;

    logsoftmax_forward(const primitive_desc &pd) : primitive(pd) {}
};

struct logsoftmax_backward : public primitive {
    struct desc {
        dnnl_logsoftmax_desc_t data;

        desc() = default;

        desc(const memory::desc &diff_data_desc, const memory::desc &data_desc,
                int logsoftmax_axis) {
            error::wrap_c_api(dnnl_logsoftmax_backward_desc_init(&data,
                                      &diff_data_desc.data, &data_desc.data,
                                      logsoftmax_axis),
                    "could not create a descriptor for a logsoftmax backward "
                    "propagation primitive");
        }
    };

    struct primitive_desc : public dnnl::primitive_desc {
        primitive_desc() = default;

        primitive_desc(const desc &desc, const engine &engine,
                const logsoftmax_forward::primitive_desc &hint_fwd_pd,
                bool allow_empty = false)
            : dnnl::primitive_desc(&desc.data, nullptr, engine,
                    hint_fwd_pd.get(), allow_empty) {}

        primitive_desc(const desc &desc, const primitive_attr &attr,
                const engine &engine,
                const logsoftmax_forward::primitive_desc &hint_fwd_pd,
                bool allow_empty = false)
            : dnnl::primitive_desc(&desc.data, &attr, engine, hint_fwd_pd.get(),
                    allow_empty) {}

        primitive_desc(dnnl_primitive_desc_t pd)
            : dnnl::primitive_desc(pd,
                    // Logsoftmax and softmax share the implementation and
                    // currently report the same primitive kind. Hence this
                    // must be softmax and not logsoftmax.
                    dnnl::primitive::kind::softmax,
                    dnnl::prop_kind::backward_data) {}

        memory::desc dst_desc() const { return base::dst_desc(0); }

        memory::desc diff_src_desc() const { return base::diff_src_desc(0); }

        memory::desc diff_dst_desc() const { return base::diff_dst_desc(0); }
    };

    logsoftmax_backward() = default;

    logsoftmax_backward(const primitive_desc &pd) : primitive(pd) {}
};



struct batch_normalization_forward : public primitive {
    struct desc {
        dnnl_batch_normalization_desc_t data;

        desc(prop_kind prop_kind, const memory::desc &data_desc, float epsilon,
                normalization_flags flags) {
            error::wrap_c_api(
                    dnnl_batch_normalization_forward_desc_init(&data,
                            dnnl::convert_to_c(prop_kind), &data_desc.data,
                            epsilon, convert_to_c(flags)),
                    "could not create a descriptor for a batch normalization "
                    "forward propagation primitive");
        }
    };

    struct primitive_desc : public dnnl::primitive_desc {
        primitive_desc() = default;

        primitive_desc(const desc &desc, const engine &engine,
                bool allow_empty = false)
            : dnnl::primitive_desc(
                    &desc.data, nullptr, engine, nullptr, allow_empty) {}

        primitive_desc(const desc &desc, const primitive_attr &attr,
                const engine &engine, bool allow_empty = false)
            : dnnl::primitive_desc(
                    &desc.data, &attr, engine, nullptr, allow_empty) {}

        primitive_desc(dnnl_primitive_desc_t pd)
            : dnnl::primitive_desc(pd,
                    dnnl::primitive::kind::batch_normalization,
                    dnnl::prop_kind::forward_training,
                    dnnl::prop_kind::forward_inference) {}

        memory::desc src_desc() const { return base::src_desc(0); }

        memory::desc dst_desc() const { return base::dst_desc(0); }

        memory::desc weights_desc() const { return base::weights_desc(0); }

        memory::desc workspace_desc() const { return base::workspace_desc(); }

        memory::desc mean_desc() const { return stat_desc(mean); }

        memory::desc variance_desc() const { return stat_desc(var); }

    private:
        enum {
            mean = 1,
            var = 2,
        };
        memory::desc stat_desc(int kind) const {
            dnnl_batch_normalization_desc_t *p;
            error::wrap_c_api(
                    dnnl_primitive_desc_query(get(),
                            dnnl::convert_to_c(query::batch_normalization_d), 0,
                            &p),
                    "could not retrieve a descriptor from a primitive "
                    "descriptor for batch normalization forward propagation "
                    "primitive");
            return query_md(p->flags & dnnl_use_global_stats ? query::src_md
                                                             : query::dst_md,
                    kind);
        }
    };

    batch_normalization_forward() = default;

    batch_normalization_forward(const primitive_desc &pd) : primitive(pd) {}
};

struct batch_normalization_backward : public primitive {
    struct desc {
        dnnl_batch_normalization_desc_t data;

        desc(prop_kind prop_kind, const memory::desc &diff_data_desc,
                const memory::desc &data_desc, float epsilon,
                normalization_flags flags) {
            error::wrap_c_api(
                    dnnl_batch_normalization_backward_desc_init(&data,
                            dnnl::convert_to_c(prop_kind), &diff_data_desc.data,
                            &data_desc.data, epsilon, convert_to_c(flags)),
                    "could not create a descriptor for a batch normalization "
                    "backward propagation primitive");
        }
    };

    struct primitive_desc : public dnnl::primitive_desc {
        primitive_desc() = default;

        primitive_desc(const desc &desc, const engine &engine,
                const batch_normalization_forward::primitive_desc &hint_fwd_pd,
                bool allow_empty = false)
            : dnnl::primitive_desc(&desc.data, nullptr, engine,
                    hint_fwd_pd.get(), allow_empty) {}

        primitive_desc(const desc &desc, const primitive_attr &attr,
                const engine &engine,
                const batch_normalization_forward::primitive_desc &hint_fwd_pd,
                bool allow_empty = false)
            : dnnl::primitive_desc(&desc.data, &attr, engine, hint_fwd_pd.get(),
                    allow_empty) {}

        primitive_desc(dnnl_primitive_desc_t pd)
            : dnnl::primitive_desc(pd,
                    dnnl::primitive::kind::batch_normalization,
                    dnnl::prop_kind::backward, dnnl::prop_kind::backward_data) {
        }

        memory::desc src_desc() const { return base::src_desc(0); }

        memory::desc weights_desc() const { return base::weights_desc(0); }

        memory::desc dst_desc() const { return base::dst_desc(0); }

        memory::desc diff_src_desc() const { return base::diff_src_desc(0); }

        memory::desc diff_dst_desc() const { return base::diff_dst_desc(0); }

        memory::desc diff_weights_desc() const {
            return base::diff_weights_desc(0);
        }

        memory::desc mean_desc() const { return query_md(query::src_md, 1); }

        memory::desc variance_desc() const {
            return query_md(query::src_md, 2);
        }

        memory::desc workspace_desc() const { return base::workspace_desc(); }
    };

    batch_normalization_backward() = default;

    batch_normalization_backward(const primitive_desc &pd) : primitive(pd) {}
};



struct layer_normalization_forward : public primitive {
    struct desc {
        dnnl_layer_normalization_desc_t data;

        desc(prop_kind prop_kind, const memory::desc &data_desc,
                const memory::desc &stat_desc, float epsilon,
                normalization_flags flags) {
            error::wrap_c_api(
                    dnnl_layer_normalization_forward_desc_init(&data,
                            dnnl::convert_to_c(prop_kind), &data_desc.data,
                            &stat_desc.data, epsilon, convert_to_c(flags)),
                    "could not create a descriptor for a layer normalization "
                    "forward propagation primitive");
        }

        desc(prop_kind prop_kind, const memory::desc &data_desc, float epsilon,
                normalization_flags flags) {
            error::wrap_c_api(
                    dnnl_layer_normalization_forward_desc_init(&data,
                            dnnl::convert_to_c(prop_kind), &data_desc.data,
                            nullptr, epsilon, convert_to_c(flags)),
                    "could not create a descriptor for a layer normalization "
                    "forward propagation primitive");
        }
    };

    struct primitive_desc : public dnnl::primitive_desc {
        primitive_desc() = default;

        primitive_desc(const desc &desc, const engine &engine,
                bool allow_empty = false)
            : dnnl::primitive_desc(
                    &desc.data, nullptr, engine, nullptr, allow_empty) {}

        primitive_desc(const desc &desc, const primitive_attr &attr,
                const engine &engine, bool allow_empty = false)
            : dnnl::primitive_desc(
                    &desc.data, &attr, engine, nullptr, allow_empty) {}

        primitive_desc(dnnl_primitive_desc_t pd)
            : dnnl::primitive_desc(pd,
                    dnnl::primitive::kind::layer_normalization,
                    dnnl::prop_kind::forward_training,
                    dnnl::prop_kind::forward_inference) {}

        memory::desc src_desc() const { return base::src_desc(0); }

        memory::desc dst_desc() const { return base::dst_desc(0); }

        memory::desc weights_desc() const { return base::weights_desc(0); }

        memory::desc workspace_desc() const { return base::workspace_desc(); }

        memory::desc mean_desc() const { return stat_desc(mean); }

        memory::desc variance_desc() const { return stat_desc(var); }

    private:
        enum {
            mean = 1,
            var = 2,
        };
        memory::desc stat_desc(int kind) const {
            dnnl_layer_normalization_desc_t *p;
            error::wrap_c_api(
                    dnnl_primitive_desc_query(get(),
                            dnnl::convert_to_c(query::layer_normalization_d), 0,
                            &p),
                    "could not retrieve a descriptor from a primitive "
                    "descriptor for layer normalization forward propagation "
                    "primitive");
            return query_md(p->flags & dnnl_use_global_stats ? query::src_md
                                                             : query::dst_md,
                    kind);
        }
    };

    layer_normalization_forward() = default;

    layer_normalization_forward(const primitive_desc &pd) : primitive(pd) {}
};

struct layer_normalization_backward : public primitive {
    struct desc {
        dnnl_layer_normalization_desc_t data;

        desc(prop_kind prop_kind, const memory::desc &diff_data_desc,
                const memory::desc &data_desc, const memory::desc &stat_desc,
                float epsilon, normalization_flags flags) {
            error::wrap_c_api(
                    dnnl_layer_normalization_backward_desc_init(&data,
                            dnnl::convert_to_c(prop_kind), &diff_data_desc.data,
                            &data_desc.data, &stat_desc.data, epsilon,
                            convert_to_c(flags)),
                    "could not create a descriptor for a batch normalization "
                    "backward propagation primitive");
        }

        desc(prop_kind prop_kind, const memory::desc &diff_data_desc,
                const memory::desc &data_desc, float epsilon,
                normalization_flags flags) {
            error::wrap_c_api(dnnl_layer_normalization_backward_desc_init(&data,
                                      dnnl::convert_to_c(prop_kind),
                                      &diff_data_desc.data, &data_desc.data,
                                      nullptr, epsilon, convert_to_c(flags)),
                    "could not create a descriptor for a batch normalization "
                    "backward propagation primitive");
        }
    };

    struct primitive_desc : public dnnl::primitive_desc {
        primitive_desc() = default;

        primitive_desc(const desc &desc, const engine &engine,
                const layer_normalization_forward::primitive_desc &hint_fwd_pd,
                bool allow_empty = false)
            : dnnl::primitive_desc(&desc.data, nullptr, engine,
                    hint_fwd_pd.get(), allow_empty) {}

        primitive_desc(const desc &desc, const primitive_attr &attr,
                const engine &engine,
                const layer_normalization_forward::primitive_desc &hint_fwd_pd,
                bool allow_empty = false)
            : dnnl::primitive_desc(&desc.data, &attr, engine, hint_fwd_pd.get(),
                    allow_empty) {}

        primitive_desc(dnnl_primitive_desc_t pd)
            : dnnl::primitive_desc(pd,
                    dnnl::primitive::kind::layer_normalization,
                    dnnl::prop_kind::backward, dnnl::prop_kind::backward_data) {
        }

        memory::desc src_desc() const { return base::src_desc(0); }

        memory::desc weights_desc() const { return base::weights_desc(0); }

        memory::desc dst_desc() const { return base::dst_desc(0); }

        memory::desc diff_src_desc() const { return base::diff_src_desc(0); }

        memory::desc diff_dst_desc() const { return base::diff_dst_desc(0); }

        memory::desc diff_weights_desc() const {
            return base::diff_weights_desc(0);
        }

        memory::desc mean_desc() const { return query_md(query::src_md, 1); }

        memory::desc variance_desc() const {
            return query_md(query::src_md, 2);
        }

        memory::desc workspace_desc() const { return base::workspace_desc(); }
    };

    layer_normalization_backward() = default;

    layer_normalization_backward(const primitive_desc &pd) : primitive(pd) {}
};



struct inner_product_forward : public primitive {
    struct desc {
        dnnl_inner_product_desc_t data;

        desc(prop_kind prop_kind, const memory::desc &src_desc,
                const memory::desc &weights_desc, const memory::desc &bias_desc,
                const memory::desc &dst_desc) {
            error::wrap_c_api(dnnl_inner_product_forward_desc_init(&data,
                                      dnnl::convert_to_c(prop_kind),
                                      &src_desc.data, &weights_desc.data,
                                      &bias_desc.data, &dst_desc.data),
                    "could not create a descriptor for an inner product "
                    "forward propagation primitive");
        }

        desc(prop_kind prop_kind, const memory::desc &src_desc,
                const memory::desc &weights_desc,
                const memory::desc &dst_desc) {
            error::wrap_c_api(
                    dnnl_inner_product_forward_desc_init(&data,
                            dnnl::convert_to_c(prop_kind), &src_desc.data,
                            &weights_desc.data, nullptr, &dst_desc.data),
                    "could not create a descriptor for an inner product "
                    "forward propagation primitive");
        }
    };

    struct primitive_desc : public dnnl::primitive_desc {
        primitive_desc() = default;

        primitive_desc(const desc &desc, const engine &engine,
                bool allow_empty = false)
            : dnnl::primitive_desc(
                    &desc.data, nullptr, engine, nullptr, allow_empty) {}

        primitive_desc(const desc &desc, const primitive_attr &attr,
                const engine &engine, bool allow_empty = false)
            : dnnl::primitive_desc(
                    &desc.data, &attr, engine, nullptr, allow_empty) {}

        primitive_desc(dnnl_primitive_desc_t pd)
            : dnnl::primitive_desc(pd, dnnl::primitive::kind::inner_product,
                    dnnl::prop_kind::forward_training,
                    dnnl::prop_kind::forward_inference) {}

        memory::desc src_desc() const { return base::src_desc(0); }

        memory::desc weights_desc() const { return base::weights_desc(0); }

        memory::desc dst_desc() const { return base::dst_desc(0); }

        memory::desc bias_desc() const { return base::weights_desc(1); }
    };

    inner_product_forward() = default;

    inner_product_forward(const primitive_desc &pd) : primitive(pd) {}
};

struct inner_product_backward_data : public primitive {
    struct desc {
        dnnl_inner_product_desc_t data;

        desc(const memory::desc &diff_src_desc,
                const memory::desc &weights_desc,
                const memory::desc &diff_dst_desc) {
            error::wrap_c_api(dnnl_inner_product_backward_data_desc_init(&data,
                                      &diff_src_desc.data, &weights_desc.data,
                                      &diff_dst_desc.data),
                    "could not create a descriptor for an inner product "
                    "backward propagation primitive");
        }
    };

    struct primitive_desc : public dnnl::primitive_desc {
        primitive_desc() = default;

        primitive_desc(const desc &desc, const engine &engine,
                const inner_product_forward::primitive_desc &hint_fwd_pd,
                bool allow_empty = false)
            : dnnl::primitive_desc(&desc.data, nullptr, engine,
                    hint_fwd_pd.get(), allow_empty) {}

        primitive_desc(const desc &desc, const primitive_attr &attr,
                const engine &engine,
                const inner_product_forward::primitive_desc &hint_fwd_pd,
                bool allow_empty = false)
            : dnnl::primitive_desc(&desc.data, &attr, engine, hint_fwd_pd.get(),
                    allow_empty) {}

        primitive_desc(dnnl_primitive_desc_t pd)
            : dnnl::primitive_desc(pd, dnnl::primitive::kind::inner_product,
                    dnnl::prop_kind::backward_data) {}

        memory::desc diff_src_desc() const { return base::diff_src_desc(0); }

        memory::desc weights_desc() const { return base::weights_desc(0); }

        memory::desc diff_dst_desc() const { return base::diff_dst_desc(0); }
    };

    inner_product_backward_data() = default;

    inner_product_backward_data(const primitive_desc &pd) : primitive(pd) {}
};

struct inner_product_backward_weights : public primitive {
    struct desc {
        dnnl_inner_product_desc_t data;

        desc(const memory::desc &src_desc,
                const memory::desc &diff_weights_desc,
                const memory::desc &diff_bias_desc,
                const memory::desc &diff_dst_desc) {
            error::wrap_c_api(
                    dnnl_inner_product_backward_weights_desc_init(&data,
                            &src_desc.data, &diff_weights_desc.data,
                            &diff_bias_desc.data, &diff_dst_desc.data),
                    "could not create a descriptor for an inner product "
                    "weights gradient primitive");
        }

        desc(const memory::desc &src_desc,
                const memory::desc &diff_weights_desc,
                const memory::desc &diff_dst_desc) {
            error::wrap_c_api(
                    dnnl_inner_product_backward_weights_desc_init(&data,
                            &src_desc.data, &diff_weights_desc.data, nullptr,
                            &diff_dst_desc.data),
                    "could not create a descriptor for an inner product "
                    "weights gradient primitive");
        }
    };

    struct primitive_desc : public dnnl::primitive_desc {
        primitive_desc() = default;

        primitive_desc(const desc &desc, const engine &engine,
                const inner_product_forward::primitive_desc &hint_fwd_pd,
                bool allow_empty = false)
            : dnnl::primitive_desc(&desc.data, nullptr, engine,
                    hint_fwd_pd.get(), allow_empty) {}

        primitive_desc(const desc &desc, const primitive_attr &attr,
                const engine &engine,
                const inner_product_forward::primitive_desc &hint_fwd_pd,
                bool allow_empty = false)
            : dnnl::primitive_desc(&desc.data, &attr, engine, hint_fwd_pd.get(),
                    allow_empty) {}

        primitive_desc(dnnl_primitive_desc_t pd)
            : dnnl::primitive_desc(pd, dnnl::primitive::kind::inner_product,
                    dnnl::prop_kind::backward_weights) {}

        memory::desc src_desc() const { return base::src_desc(0); }

        memory::desc diff_weights_desc() const {
            return base::diff_weights_desc(0);
        }

        memory::desc diff_dst_desc() const { return base::diff_dst_desc(0); }

        memory::desc diff_bias_desc() const {
            return base::diff_weights_desc(1);
        }
    };

    inner_product_backward_weights() = default;

    inner_product_backward_weights(const primitive_desc &pd) : primitive(pd) {}
};



struct rnn_primitive_desc_base : public primitive_desc {
    using primitive_desc::primitive_desc;

    rnn_primitive_desc_base() = default;

    rnn_primitive_desc_base(dnnl_primitive_desc_t pd, dnnl::prop_kind prop_kind,
            dnnl::algorithm cell_kind)
        : rnn_primitive_desc_base(pd, prop_kind, prop_kind, cell_kind) {}

    memory::desc src_layer_desc() const {
        return base::query_md(query::exec_arg_md, DNNL_ARG_SRC_LAYER);
    }

    memory::desc src_iter_desc() const {
        return base::query_md(query::exec_arg_md, DNNL_ARG_SRC_ITER);
    }

    memory::desc src_iter_c_desc() const {
        return base::query_md(query::exec_arg_md, DNNL_ARG_SRC_ITER_C);
    }

    memory::desc weights_layer_desc() const {
        return base::query_md(query::exec_arg_md, DNNL_ARG_WEIGHTS_LAYER);
    }

    memory::desc weights_iter_desc() const {
        return base::query_md(query::exec_arg_md, DNNL_ARG_WEIGHTS_ITER);
    }

    memory::desc weights_peephole_desc() const {
        return base::query_md(query::exec_arg_md, DNNL_ARG_WEIGHTS_PEEPHOLE);
    }

    memory::desc weights_projection_desc() const {
        return base::query_md(query::exec_arg_md, DNNL_ARG_WEIGHTS_PROJECTION);
    }

    memory::desc bias_desc() const {
        return base::query_md(query::exec_arg_md, DNNL_ARG_BIAS);
    }

    memory::desc dst_layer_desc() const {
        return base::query_md(query::exec_arg_md, DNNL_ARG_DST_LAYER);
    }

    memory::desc dst_iter_desc() const {
        return base::query_md(query::exec_arg_md, DNNL_ARG_DST_ITER);
    }

    memory::desc dst_iter_c_desc() const {
        return base::query_md(query::exec_arg_md, DNNL_ARG_DST_ITER_C);
    }

    memory::desc diff_src_layer_desc() const {
        return base::query_md(query::exec_arg_md, DNNL_ARG_DIFF_SRC_LAYER);
    }

    memory::desc diff_src_iter_desc() const {
        return base::query_md(query::exec_arg_md, DNNL_ARG_DIFF_SRC_ITER);
    }

    memory::desc diff_src_iter_c_desc() const {
        return base::query_md(query::exec_arg_md, DNNL_ARG_DIFF_SRC_ITER_C);
    }

    memory::desc diff_weights_layer_desc() const {
        return base::query_md(query::exec_arg_md, DNNL_ARG_DIFF_WEIGHTS_LAYER);
    }

    memory::desc diff_weights_iter_desc() const {
        return base::query_md(query::exec_arg_md, DNNL_ARG_DIFF_WEIGHTS_ITER);
    }

    memory::desc diff_weights_peephole_desc() const {
        return base::query_md(
                query::exec_arg_md, DNNL_ARG_DIFF_WEIGHTS_PEEPHOLE);
    }

    memory::desc diff_weights_projection_desc() const {
        return base::query_md(
                query::exec_arg_md, DNNL_ARG_DIFF_WEIGHTS_PROJECTION);
    }

    memory::desc diff_bias_desc() const {
        return base::query_md(query::exec_arg_md, DNNL_ARG_DIFF_BIAS);
    }

    memory::desc diff_dst_layer_desc() const {
        return base::query_md(query::exec_arg_md, DNNL_ARG_DIFF_DST_LAYER);
    }

    memory::desc diff_dst_iter_desc() const {
        return base::query_md(query::exec_arg_md, DNNL_ARG_DIFF_DST_ITER);
    }

    memory::desc diff_dst_iter_c_desc() const {
        return base::query_md(query::exec_arg_md, DNNL_ARG_DIFF_DST_ITER_C);
    }

protected:
    using rnn_base = rnn_primitive_desc_base;

    // (Deliberately not using doxygen comments)
    //
    // Constructs an RNN primitive descriptor base from a C API primitive
    // descriptor while checking that it actually describes the expected
    // primitive by comparing propagation and primitive kinds. Caller can
    // pass two options propagation kinds. This is typically used to check
    // that propagation kind is inference or training forward propagation.
    //
    // @param pd C API primitive descriptor.
    // @param prop_kind1 Expected propagation kind.
    // @param prop_kind2 Expected propagation kind.
    // @param cell_kind Expected cell kind.
    rnn_primitive_desc_base(dnnl_primitive_desc_t pd,
            dnnl::prop_kind prop_kind1, dnnl::prop_kind prop_kind2,
            dnnl::algorithm cell_kind) {
        dnnl_rnn_desc_t *rnn_d;
        dnnl_status_t rc;
        rc = dnnl_primitive_desc_query(pd, dnnl_query_rnn_d, 0, &rnn_d);
        error::wrap_c_api(rc,
                "could not retrieve a descriptor from a primitive descriptor "
                "for an RNN primitive");

        dnnl_prop_kind_t c_prop_kind1 = convert_to_c(prop_kind1);
        dnnl_prop_kind_t c_prop_kind2 = convert_to_c(prop_kind2);
        dnnl_alg_kind_t c_cell_kind = convert_to_c(cell_kind);

        bool ok = rnn_d->primitive_kind == dnnl_rnn
                && (rnn_d->prop_kind == c_prop_kind1
                        || rnn_d->prop_kind == c_prop_kind2)
                && rnn_d->cell_kind == c_cell_kind;

        if (!ok)
            DNNL_THROW_ERROR(dnnl_invalid_arguments,
                    "mismatch between expected and provided descriptors for an "
                    "RNN primitive");

        reset_with_clone(pd);
    }
};

struct vanilla_rnn_forward : public primitive {
    struct desc {
        dnnl_rnn_desc_t data;

        desc(prop_kind prop_kind, algorithm activation, rnn_direction direction,
                const memory::desc &src_layer_desc,
                const memory::desc &src_iter_desc,
                const memory::desc &weights_layer_desc,
                const memory::desc &weights_iter_desc,
                const memory::desc &bias_desc,
                const memory::desc &dst_layer_desc,
                const memory::desc &dst_iter_desc,
                rnn_flags flags = rnn_flags::undef, float alpha = 0.0f,
                float beta = 0.0f) {
            error::wrap_c_api(
                    dnnl_vanilla_rnn_forward_desc_init(&data,
                            dnnl::convert_to_c(prop_kind),
                            dnnl::convert_to_c(activation),
                            dnnl::convert_to_c(direction), &src_layer_desc.data,
                            &src_iter_desc.data, &weights_layer_desc.data,
                            &weights_iter_desc.data, &bias_desc.data,
                            &dst_layer_desc.data, &dst_iter_desc.data,
                            dnnl::convert_to_c(flags), alpha, beta),
                    "could not create a descriptor for a vanilla RNN forward "
                    "propagation primitive");
        }
    };

    struct primitive_desc : public rnn_primitive_desc_base {
        primitive_desc() = default;

        primitive_desc(const desc &desc, const engine &engine,
                bool allow_empty = false)
            : rnn_primitive_desc_base(
                    &desc.data, nullptr, engine, nullptr, allow_empty) {}

        primitive_desc(const desc &desc, const primitive_attr &attr,
                const engine &engine, bool allow_empty = false)
            : rnn_primitive_desc_base(
                    &desc.data, &attr, engine, nullptr, allow_empty) {}

        primitive_desc(dnnl_primitive_desc_t pd)
            : rnn_primitive_desc_base(pd, dnnl::prop_kind::forward_training,
                    dnnl::prop_kind::forward_inference,
                    dnnl::algorithm::vanilla_rnn) {}

        memory::desc src_layer_desc() const {
            return rnn_base::src_layer_desc();
        }

        memory::desc src_iter_desc() const { return rnn_base::src_iter_desc(); }

        memory::desc weights_layer_desc() const {
            return rnn_base::weights_layer_desc();
        }

        memory::desc weights_iter_desc() const {
            return rnn_base::weights_iter_desc();
        }

        memory::desc bias_desc() const { return rnn_base::bias_desc(); }

        memory::desc dst_layer_desc() const {
            return rnn_base::dst_layer_desc();
        }

        memory::desc dst_iter_desc() const { return rnn_base::dst_iter_desc(); }

        memory::desc workspace_desc() const {
            return rnn_base::workspace_desc();
        }
    };

    vanilla_rnn_forward() = default;

    vanilla_rnn_forward(const primitive_desc &pd) : primitive(pd) {}
};

struct vanilla_rnn_backward : public primitive {
    struct desc {
        dnnl_rnn_desc_t data;

        desc(prop_kind prop_kind, algorithm activation, rnn_direction direction,
                const memory::desc &src_layer_desc,
                const memory::desc &src_iter_desc,
                const memory::desc &weights_layer_desc,
                const memory::desc &weights_iter_desc,
                const memory::desc &bias_desc,
                const memory::desc &dst_layer_desc,
                const memory::desc &dst_iter_desc,
                const memory::desc &diff_src_layer_desc,
                const memory::desc &diff_src_iter_desc,
                const memory::desc &diff_weights_layer_desc,
                const memory::desc &diff_weights_iter_desc,
                const memory::desc &diff_bias_desc,
                const memory::desc &diff_dst_layer_desc,
                const memory::desc &diff_dst_iter_desc,
                rnn_flags flags = rnn_flags::undef, float alpha = 0.0f,
                float beta = 0.0f) {
            error::wrap_c_api(
                    dnnl_vanilla_rnn_backward_desc_init(&data,
                            dnnl::convert_to_c(prop_kind),
                            dnnl::convert_to_c(activation),
                            dnnl::convert_to_c(direction), &src_layer_desc.data,
                            &src_iter_desc.data, &weights_layer_desc.data,
                            &weights_iter_desc.data, &bias_desc.data,
                            &dst_layer_desc.data, &dst_iter_desc.data,
                            &diff_src_layer_desc.data, &diff_src_iter_desc.data,
                            &diff_weights_layer_desc.data,
                            &diff_weights_iter_desc.data, &diff_bias_desc.data,
                            &diff_dst_layer_desc.data, &diff_dst_iter_desc.data,
                            dnnl::convert_to_c(flags), alpha, beta),
                    "could not create a descriptor for a vanilla RNN backward "
                    "propagation primitive");
        }
    };

    struct primitive_desc : public rnn_primitive_desc_base {
        primitive_desc() = default;

        primitive_desc(const desc &desc, const engine &engine,
                const vanilla_rnn_forward::primitive_desc &hint_fwd_pd,
                bool allow_empty = false)
            : rnn_primitive_desc_base(&desc.data, nullptr, engine,
                    hint_fwd_pd.get(), allow_empty) {}

        primitive_desc(const desc &desc, const primitive_attr &attr,
                const engine &engine,
                const vanilla_rnn_forward::primitive_desc &hint_fwd_pd,
                bool allow_empty = false)
            : rnn_primitive_desc_base(&desc.data, &attr, engine,
                    hint_fwd_pd.get(), allow_empty) {}

        primitive_desc(dnnl_primitive_desc_t pd)
            : rnn_primitive_desc_base(pd, dnnl::prop_kind::backward,
                    dnnl::algorithm::vanilla_rnn) {}

        memory::desc src_layer_desc() const {
            return rnn_base::src_layer_desc();
        }

        memory::desc src_iter_desc() const { return rnn_base::src_iter_desc(); }

        memory::desc weights_layer_desc() const {
            return rnn_base::weights_layer_desc();
        }

        memory::desc weights_iter_desc() const {
            return rnn_base::weights_iter_desc();
        }

        memory::desc bias_desc() const { return rnn_base::bias_desc(); }

        memory::desc dst_layer_desc() const {
            return rnn_base::dst_layer_desc();
        }

        memory::desc dst_iter_desc() const { return rnn_base::dst_iter_desc(); }

        memory::desc workspace_desc() const {
            return rnn_base::workspace_desc();
        }

        memory::desc diff_src_layer_desc() const {
            return rnn_base::diff_src_layer_desc();
        }

        memory::desc diff_src_iter_desc() const {
            return rnn_base::diff_src_iter_desc();
        }

        memory::desc diff_weights_layer_desc() const {
            return rnn_base::diff_weights_layer_desc();
        }

        memory::desc diff_weights_iter_desc() const {
            return rnn_base::diff_weights_iter_desc();
        }

        memory::desc diff_bias_desc() const {
            return rnn_base::diff_bias_desc();
        }

        memory::desc diff_dst_layer_desc() const {
            return rnn_base::diff_dst_layer_desc();
        }

        memory::desc diff_dst_iter_desc() const {
            return rnn_base::diff_dst_iter_desc();
        }
    };

    vanilla_rnn_backward() = default;

    vanilla_rnn_backward(const primitive_desc &pd) : primitive(pd) {}
};

struct lstm_forward : public primitive {
    struct desc {
        dnnl_rnn_desc_t data;

        desc(prop_kind prop_kind, rnn_direction direction,
                const memory::desc &src_layer_desc,
                const memory::desc &src_iter_desc,
                const memory::desc &src_iter_c_desc,
                const memory::desc &weights_layer_desc,
                const memory::desc &weights_iter_desc,
                const memory::desc &weights_peephole_desc,
                const memory::desc &weights_projection_desc,
                const memory::desc &bias_desc,
                const memory::desc &dst_layer_desc,
                const memory::desc &dst_iter_desc,
                const memory::desc &dst_iter_c_desc,
                rnn_flags flags = rnn_flags::undef) {
            error::wrap_c_api(
                    dnnl_lstm_forward_desc_init_v3(&data,
                            dnnl::convert_to_c(prop_kind),
                            dnnl::convert_to_c(direction), &src_layer_desc.data,
                            &src_iter_desc.data, &src_iter_c_desc.data,
                            &weights_layer_desc.data, &weights_iter_desc.data,
                            &weights_peephole_desc.data,
                            &weights_projection_desc.data, &bias_desc.data,
                            &dst_layer_desc.data, &dst_iter_desc.data,
                            &dst_iter_c_desc.data, dnnl::convert_to_c(flags)),
                    "could not create a descriptor for an LSTM forward "
                    "propagation primitive");
        }

        desc(prop_kind prop_kind, rnn_direction direction,
                const memory::desc &src_layer_desc,
                const memory::desc &src_iter_desc,
                const memory::desc &src_iter_c_desc,
                const memory::desc &weights_layer_desc,
                const memory::desc &weights_iter_desc,
                const memory::desc &weights_peephole_desc,
                const memory::desc &bias_desc,
                const memory::desc &dst_layer_desc,
                const memory::desc &dst_iter_desc,
                const memory::desc &dst_iter_c_desc,
                rnn_flags flags = rnn_flags::undef) {
            error::wrap_c_api(
                    dnnl_lstm_forward_desc_init_v2(&data,
                            dnnl::convert_to_c(prop_kind),
                            dnnl::convert_to_c(direction), &src_layer_desc.data,
                            &src_iter_desc.data, &src_iter_c_desc.data,
                            &weights_layer_desc.data, &weights_iter_desc.data,
                            &weights_peephole_desc.data, &bias_desc.data,
                            &dst_layer_desc.data, &dst_iter_desc.data,
                            &dst_iter_c_desc.data, dnnl::convert_to_c(flags)),
                    "could not create a descriptor for an LSTM forward "
                    "propagation primitive");
        }

        desc(prop_kind prop_kind, rnn_direction direction,
                const memory::desc &src_layer_desc,
                const memory::desc &src_iter_desc,
                const memory::desc &src_iter_c_desc,
                const memory::desc &weights_layer_desc,
                const memory::desc &weights_iter_desc,
                const memory::desc &bias_desc,
                const memory::desc &dst_layer_desc,
                const memory::desc &dst_iter_desc,
                const memory::desc &dst_iter_c_desc,
                rnn_flags flags = rnn_flags::undef) {
            error::wrap_c_api(
                    dnnl_lstm_forward_desc_init(&data,
                            dnnl::convert_to_c(prop_kind),
                            dnnl::convert_to_c(direction), &src_layer_desc.data,
                            &src_iter_desc.data, &src_iter_c_desc.data,
                            &weights_layer_desc.data, &weights_iter_desc.data,
                            &bias_desc.data, &dst_layer_desc.data,
                            &dst_iter_desc.data, &dst_iter_c_desc.data,
                            dnnl::convert_to_c(flags)),
                    "could not create a descriptor for an LSTM forward "
                    "propagation primitive");
        }
    };

    struct primitive_desc : public rnn_primitive_desc_base {
        primitive_desc() = default;

        primitive_desc(const desc &desc, const engine &engine,
                bool allow_empty = false)
            : rnn_primitive_desc_base(
                    &desc.data, nullptr, engine, nullptr, allow_empty) {}

        primitive_desc(const desc &desc, const primitive_attr &attr,
                const engine &engine, bool allow_empty = false)
            : rnn_primitive_desc_base(
                    &desc.data, &attr, engine, nullptr, allow_empty) {}

        primitive_desc(dnnl_primitive_desc_t pd)
            : rnn_primitive_desc_base(pd, dnnl::prop_kind::forward_training,
                    dnnl::prop_kind::forward_inference,
                    dnnl::algorithm::vanilla_lstm) {}

        memory::desc src_layer_desc() const {
            return rnn_base::src_layer_desc();
        }

        memory::desc src_iter_desc() const { return rnn_base::src_iter_desc(); }

        memory::desc src_iter_c_desc() const {
            return rnn_base::src_iter_c_desc();
        }

        memory::desc weights_layer_desc() const {
            return rnn_base::weights_layer_desc();
        }

        memory::desc weights_iter_desc() const {
            return rnn_base::weights_iter_desc();
        }

        memory::desc weights_peephole_desc() const {
            return rnn_base::weights_peephole_desc();
        }

        memory::desc weights_projection_desc() const {
            return rnn_base::weights_projection_desc();
        }

        memory::desc bias_desc() const { return rnn_base::bias_desc(); }

        memory::desc dst_layer_desc() const {
            return rnn_base::dst_layer_desc();
        }

        memory::desc dst_iter_desc() const { return rnn_base::dst_iter_desc(); }

        memory::desc dst_iter_c_desc() const {
            return rnn_base::dst_iter_c_desc();
        }

        memory::desc workspace_desc() const {
            return rnn_base::workspace_desc();
        }
    };

    lstm_forward() = default;

    lstm_forward(const primitive_desc &pd) : primitive(pd) {}
};

struct lstm_backward : public primitive {
    struct desc {
        dnnl_rnn_desc_t data;

        desc(prop_kind prop_kind, rnn_direction direction,
                const memory::desc &src_layer_desc,
                const memory::desc &src_iter_desc,
                const memory::desc &src_iter_c_desc,
                const memory::desc &weights_layer_desc,
                const memory::desc &weights_iter_desc,
                const memory::desc &weights_peephole_desc,
                const memory::desc &weights_projection_desc,
                const memory::desc &bias_desc,
                const memory::desc &dst_layer_desc,
                const memory::desc &dst_iter_desc,
                const memory::desc &dst_iter_c_desc,
                const memory::desc &diff_src_layer_desc,
                const memory::desc &diff_src_iter_desc,
                const memory::desc &diff_src_iter_c_desc,
                const memory::desc &diff_weights_layer_desc,
                const memory::desc &diff_weights_iter_desc,
                const memory::desc &diff_weights_peephole_desc,
                const memory::desc &diff_weights_projection_desc,
                const memory::desc &diff_bias_desc,
                const memory::desc &diff_dst_layer_desc,
                const memory::desc &diff_dst_iter_desc,
                const memory::desc &diff_dst_iter_c_desc,
                rnn_flags flags = rnn_flags::undef) {
            error::wrap_c_api(
                    dnnl_lstm_backward_desc_init_v3(&data,
                            dnnl::convert_to_c(prop_kind),
                            dnnl::convert_to_c(direction), &src_layer_desc.data,
                            &src_iter_desc.data, &src_iter_c_desc.data,
                            &weights_layer_desc.data, &weights_iter_desc.data,
                            &weights_peephole_desc.data,
                            &weights_projection_desc.data, &bias_desc.data,
                            &dst_layer_desc.data, &dst_iter_desc.data,
                            &dst_iter_c_desc.data, &diff_src_layer_desc.data,
                            &diff_src_iter_desc.data,
                            &diff_src_iter_c_desc.data,
                            &diff_weights_layer_desc.data,
                            &diff_weights_iter_desc.data,
                            &diff_weights_peephole_desc.data,
                            &diff_weights_projection_desc.data,
                            &diff_bias_desc.data, &diff_dst_layer_desc.data,
                            &diff_dst_iter_desc.data,
                            &diff_dst_iter_c_desc.data,
                            dnnl::convert_to_c(flags)),
                    "could not create a descriptor for an LSTM backward "
                    "propagation primitive");
        }

        desc(prop_kind prop_kind, rnn_direction direction,
                const memory::desc &src_layer_desc,
                const memory::desc &src_iter_desc,
                const memory::desc &src_iter_c_desc,
                const memory::desc &weights_layer_desc,
                const memory::desc &weights_iter_desc,
                const memory::desc &weights_peephole_desc,
                const memory::desc &bias_desc,
                const memory::desc &dst_layer_desc,
                const memory::desc &dst_iter_desc,
                const memory::desc &dst_iter_c_desc,
                const memory::desc &diff_src_layer_desc,
                const memory::desc &diff_src_iter_desc,
                const memory::desc &diff_src_iter_c_desc,
                const memory::desc &diff_weights_layer_desc,
                const memory::desc &diff_weights_iter_desc,
                const memory::desc &diff_weights_peephole_desc,
                const memory::desc &diff_bias_desc,
                const memory::desc &diff_dst_layer_desc,
                const memory::desc &diff_dst_iter_desc,
                const memory::desc &diff_dst_iter_c_desc,
                rnn_flags flags = rnn_flags::undef) {
            error::wrap_c_api(
                    dnnl_lstm_backward_desc_init_v2(&data,
                            dnnl::convert_to_c(prop_kind),
                            dnnl::convert_to_c(direction), &src_layer_desc.data,
                            &src_iter_desc.data, &src_iter_c_desc.data,
                            &weights_layer_desc.data, &weights_iter_desc.data,
                            &weights_peephole_desc.data, &bias_desc.data,
                            &dst_layer_desc.data, &dst_iter_desc.data,
                            &dst_iter_c_desc.data, &diff_src_layer_desc.data,
                            &diff_src_iter_desc.data,
                            &diff_src_iter_c_desc.data,
                            &diff_weights_layer_desc.data,
                            &diff_weights_iter_desc.data,
                            &diff_weights_peephole_desc.data,
                            &diff_bias_desc.data, &diff_dst_layer_desc.data,
                            &diff_dst_iter_desc.data,
                            &diff_dst_iter_c_desc.data,
                            dnnl::convert_to_c(flags)),
                    "could not create a descriptor for an LSTM backward "
                    "propagation primitive");
        }

        desc(prop_kind prop_kind, rnn_direction direction,
                const memory::desc &src_layer_desc,
                const memory::desc &src_iter_desc,
                const memory::desc &src_iter_c_desc,
                const memory::desc &weights_layer_desc,
                const memory::desc &weights_iter_desc,
                const memory::desc &bias_desc,
                const memory::desc &dst_layer_desc,
                const memory::desc &dst_iter_desc,
                const memory::desc &dst_iter_c_desc,
                const memory::desc &diff_src_layer_desc,
                const memory::desc &diff_src_iter_desc,
                const memory::desc &diff_src_iter_c_desc,
                const memory::desc &diff_weights_layer_desc,
                const memory::desc &diff_weights_iter_desc,
                const memory::desc &diff_bias_desc,
                const memory::desc &diff_dst_layer_desc,
                const memory::desc &diff_dst_iter_desc,
                const memory::desc &diff_dst_iter_c_desc,
                rnn_flags flags = rnn_flags::undef) {
            error::wrap_c_api(
                    dnnl_lstm_backward_desc_init(&data,
                            dnnl::convert_to_c(prop_kind),
                            dnnl::convert_to_c(direction), &src_layer_desc.data,
                            &src_iter_desc.data, &src_iter_c_desc.data,
                            &weights_layer_desc.data, &weights_iter_desc.data,
                            &bias_desc.data, &dst_layer_desc.data,
                            &dst_iter_desc.data, &dst_iter_c_desc.data,
                            &diff_src_layer_desc.data, &diff_src_iter_desc.data,
                            &diff_src_iter_c_desc.data,
                            &diff_weights_layer_desc.data,
                            &diff_weights_iter_desc.data, &diff_bias_desc.data,
                            &diff_dst_layer_desc.data, &diff_dst_iter_desc.data,
                            &diff_dst_iter_c_desc.data,
                            dnnl::convert_to_c(flags)),
                    "could not create a descriptor for an LSTM backward "
                    "propagation primitive");
        }
    };

    struct primitive_desc : public rnn_primitive_desc_base {
        primitive_desc() = default;

        primitive_desc(const desc &desc, const engine &engine,
                const lstm_forward::primitive_desc &hint_fwd_pd,
                bool allow_empty = false)
            : rnn_primitive_desc_base(&desc.data, nullptr, engine,
                    hint_fwd_pd.get(), allow_empty) {}

        primitive_desc(const desc &desc, const primitive_attr &attr,
                const engine &engine,
                const lstm_forward::primitive_desc &hint_fwd_pd,
                bool allow_empty = false)
            : rnn_primitive_desc_base(&desc.data, &attr, engine,
                    hint_fwd_pd.get(), allow_empty) {}

        primitive_desc(dnnl_primitive_desc_t pd)
            : rnn_primitive_desc_base(pd, dnnl::prop_kind::backward,
                    dnnl::algorithm::vanilla_lstm) {}

        memory::desc src_layer_desc() const {
            return rnn_base::src_layer_desc();
        }

        memory::desc src_iter_desc() const { return rnn_base::src_iter_desc(); }

        memory::desc src_iter_c_desc() const {
            return rnn_base::src_iter_c_desc();
        }

        memory::desc weights_layer_desc() const {
            return rnn_base::weights_layer_desc();
        }

        memory::desc weights_iter_desc() const {
            return rnn_base::weights_iter_desc();
        }

        memory::desc weights_peephole_desc() const {
            return rnn_base::weights_peephole_desc();
        }

        memory::desc weights_projection_desc() const {
            return rnn_base::weights_projection_desc();
        }

        memory::desc bias_desc() const { return rnn_base::bias_desc(); }

        memory::desc dst_layer_desc() const {
            return rnn_base::dst_layer_desc();
        }

        memory::desc dst_iter_desc() const { return rnn_base::dst_iter_desc(); }

        memory::desc dst_iter_c_desc() const {
            return rnn_base::dst_iter_c_desc();
        }

        memory::desc workspace_desc() const {
            return rnn_base::workspace_desc();
        }

        memory::desc diff_src_layer_desc() const {
            return rnn_base::diff_src_layer_desc();
        }

        memory::desc diff_src_iter_desc() const {
            return rnn_base::diff_src_iter_desc();
        }

        memory::desc diff_src_iter_c_desc() const {
            return rnn_base::diff_src_iter_c_desc();
        }

        memory::desc diff_weights_layer_desc() const {
            return rnn_base::diff_weights_layer_desc();
        }

        memory::desc diff_weights_iter_desc() const {
            return rnn_base::diff_weights_iter_desc();
        }

        memory::desc diff_weights_peephole_desc() const {
            return rnn_base::diff_weights_peephole_desc();
        }

        memory::desc diff_weights_projection_desc() const {
            return rnn_base::diff_weights_projection_desc();
        }

        memory::desc diff_bias_desc() const {
            return rnn_base::diff_bias_desc();
        }

        memory::desc diff_dst_layer_desc() const {
            return rnn_base::diff_dst_layer_desc();
        }

        memory::desc diff_dst_iter_desc() const {
            return rnn_base::diff_dst_iter_desc();
        }

        memory::desc diff_dst_iter_c_desc() const {
            return rnn_base::diff_dst_iter_c_desc();
        }
    };

    lstm_backward() = default;

    lstm_backward(const primitive_desc &pd) : primitive(pd) {}
};

struct gru_forward : public primitive {
    struct desc {
        dnnl_rnn_desc_t data;

        desc(prop_kind prop_kind, rnn_direction direction,
                const memory::desc &src_layer_desc,
                const memory::desc &src_iter_desc,
                const memory::desc &weights_layer_desc,
                const memory::desc &weights_iter_desc,
                const memory::desc &bias_desc,
                const memory::desc &dst_layer_desc,
                const memory::desc &dst_iter_desc,
                rnn_flags flags = rnn_flags::undef) {
            error::wrap_c_api(
                    dnnl_gru_forward_desc_init(&data,
                            dnnl::convert_to_c(prop_kind),
                            dnnl::convert_to_c(direction), &src_layer_desc.data,
                            &src_iter_desc.data, &weights_layer_desc.data,
                            &weights_iter_desc.data, &bias_desc.data,
                            &dst_layer_desc.data, &dst_iter_desc.data,
                            dnnl::convert_to_c(flags)),
                    "could not create a descriptor for a GRU forward "
                    "propagation primitive");
        }
    };

    struct primitive_desc : public rnn_primitive_desc_base {
        primitive_desc() = default;

        primitive_desc(const desc &desc, const engine &engine,
                bool allow_empty = false)
            : rnn_primitive_desc_base(
                    &desc.data, nullptr, engine, nullptr, allow_empty) {}

        primitive_desc(const desc &desc, const primitive_attr &attr,
                const engine &engine, bool allow_empty = false)
            : rnn_primitive_desc_base(
                    &desc.data, &attr, engine, nullptr, allow_empty) {}

        primitive_desc(dnnl_primitive_desc_t pd)
            : rnn_primitive_desc_base(pd, dnnl::prop_kind::forward_training,
                    dnnl::prop_kind::forward_inference,
                    dnnl::algorithm::vanilla_gru) {}

        memory::desc src_layer_desc() const {
            return rnn_base::src_layer_desc();
        }

        memory::desc src_iter_desc() const { return rnn_base::src_iter_desc(); }

        memory::desc weights_layer_desc() const {
            return rnn_base::weights_layer_desc();
        }

        memory::desc weights_iter_desc() const {
            return rnn_base::weights_iter_desc();
        }

        memory::desc bias_desc() const { return rnn_base::bias_desc(); }

        memory::desc dst_layer_desc() const {
            return rnn_base::dst_layer_desc();
        }

        memory::desc dst_iter_desc() const { return rnn_base::dst_iter_desc(); }

        memory::desc workspace_desc() const {
            return rnn_base::workspace_desc();
        }
    };

    gru_forward() = default;

    gru_forward(const primitive_desc &pd) : primitive(pd) {}
};

struct gru_backward : public primitive {
    struct desc {
        dnnl_rnn_desc_t data;

        desc(prop_kind prop_kind, rnn_direction direction,
                const memory::desc &src_layer_desc,
                const memory::desc &src_iter_desc,
                const memory::desc &weights_layer_desc,
                const memory::desc &weights_iter_desc,
                const memory::desc &bias_desc,
                const memory::desc &dst_layer_desc,
                const memory::desc &dst_iter_desc,
                const memory::desc &diff_src_layer_desc,
                const memory::desc &diff_src_iter_desc,
                const memory::desc &diff_weights_layer_desc,
                const memory::desc &diff_weights_iter_desc,
                const memory::desc &diff_bias_desc,
                const memory::desc &diff_dst_layer_desc,
                const memory::desc &diff_dst_iter_desc,
                rnn_flags flags = rnn_flags::undef) {
            error::wrap_c_api(
                    dnnl_gru_backward_desc_init(&data,
                            dnnl::convert_to_c(prop_kind),
                            dnnl::convert_to_c(direction), &src_layer_desc.data,
                            &src_iter_desc.data, &weights_layer_desc.data,
                            &weights_iter_desc.data, &bias_desc.data,
                            &dst_layer_desc.data, &dst_iter_desc.data,
                            &diff_src_layer_desc.data, &diff_src_iter_desc.data,
                            &diff_weights_layer_desc.data,
                            &diff_weights_iter_desc.data, &diff_bias_desc.data,
                            &diff_dst_layer_desc.data, &diff_dst_iter_desc.data,
                            dnnl::convert_to_c(flags)),
                    "could not create a descriptor for a GRU backward "
                    "propagation primitive");
        }
    };

    struct primitive_desc : public rnn_primitive_desc_base {
        primitive_desc() = default;

        primitive_desc(const desc &desc, const engine &engine,
                const gru_forward::primitive_desc &hint_fwd_pd,
                bool allow_empty = false)
            : rnn_primitive_desc_base(&desc.data, nullptr, engine,
                    hint_fwd_pd.get(), allow_empty) {}

        primitive_desc(const desc &desc, const primitive_attr &attr,
                const engine &engine,
                const gru_forward::primitive_desc &hint_fwd_pd,
                bool allow_empty = false)
            : rnn_primitive_desc_base(&desc.data, &attr, engine,
                    hint_fwd_pd.get(), allow_empty) {}

        primitive_desc(dnnl_primitive_desc_t pd)
            : rnn_primitive_desc_base(pd, dnnl::prop_kind::backward,
                    dnnl::algorithm::vanilla_gru) {}

        memory::desc src_layer_desc() const {
            return rnn_base::src_layer_desc();
        }

        memory::desc src_iter_desc() const { return rnn_base::src_iter_desc(); }

        memory::desc weights_layer_desc() const {
            return rnn_base::weights_layer_desc();
        }

        memory::desc weights_iter_desc() const {
            return rnn_base::weights_iter_desc();
        }

        memory::desc bias_desc() const { return rnn_base::bias_desc(); }

        memory::desc dst_layer_desc() const {
            return rnn_base::dst_layer_desc();
        }

        memory::desc dst_iter_desc() const { return rnn_base::dst_iter_desc(); }

        memory::desc workspace_desc() const {
            return rnn_base::workspace_desc();
        }

        memory::desc diff_src_layer_desc() const {
            return rnn_base::diff_src_layer_desc();
        }

        memory::desc diff_src_iter_desc() const {
            return rnn_base::diff_src_iter_desc();
        }

        memory::desc diff_weights_layer_desc() const {
            return rnn_base::diff_weights_layer_desc();
        }

        memory::desc diff_weights_iter_desc() const {
            return rnn_base::diff_weights_iter_desc();
        }

        memory::desc diff_bias_desc() const {
            return rnn_base::diff_bias_desc();
        }

        memory::desc diff_dst_layer_desc() const {
            return rnn_base::diff_dst_layer_desc();
        }

        memory::desc diff_dst_iter_desc() const {
            return rnn_base::diff_dst_iter_desc();
        }
    };

    gru_backward() = default;

    gru_backward(const primitive_desc &pd) : primitive(pd) {}
};

struct lbr_gru_forward : public primitive {
    struct desc {
        dnnl_rnn_desc_t data;

        desc(prop_kind prop_kind, rnn_direction direction,
                const memory::desc &src_layer_desc,
                const memory::desc &src_iter_desc,
                const memory::desc &weights_layer_desc,
                const memory::desc &weights_iter_desc,
                const memory::desc &bias_desc,
                const memory::desc &dst_layer_desc,
                const memory::desc &dst_iter_desc,
                rnn_flags flags = rnn_flags::undef) {
            error::wrap_c_api(
                    dnnl_lbr_gru_forward_desc_init(&data,
                            dnnl::convert_to_c(prop_kind),
                            dnnl::convert_to_c(direction), &src_layer_desc.data,
                            &src_iter_desc.data, &weights_layer_desc.data,
                            &weights_iter_desc.data, &bias_desc.data,
                            &dst_layer_desc.data, &dst_iter_desc.data,
                            dnnl::convert_to_c(flags)),
                    "could not create a descriptor for an LBR GRU forward "
                    "propagation primitive");
        }
    };

    struct primitive_desc : public rnn_primitive_desc_base {
        primitive_desc() = default;

        primitive_desc(const desc &desc, const engine &engine,
                bool allow_empty = false)
            : rnn_primitive_desc_base(
                    &desc.data, nullptr, engine, nullptr, allow_empty) {}

        primitive_desc(const desc &desc, const primitive_attr &attr,
                const engine &engine, bool allow_empty = false)
            : rnn_primitive_desc_base(
                    &desc.data, &attr, engine, nullptr, allow_empty) {}

        primitive_desc(dnnl_primitive_desc_t pd)
            : rnn_primitive_desc_base(pd, dnnl::prop_kind::forward_training,
                    dnnl::prop_kind::forward_inference,
                    dnnl::algorithm::lbr_gru) {}

        memory::desc src_layer_desc() const {
            return rnn_base::src_layer_desc();
        }

        memory::desc src_iter_desc() const { return rnn_base::src_iter_desc(); }

        memory::desc weights_layer_desc() const {
            return rnn_base::weights_layer_desc();
        }

        memory::desc weights_iter_desc() const {
            return rnn_base::weights_iter_desc();
        }

        memory::desc bias_desc() const { return rnn_base::bias_desc(); }

        memory::desc dst_layer_desc() const {
            return rnn_base::dst_layer_desc();
        }

        memory::desc dst_iter_desc() const { return rnn_base::dst_iter_desc(); }

        memory::desc workspace_desc() const {
            return rnn_base::workspace_desc();
        }
    };

    lbr_gru_forward() = default;

    lbr_gru_forward(const primitive_desc &pd) : primitive(pd) {}
};

struct lbr_gru_backward : public primitive {
    struct desc {
        dnnl_rnn_desc_t data;

        desc(prop_kind prop_kind, rnn_direction direction,
                const memory::desc &src_layer_desc,
                const memory::desc &src_iter_desc,
                const memory::desc &weights_layer_desc,
                const memory::desc &weights_iter_desc,
                const memory::desc &bias_desc,
                const memory::desc &dst_layer_desc,
                const memory::desc &dst_iter_desc,
                const memory::desc &diff_src_layer_desc,
                const memory::desc &diff_src_iter_desc,
                const memory::desc &diff_weights_layer_desc,
                const memory::desc &diff_weights_iter_desc,
                const memory::desc &diff_bias_desc,
                const memory::desc &diff_dst_layer_desc,
                const memory::desc &diff_dst_iter_desc,
                rnn_flags flags = rnn_flags::undef) {
            error::wrap_c_api(
                    dnnl_lbr_gru_backward_desc_init(&data,
                            dnnl::convert_to_c(prop_kind),
                            dnnl::convert_to_c(direction), &src_layer_desc.data,
                            &src_iter_desc.data, &weights_layer_desc.data,
                            &weights_iter_desc.data, &bias_desc.data,
                            &dst_layer_desc.data, &dst_iter_desc.data,
                            &diff_src_layer_desc.data, &diff_src_iter_desc.data,
                            &diff_weights_layer_desc.data,
                            &diff_weights_iter_desc.data, &diff_bias_desc.data,
                            &diff_dst_layer_desc.data, &diff_dst_iter_desc.data,
                            dnnl::convert_to_c(flags)),
                    "could not create a descriptor for an LBR GRU backward "
                    "propagation primitive");
        }
    };

    struct primitive_desc : public rnn_primitive_desc_base {
        primitive_desc() = default;

        primitive_desc(const desc &desc, const engine &engine,
                const lbr_gru_forward::primitive_desc &hint_fwd_pd,
                bool allow_empty = false)
            : rnn_primitive_desc_base(&desc.data, nullptr, engine,
                    hint_fwd_pd.get(), allow_empty) {}

        primitive_desc(const desc &desc, const primitive_attr &attr,
                const engine &engine,
                const lbr_gru_forward::primitive_desc &hint_fwd_pd,
                bool allow_empty = false)
            : rnn_primitive_desc_base(&desc.data, &attr, engine,
                    hint_fwd_pd.get(), allow_empty) {}

        primitive_desc(dnnl_primitive_desc_t pd)
            : rnn_primitive_desc_base(
                    pd, dnnl::prop_kind::backward, dnnl::algorithm::lbr_gru) {}

        memory::desc src_layer_desc() const {
            return rnn_base::src_layer_desc();
        }

        memory::desc src_iter_desc() const { return rnn_base::src_iter_desc(); }

        memory::desc weights_layer_desc() const {
            return rnn_base::weights_layer_desc();
        }

        memory::desc weights_iter_desc() const {
            return rnn_base::weights_iter_desc();
        }

        memory::desc bias_desc() const { return rnn_base::bias_desc(); }

        memory::desc dst_layer_desc() const {
            return rnn_base::dst_layer_desc();
        }

        memory::desc dst_iter_desc() const { return rnn_base::dst_iter_desc(); }

        memory::desc workspace_desc() const {
            return rnn_base::workspace_desc();
        }

        memory::desc diff_src_layer_desc() const {
            return rnn_base::diff_src_layer_desc();
        }

        memory::desc diff_src_iter_desc() const {
            return rnn_base::diff_src_iter_desc();
        }

        memory::desc diff_weights_layer_desc() const {
            return rnn_base::diff_weights_layer_desc();
        }

        memory::desc diff_weights_iter_desc() const {
            return rnn_base::diff_weights_iter_desc();
        }

        memory::desc diff_bias_desc() const {
            return rnn_base::diff_bias_desc();
        }

        memory::desc diff_dst_layer_desc() const {
            return rnn_base::diff_dst_layer_desc();
        }

        memory::desc diff_dst_iter_desc() const {
            return rnn_base::diff_dst_iter_desc();
        }
    };

    lbr_gru_backward() = default;

    lbr_gru_backward(const primitive_desc &pd) : primitive(pd) {}
};



struct shuffle_forward : public primitive {
    struct desc {
        dnnl_shuffle_desc_t data;

        desc(prop_kind prop_kind, const memory::desc &data_desc, int axis,
                int group_size) {
            error::wrap_c_api(dnnl_shuffle_forward_desc_init(&data,
                                      dnnl::convert_to_c(prop_kind),
                                      &data_desc.data, axis, group_size),
                    "could not create a descriptor for a shuffle forward "
                    "propagation primitive");
        }
    };

    struct primitive_desc : public dnnl::primitive_desc {
        primitive_desc() = default;

        primitive_desc(const desc &desc, const engine &engine,
                const primitive_attr &attr = primitive_attr(),
                bool allow_empty = false)
            : dnnl::primitive_desc(
                    &desc.data, &attr, engine, nullptr, allow_empty) {}

        primitive_desc(dnnl_primitive_desc_t pd)
            : dnnl::primitive_desc(pd, dnnl::primitive::kind::shuffle,
                    dnnl::prop_kind::forward_training,
                    dnnl::prop_kind::forward_inference) {}

        memory::desc src_desc() const { return base::src_desc(0); }

        memory::desc dst_desc() const { return base::dst_desc(0); }
    };

    shuffle_forward() = default;

    shuffle_forward(const primitive_desc &pd) : primitive(pd) {}
};

struct shuffle_backward : public primitive {
    struct desc {
        dnnl_shuffle_desc_t data;

        desc(const memory::desc &diff_data_desc, int axis, int group_size) {
            error::wrap_c_api(dnnl_shuffle_backward_desc_init(&data,
                                      &diff_data_desc.data, axis, group_size),
                    "could not create a descriptor for a shuffle backward "
                    "propagation primitive");
        }
    };

    struct primitive_desc : public dnnl::primitive_desc {
        primitive_desc() = default;

        primitive_desc(const desc &desc, const engine &engine,
                const shuffle_forward::primitive_desc &hint_fwd_pd,
                const primitive_attr &attr = primitive_attr(),
                bool allow_empty = false)
            : dnnl::primitive_desc(&desc.data, &attr, engine, hint_fwd_pd.get(),
                    allow_empty) {}

        primitive_desc(dnnl_primitive_desc_t pd)
            : dnnl::primitive_desc(pd, dnnl::primitive::kind::shuffle,
                    dnnl::prop_kind::backward_data) {}

        memory::desc diff_src_desc() const { return base::diff_src_desc(0); }

        memory::desc diff_dst_desc() const { return base::diff_dst_desc(0); }
    };

    shuffle_backward() = default;

    shuffle_backward(const primitive_desc &pd) : primitive(pd) {}
};



struct binary : public primitive {
    struct desc {
        dnnl_binary_desc_t data;

        desc() = default;

        desc(algorithm algorithm, const memory::desc &src0,
                const memory::desc &src1, const memory::desc &dst) {
            error::wrap_c_api(
                    dnnl_binary_desc_init(&data, dnnl::convert_to_c(algorithm),
                            &src0.data, &src1.data, &dst.data),
                    "could not create a descriptor for a binary operation "
                    "primitive");
        }
    };

    struct primitive_desc : public dnnl::primitive_desc {
        primitive_desc() = default;

        primitive_desc(const desc &desc, const engine &engine,
                bool allow_empty = false)
            : dnnl::primitive_desc(
                    &desc.data, nullptr, engine, nullptr, allow_empty) {}

        primitive_desc(const desc &desc, const primitive_attr &attr,
                const engine &engine, bool allow_empty = false)
            : dnnl::primitive_desc(
                    &desc.data, &attr, engine, nullptr, allow_empty) {}

        primitive_desc(dnnl_primitive_desc_t pd)
            : dnnl::primitive_desc(pd, dnnl::primitive::kind::binary) {}

        memory::desc src_desc(int idx = 0) const { return base::src_desc(idx); }

        memory::desc src0_desc() const { return base::src_desc(0); }

        memory::desc src1_desc() const { return base::src_desc(1); }

        memory::desc dst_desc() const { return base::dst_desc(0); }
    };

    binary() = default;

    binary(const primitive_desc &pd) : primitive(pd) {}
};



struct matmul : public primitive {
    struct desc {
        dnnl_matmul_desc_t data;

        desc(const memory::desc &src_desc, const memory::desc &weights_desc,
                const memory::desc &dst_desc) {
            error::wrap_c_api(
                    dnnl_matmul_desc_init(&data, &src_desc.data,
                            &weights_desc.data, nullptr, &dst_desc.data),
                    "could not create a descriptor for a matmul primitive");
        }

        desc(const memory::desc &src_desc, const memory::desc &weights_desc,
                const memory::desc &bias_desc, const memory::desc &dst_desc) {
            error::wrap_c_api(dnnl_matmul_desc_init(&data, &src_desc.data,
                                      &weights_desc.data, &bias_desc.data,
                                      &dst_desc.data),
                    "could not create a descriptor for a matmul primitive");
        }
    };

    struct primitive_desc : public dnnl::primitive_desc {
        primitive_desc() = default;

        primitive_desc(const desc &desc, const engine &engine,
                bool allow_empty = false)
            : dnnl::primitive_desc(
                    &desc.data, nullptr, engine, nullptr, allow_empty) {}

        primitive_desc(const desc &desc, const primitive_attr &attr,
                const engine &engine, bool allow_empty = false)
            : dnnl::primitive_desc(
                    &desc.data, &attr, engine, nullptr, allow_empty) {}

        primitive_desc(dnnl_primitive_desc_t pd)
            : dnnl::primitive_desc(pd, dnnl::primitive::kind::matmul) {}

        memory::desc src_desc() const { return query_md(query::src_md, 0); }

        memory::desc weights_desc() const {
            return query_md(query::weights_md, 0);
        }

        memory::desc bias_desc() const {
            return query_md(query::weights_md, 1);
        }

        memory::desc dst_desc() const { return query_md(query::dst_md, 0); }
    };

    matmul() = default;

    matmul(const primitive_desc &pd) : primitive(pd) {}
};



struct resampling_forward : public primitive {
    struct desc {
        dnnl_resampling_desc_t data;

        //
        desc(prop_kind prop_kind, algorithm algorithm,
                const memory::desc &src_desc, const memory::desc &dst_desc) {
            error::wrap_c_api(dnnl_resampling_forward_desc_init(&data,
                                      dnnl::convert_to_c(prop_kind),
                                      convert_to_c(algorithm), nullptr,
                                      &src_desc.data, &dst_desc.data),
                    "could not create a resampling forward descriptor");
        }

        desc(prop_kind prop_kind, algorithm algorithm,
                const std::vector<float> &factors,
                const memory::desc &src_desc) {
            memory::validate_dims(factors, src_desc.data.ndims - 2);
            error::wrap_c_api(dnnl_resampling_forward_desc_init(&data,
                                      dnnl::convert_to_c(prop_kind),
                                      convert_to_c(algorithm), &factors[0],
                                      &src_desc.data, nullptr),
                    "could not create a resampling forward descriptor");
        }

        //
        desc(prop_kind prop_kind, algorithm algorithm,
                const std::vector<float> &factors, const memory::desc &src_desc,
                const memory::desc &dst_desc) {
            if (!factors.empty())
                memory::validate_dims(factors, src_desc.data.ndims - 2);
            error::wrap_c_api(dnnl_resampling_forward_desc_init(&data,
                                      dnnl::convert_to_c(prop_kind),
                                      convert_to_c(algorithm), factors.data(),
                                      &src_desc.data, &dst_desc.data),
                    "could not create a resampling forward descriptor");
        }
    };

    struct primitive_desc : public dnnl::primitive_desc {
        primitive_desc() = default;

        primitive_desc(const desc &desc, const engine &engine,
                bool allow_empty = false)
            : dnnl::primitive_desc(
                    &desc.data, nullptr, engine, nullptr, allow_empty) {}

        primitive_desc(const desc &desc, const primitive_attr &attr,
                const engine &engine, bool allow_empty = false)
            : dnnl::primitive_desc(
                    &desc.data, &attr, engine, nullptr, allow_empty) {}

        primitive_desc(dnnl_primitive_desc_t pd)
            : dnnl::primitive_desc(pd, dnnl::primitive::kind::resampling,
                    dnnl::prop_kind::forward_training,
                    dnnl::prop_kind::forward_inference) {}

        memory::desc src_desc() const { return base::src_desc(0); }

        memory::desc dst_desc() const { return base::dst_desc(0); }
    };

    resampling_forward() = default;

    resampling_forward(const primitive_desc &pd) : primitive(pd) {}
};

struct resampling_backward : public primitive {
    struct desc {
        dnnl_resampling_desc_t data;

        desc(algorithm algorithm, const memory::desc &diff_src_desc,
                const memory::desc &diff_dst_desc) {
            error::wrap_c_api(dnnl_resampling_backward_desc_init(&data,
                                      convert_to_c(algorithm), nullptr,
                                      &diff_src_desc.data, &diff_dst_desc.data),
                    "could not create a resampling backward data descriptor");
        }

        desc(algorithm algorithm, const std::vector<float> &factors,
                const memory::desc &diff_src_desc,
                const memory::desc &diff_dst_desc) {
            if (!factors.empty())
                memory::validate_dims(factors, diff_src_desc.data.ndims - 2);
            error::wrap_c_api(dnnl_resampling_backward_desc_init(&data,
                                      convert_to_c(algorithm), factors.data(),
                                      &diff_src_desc.data, &diff_dst_desc.data),
                    "could not create a resampling backward data descriptor");
        }
    };

    struct primitive_desc : public dnnl::primitive_desc {
        primitive_desc() = default;

        primitive_desc(const desc &desc, const engine &engine,
                const resampling_forward::primitive_desc &hint_fwd_pd,
                bool allow_empty = false)
            : dnnl::primitive_desc(&desc.data, nullptr, engine,
                    hint_fwd_pd.get(), allow_empty) {}

        primitive_desc(const desc &desc, const primitive_attr &attr,
                const engine &engine,
                const resampling_forward::primitive_desc &hint_fwd_pd,
                bool allow_empty = false)
            : dnnl::primitive_desc(&desc.data, &attr, engine, hint_fwd_pd.get(),
                    allow_empty) {}

        primitive_desc(dnnl_primitive_desc_t pd)
            : dnnl::primitive_desc(pd, dnnl::primitive::kind::resampling,
                    dnnl::prop_kind::backward_data) {}

        memory::desc diff_src_desc() const { return base::diff_src_desc(0); }

        memory::desc diff_dst_desc() const { return base::diff_dst_desc(0); }
    };

    resampling_backward() = default;

    resampling_backward(const primitive_desc &pd) : primitive(pd) {}
};




using version_t = dnnl_version_t;

enum class status {
    success = dnnl_success,
    out_of_memory = dnnl_out_of_memory,
    invalid_arguments = dnnl_invalid_arguments,
    unimplemented = dnnl_unimplemented,
    iterator_ends = dnnl_iterator_ends,
    runtime_error = dnnl_runtime_error,
    not_required = dnnl_not_required,
};

inline status set_verbose(int level) {
    return static_cast<status>(dnnl_set_verbose(level));
}

inline const version_t *version() {
    return dnnl_version();
}

inline status set_jit_dump(int enable) {
    return static_cast<status>(dnnl_set_jit_dump(enable));
}

inline status set_jit_profiling_flags(unsigned flags) {
    return static_cast<status>(dnnl_set_jit_profiling_flags(flags));
}

inline status set_jit_profiling_jitdumpdir(const std::string &dir) {
    return static_cast<status>(dnnl_set_jit_profiling_jitdumpdir(dir.c_str()));
}

enum class cpu_isa {
    all = dnnl_cpu_isa_all,
    sse41 = dnnl_cpu_isa_sse41,
    avx = dnnl_cpu_isa_avx,
    avx2 = dnnl_cpu_isa_avx2,
    avx512_mic = dnnl_cpu_isa_avx512_mic,
    avx512_mic_4ops = dnnl_cpu_isa_avx512_mic_4ops,
    avx512_core = dnnl_cpu_isa_avx512_core,
    avx512_core_vnni = dnnl_cpu_isa_avx512_core_vnni,
    avx512_core_bf16 = dnnl_cpu_isa_avx512_core_bf16,
};

inline status set_max_cpu_isa(cpu_isa isa) {
    return static_cast<status>(
            dnnl_set_max_cpu_isa(static_cast<dnnl_cpu_isa_t>(isa)));
}



inline status sgemm(char transa, char transb, dnnl_dim_t M, dnnl_dim_t N,
        dnnl_dim_t K, float alpha, const float *A, dnnl_dim_t lda,
        const float *B, dnnl_dim_t ldb, float beta, float *C, dnnl_dim_t ldc) {
    return static_cast<status>(dnnl_sgemm(
            transa, transb, M, N, K, alpha, A, lda, B, ldb, beta, C, ldc));
}

inline status gemm_u8s8s32(char transa, char transb, char offsetc, dnnl_dim_t M,
        dnnl_dim_t N, dnnl_dim_t K, float alpha, const uint8_t *A,
        dnnl_dim_t lda, uint8_t ao, const int8_t *B, dnnl_dim_t ldb, int8_t bo,
        float beta, int32_t *C, dnnl_dim_t ldc, const int32_t *co) {
    return static_cast<status>(dnnl_gemm_u8s8s32(transa, transb, offsetc, M, N,
            K, alpha, A, lda, ao, B, ldb, bo, beta, C, ldc, co));
}

inline status gemm_s8s8s32(char transa, char transb, char offsetc, dnnl_dim_t M,
        dnnl_dim_t N, dnnl_dim_t K, float alpha, const int8_t *A,
        dnnl_dim_t lda, int8_t ao, const int8_t *B, dnnl_dim_t ldb, int8_t bo,
        float beta, int32_t *C, dnnl_dim_t ldc, const int32_t *co) {
    return static_cast<status>(dnnl_gemm_s8s8s32(transa, transb, offsetc, M, N,
            K, alpha, A, lda, ao, B, ldb, bo, beta, C, ldc, co));
}

#if DNNL_CPU_RUNTIME == DNNL_RUNTIME_THREADPOOL
inline status sgemm(char transa, char transb, dnnl_dim_t M, dnnl_dim_t N,
        dnnl_dim_t K, float alpha, const float *A, dnnl_dim_t lda,
        const float *B, dnnl_dim_t ldb, float beta, float *C, dnnl_dim_t ldc,
        dnnl::threadpool_iface *tp) {
    return static_cast<status>(dnnl_sgemm_tp(
            transa, transb, M, N, K, alpha, A, lda, B, ldb, beta, C, ldc, tp));
}
inline status gemm_u8s8s32(char transa, char transb, char offsetc, dnnl_dim_t M,
        dnnl_dim_t N, dnnl_dim_t K, float alpha, const uint8_t *A,
        dnnl_dim_t lda, uint8_t ao, const int8_t *B, dnnl_dim_t ldb, int8_t bo,
        float beta, int32_t *C, dnnl_dim_t ldc, const int32_t *co,
        dnnl::threadpool_iface *tp) {
    return static_cast<status>(dnnl_gemm_u8s8s32_tp(transa, transb, offsetc, M,
            N, K, alpha, A, lda, ao, B, ldb, bo, beta, C, ldc, co, tp));
}

inline status gemm_s8s8s32(char transa, char transb, char offsetc, dnnl_dim_t M,
        dnnl_dim_t N, dnnl_dim_t K, float alpha, const int8_t *A,
        dnnl_dim_t lda, int8_t ao, const int8_t *B, dnnl_dim_t ldb, int8_t bo,
        float beta, int32_t *C, dnnl_dim_t ldc, const int32_t *co,
        dnnl::threadpool_iface *tp) {
    return static_cast<status>(dnnl_gemm_s8s8s32_tp(transa, transb, offsetc, M,
            N, K, alpha, A, lda, ao, B, ldb, bo, beta, C, ldc, co, tp));
}
#endif


// implementation section

inline primitive::primitive(const_dnnl_primitive_desc_t c_pd) {
    dnnl_primitive_t result;
    error::wrap_c_api(dnnl_primitive_create(&result, c_pd),
            "could not create a primitive");
    reset(result);
}

inline primitive::primitive(const primitive_desc &pd) : primitive(pd.get()) {}

inline void primitive::execute(const stream &stream,
        const std::unordered_map<int, memory> &args) const {
    std::vector<dnnl_exec_arg_t> c_args;
    c_args.reserve(args.size());
    for (const auto &a : args)
        c_args.push_back({a.first, a.second.get(true)});

    error::wrap_c_api(dnnl_primitive_execute(get(), stream.get(),
                              (int)c_args.size(), c_args.data()),
            "could not execute a primitive");
}

#undef DNNL_DEFINE_BITMASK_OPS

} // namespace dnnl


#endif