*******************************************************************************
 Copyright 2023 Arm Limited and affiliates.
 SPDX-License-Identifier: Apache-2.0

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 *******************************************************************************
diff --git a/src/common/matmul_pd.hpp b/src/common/matmul_pd.hpp
index 4330ad938b..df16c5fcca 100644
--- a/src/common/matmul_pd.hpp
+++ b/src/common/matmul_pd.hpp
@@ -159,6 +159,19 @@ protected:
 
         return true;
     }
+
+    // All implementations that do not support sparse inputs/outputs should
+    // call this function.
+    bool is_dense_data() {
+#ifdef DNNL_EXPERIMENTAL_SPARSE
+        for (auto md : {&src_md_, &weights_md_, &bias_md_, &dst_md_}) {
+            if (memory_desc_wrapper(md).format_kind() == format_kind::sparse)
+                return false;
+        }
+#endif
+        return true;
+    }
+
 };
 
 } // namespace impl
diff --git a/src/cpu/aarch64/acl_convolution_utils.cpp b/src/cpu/aarch64/acl_convolution_utils.cpp
index 37f8ecbc06..6b57374643 100644
--- a/src/cpu/aarch64/acl_convolution_utils.cpp
+++ b/src/cpu/aarch64/acl_convolution_utils.cpp
@@ -41,25 +41,23 @@ status_t acl_init_conf(acl_conv_conf_t &acp, memory_desc_t &src_md,
     const memory_desc_wrapper dst_d(&dst_md);
     const memory_desc_wrapper bia_d(&bias_md);
 
-    auto math_mode = get_fpmath_mode();
-    acp.fast_math = one_of(math_mode, fpmath_mode::bf16, fpmath_mode::any);
-
     // Compute Library currently supports forward propagation only
     const prop_kind_t prop_kind = cd.prop_kind;
     const bool is_fwd = (prop_kind == dnnl_forward_training)
             || (prop_kind == dnnl_forward_inference);
     if (!is_fwd) return status::unimplemented;
 
-    const int with_groups = wei_d.ndims() == src_d.ndims() + 1;
     const int ndims = src_d.ndims();
-    const bool is_1d = ndims == 3;
-    const bool is_3d = ndims == 5;
-    bool is_nspc;
 
-    // Compute Library unsupported shape scenarios
-    if (one_of(true, is_3d, is_1d, with_groups)) {
-        return status::unimplemented;
-    }
+    ACL_CHECK_SUPPORT(ndims != 4, " only supports 2 spatial dimensions");
+
+    const int with_groups = wei_d.ndims() == src_d.ndims() + 1;
+    ACL_CHECK_SUPPORT(with_groups, " does not support groups");
+
+    ACL_CHECK_SUPPORT(src_d.data_type() != data_type::f32
+                    || wei_d.data_type() != data_type::f32
+                    || dst_d.data_type() != data_type::f32,
+            " src, dst and wei must be fp32");
 
     // batch size
     const int mb = src_d.dims()[0];
@@ -110,108 +108,143 @@ status_t acl_init_conf(acl_conv_conf_t &acp, memory_desc_t &src_md,
 
     acp.with_bias = cd.bias_desc.format_kind != format_kind::undef;
 
-    auto set_or_check_tags = [&](format_tag_t desired_src_tag,
-                                     format_tag_t desired_dst_tag) -> status_t {
-        using namespace format_tag;
-        auto src_tag = any, dst_tag = any;
-
-        if (src_d.format_kind() == format_kind::any) {
-            CHECK(memory_desc_init_by_tag(src_md, desired_src_tag));
-            src_tag = desired_src_tag;
-        } else {
-            src_tag = memory_desc_matches_one_of_tag(src_md, nhwc, nchw);
-        }
-
-        if (dst_d.format_kind() == format_kind::any) {
-            CHECK(memory_desc_init_by_tag(dst_md, desired_dst_tag));
-            dst_tag = desired_dst_tag;
-        } else {
-            dst_tag = memory_desc_matches_one_of_tag(dst_md, nhwc, nchw);
-        }
-
-        if (acp.with_bias && bias_md.format_kind == format_kind::any)
-            CHECK(memory_desc_init_by_tag(bias_md, x));
-
-        is_nspc = utils::one_of(src_tag, nhwc);
-
-        memory_desc_t want_wei_md = weights_md;
-        auto wei_tag = is_nspc ? ohwi : oihw;
-        CHECK(memory_desc_init_by_tag(want_wei_md, wei_tag));
-
-        // Compute Library does not support mismatching layouts
-        if ((src_tag != wei_tag) || (src_tag != dst_tag))
-            return status::unimplemented;
+    if (wei_d.format_kind() != format_kind::any) return status::unimplemented;
+
+    auto src_tag = memory_desc_matches_one_of_tag(
+            src_md, format_tag::nhwc, format_tag::nchw);
+    auto dst_tag = memory_desc_matches_one_of_tag(
+            dst_md, format_tag::nhwc, format_tag::nchw);
+
+    // We want src and dst to match, preferrably both to be NHWC
+    if (src_d.format_kind() == format_kind::any
+            && dst_d.format_kind() == format_kind::any) {
+        CHECK(memory_desc_init_by_tag(src_md, format_tag::nhwc));
+        CHECK(memory_desc_init_by_tag(dst_md, format_tag::nhwc));
+    } else if (src_d.format_kind() == format_kind::any
+            && dst_tag != format_tag::undef) {
+        CHECK(memory_desc_init_by_tag(src_md, dst_tag));
+    } else if (dst_d.format_kind() == format_kind::any
+            && src_tag != format_tag::undef) {
+        CHECK(memory_desc_init_by_tag(dst_md, src_tag));
+    }
 
-        if (weights_md.format_kind == format_kind::any) {
-            weights_md = want_wei_md;
-        }
-        return (want_wei_md == weights_md) ? status::success
-                                           : status::unimplemented;
-    };
+    // Recompute tags after potentially running memory desc init
+    src_tag = memory_desc_matches_one_of_tag(
+            src_md, format_tag::nhwc, format_tag::nchw);
+    dst_tag = memory_desc_matches_one_of_tag(
+            dst_md, format_tag::nhwc, format_tag::nchw);
 
-    auto default_dat_tag = format_tag::nhwc;
-    if (set_or_check_tags(default_dat_tag, default_dat_tag) != status::success)
+    if (src_tag == format_tag::undef || dst_tag == format_tag::undef
+            || src_tag != dst_tag)
         return status::unimplemented;
 
-    const auto acl_layout = is_nspc ? arm_compute::DataLayout::NHWC
-                                    : arm_compute::DataLayout::NCHW;
+    // Set weights to initially be the same as src
+    CHECK(memory_desc_init_by_tag(weights_md, src_tag));
 
-    // For convolutions, int8 datatypes imply quantized types in ACL
-    acp.is_int8 = utils::one_of(src_d.data_type(), s8, u8)
-            && wei_d.data_type() == s8;
+    // Bias is just 1D, set to be the obvious format
+    if (acp.with_bias && bias_md.format_kind == format_kind::any)
+        CHECK(memory_desc_init_by_tag(bias_md, format_tag::x));
 
-    auto acl_src_data_t
-            = acl_utils::get_acl_data_t(src_d.data_type(), acp.is_int8);
-    auto acl_wei_data_t
-            = acl_utils::get_acl_data_t(wei_d.data_type(), acp.is_int8);
-    auto acl_dst_data_t
-            = acl_utils::get_acl_data_t(dst_d.data_type(), acp.is_int8);
-    auto acl_bia_data_t
-            = acl_utils::get_acl_data_t(bia_d.data_type(), acp.is_int8);
+    bool is_nhwc = src_tag == format_tag::nhwc;
+    // The layouts have to match (although we may later modify the weights)
+    const auto acl_layout = is_nhwc ? arm_compute::DataLayout::NHWC
+                                    : arm_compute::DataLayout::NCHW;
 
-    if (acl_bia_data_t == arm_compute::DataType::UNKNOWN)
-        acl_bia_data_t = arm_compute::DataType::F32;
+    auto acl_data_type = arm_compute::DataType::F32;
 
     // clang-format off
-    acp.src_info = arm_compute::TensorInfo(
-            is_nspc ? arm_compute::TensorShape(ic, iw, ih, mb) :
+    acp.src_tensor_info = arm_compute::TensorInfo(
+            is_nhwc ? arm_compute::TensorShape(ic, iw, ih, mb) :
             arm_compute::TensorShape(iw, ih, ic, mb),
             1,
-            acl_src_data_t,
+            acl_data_type,
             acl_layout);
 
-    acp.wei_info = arm_compute::TensorInfo(
-            is_nspc ? arm_compute::TensorShape(ic, kw, kh, oc) :
+    acp.wei_tensor_info = arm_compute::TensorInfo(
+            is_nhwc ? arm_compute::TensorShape(ic, kw, kh, oc) :
             arm_compute::TensorShape(kw, kh, ic, oc),
             1,
-            acl_wei_data_t,
+            acl_data_type,
             acl_layout);
 
-    acp.dst_info = arm_compute::TensorInfo(
-            is_nspc ? arm_compute::TensorShape(oc, ow, oh, mb) :
+    acp.dst_tensor_info = arm_compute::TensorInfo(
+            is_nhwc ? arm_compute::TensorShape(oc, ow, oh, mb) :
             arm_compute::TensorShape(ow, oh, oc, mb),
             1,
-            acl_dst_data_t,
+            acl_data_type,
             acl_layout);
 
-    acp.bia_info = arm_compute::TensorInfo(
+    acp.bia_tensor_info = arm_compute::TensorInfo(
             acp.with_bias ? arm_compute::TensorShape(oc)
                           : arm_compute::TensorShape(),
             1,
-            acl_bia_data_t,
+            acl_data_type,
             acl_layout);
     // clang-format on
 
-    // Add quantization info to tensors
-    if (acp.is_int8) {
-        const float *scales = attr.output_scales_.scales_;
-        acp.src_info.set_quantization_info(arm_compute::QuantizationInfo(1, 0));
-        acp.bia_info.set_quantization_info(arm_compute::QuantizationInfo(1, 0));
-        acp.wei_info.set_quantization_info(arm_compute::QuantizationInfo(1, 0));
-        acp.dst_info.set_quantization_info(
-                arm_compute::QuantizationInfo(1.0f / scales[0], 0));
+    // Are we allowed to cast down to bf16 or not?
+    acp.fast_math
+            = one_of(attr.fpmath_mode_, fpmath_mode::bf16, fpmath_mode::any);
+
+    // WeightFormat::ANY tells ACL we can handle any format
+    acp.weights_info = arm_compute::WeightsInfo(
+            false, kw, kh, oc, false, arm_compute::WeightFormat::ANY);
+
+    // Get the format that the ACL kernel will expect the weights to be
+    // in (if a kernel exists). Note that these are referred to as fixed format
+    // kernels, because they require one specific weights format
+    arm_compute::WeightFormat expected_weight_format;
+    ACL_CHECK_VALID(arm_compute::NEGEMMConvolutionLayer::has_opt_impl(
+            expected_weight_format, &acp.src_tensor_info, &acp.wei_tensor_info,
+            acp.with_bias ? &acp.bia_tensor_info : nullptr,
+            &acp.dst_tensor_info, acp.padstride_info, acp.weights_info,
+            acp.dilation_info, acp.act_info, acp.fast_math));
+
+    // Set weights info to the one returned by has_opt_impl
+    acp.weights_info.set_weight_format(expected_weight_format);
+
+    // has_opt_impl may return a non fast math kernel, even if we requested one
+    acp.fast_math
+            = arm_compute::is_fixed_format_fast_math(expected_weight_format);
+
+    // Map OIHW used in ACL WeightFormat to the logical dimensions of the memory descriptor
+    dim_t O_dim = 0;
+    dim_t I_dim = 1;
+    dim_t H_dim = 2;
+    dim_t W_dim = 3;
+
+    if (!is_nhwc) {
+        // We can try to support NCHW by swapping IHW around, note that this
+        // requires weights_md.dims[I_dim] % block_by != 0 (see next block)
+        O_dim = 0;
+        I_dim = 3;
+        H_dim = 1;
+        W_dim = 2;
     }
 
+    // We can't currently support nchw and block_by != 1. If this is the case,
+    // try a non fast math kernel, which currently have no blocking
+    int block_by = arm_compute::block_by(acp.weights_info.weight_format());
+    if (!is_nhwc && weights_md.dims[I_dim] % block_by != 0 && acp.fast_math) {
+        acp.fast_math = false;
+        acp.weights_info.set_weight_format(arm_compute::WeightFormat::ANY);
+        ACL_CHECK_VALID(arm_compute::NEGEMMConvolutionLayer::has_opt_impl(
+                expected_weight_format, &acp.src_tensor_info,
+                &acp.wei_tensor_info,
+                acp.with_bias ? &acp.bia_tensor_info : nullptr,
+                &acp.dst_tensor_info, acp.padstride_info, acp.weights_info,
+                acp.dilation_info, acp.act_info, acp.fast_math));
+        acp.weights_info.set_weight_format(expected_weight_format);
+        block_by = arm_compute::block_by(expected_weight_format);
+        // This shouldn't happen, because non-fastmath have no blocking, but
+        // guard against it because it would silently return incorrect results
+        if (weights_md.dims[I_dim] % block_by != 0)
+            return status::unimplemented;
+    }
+
+    acl_utils::reorder_to_weight_format(acp.wei_tensor_info, weights_md,
+            expected_weight_format, I_dim, O_dim, {W_dim, H_dim}, {});
+
     return status::success;
 }
 
@@ -226,10 +259,10 @@ status_t init_conf_gemm(acl_conv_conf_t &acp, memory_desc_t &src_md,
     // clang-format off
     // Validate convolution manually to check for return status
     ACL_CHECK_VALID(arm_compute::NEGEMMConvolutionLayer::validate(
-        &acp.src_info,
-        &acp.wei_info,
-        acp.with_bias ? &acp.bia_info : nullptr,
-        &acp.dst_info,
+        &acp.src_tensor_info,
+        &acp.wei_tensor_info,
+        acp.with_bias ? &acp.bia_tensor_info : nullptr,
+        &acp.dst_tensor_info,
         acp.padstride_info,
         acp.weights_info,
         acp.dilation_info,
@@ -244,28 +277,38 @@ status_t init_conf_indirect_gemm(acl_conv_conf_t &acp, memory_desc_t &src_md,
         memory_desc_t &weights_md, memory_desc_t &dst_md,
         memory_desc_t &bias_md, const convolution_desc_t &cd,
         const primitive_attr_t &attr) {
-    // Indirect convolution results in slowdown for low thread count or 1x1
-    // kernels, so fall back to GEMM-based convolution in these cases
-    if (one_of(true, weights_md.dims[2] == 1, // kh
-                weights_md.dims[3] == 1, // kw
-                dnnl_get_max_threads() < 28)) {
+
+    // Indirect is slower for small convolution kernels
+    if (weights_md.dims[2] == 1 && weights_md.dims[3] == 1)
         return status::unimplemented;
-    }
 
     CHECK(acl_init_conf(acp, src_md, weights_md, dst_md, bias_md, cd, attr));
 
+    // Indirect is slower than gemm for low thread counts, except for fast math
+    if (dnnl_get_max_threads() < 28 && !acp.fast_math)
+        return status::unimplemented;
+
+    // If we do not need to pad input channels for fast math mode then it would
+    // be faster to run convolution with im2row instead of using indirect kernel
+    int block_by = arm_compute::block_by(acp.weights_info.weight_format());
+    int ic = src_md.dims[1];
+    if (acp.fast_math && ic % block_by == 0) return status::unimplemented;
+
+    // TODO: remove this once NEGEMMConv2d::validate allows src and weights to mismatch
+    acp.wei_tensor_info.set_data_layout(arm_compute::DataLayout::NHWC);
+
     // clang-format off
     // NOTE: indirect convolution method supports only nhwc layout.
     ACL_CHECK_VALID(arm_compute::NEGEMMConv2d::validate(
-        &acp.src_info,
-        &acp.wei_info,
-        acp.with_bias ? &acp.bia_info : nullptr,
-        &acp.dst_info,
+        &acp.src_tensor_info,
+        &acp.wei_tensor_info,
+        acp.with_bias ? &acp.bia_tensor_info : nullptr,
+        &acp.dst_tensor_info,
         arm_compute::Conv2dInfo(acp.padstride_info,
                                 acp.dilation_info,
                                 acp.act_info,
                                 acp.fast_math,
-                                1)));
+                                1, {}, acp.weights_info)));
     // clang-format on
 
     return status::success;
diff --git a/src/cpu/aarch64/acl_convolution_utils.hpp b/src/cpu/aarch64/acl_convolution_utils.hpp
index 0398ab06b9..e3d40a5e75 100644
--- a/src/cpu/aarch64/acl_convolution_utils.hpp
+++ b/src/cpu/aarch64/acl_convolution_utils.hpp
@@ -38,17 +38,17 @@ struct acl_obj_t {
 
 struct acl_conv_conf_t {
     bool with_bias;
-    bool is_int8;
     bool fast_math;
     // If this is true, the result of the convolution goes into a temporarily
     // allocated ACL tensor to be accumulated into the oneDNN dst during postops
     bool use_dst_acc;
-    arm_compute::TensorInfo src_info;
-    arm_compute::TensorInfo wei_info;
-    arm_compute::TensorInfo bia_info;
-    arm_compute::TensorInfo dst_info;
+    arm_compute::TensorInfo src_tensor_info;
+    arm_compute::TensorInfo wei_tensor_info;
+    arm_compute::TensorInfo bia_tensor_info;
+    arm_compute::TensorInfo dst_tensor_info;
     arm_compute::PadStrideInfo padstride_info;
     arm_compute::Size2D dilation_info;
+    // Additional information about the weights not included in wei_tensor_info
     arm_compute::WeightsInfo weights_info;
     // Note: this will default to not enabled, and will do nothing
     arm_compute::ActivationLayerInfo act_info;
diff --git a/src/cpu/aarch64/acl_gemm_convolution.hpp b/src/cpu/aarch64/acl_gemm_convolution.hpp
index 485db954ea..da58e4f610 100644
--- a/src/cpu/aarch64/acl_gemm_convolution.hpp
+++ b/src/cpu/aarch64/acl_gemm_convolution.hpp
@@ -1,5 +1,5 @@
 /*******************************************************************************
-* Copyright 2020-2022 Arm Ltd. and affiliates
+* Copyright 2020-2023 Arm Ltd. and affiliates
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
@@ -36,10 +36,10 @@ struct acl_resource_t : public resource_t {
         if (!acl_obj_) return status::out_of_memory;
 
         // Init Compute Library tensors based on info from descriptor
-        acl_obj_->src_tensor.allocator()->init(acp.src_info);
-        acl_obj_->wei_tensor.allocator()->init(acp.wei_info);
-        acl_obj_->dst_tensor.allocator()->init(acp.dst_info);
-        acl_obj_->bia_tensor.allocator()->init(acp.bia_info);
+        acl_obj_->src_tensor.allocator()->init(acp.src_tensor_info);
+        acl_obj_->wei_tensor.allocator()->init(acp.wei_tensor_info);
+        acl_obj_->dst_tensor.allocator()->init(acp.dst_tensor_info);
+        acl_obj_->bia_tensor.allocator()->init(acp.bia_tensor_info);
 
         acl_obj_->conv.configure(&acl_obj_->src_tensor, &acl_obj_->wei_tensor,
                 acp.with_bias ? &acl_obj_->bia_tensor : nullptr,
diff --git a/src/cpu/aarch64/acl_indirect_gemm_convolution.hpp b/src/cpu/aarch64/acl_indirect_gemm_convolution.hpp
index bcf031a771..b7c8dce894 100644
--- a/src/cpu/aarch64/acl_indirect_gemm_convolution.hpp
+++ b/src/cpu/aarch64/acl_indirect_gemm_convolution.hpp
@@ -1,5 +1,5 @@
 /*******************************************************************************
-* Copyright 2021-2022 Arm Ltd. and affiliates
+* Copyright 2021-2023 Arm Ltd. and affiliates
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
@@ -35,10 +35,10 @@ struct acl_indirect_gemm_resource_t : public resource_t {
         if (!acl_obj_) return status::out_of_memory;
 
         // Init Compute Library tensors based on info from descriptor
-        acl_obj_->src_tensor.allocator()->init(acp.src_info);
-        acl_obj_->wei_tensor.allocator()->init(acp.wei_info);
-        acl_obj_->dst_tensor.allocator()->init(acp.dst_info);
-        acl_obj_->bia_tensor.allocator()->init(acp.bia_info);
+        acl_obj_->src_tensor.allocator()->init(acp.src_tensor_info);
+        acl_obj_->wei_tensor.allocator()->init(acp.wei_tensor_info);
+        acl_obj_->dst_tensor.allocator()->init(acp.dst_tensor_info);
+        acl_obj_->bia_tensor.allocator()->init(acp.bia_tensor_info);
 
         // clang-format off
         acl_obj_->conv.configure(
@@ -50,7 +50,9 @@ struct acl_indirect_gemm_resource_t : public resource_t {
                                     acp.dilation_info,
                                     acp.act_info,
                                     acp.fast_math,
-                                    1));
+                                    1,
+                                    {},
+                                    acp.weights_info));
         // clang-format on
 
         return status::success;
diff --git a/src/cpu/aarch64/acl_inner_product.hpp b/src/cpu/aarch64/acl_inner_product.hpp
index c5e507085f..a27df640fb 100644
--- a/src/cpu/aarch64/acl_inner_product.hpp
+++ b/src/cpu/aarch64/acl_inner_product.hpp
@@ -40,11 +40,13 @@ struct acl_ip_conf_t {
     // If this is true, the result of the inner product goes into a temporarily
     // allocated ACL tensor to be accumulated into the oneDNN dst during postops
     bool use_dst_acc;
-    arm_compute::TensorInfo src_info;
-    arm_compute::TensorInfo wei_info;
-    arm_compute::TensorInfo bia_info;
-    arm_compute::TensorInfo dst_info;
+    arm_compute::TensorInfo src_tensor_info;
+    arm_compute::TensorInfo wei_tensor_info;
+    arm_compute::TensorInfo bia_tensor_info;
+    arm_compute::TensorInfo dst_tensor_info;
     arm_compute::FullyConnectedLayerInfo fc_info;
+    // Additional information about the weights not included in wei_tensor_info
+    arm_compute::WeightsInfo weights_info;
 };
 struct acl_ip_resource_t : public resource_t {
     acl_ip_resource_t() : acl_ip_obj_(utils::make_unique<acl_ip_obj_t>()) {}
@@ -53,10 +55,10 @@ struct acl_ip_resource_t : public resource_t {
         if (!acl_ip_obj_) return status::out_of_memory;
 
         // Init Compute Library tensors based on info from descriptor
-        acl_ip_obj_->src_tensor.allocator()->init(aip.src_info);
-        acl_ip_obj_->wei_tensor.allocator()->init(aip.wei_info);
-        acl_ip_obj_->dst_tensor.allocator()->init(aip.dst_info);
-        acl_ip_obj_->bia_tensor.allocator()->init(aip.bia_info);
+        acl_ip_obj_->src_tensor.allocator()->init(aip.src_tensor_info);
+        acl_ip_obj_->wei_tensor.allocator()->init(aip.wei_tensor_info);
+        acl_ip_obj_->dst_tensor.allocator()->init(aip.dst_tensor_info);
+        acl_ip_obj_->bia_tensor.allocator()->init(aip.bia_tensor_info);
 
         // clang-format off
         acl_ip_obj_->fc.configure(
@@ -64,7 +66,8 @@ struct acl_ip_resource_t : public resource_t {
             &acl_ip_obj_->wei_tensor,
             aip.with_bias ? &acl_ip_obj_->bia_tensor : nullptr,
             &acl_ip_obj_->dst_tensor,
-            aip.fc_info);
+            aip.fc_info,
+            aip.weights_info);
         // clang-format on
 
         return status::success;
@@ -89,12 +92,16 @@ struct acl_inner_product_fwd_t : public primitive_t {
         DECLARE_COMMON_PD_T("acl", acl_inner_product_fwd_t);
 
         status_t init(engine_t *engine) {
-            const bool ok = is_fwd() && !has_zero_dim_memory()
-                    && expect_data_types(data_type::f32, data_type::f32,
-                            data_type::f32, data_type::f32, data_type::f32)
+            using namespace data_type;
+            const bool is_fp16_ok = expect_data_types(f16, f16, f16, f16, undef)
+                && attr()->has_default_values(
+                        primitive_attr_t::skip_mask_t::post_ops, f16);
+            const bool is_fp32_ok = expect_data_types(f32, f32, f32, f32, undef)
                     && attr()->has_default_values(
-                            primitive_attr_t::skip_mask_t::post_ops,
-                            data_type::f32)
+                            primitive_attr_t::skip_mask_t::post_ops, f32);
+            const bool ok = is_fwd() && !has_zero_dim_memory()
+                    && utils::one_of(true, is_fp16_ok, is_fp32_ok)
+                    && weights_md_.format_kind == format_kind::any
                     && set_default_params() == status::success;
 
             if (!ok) return status::unimplemented;
@@ -121,88 +128,46 @@ struct acl_inner_product_fwd_t : public primitive_t {
             ACL_CHECK_SUPPORT(
                     !(is_2d || is_4d), "ACL supports only 2d or 4d cases");
 
-            // batch size
-            const int n = src_md()->dims[0];
-
-            // input and output channels
-            const int ic = src_md()->dims[1];
-            const int oc = dst_md()->dims[1];
-
-            // source spatial dimensions
-            const int ih = is_4d ? src_md()->dims[ndims - 2] : 0;
-            const int iw = is_4d ? src_md()->dims[ndims - 1] : 0;
-
-            // weights spatial dimensions
-            const int kh = is_4d ? weights_md()->dims[ndims - 2] : 0;
-            const int kw = is_4d ? weights_md()->dims[ndims - 1] : 0;
-
-            // Only NCHW or NHWC derivatives supported by ACL kernels
             using namespace format_tag;
-            auto src_tag = memory_desc_matches_one_of_tag(
-                    src_md_, nhwc, nchw, nc, cn);
-            auto wei_tag = memory_desc_matches_one_of_tag(
-                    weights_md_, ohwi, oihw, oi, io);
-            auto dst_tag = memory_desc_matches_one_of_tag(dst_md_, nc, cn);
+            auto src_tag
+                    = memory_desc_matches_one_of_tag(src_md_, nhwc, nchw, nc);
+            auto dst_tag = memory_desc_matches_one_of_tag(dst_md_, nc);
 
             ACL_CHECK_SUPPORT(
-                    utils::one_of(format_tag::undef, src_tag, wei_tag, dst_tag),
+                    utils::one_of(format_tag::undef, src_tag, dst_tag),
                     "unsupported memory layout");
 
             ACL_CHECK_SUPPORT(is_2d && src_tag != dst_tag,
                     "for src and dst layouts must match");
 
-            arm_compute::TensorShape src_shape, wei_shape;
-            if (is_2d) {
-                src_shape = (src_tag == nc) ? arm_compute::TensorShape(ic, n)
-                                            : arm_compute::TensorShape(n, ic);
-
-                wei_shape = (wei_tag == io) ? arm_compute::TensorShape(oc, ic)
-                                            : arm_compute::TensorShape(ic, oc);
-            }
-            if (is_4d) {
-                src_shape = (src_tag == nhwc)
-                        ? arm_compute::TensorShape(ic, iw, ih, n)
-                        : arm_compute::TensorShape(iw, ih, ic, n);
-
-                // ACL requires the weights to be in 2D flattened shape
-                const int flattened_ic = is_4d ? ic * kh * kw : ic;
-                wei_shape = arm_compute::TensorShape(flattened_ic, oc);
-            }
-
-            arm_compute::DataLayout src_layout = (src_tag == nhwc)
-                    ? arm_compute::DataLayout::NHWC
-                    : arm_compute::DataLayout::NCHW;
+            const dim_t ic_total = IC_total();
+            const dim_t n = MB();
+            const dim_t oc = OC();
 
-            arm_compute::DataLayout wei_layout = (wei_tag == ohwi)
-                    ? arm_compute::DataLayout::NHWC
-                    : arm_compute::DataLayout::NCHW;
+            aip.src_tensor_info = arm_compute::TensorInfo(
+                    arm_compute::TensorShape(ic_total, n), 1,
+                    acl_utils::get_acl_data_t(src_md()->data_type));
 
-            aip.src_info = arm_compute::TensorInfo(
-                    src_shape, 1, arm_compute::DataType::F32, src_layout);
+            // ACL requires the weights to be in 2D flattened shape
+            aip.wei_tensor_info = arm_compute::TensorInfo(
+                    arm_compute::TensorShape(oc, ic_total), 1,
+                    acl_utils::get_acl_data_t(weights_md(0)->data_type));
 
-            aip.wei_info = arm_compute::TensorInfo(
-                    wei_shape, 1, arm_compute::DataType::F32, wei_layout);
-
-            aip.dst_info
-                    = arm_compute::TensorInfo(arm_compute::TensorShape(oc, n),
-                            1, arm_compute::DataType::F32);
+            auto acl_dst_data_t
+                    = acl_utils::get_acl_data_t(dst_md()->data_type);
+            aip.dst_tensor_info = arm_compute::TensorInfo(
+                    arm_compute::TensorShape(oc, n), 1, acl_dst_data_t);
 
             aip.with_bias = desc()->bias_desc.format_kind != format_kind::undef;
-            aip.bia_info = arm_compute::TensorInfo(aip.with_bias
+            auto acl_bia_data_t = aip.with_bias
+                    ? acl_utils::get_acl_data_t(weights_md(1)->data_type)
+                    : acl_dst_data_t;
+            aip.bia_tensor_info = arm_compute::TensorInfo(aip.with_bias
                             ? arm_compute::TensorShape(oc)
                             : arm_compute::TensorShape(),
                     1, arm_compute::DataType::F32);
 
-            aip.fc_info.weights_trained_layout = wei_layout;
-            if (is_2d && wei_tag != src_tag) {
-                // weights are already transposed
-                aip.fc_info.transpose_weights = false;
-
-                if (desc()->prop_kind == dnnl_forward_training) {
-                    aip.wei_info.set_are_values_constant(false);
-                    aip.fc_info.are_weights_reshaped = true;
-                }
-            }
+            aip.fc_info.transpose_weights = false;
 
             // Fast math mode
             auto math_mode = get_fpmath_mode();
@@ -214,15 +179,103 @@ struct acl_inner_product_fwd_t : public primitive_t {
                     aip.fc_info.activation_info));
             aip.use_dst_acc = post_ops.has_sum();
 
+            // WeightFormat::ANY tells ACL we can handle any format
+            aip.weights_info = arm_compute::WeightsInfo(false, 1, 1, ic_total,
+                    false, arm_compute::WeightFormat::ANY);
+
+            // Get the format that the ACL kernel will expect the weights to be
+            // in (if a kernel exists) Note that these are referred to as fixed
+            // format kernels, because they require one specific weights format
+            arm_compute::WeightFormat expected_weight_format;
+            ACL_CHECK_VALID(arm_compute::NEFullyConnectedLayer::has_opt_impl(
+                    expected_weight_format, &aip.src_tensor_info,
+                    &aip.wei_tensor_info,
+                    aip.with_bias ? &aip.bia_tensor_info : nullptr,
+                    &aip.dst_tensor_info, aip.fc_info, aip.weights_info));
+
+            // Set weights info to the one returned by has_opt_impl
+            aip.weights_info.set_weight_format(expected_weight_format);
+
+            // has_opt_impl may return a non fast math kernel, even if requested
+            aip.fc_info.enable_fast_math
+                    = arm_compute::is_fixed_format_fast_math(
+                            expected_weight_format);
+
+            // Inner product is the same as the matmul n x (chw) * (ihw) x o
+            // (note that the src c and weights i both correspond to the input
+            // channel). ACL FullyConnectedLayer assumes the chw dimensions of
+            // src and ihw dimensions of weights are collapsed, so we need to
+            // make sure that they have the same layout. Given that weights are
+            // more often fixed, (so reorders can be hoisted) it makes sense to
+            // reorder the weights to fit the src.
+
+            // For 4D tensors we need to:
+            // - reorder the ihw of the weights to match the src chw
+            // - collapse ihw
+            // - pad the collapsed ihw
+            // But there is not yet a way to express this collapse+pad as a
+            // reorder. So we try to reorder the weights to match the src,
+            // implicitly collapse ihw in our definition of the weights
+            // TensorInfo and hope that the inner_dim has zero padding
+            // (weights_md_.dims[inner_dim] % block_by == 0). If it does, we
+            // fall back to a kernel without blocking (currently this is
+            // equivalent to non-fastmath).
+
+            // 2D just works because we just pad the only dimension.
+
+            // o_dim is always the first logical dimension (oihw, ohwi, oi)
+            dim_t o_dim = 0;
+            dim_t inner_dim;
+            // Rest of logical dimensions in order of innermost to outermost
+            std::vector<dim_t> remaining_dims = {};
+
+            if (src_tag == nchw) {
+                inner_dim = 3; // w
+                remaining_dims = {2, 1}; // h, i
+            } else if (src_tag == nhwc) {
+                inner_dim = 1; // i
+                remaining_dims = {3, 2}; // w, h
+            } else { // Only remaining case is 2D (nc)
+                inner_dim = 1; // i
+                remaining_dims = {}; // No other dimensions for 2D
+            }
+
+            // Fallback
+            int block_by = arm_compute::block_by(expected_weight_format);
+            if (is_4d && weights_md_.dims[inner_dim] % block_by != 0
+                    && aip.fc_info.enable_fast_math) {
+                aip.fc_info.enable_fast_math = false;
+                aip.weights_info.set_weight_format(
+                        arm_compute::WeightFormat::ANY);
+                ACL_CHECK_VALID(
+                        arm_compute::NEFullyConnectedLayer::has_opt_impl(
+                                expected_weight_format, &aip.src_tensor_info,
+                                &aip.wei_tensor_info,
+                                aip.with_bias ? &aip.bia_tensor_info : nullptr,
+                                &aip.dst_tensor_info, aip.fc_info,
+                                aip.weights_info));
+                aip.weights_info.set_weight_format(expected_weight_format);
+                block_by = arm_compute::block_by(expected_weight_format);
+                if (weights_md_.dims[inner_dim] % block_by != 0)
+                    return status::unimplemented;
+            }
+
+            acl_utils::reorder_to_weight_format(aip.wei_tensor_info,
+                    weights_md_, expected_weight_format, inner_dim, o_dim,
+                    remaining_dims, {});
+
             // clang-format off
+
             // Validate fully connected layer manually to check for return status
             ACL_CHECK_VALID(arm_compute::NEFullyConnectedLayer::validate(
-                &aip.src_info,
-                &aip.wei_info,
-                aip.with_bias ? &aip.bia_info : nullptr,
-                &aip.dst_info,
-                aip.fc_info));
+                &aip.src_tensor_info,
+                &aip.wei_tensor_info,
+                aip.with_bias ? &aip.bia_tensor_info : nullptr,
+                &aip.dst_tensor_info,
+                aip.fc_info,
+                aip.weights_info));
             // clang-format on
+
             return status::success;
         }
     }; // pd_t
diff --git a/src/cpu/aarch64/acl_utils.cpp b/src/cpu/aarch64/acl_utils.cpp
index 79ea775d6d..5792fd4911 100644
--- a/src/cpu/aarch64/acl_utils.cpp
+++ b/src/cpu/aarch64/acl_utils.cpp
@@ -1,5 +1,5 @@
 /*******************************************************************************
-* Copyright 2021-2022 Arm Ltd. and affiliates
+* Copyright 2021-2023 Arm Ltd. and affiliates
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
@@ -261,6 +261,75 @@ int reorder_dimensions_by_stride(std::vector<memory_desc_t *> permuted_mds,
     return reordered_dims;
 }
 
+void reorder_to_weight_format(arm_compute::TensorInfo &info, memory_desc_t &md,
+        arm_compute::WeightFormat wf, dim_t I_dim, dim_t O_dim,
+        std::vector<dim_t> spatial_dims, std::vector<dim_t> batch_dims) {
+
+    md.format_kind = format_kind::blocked;
+    md.format_desc.blocking = blocking_desc_t {};
+    const int interleaved_by = arm_compute::interleave_by(wf);
+    const int block_by = arm_compute::block_by(wf);
+
+    // I dimension becomes densest (apart from blocking)
+    md.format_desc.blocking.strides[I_dim] = interleaved_by * block_by;
+    md.padded_dims[I_dim] = utils::rnd_up(md.dims[I_dim], block_by);
+
+    // Then any spatial dimensions (e.g. HW)
+    dim_t ldb = interleaved_by * md.padded_dims[I_dim];
+    for (dim_t sd : spatial_dims) {
+        md.format_desc.blocking.strides[sd] = ldb;
+        ldb *= md.padded_dims[sd];
+    }
+
+    // O dim (which was the innermost) becomes the outermost (apart from batching)
+    md.format_desc.blocking.strides[O_dim] = ldb;
+    md.padded_dims[O_dim] = utils::rnd_up(md.dims[O_dim], interleaved_by);
+
+    // Update the batch dimensions, starting with stride of the innermost batch
+    const dim_t innermost_batch_stride
+            = md.padded_dims[I_dim] * md.padded_dims[O_dim];
+    dim_t batch_stride = innermost_batch_stride;
+    for (dim_t bd : batch_dims) {
+        md.format_desc.blocking.strides[bd] = batch_stride;
+        batch_stride *= md.padded_dims[bd];
+    }
+
+    // Weights can only be blocked if they are also interleaved
+    if (interleaved_by > 1) {
+        md.format_desc.blocking.inner_nblks = 1 + (block_by > 1);
+
+        md.format_desc.blocking.inner_idxs[0] = O_dim;
+        md.format_desc.blocking.inner_blks[0] = interleaved_by;
+        if (block_by > 1) {
+            md.format_desc.blocking.inner_idxs[1] = I_dim;
+            md.format_desc.blocking.inner_blks[1] = block_by;
+        }
+    }
+
+    if (arm_compute::is_fixed_format_fast_math(wf)) {
+        md.data_type = dnnl_bf16;
+        info.set_data_type(arm_compute::DataType::BFLOAT16);
+    }
+
+    // The data layout is now determined by the manually set strides
+    info.set_data_layout(arm_compute::DataLayout::UNKNOWN);
+
+    // x is ignored in fixed format kernels
+    // y is the leading dimension of b (ldb) in the GEMM d = a*b + c
+    //   This is the stride of O_dim in the md
+    // z is the batch dimension (not strictly needed if there's only 1 batch)
+    //   i.e. how much do I need to stride to get to the next matmul (ignoring
+    //   the interleaving). Note that we use the innermost_batch_stride
+    //   because all the batched dimensions are collapsed (as required by ACL).
+    arm_compute::Strides new_strides_in_bytes = info.strides_in_bytes();
+    new_strides_in_bytes.set(1, ldb * info.element_size());
+    new_strides_in_bytes.set(2, innermost_batch_stride * info.element_size());
+
+    info.init(info.tensor_shape(), info.num_channels(), info.data_type(),
+            new_strides_in_bytes, info.offset_first_element_in_bytes(),
+            memory_desc_wrapper(md).size());
+}
+
 } // namespace acl_utils
 
 } // namespace aarch64
diff --git a/src/cpu/aarch64/acl_utils.hpp b/src/cpu/aarch64/acl_utils.hpp
index 28693bb167..d9affe1c8f 100644
--- a/src/cpu/aarch64/acl_utils.hpp
+++ b/src/cpu/aarch64/acl_utils.hpp
@@ -1,5 +1,5 @@
 /*******************************************************************************
-* Copyright 2021-2022 Arm Ltd. and affiliates
+* Copyright 2021-2023 Arm Ltd. and affiliates
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
@@ -74,6 +74,28 @@ status_t insert_singleton_dimension(arm_compute::TensorInfo &ti, size_t dim_i);
 int reorder_dimensions_by_stride(std::vector<memory_desc_t *> permuted_mds,
         std::vector<const memory_desc_t *> mds);
 
+// Reorder a memory_desc_t and set the strides on a arm_compute::TensorInfo to
+// match an arm_compute::WeightFormat. You are required to specify how various
+// logical dimensions in oneDNN correspond to logical dimensions in arm_compute.
+// info  TensorInfo where the strides will be changed to match the reordering
+// md    memory descriptor where the stride and padded dimensions will be
+//       changed or reordering
+// wf    Describes the memory format/layout of the weights
+// I_dim The logical dimension of md corresponding to the input channel of
+//       a convolution or the K dimension in a matmul
+// O_dim The logical dimension of md corresponding to the output channel of a
+//       convolution or the N dimension in a matmul
+// spatial_dims The logical dimensions of md corresponding to the spatial
+//              dimensions of the weights (H, W, D for example). These will be
+//              the next densest after the inner blocks and the input channel.
+// batch_dims The logical dimensions of md related to the batch in a batched
+//            matmul, ordered from innermost to outermost. ACL calls these
+//            the multi_stride_b. These will become the outermost (least dense)
+//            dimensions and will be collapsed.
+void reorder_to_weight_format(arm_compute::TensorInfo &info, memory_desc_t &md,
+        arm_compute::WeightFormat wf, dim_t I_dim, dim_t O_dim,
+        std::vector<dim_t> spatial_dims, std::vector<dim_t> batch_dims = {});
+
 // Logs a custom 'info' line describing an unsupported case
 #define LOG_ACL_UNSUPPORTED(msg) \
     do { \
diff --git a/src/cpu/aarch64/matmul/acl_matmul.cpp b/src/cpu/aarch64/matmul/acl_matmul.cpp
index dce220fb6e..ca1c7eb47e 100644
--- a/src/cpu/aarch64/matmul/acl_matmul.cpp
+++ b/src/cpu/aarch64/matmul/acl_matmul.cpp
@@ -1,5 +1,5 @@
 /*******************************************************************************
-* Copyright 2021-2022 Arm Ltd. and affiliates
+* Copyright 2021-2023 Arm Ltd. and affiliates
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
@@ -31,36 +31,19 @@ status_t acl_matmul_t::execute_forward(const exec_ctx_t &ctx) const {
     auto wei_base = CTX_IN_MEM(const data_t *, DNNL_ARG_WEIGHTS);
 
     bool is_transA = pd()->amp_.is_transA;
-    bool is_transB = pd()->amp_.is_transB;
     bool use_dst_acc = pd()->amp_.use_dst_acc;
 
     std::lock_guard<std::mutex> _lock {this->mtx};
     auto *acl_resource = ctx.get_resource_mapper()->get<acl_resource_t>(this);
     acl_matmul_obj_t &acl_obj = acl_resource->get_acl_obj();
     // Run transpose kernel
-    if (is_transA && !is_transB) {
+    if (is_transA) {
         acl_obj.src_tensor.allocator()->allocate();
         acl_obj.src_acc_tensor.allocator()->import_memory(
                 const_cast<data_t *>(src_base));
         acl_obj.transA.run();
         acl_obj.wei_tensor.allocator()->import_memory(
                 const_cast<data_t *>(wei_base));
-    } else if (is_transB && !is_transA) {
-        acl_obj.wei_tensor.allocator()->allocate();
-        acl_obj.wei_acc_tensor.allocator()->import_memory(
-                const_cast<data_t *>(wei_base));
-        acl_obj.transB.run();
-        acl_obj.src_tensor.allocator()->import_memory(
-                const_cast<data_t *>(src_base));
-    } else if (is_transA && is_transB) {
-        acl_obj.src_tensor.allocator()->allocate();
-        acl_obj.src_acc_tensor.allocator()->import_memory(
-                const_cast<data_t *>(src_base));
-        acl_obj.wei_tensor.allocator()->allocate();
-        acl_obj.wei_acc_tensor.allocator()->import_memory(
-                const_cast<data_t *>(wei_base));
-        acl_obj.transA.run();
-        acl_obj.transB.run();
     } else {
         acl_obj.src_tensor.allocator()->import_memory(
                 const_cast<data_t *>(src_base));
@@ -69,7 +52,7 @@ status_t acl_matmul_t::execute_forward(const exec_ctx_t &ctx) const {
     }
 
     if (use_dst_acc) {
-        // Put the result in a new tensor, it will be accumalated to the dst
+        // Put the result in a new tensor, it will be accumulated to the dst
         // during the post ops
         acl_obj.dst_tensor.allocator()->allocate();
     } else {
@@ -82,7 +65,6 @@ status_t acl_matmul_t::execute_forward(const exec_ctx_t &ctx) const {
     acl_obj.src_tensor.allocator()->free();
     acl_obj.wei_tensor.allocator()->free();
     if (is_transA) acl_obj.src_acc_tensor.allocator()->free();
-    if (is_transB) acl_obj.wei_acc_tensor.allocator()->free();
 
     void *dst = acl_obj.dst_tensor.buffer();
     pd()->post_ops.execute(ctx, dst);
diff --git a/src/cpu/aarch64/matmul/acl_matmul.hpp b/src/cpu/aarch64/matmul/acl_matmul.hpp
index cdc942e995..832b1dbb68 100644
--- a/src/cpu/aarch64/matmul/acl_matmul.hpp
+++ b/src/cpu/aarch64/matmul/acl_matmul.hpp
@@ -32,20 +32,15 @@ struct acl_resource_t : public resource_t {
 
     status_t configure(const acl_matmul_conf_t &amp) {
         if (!acl_obj_) return status::out_of_memory;
-        acl_obj_->src_tensor.allocator()->init(amp.src_info);
-        acl_obj_->wei_tensor.allocator()->init(amp.wei_info);
-        acl_obj_->dst_tensor.allocator()->init(amp.dst_info);
+        acl_obj_->src_tensor.allocator()->init(amp.src_tensor_info);
+        acl_obj_->wei_tensor.allocator()->init(amp.wei_tensor_info);
+        acl_obj_->dst_tensor.allocator()->init(amp.dst_tensor_info);
         // Configure transpose kernel for src, wei or both
         if (amp.is_transA) {
             acl_obj_->src_acc_tensor.allocator()->init(amp.src_acc_info);
             acl_obj_->transA.configure(
                     &acl_obj_->src_acc_tensor, &acl_obj_->src_tensor);
         }
-        if (amp.is_transB) {
-            acl_obj_->wei_acc_tensor.allocator()->init(amp.wei_acc_info);
-            acl_obj_->transB.configure(
-                    &acl_obj_->wei_acc_tensor, &acl_obj_->wei_tensor);
-        }
         // Configure GEMM
         acl_obj_->gemm.configure(&acl_obj_->src_tensor, &acl_obj_->wei_tensor,
                 nullptr, &acl_obj_->dst_tensor, amp.alpha, 0.0f, amp.gemm_info);
@@ -72,12 +67,20 @@ struct acl_matmul_t : public primitive_t {
 
         status_t init(engine_t *engine) {
             using smask_t = primitive_attr_t::skip_mask_t;
-            bool ok = src_md()->data_type == data_type::f32
-                    && weights_md()->data_type == data_type::f32
-                    && desc()->accum_data_type == data_type::f32
-                    && dst_md()->data_type == data_type::f32
-                    && platform::has_data_type_support(data_type::f32)
+            const bool is_fp32_ok
+                    = utils::everyone_is(data_type::f32, src_md()->data_type,
+                              weights_md()->data_type, dst_md()->data_type,
+                              desc()->accum_data_type)
+                    && platform::has_data_type_support(data_type::f32);
+            const bool is_fp16_ok
+                    = utils::everyone_is(data_type::f16, src_md()->data_type,
+                              weights_md()->data_type, dst_md()->data_type)
+                    && platform::has_data_type_support(data_type::f16);
+            bool ok = is_dense_data()
+                    && utils::one_of(true, is_fp32_ok, is_fp16_ok)
                     && !has_zero_dim_memory()
+                    && weights_md_.format_kind == format_kind::any
+                    && set_default_formats()
                     && attr()->has_default_values(
                             smask_t::oscale | smask_t::post_ops)
                     && attr_oscale_ok() && !has_runtime_dims_or_strides();
@@ -92,9 +95,9 @@ struct acl_matmul_t : public primitive_t {
             amp_.use_dst_acc = post_ops.has_sum();
 
             // Validate ACL GEMM
-            ACL_CHECK_VALID(arm_compute::NEGEMM::validate(&amp_.src_info,
-                    &amp_.wei_info, nullptr, &amp_.dst_info, amp_.alpha, 0.0f,
-                    amp_.gemm_info));
+            ACL_CHECK_VALID(arm_compute::NEGEMM::validate(&amp_.src_tensor_info,
+                    &amp_.wei_tensor_info, nullptr, &amp_.dst_tensor_info,
+                    amp_.alpha, 0.0f, amp_.gemm_info));
 
             return status::success;
         }
diff --git a/src/cpu/aarch64/matmul/acl_matmul_utils.cpp b/src/cpu/aarch64/matmul/acl_matmul_utils.cpp
index 679baec3a4..30bc2c1443 100644
--- a/src/cpu/aarch64/matmul/acl_matmul_utils.cpp
+++ b/src/cpu/aarch64/matmul/acl_matmul_utils.cpp
@@ -41,6 +41,7 @@ status_t init_conf_matmul(acl_matmul_conf_t &amp, memory_desc_t &src_md,
     const dim_t src_batch = helper.src_batch();
     const dim_t wei_batch = helper.wei_batch();
 
+    // We can only broadcast on one of src or wei at once
     // ACL supports broadcast for 3D shapes, and 4D shapes
     // for e.g when ab in abcd is 1x1
     bool batch_ok = IMPLICATION(src_batch > 1, wei_batch == 1)
@@ -53,44 +54,33 @@ status_t init_conf_matmul(acl_matmul_conf_t &amp, memory_desc_t &src_md,
     bool with_bias = md.bias_desc.format_kind != format_kind::undef;
     ACL_CHECK_SUPPORT(with_bias, "ACL does not support bias for matmul");
 
+    // The two innermost dimensions can be transposed, but the batch dimensions
+    // must be the outermost
     using namespace format_tag;
     auto src_tag = memory_desc_matches_one_of_tag(
             src_md, abcd, abdc, abc, acb, ab, ba);
-    auto wei_tag = memory_desc_matches_one_of_tag(
-            wei_md, abcd, abdc, abc, acb, ab, ba);
-    auto dst_tag
-            = memory_desc_matches_one_of_tag(dst_md, abcd, abc, acb, ab, ba);
-    ACL_CHECK_SUPPORT(
-            utils::one_of(format_tag::undef, src_tag, wei_tag, dst_tag),
+    auto dst_tag = memory_desc_matches_one_of_tag(dst_md, abcd, abc, ab, ba);
+    ACL_CHECK_SUPPORT(utils::one_of(format_tag::undef, src_tag, dst_tag),
             "Format tag is undefined");
 
-    // Transpose A (src) or B (wei)
+    // Transpose A (src)
     amp.is_transA = helper.transA() == 'T';
-    amp.is_transB = helper.transB() == 'T';
+
+    auto acl_src_data_t = acl_utils::get_acl_data_t(src_md.data_type);
+    auto acl_wei_data_t = acl_utils::get_acl_data_t(wei_md.data_type);
+    auto acl_dst_data_t = acl_utils::get_acl_data_t(dst_md.data_type);
+
     if (amp.is_transA)
         amp.src_acc_info = arm_compute::TensorInfo(
                 arm_compute::TensorShape(M, K, 1, src_batch), 1,
-                arm_compute::DataType::F32);
-    if (amp.is_transB)
-        amp.wei_acc_info = arm_compute::TensorInfo(
-                arm_compute::TensorShape(K, N, wei_batch), 1,
-                arm_compute::DataType::F32);
-
-    amp.src_info = arm_compute::TensorInfo(
-            arm_compute::TensorShape(K, M, 1, src_batch), 1,
-            arm_compute::DataType::F32);
-    amp.wei_info
-            = arm_compute::TensorInfo(arm_compute::TensorShape(N, K, wei_batch),
-                    1, arm_compute::DataType::F32);
-    amp.dst_info = arm_compute::TensorInfo(
-            arm_compute::TensorShape(N, M, 1, dst_batch), 1,
-            arm_compute::DataType::F32);
-
-    // Fast-math mode
-    auto math_mode = get_fpmath_mode();
-    bool is_fastmath_enabled
-            = utils::one_of(math_mode, fpmath_mode::bf16, fpmath_mode::any);
-    amp.gemm_info.set_fast_math(is_fastmath_enabled);
+                acl_src_data_t);
+
+    amp.src_tensor_info = arm_compute::TensorInfo(
+            arm_compute::TensorShape(K, M, 1, src_batch), 1, acl_src_data_t);
+    amp.wei_tensor_info = arm_compute::TensorInfo(
+            arm_compute::TensorShape(N, K, wei_batch), 1, acl_wei_data_t);
+    amp.dst_tensor_info = arm_compute::TensorInfo(
+            arm_compute::TensorShape(N, M, 1, dst_batch), 1, acl_dst_data_t);
 
     // Set alpha (output scaling)
     amp.alpha = attr.output_scales_.scales_[0];
@@ -98,10 +88,45 @@ status_t init_conf_matmul(acl_matmul_conf_t &amp, memory_desc_t &src_md,
     // Validate ACL transpose
     if (amp.is_transA)
         ACL_CHECK_VALID(arm_compute::NETranspose::validate(
-                &amp.src_acc_info, &amp.src_info));
-    if (amp.is_transB)
-        ACL_CHECK_VALID(arm_compute::NETranspose::validate(
-                &amp.wei_acc_info, &amp.wei_info));
+                &amp.src_acc_info, &amp.src_tensor_info));
+
+    bool is_fastmath_enabled = utils::one_of(
+            attr.fpmath_mode_, fpmath_mode::bf16, fpmath_mode::any);
+    amp.gemm_info.set_fast_math(is_fastmath_enabled);
+
+    amp.gemm_info.set_fixed_format(true);
+
+    // WeightFormat::ANY tells ACL we can handle any format
+    amp.gemm_info.set_weight_format(arm_compute::WeightFormat::ANY);
+
+    // Get the format that the ACL kernel will expect the weights to be
+    // in (if a kernel exists). Note that these are referred to as fixed format
+    // kernels, because they require one specific weights format
+    arm_compute::WeightFormat expected_weight_format;
+    ACL_CHECK_VALID(arm_compute::NEGEMM::has_opt_impl(expected_weight_format,
+            &amp.src_tensor_info, &amp.wei_tensor_info, nullptr,
+            &amp.dst_tensor_info, amp.alpha, 0.0f, amp.gemm_info));
+
+    // Set gemm weights info to the one returned by has_opt_impl
+    amp.gemm_info.set_weight_format(expected_weight_format);
+
+    // has_opt_impl may return a non fast math kernel, even if we requested one
+    amp.gemm_info.set_fast_math(
+            arm_compute::is_fixed_format_fast_math(expected_weight_format));
+
+    // Logical dimension indices
+    dim_t innermost_dim = wei_md.ndims - 1;
+    dim_t N_dim = innermost_dim;
+    dim_t K_dim = innermost_dim - 1;
+
+    // The logical indices of dimensions related to the batch, ordered from
+    // innermost to outermost
+    std::vector<dim_t> batch_dims = {};
+    for (dim_t i = K_dim - 1; i >= 0; --i)
+        batch_dims.push_back(i);
+
+    acl_utils::reorder_to_weight_format(amp.wei_tensor_info, wei_md,
+            expected_weight_format, K_dim, N_dim, {}, batch_dims);
 
     return status::success;
 }
diff --git a/src/cpu/aarch64/matmul/acl_matmul_utils.hpp b/src/cpu/aarch64/matmul/acl_matmul_utils.hpp
index 0a5ee6a987..67bb2e78eb 100644
--- a/src/cpu/aarch64/matmul/acl_matmul_utils.hpp
+++ b/src/cpu/aarch64/matmul/acl_matmul_utils.hpp
@@ -1,5 +1,5 @@
 /*******************************************************************************
-* Copyright 2021-2022 Arm Ltd. and affiliates
+* Copyright 2021-2023 Arm Ltd. and affiliates
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
@@ -29,25 +29,21 @@ namespace aarch64 {
 struct acl_matmul_obj_t {
     arm_compute::NEGEMM gemm;
     arm_compute::NETranspose transA;
-    arm_compute::NETranspose transB;
     arm_compute::Tensor src_tensor;
     arm_compute::Tensor src_acc_tensor;
     arm_compute::Tensor wei_tensor;
-    arm_compute::Tensor wei_acc_tensor;
     arm_compute::Tensor dst_tensor;
 };
 
 struct acl_matmul_conf_t {
     bool is_transA;
-    bool is_transB;
     // If this is true, the result of the matmul goes into a temporarily
     // allocated ACL tensor to be accumulated into the oneDNN dst during postops
     bool use_dst_acc;
-    arm_compute::TensorInfo src_info;
+    arm_compute::TensorInfo src_tensor_info;
     arm_compute::TensorInfo src_acc_info;
-    arm_compute::TensorInfo wei_info;
-    arm_compute::TensorInfo wei_acc_info;
-    arm_compute::TensorInfo dst_info;
+    arm_compute::TensorInfo wei_tensor_info;
+    arm_compute::TensorInfo dst_tensor_info;
     arm_compute::GEMMInfo gemm_info;
     float alpha;
 };