File: UnifyAliasedResourcePass.cpp

package info (click to toggle)
llvm-toolchain-17 1%3A17.0.6-22
  • links: PTS, VCS
  • area: main
  • in suites: forky, sid, trixie
  • size: 1,799,624 kB
  • sloc: cpp: 6,428,607; ansic: 1,383,196; asm: 793,408; python: 223,504; objc: 75,364; f90: 60,502; lisp: 33,869; pascal: 15,282; sh: 9,684; perl: 7,453; ml: 4,937; awk: 3,523; makefile: 2,889; javascript: 2,149; xml: 888; fortran: 619; cs: 573
file content (629 lines) | stat: -rw-r--r-- 25,144 bytes parent folder | download | duplicates (3)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
//===- UnifyAliasedResourcePass.cpp - Pass to Unify Aliased Resources -----===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This file implements a pass that unifies access of multiple aliased resources
// into access of one single resource.
//
//===----------------------------------------------------------------------===//

#include "mlir/Dialect/SPIRV/Transforms/Passes.h"

#include "mlir/Dialect/SPIRV/IR/SPIRVDialect.h"
#include "mlir/Dialect/SPIRV/IR/SPIRVOps.h"
#include "mlir/Dialect/SPIRV/IR/SPIRVTypes.h"
#include "mlir/Dialect/SPIRV/IR/TargetAndABI.h"
#include "mlir/IR/Builders.h"
#include "mlir/IR/BuiltinAttributes.h"
#include "mlir/IR/BuiltinTypes.h"
#include "mlir/IR/SymbolTable.h"
#include "mlir/Transforms/DialectConversion.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/Support/Debug.h"
#include <algorithm>
#include <iterator>

namespace mlir {
namespace spirv {
#define GEN_PASS_DEF_SPIRVUNIFYALIASEDRESOURCEPASS
#include "mlir/Dialect/SPIRV/Transforms/Passes.h.inc"
} // namespace spirv
} // namespace mlir

#define DEBUG_TYPE "spirv-unify-aliased-resource"

using namespace mlir;

//===----------------------------------------------------------------------===//
// Utility functions
//===----------------------------------------------------------------------===//

using Descriptor = std::pair<uint32_t, uint32_t>; // (set #, binding #)
using AliasedResourceMap =
    DenseMap<Descriptor, SmallVector<spirv::GlobalVariableOp>>;

/// Collects all aliased resources in the given SPIR-V `moduleOp`.
static AliasedResourceMap collectAliasedResources(spirv::ModuleOp moduleOp) {
  AliasedResourceMap aliasedResources;
  moduleOp->walk([&aliasedResources](spirv::GlobalVariableOp varOp) {
    if (varOp->getAttrOfType<UnitAttr>("aliased")) {
      std::optional<uint32_t> set = varOp.getDescriptorSet();
      std::optional<uint32_t> binding = varOp.getBinding();
      if (set && binding)
        aliasedResources[{*set, *binding}].push_back(varOp);
    }
  });
  return aliasedResources;
}

/// Returns the element type if the given `type` is a runtime array resource:
/// `!spirv.ptr<!spirv.struct<!spirv.rtarray<...>>>`. Returns null type
/// otherwise.
static Type getRuntimeArrayElementType(Type type) {
  auto ptrType = dyn_cast<spirv::PointerType>(type);
  if (!ptrType)
    return {};

  auto structType = dyn_cast<spirv::StructType>(ptrType.getPointeeType());
  if (!structType || structType.getNumElements() != 1)
    return {};

  auto rtArrayType =
      dyn_cast<spirv::RuntimeArrayType>(structType.getElementType(0));
  if (!rtArrayType)
    return {};

  return rtArrayType.getElementType();
}

/// Given a list of resource element `types`, returns the index of the canonical
/// resource that all resources should be unified into. Returns std::nullopt if
/// unable to unify.
static std::optional<int>
deduceCanonicalResource(ArrayRef<spirv::SPIRVType> types) {
  // scalarNumBits: contains all resources' scalar types' bit counts.
  // vectorNumBits: only contains resources whose element types are vectors.
  // vectorIndices: each vector's original index in `types`.
  SmallVector<int> scalarNumBits, vectorNumBits, vectorIndices;
  scalarNumBits.reserve(types.size());
  vectorNumBits.reserve(types.size());
  vectorIndices.reserve(types.size());

  for (const auto &indexedTypes : llvm::enumerate(types)) {
    spirv::SPIRVType type = indexedTypes.value();
    assert(type.isScalarOrVector());
    if (auto vectorType = dyn_cast<VectorType>(type)) {
      if (vectorType.getNumElements() % 2 != 0)
        return std::nullopt; // Odd-sized vector has special layout
                             // requirements.

      std::optional<int64_t> numBytes = type.getSizeInBytes();
      if (!numBytes)
        return std::nullopt;

      scalarNumBits.push_back(
          vectorType.getElementType().getIntOrFloatBitWidth());
      vectorNumBits.push_back(*numBytes * 8);
      vectorIndices.push_back(indexedTypes.index());
    } else {
      scalarNumBits.push_back(type.getIntOrFloatBitWidth());
    }
  }

  if (!vectorNumBits.empty()) {
    // Choose the *vector* with the smallest bitwidth as the canonical resource,
    // so that we can still keep vectorized load/store and avoid partial updates
    // to large vectors.
    auto *minVal = std::min_element(vectorNumBits.begin(), vectorNumBits.end());
    // Make sure that the canonical resource's bitwidth is divisible by others.
    // With out this, we cannot properly adjust the index later.
    if (llvm::any_of(vectorNumBits,
                     [&](int bits) { return bits % *minVal != 0; }))
      return std::nullopt;

    // Require all scalar type bit counts to be a multiple of the chosen
    // vector's primitive type to avoid reading/writing subcomponents.
    int index = vectorIndices[std::distance(vectorNumBits.begin(), minVal)];
    int baseNumBits = scalarNumBits[index];
    if (llvm::any_of(scalarNumBits,
                     [&](int bits) { return bits % baseNumBits != 0; }))
      return std::nullopt;

    return index;
  }

  // All element types are scalars. Then choose the smallest bitwidth as the
  // cannonical resource to avoid subcomponent load/store.
  auto *minVal = std::min_element(scalarNumBits.begin(), scalarNumBits.end());
  if (llvm::any_of(scalarNumBits,
                   [minVal](int64_t bit) { return bit % *minVal != 0; }))
    return std::nullopt;
  return std::distance(scalarNumBits.begin(), minVal);
}

static bool areSameBitwidthScalarType(Type a, Type b) {
  return a.isIntOrFloat() && b.isIntOrFloat() &&
         a.getIntOrFloatBitWidth() == b.getIntOrFloatBitWidth();
}

//===----------------------------------------------------------------------===//
// Analysis
//===----------------------------------------------------------------------===//

namespace {
/// A class for analyzing aliased resources.
///
/// Resources are expected to be spirv.GlobalVarible that has a descriptor set
/// and binding number. Such resources are of the type
/// `!spirv.ptr<!spirv.struct<...>>` per Vulkan requirements.
///
/// Right now, we only support the case that there is a single runtime array
/// inside the struct.
class ResourceAliasAnalysis {
public:
  MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(ResourceAliasAnalysis)

  explicit ResourceAliasAnalysis(Operation *);

  /// Returns true if the given `op` can be rewritten to use a canonical
  /// resource.
  bool shouldUnify(Operation *op) const;

  /// Returns all descriptors and their corresponding aliased resources.
  const AliasedResourceMap &getResourceMap() const { return resourceMap; }

  /// Returns the canonical resource for the given descriptor/variable.
  spirv::GlobalVariableOp
  getCanonicalResource(const Descriptor &descriptor) const;
  spirv::GlobalVariableOp
  getCanonicalResource(spirv::GlobalVariableOp varOp) const;

  /// Returns the element type for the given variable.
  spirv::SPIRVType getElementType(spirv::GlobalVariableOp varOp) const;

private:
  /// Given the descriptor and aliased resources bound to it, analyze whether we
  /// can unify them and record if so.
  void recordIfUnifiable(const Descriptor &descriptor,
                         ArrayRef<spirv::GlobalVariableOp> resources);

  /// Mapping from a descriptor to all aliased resources bound to it.
  AliasedResourceMap resourceMap;

  /// Mapping from a descriptor to the chosen canonical resource.
  DenseMap<Descriptor, spirv::GlobalVariableOp> canonicalResourceMap;

  /// Mapping from an aliased resource to its descriptor.
  DenseMap<spirv::GlobalVariableOp, Descriptor> descriptorMap;

  /// Mapping from an aliased resource to its element (scalar/vector) type.
  DenseMap<spirv::GlobalVariableOp, spirv::SPIRVType> elementTypeMap;
};
} // namespace

ResourceAliasAnalysis::ResourceAliasAnalysis(Operation *root) {
  // Collect all aliased resources first and put them into different sets
  // according to the descriptor.
  AliasedResourceMap aliasedResources =
      collectAliasedResources(cast<spirv::ModuleOp>(root));

  // For each resource set, analyze whether we can unify; if so, try to identify
  // a canonical resource, whose element type has the largest bitwidth.
  for (const auto &descriptorResource : aliasedResources) {
    recordIfUnifiable(descriptorResource.first, descriptorResource.second);
  }
}

bool ResourceAliasAnalysis::shouldUnify(Operation *op) const {
  if (!op)
    return false;

  if (auto varOp = dyn_cast<spirv::GlobalVariableOp>(op)) {
    auto canonicalOp = getCanonicalResource(varOp);
    return canonicalOp && varOp != canonicalOp;
  }
  if (auto addressOp = dyn_cast<spirv::AddressOfOp>(op)) {
    auto moduleOp = addressOp->getParentOfType<spirv::ModuleOp>();
    auto *varOp =
        SymbolTable::lookupSymbolIn(moduleOp, addressOp.getVariable());
    return shouldUnify(varOp);
  }

  if (auto acOp = dyn_cast<spirv::AccessChainOp>(op))
    return shouldUnify(acOp.getBasePtr().getDefiningOp());
  if (auto loadOp = dyn_cast<spirv::LoadOp>(op))
    return shouldUnify(loadOp.getPtr().getDefiningOp());
  if (auto storeOp = dyn_cast<spirv::StoreOp>(op))
    return shouldUnify(storeOp.getPtr().getDefiningOp());

  return false;
}

spirv::GlobalVariableOp ResourceAliasAnalysis::getCanonicalResource(
    const Descriptor &descriptor) const {
  auto varIt = canonicalResourceMap.find(descriptor);
  if (varIt == canonicalResourceMap.end())
    return {};
  return varIt->second;
}

spirv::GlobalVariableOp ResourceAliasAnalysis::getCanonicalResource(
    spirv::GlobalVariableOp varOp) const {
  auto descriptorIt = descriptorMap.find(varOp);
  if (descriptorIt == descriptorMap.end())
    return {};
  return getCanonicalResource(descriptorIt->second);
}

spirv::SPIRVType
ResourceAliasAnalysis::getElementType(spirv::GlobalVariableOp varOp) const {
  auto it = elementTypeMap.find(varOp);
  if (it == elementTypeMap.end())
    return {};
  return it->second;
}

void ResourceAliasAnalysis::recordIfUnifiable(
    const Descriptor &descriptor, ArrayRef<spirv::GlobalVariableOp> resources) {
  // Collect the element types for all resources in the current set.
  SmallVector<spirv::SPIRVType> elementTypes;
  for (spirv::GlobalVariableOp resource : resources) {
    Type elementType = getRuntimeArrayElementType(resource.getType());
    if (!elementType)
      return; // Unexpected resource variable type.

    auto type = cast<spirv::SPIRVType>(elementType);
    if (!type.isScalarOrVector())
      return; // Unexpected resource element type.

    elementTypes.push_back(type);
  }

  std::optional<int> index = deduceCanonicalResource(elementTypes);
  if (!index)
    return;

  // Update internal data structures for later use.
  resourceMap[descriptor].assign(resources.begin(), resources.end());
  canonicalResourceMap[descriptor] = resources[*index];
  for (const auto &resource : llvm::enumerate(resources)) {
    descriptorMap[resource.value()] = descriptor;
    elementTypeMap[resource.value()] = elementTypes[resource.index()];
  }
}

//===----------------------------------------------------------------------===//
// Patterns
//===----------------------------------------------------------------------===//

template <typename OpTy>
class ConvertAliasResource : public OpConversionPattern<OpTy> {
public:
  ConvertAliasResource(const ResourceAliasAnalysis &analysis,
                       MLIRContext *context, PatternBenefit benefit = 1)
      : OpConversionPattern<OpTy>(context, benefit), analysis(analysis) {}

protected:
  const ResourceAliasAnalysis &analysis;
};

struct ConvertVariable : public ConvertAliasResource<spirv::GlobalVariableOp> {
  using ConvertAliasResource::ConvertAliasResource;

  LogicalResult
  matchAndRewrite(spirv::GlobalVariableOp varOp, OpAdaptor adaptor,
                  ConversionPatternRewriter &rewriter) const override {
    // Just remove the aliased resource. Users will be rewritten to use the
    // canonical one.
    rewriter.eraseOp(varOp);
    return success();
  }
};

struct ConvertAddressOf : public ConvertAliasResource<spirv::AddressOfOp> {
  using ConvertAliasResource::ConvertAliasResource;

  LogicalResult
  matchAndRewrite(spirv::AddressOfOp addressOp, OpAdaptor adaptor,
                  ConversionPatternRewriter &rewriter) const override {
    // Rewrite the AddressOf op to get the address of the canoncical resource.
    auto moduleOp = addressOp->getParentOfType<spirv::ModuleOp>();
    auto srcVarOp = cast<spirv::GlobalVariableOp>(
        SymbolTable::lookupSymbolIn(moduleOp, addressOp.getVariable()));
    auto dstVarOp = analysis.getCanonicalResource(srcVarOp);
    rewriter.replaceOpWithNewOp<spirv::AddressOfOp>(addressOp, dstVarOp);
    return success();
  }
};

struct ConvertAccessChain : public ConvertAliasResource<spirv::AccessChainOp> {
  using ConvertAliasResource::ConvertAliasResource;

  LogicalResult
  matchAndRewrite(spirv::AccessChainOp acOp, OpAdaptor adaptor,
                  ConversionPatternRewriter &rewriter) const override {
    auto addressOp = acOp.getBasePtr().getDefiningOp<spirv::AddressOfOp>();
    if (!addressOp)
      return rewriter.notifyMatchFailure(acOp, "base ptr not addressof op");

    auto moduleOp = acOp->getParentOfType<spirv::ModuleOp>();
    auto srcVarOp = cast<spirv::GlobalVariableOp>(
        SymbolTable::lookupSymbolIn(moduleOp, addressOp.getVariable()));
    auto dstVarOp = analysis.getCanonicalResource(srcVarOp);

    spirv::SPIRVType srcElemType = analysis.getElementType(srcVarOp);
    spirv::SPIRVType dstElemType = analysis.getElementType(dstVarOp);

    if (srcElemType == dstElemType ||
        areSameBitwidthScalarType(srcElemType, dstElemType)) {
      // We have the same bitwidth for source and destination element types.
      // Thie indices keep the same.
      rewriter.replaceOpWithNewOp<spirv::AccessChainOp>(
          acOp, adaptor.getBasePtr(), adaptor.getIndices());
      return success();
    }

    Location loc = acOp.getLoc();

    if (srcElemType.isIntOrFloat() && isa<VectorType>(dstElemType)) {
      // The source indices are for a buffer with scalar element types. Rewrite
      // them into a buffer with vector element types. We need to scale the last
      // index for the vector as a whole, then add one level of index for inside
      // the vector.
      int srcNumBytes = *srcElemType.getSizeInBytes();
      int dstNumBytes = *dstElemType.getSizeInBytes();
      assert(dstNumBytes >= srcNumBytes && dstNumBytes % srcNumBytes == 0);

      auto indices = llvm::to_vector<4>(acOp.getIndices());
      Value oldIndex = indices.back();
      Type indexType = oldIndex.getType();

      int ratio = dstNumBytes / srcNumBytes;
      auto ratioValue = rewriter.create<spirv::ConstantOp>(
          loc, indexType, rewriter.getIntegerAttr(indexType, ratio));

      indices.back() =
          rewriter.create<spirv::SDivOp>(loc, indexType, oldIndex, ratioValue);
      indices.push_back(
          rewriter.create<spirv::SModOp>(loc, indexType, oldIndex, ratioValue));

      rewriter.replaceOpWithNewOp<spirv::AccessChainOp>(
          acOp, adaptor.getBasePtr(), indices);
      return success();
    }

    if ((srcElemType.isIntOrFloat() && dstElemType.isIntOrFloat()) ||
        (isa<VectorType>(srcElemType) && isa<VectorType>(dstElemType))) {
      // The source indices are for a buffer with larger bitwidth scalar/vector
      // element types. Rewrite them into a buffer with smaller bitwidth element
      // types. We only need to scale the last index.
      int srcNumBytes = *srcElemType.getSizeInBytes();
      int dstNumBytes = *dstElemType.getSizeInBytes();
      assert(srcNumBytes >= dstNumBytes && srcNumBytes % dstNumBytes == 0);

      auto indices = llvm::to_vector<4>(acOp.getIndices());
      Value oldIndex = indices.back();
      Type indexType = oldIndex.getType();

      int ratio = srcNumBytes / dstNumBytes;
      auto ratioValue = rewriter.create<spirv::ConstantOp>(
          loc, indexType, rewriter.getIntegerAttr(indexType, ratio));

      indices.back() =
          rewriter.create<spirv::IMulOp>(loc, indexType, oldIndex, ratioValue);

      rewriter.replaceOpWithNewOp<spirv::AccessChainOp>(
          acOp, adaptor.getBasePtr(), indices);
      return success();
    }

    return rewriter.notifyMatchFailure(
        acOp, "unsupported src/dst types for spirv.AccessChain");
  }
};

struct ConvertLoad : public ConvertAliasResource<spirv::LoadOp> {
  using ConvertAliasResource::ConvertAliasResource;

  LogicalResult
  matchAndRewrite(spirv::LoadOp loadOp, OpAdaptor adaptor,
                  ConversionPatternRewriter &rewriter) const override {
    auto srcPtrType = cast<spirv::PointerType>(loadOp.getPtr().getType());
    auto srcElemType = cast<spirv::SPIRVType>(srcPtrType.getPointeeType());
    auto dstPtrType = cast<spirv::PointerType>(adaptor.getPtr().getType());
    auto dstElemType = cast<spirv::SPIRVType>(dstPtrType.getPointeeType());

    Location loc = loadOp.getLoc();
    auto newLoadOp = rewriter.create<spirv::LoadOp>(loc, adaptor.getPtr());
    if (srcElemType == dstElemType) {
      rewriter.replaceOp(loadOp, newLoadOp->getResults());
      return success();
    }

    if (areSameBitwidthScalarType(srcElemType, dstElemType)) {
      auto castOp = rewriter.create<spirv::BitcastOp>(loc, srcElemType,
                                                      newLoadOp.getValue());
      rewriter.replaceOp(loadOp, castOp->getResults());

      return success();
    }

    if ((srcElemType.isIntOrFloat() && dstElemType.isIntOrFloat()) ||
        (isa<VectorType>(srcElemType) && isa<VectorType>(dstElemType))) {
      // The source and destination have scalar types of different bitwidths, or
      // vector types of different component counts. For such cases, we load
      // multiple smaller bitwidth values and construct a larger bitwidth one.

      int srcNumBytes = *srcElemType.getSizeInBytes();
      int dstNumBytes = *dstElemType.getSizeInBytes();
      assert(srcNumBytes > dstNumBytes && srcNumBytes % dstNumBytes == 0);
      int ratio = srcNumBytes / dstNumBytes;
      if (ratio > 4)
        return rewriter.notifyMatchFailure(loadOp, "more than 4 components");

      SmallVector<Value> components;
      components.reserve(ratio);
      components.push_back(newLoadOp);

      auto acOp = adaptor.getPtr().getDefiningOp<spirv::AccessChainOp>();
      if (!acOp)
        return rewriter.notifyMatchFailure(loadOp, "ptr not spirv.AccessChain");

      auto i32Type = rewriter.getI32Type();
      Value oneValue = spirv::ConstantOp::getOne(i32Type, loc, rewriter);
      auto indices = llvm::to_vector<4>(acOp.getIndices());
      for (int i = 1; i < ratio; ++i) {
        // Load all subsequent components belonging to this element.
        indices.back() = rewriter.create<spirv::IAddOp>(
            loc, i32Type, indices.back(), oneValue);
        auto componentAcOp = rewriter.create<spirv::AccessChainOp>(
            loc, acOp.getBasePtr(), indices);
        // Assuming little endian, this reads lower-ordered bits of the number
        // to lower-numbered components of the vector.
        components.push_back(
            rewriter.create<spirv::LoadOp>(loc, componentAcOp));
      }

      // Create a vector of the components and then cast back to the larger
      // bitwidth element type. For spirv.bitcast, the lower-numbered components
      // of the vector map to lower-ordered bits of the larger bitwidth element
      // type.

      Type vectorType = srcElemType;
      if (!isa<VectorType>(srcElemType))
        vectorType = VectorType::get({ratio}, dstElemType);

      // If both the source and destination are vector types, we need to make
      // sure the scalar type is the same for composite construction later.
      if (auto srcElemVecType = dyn_cast<VectorType>(srcElemType))
        if (auto dstElemVecType = dyn_cast<VectorType>(dstElemType)) {
          if (srcElemVecType.getElementType() !=
              dstElemVecType.getElementType()) {
            int64_t count =
                dstNumBytes / (srcElemVecType.getElementTypeBitWidth() / 8);

            // Make sure not to create 1-element vectors, which are illegal in
            // SPIR-V.
            Type castType = srcElemVecType.getElementType();
            if (count > 1)
              castType = VectorType::get({count}, castType);

            for (Value &c : components)
              c = rewriter.create<spirv::BitcastOp>(loc, castType, c);
          }
        }
      Value vectorValue = rewriter.create<spirv::CompositeConstructOp>(
          loc, vectorType, components);

      if (!isa<VectorType>(srcElemType))
        vectorValue =
            rewriter.create<spirv::BitcastOp>(loc, srcElemType, vectorValue);
      rewriter.replaceOp(loadOp, vectorValue);
      return success();
    }

    return rewriter.notifyMatchFailure(
        loadOp, "unsupported src/dst types for spirv.Load");
  }
};

struct ConvertStore : public ConvertAliasResource<spirv::StoreOp> {
  using ConvertAliasResource::ConvertAliasResource;

  LogicalResult
  matchAndRewrite(spirv::StoreOp storeOp, OpAdaptor adaptor,
                  ConversionPatternRewriter &rewriter) const override {
    auto srcElemType =
        cast<spirv::PointerType>(storeOp.getPtr().getType()).getPointeeType();
    auto dstElemType =
        cast<spirv::PointerType>(adaptor.getPtr().getType()).getPointeeType();
    if (!srcElemType.isIntOrFloat() || !dstElemType.isIntOrFloat())
      return rewriter.notifyMatchFailure(storeOp, "not scalar type");
    if (!areSameBitwidthScalarType(srcElemType, dstElemType))
      return rewriter.notifyMatchFailure(storeOp, "different bitwidth");

    Location loc = storeOp.getLoc();
    Value value = adaptor.getValue();
    if (srcElemType != dstElemType)
      value = rewriter.create<spirv::BitcastOp>(loc, dstElemType, value);
    rewriter.replaceOpWithNewOp<spirv::StoreOp>(storeOp, adaptor.getPtr(),
                                                value, storeOp->getAttrs());
    return success();
  }
};

//===----------------------------------------------------------------------===//
// Pass
//===----------------------------------------------------------------------===//

namespace {
class UnifyAliasedResourcePass final
    : public spirv::impl::SPIRVUnifyAliasedResourcePassBase<
          UnifyAliasedResourcePass> {
public:
  explicit UnifyAliasedResourcePass(spirv::GetTargetEnvFn getTargetEnv)
      : getTargetEnvFn(std::move(getTargetEnv)) {}

  void runOnOperation() override;

private:
  spirv::GetTargetEnvFn getTargetEnvFn;
};

void UnifyAliasedResourcePass::runOnOperation() {
  spirv::ModuleOp moduleOp = getOperation();
  MLIRContext *context = &getContext();

  if (getTargetEnvFn) {
    // This pass is only needed for targeting WebGPU, Metal, or layering
    // Vulkan on Metal via MoltenVK, where we need to translate SPIR-V into
    // WGSL or MSL. The translation has limitations.
    spirv::TargetEnvAttr targetEnv = getTargetEnvFn(moduleOp);
    spirv::ClientAPI clientAPI = targetEnv.getClientAPI();
    bool isVulkanOnAppleDevices =
        clientAPI == spirv::ClientAPI::Vulkan &&
        targetEnv.getVendorID() == spirv::Vendor::Apple;
    if (clientAPI != spirv::ClientAPI::WebGPU &&
        clientAPI != spirv::ClientAPI::Metal && !isVulkanOnAppleDevices)
      return;
  }

  // Analyze aliased resources first.
  ResourceAliasAnalysis &analysis = getAnalysis<ResourceAliasAnalysis>();

  ConversionTarget target(*context);
  target.addDynamicallyLegalOp<spirv::GlobalVariableOp, spirv::AddressOfOp,
                               spirv::AccessChainOp, spirv::LoadOp,
                               spirv::StoreOp>(
      [&analysis](Operation *op) { return !analysis.shouldUnify(op); });
  target.addLegalDialect<spirv::SPIRVDialect>();

  // Run patterns to rewrite usages of non-canonical resources.
  RewritePatternSet patterns(context);
  patterns.add<ConvertVariable, ConvertAddressOf, ConvertAccessChain,
               ConvertLoad, ConvertStore>(analysis, context);
  if (failed(applyPartialConversion(moduleOp, target, std::move(patterns))))
    return signalPassFailure();

  // Drop aliased attribute if we only have one single bound resource for a
  // descriptor. We need to re-collect the map here given in the above the
  // conversion is best effort; certain sets may not be converted.
  AliasedResourceMap resourceMap =
      collectAliasedResources(cast<spirv::ModuleOp>(moduleOp));
  for (const auto &dr : resourceMap) {
    const auto &resources = dr.second;
    if (resources.size() == 1)
      resources.front()->removeAttr("aliased");
  }
}
} // namespace

std::unique_ptr<mlir::OperationPass<spirv::ModuleOp>>
spirv::createUnifyAliasedResourcePass(spirv::GetTargetEnvFn getTargetEnv) {
  return std::make_unique<UnifyAliasedResourcePass>(std::move(getTargetEnv));
}