//===----------------------------------------------------------------------===//
//
// This source file is part of the Swift.org open source project
//
// Copyright (c) 2023 Apple Inc. and the Swift project authors
// Licensed under Apache License v2.0 with Runtime Library Exception
//
// See https://swift.org/LICENSE.txt for license information
// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
//
//===----------------------------------------------------------------------===//

%{
  from gyb_utils import *
}%
${autogenerated_warning()}

// FIXME: The conditionals below have been carefully constructed to
// avoid confusing Swift 5.7; they can be sanitized once we drop support
// for that version.
#if compiler(>=5.9)
#if !ATOMICS_NATIVE_BUILTINS
#error("swift-atomics requires native builtins on Swift 5.9")
#endif
import Builtin

@_alwaysEmitIntoClient
@_transparent
internal func _atomicMemoryFence(
  ordering: AtomicUpdateOrdering
) {
  switch ordering {
    case .relaxed:
      break
% for (enumOrder, apiOrder, docOrder, llvmOrder, _) in updateOrderings:
%   if enumOrder != "relaxed":
    case .${enumOrder}:
      Builtin.fence_${llvmOrder}()
%   end
% end
    default:
      fatalError("Unsupported ordering")
  }
}
% for (swiftType, builtinType, alignment) in nativePrimitives:

%   if builtinType == "Int128":
#if _pointerBitWidth(_64)
%   end
@usableFromInline
@frozen
@_alignment(${alignment})
internal struct ${swiftType} {
  @usableFromInline
  internal var _value: Builtin.${builtinType}

  @_alwaysEmitIntoClient @_transparent
  internal init(_ value: Builtin.${builtinType}) {
    self._value = value
  }
}

extension UnsafeMutablePointer where Pointee == ${swiftType} {
  /// Atomically loads a word starting at this address with the specified
  /// memory ordering.
  @_semantics("atomics.requires_constant_orderings")
  @_alwaysEmitIntoClient
  @_transparent // Debug performance
  internal func _atomicLoad(ordering: AtomicLoadOrdering) -> ${swiftType} {
    switch ordering {
% for (enumOrder, apiOrder, _, llvmOrder) in loadOrderings:
    case .${enumOrder}:
      return ${swiftType}(Builtin.atomicload_${llvmOrder}_${builtinType}(_rawValue))
% end
    default:
      fatalError("Unsupported ordering")
    }
  }

  /// Atomically stores the specified value starting at the memory referenced by
  /// this pointer, with the specified memory ordering.
  @_semantics("atomics.requires_constant_orderings")
  @_alwaysEmitIntoClient
  @_transparent // Debug performance
  internal func _atomicStore(
    _ desired: ${swiftType},
    ordering: AtomicStoreOrdering
  ) {
    switch ordering {
% for (enumOrder, apiOrder, _, llvmOrder) in storeOrderings:
    case .${enumOrder}:
      Builtin.atomicstore_${llvmOrder}_${builtinType}(_rawValue, desired._value)
% end
    default:
      fatalError("Unsupported ordering")
    }
  }

  /// Atomically stores the specified value starting at the memory referenced by
  /// this pointer, with the specified memory ordering.
  @_semantics("atomics.requires_constant_orderings")
  @_alwaysEmitIntoClient
  @_transparent // Debug performance
  internal func _atomicExchange(
    _ desired: ${swiftType},
    ordering: AtomicUpdateOrdering
  ) -> ${swiftType} {
    switch ordering {
% for (enumOrder, apiOrder, _, llvmOrder, failureOrder) in updateOrderings:
    case .${enumOrder}:
      let oldValue = Builtin.atomicrmw_xchg_${llvmOrder}_${builtinType}(
        _rawValue, desired._value)
      return ${swiftType}(oldValue)
% end
    default:
      fatalError("Unsupported ordering")
    }
  }

  /// Perform an atomic compare and exchange operation with the specified memory
  /// ordering.
  ///
  /// This operation is equivalent to the following pseudocode:
  ///
  /// ```
  /// atomic(self, ordering) { currentValue in
  ///   let original = currentValue
  ///   guard original == expected else { return (false, original) }
  ///   currentValue = desired
  ///   return (true, original)
  /// }
  /// ```
  @_semantics("atomics.requires_constant_orderings")
  @_alwaysEmitIntoClient
  @_transparent // Debug performance
  internal func _atomicCompareExchange(
    expected: ${swiftType},
    desired: ${swiftType},
    ordering: AtomicUpdateOrdering
  ) -> (exchanged: Bool, original: ${swiftType}) {
    switch ordering {
% for (enumOrder, apiOrder, _, llvmOrder, failureOrder) in updateOrderings:
    case .${enumOrder}:
      let (oldValue, won) = Builtin.cmpxchg_${llvmOrder}_${failureOrder}_${builtinType}(
        _rawValue, expected._value, desired._value)
      return (Bool(won), ${swiftType}(oldValue))
% end
    default:
      fatalError("Unsupported ordering")
    }
  }

  /// Perform an atomic compare and exchange operation with the specified
  /// success/failure memory orderings.
  ///
  /// This operation is equivalent to the following pseudocode:
  ///
  /// ```
  /// atomic(self, ordering, failureOrdering) { currentValue in
  ///   let original = currentValue
  ///   guard original == expected else { return (false, original) }
  ///   currentValue = desired
  ///   return (true, original)
  /// }
  /// ```
  ///
  /// The `ordering` argument specifies the memory ordering to use when the
  /// operation manages to update the current value, while `failureOrdering`
  /// will be used when the operation leaves the value intact.
  @_semantics("atomics.requires_constant_orderings")
  @_alwaysEmitIntoClient
  @_transparent // Debug performance
  internal func _atomicCompareExchange(
    expected: ${swiftType},
    desired: ${swiftType},
    successOrdering: AtomicUpdateOrdering,
    failureOrdering: AtomicLoadOrdering
  ) -> (exchanged: Bool, original: ${swiftType}) {
    // FIXME: LLVM doesn't support arbitrary ordering combinations
    // yet, so upgrade the success ordering when necessary so that it
    // is at least as "strong" as the failure case.
    switch (successOrdering, failureOrdering) {
% for (swiftSuccess, apiSuccess, _, llvmOrder, _) in updateOrderings:
%   for (swiftFailure, apiFailure, _, llvmFailOrder) in loadOrderings:
    case (.${swiftSuccess}, .${swiftFailure}):
      let (oldValue, won) = Builtin.cmpxchg_${actualOrders(llvmOrder, llvmFailOrder)}_${builtinType}(
        _rawValue, expected._value, desired._value)
      return (Bool(won), ${swiftType}(oldValue))
%   end
% end
    default:
      preconditionFailure("Unsupported orderings")
    }
  }

  /// Perform an atomic compare and exchange operation with the specified
  /// success/failure memory orderings.
  ///
  /// This operation is equivalent to the following pseudocode:
  ///
  /// ```
  /// atomic(self, ordering, failureOrdering) { currentValue in
  ///   let original = currentValue
  ///   guard original == expected else { return (false, original) }
  ///   currentValue = desired
  ///   return (true, original)
  /// }
  /// ```
  ///
  /// The `ordering` argument specifies the memory ordering to use when the
  /// operation manages to update the current value, while `failureOrdering`
  /// will be used when the operation leaves the value intact.
  @_semantics("atomics.requires_constant_orderings")
  @_alwaysEmitIntoClient
  @_transparent // Debug performance
  internal func _atomicWeakCompareExchange(
    expected: ${swiftType},
    desired: ${swiftType},
    successOrdering: AtomicUpdateOrdering,
    failureOrdering: AtomicLoadOrdering
  ) -> (exchanged: Bool, original: ${swiftType}) {
    // FIXME: LLVM doesn't support arbitrary ordering combinations
    // yet, so upgrade the success ordering when necessary so that it
    // is at least as "strong" as the failure case.
    switch (successOrdering, failureOrdering) {
% for (swiftSuccess, apiSuccess, _, llvmOrder, _) in updateOrderings:
%   for (swiftFailure, apiFailure, _, llvmFailOrder) in loadOrderings:
    case (.${swiftSuccess}, .${swiftFailure}):
      let (oldValue, won) = Builtin.cmpxchg_${actualOrders(llvmOrder, llvmFailOrder)}_weak_${builtinType}(
        _rawValue, expected._value, desired._value)
      return (Bool(won), ${swiftType}(oldValue))
%   end
% end
    default:
      preconditionFailure("Unsupported orderings")
    }
  }
% if builtinType != "Int128":
%   for (swiftName, builtinName, operator, label, doc) in integerOperations:

  /// Perform an atomic ${doc} operation and return the new value,
  /// with the specified memory ordering.
  ///
  % if "Wrapping" in swiftName:
  /// - Note: This operation silently wraps around on overflow, like the
  /// `${operator}` operator does on `UInt` values.
  ///
  % end
  /// - Returns: The original value before the operation.
  @_semantics("atomics.requires_constant_orderings")
  @_alwaysEmitIntoClient
  @_transparent // Debug performance
  internal func _atomicLoadThen${swiftName}(
    ${label} operand: ${swiftType},
    ordering: AtomicUpdateOrdering
  ) -> ${swiftType} {
    switch ordering {
% for (enumOrder, apiOrder, _, llvmOrder, failureOrder) in updateOrderings:
    case .${enumOrder}:
      let value = Builtin.atomicrmw_${builtinName}_${llvmOrder}_${builtinType}(
        _rawValue, operand._value)
      return ${swiftType}(value)
% end
    default:
      preconditionFailure("Unsupported ordering")
    }
  }
%   end
% end
}
% if builtinType == "Int128":
#endif
% end
% end

#if _pointerBitWidth(_64)
@usableFromInline internal typealias _AtomicIntStorage = _AtomicInt64Storage
@usableFromInline internal typealias _AtomicDoubleWordStorage = _AtomicInt128Storage
#elseif _pointerBitWidth(_32)
@usableFromInline internal typealias _AtomicIntStorage = _AtomicInt32Storage
@usableFromInline internal typealias _AtomicDoubleWordStorage = _AtomicInt64Storage
#else
#error("Unexpected pointer bit width")
#endif

#else // compiler(>=5.9)
#if ATOMICS_NATIVE_BUILTINS
#error("swift-atomics requires C shims on Swift versions below 5.9")
#endif
#endif // compiler(>=5.9)