//===- AttributeDetail.h - MLIR Affine Map details Class --------*- C++ -*-===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // // This holds implementation details of Attribute. // //===----------------------------------------------------------------------===// #ifndef ATTRIBUTEDETAIL_H_ #define ATTRIBUTEDETAIL_H_ #include "mlir/IR/AffineMap.h" #include "mlir/IR/BuiltinAttributes.h" #include "mlir/IR/BuiltinTypes.h" #include "mlir/IR/Identifier.h" #include "mlir/IR/IntegerSet.h" #include "mlir/IR/MLIRContext.h" #include "mlir/Support/StorageUniquer.h" #include "llvm/ADT/APFloat.h" #include "llvm/ADT/PointerIntPair.h" #include "llvm/Support/TrailingObjects.h" namespace mlir { namespace detail { // An attribute representing a reference to an affine map. struct AffineMapAttributeStorage : public AttributeStorage { using KeyTy = AffineMap; AffineMapAttributeStorage(AffineMap value) : AttributeStorage(IndexType::get(value.getContext())), value(value) {} /// Key equality function. bool operator==(const KeyTy &key) const { return key == value; } /// Construct a new storage instance. static AffineMapAttributeStorage * construct(AttributeStorageAllocator &allocator, KeyTy key) { return new (allocator.allocate()) AffineMapAttributeStorage(key); } AffineMap value; }; /// An attribute representing an array of other attributes. struct ArrayAttributeStorage : public AttributeStorage { using KeyTy = ArrayRef; ArrayAttributeStorage(ArrayRef value) : value(value) {} /// Key equality function. bool operator==(const KeyTy &key) const { return key == value; } /// Construct a new storage instance. static ArrayAttributeStorage *construct(AttributeStorageAllocator &allocator, const KeyTy &key) { return new (allocator.allocate()) ArrayAttributeStorage(allocator.copyInto(key)); } ArrayRef value; }; /// An attribute representing a dictionary of sorted named attributes. struct DictionaryAttributeStorage final : public AttributeStorage, private llvm::TrailingObjects { using KeyTy = ArrayRef; /// Given a list of NamedAttribute's, canonicalize the list (sorting /// by name) and return the unique'd result. static DictionaryAttributeStorage *get(ArrayRef attrs); /// Key equality function. bool operator==(const KeyTy &key) const { return key == getElements(); } /// Construct a new storage instance. static DictionaryAttributeStorage * construct(AttributeStorageAllocator &allocator, const KeyTy &key) { auto size = DictionaryAttributeStorage::totalSizeToAlloc( key.size()); auto rawMem = allocator.allocate(size, alignof(DictionaryAttributeStorage)); // Initialize the storage and trailing attribute list. auto result = ::new (rawMem) DictionaryAttributeStorage(key.size()); std::uninitialized_copy(key.begin(), key.end(), result->getTrailingObjects()); return result; } /// Return the elements of this dictionary attribute. ArrayRef getElements() const { return {getTrailingObjects(), numElements}; } private: friend class llvm::TrailingObjects; // This is used by the llvm::TrailingObjects base class. size_t numTrailingObjects(OverloadToken) const { return numElements; } DictionaryAttributeStorage(unsigned numElements) : numElements(numElements) {} /// This is the number of attributes. const unsigned numElements; }; /// An attribute representing a floating point value. struct FloatAttributeStorage final : public AttributeStorage, public llvm::TrailingObjects { using KeyTy = std::pair; FloatAttributeStorage(const llvm::fltSemantics &semantics, Type type, size_t numObjects) : AttributeStorage(type), semantics(semantics), numObjects(numObjects) {} /// Key equality and hash functions. bool operator==(const KeyTy &key) const { return key.first == getType() && key.second.bitwiseIsEqual(getValue()); } static unsigned hashKey(const KeyTy &key) { return llvm::hash_combine(key.first, llvm::hash_value(key.second)); } /// Construct a key with a type and double. static KeyTy getKey(Type type, double value) { if (type.isF64()) return KeyTy(type, APFloat(value)); // This handles, e.g., F16 because there is no APFloat constructor for it. bool unused; APFloat val(value); val.convert(type.cast().getFloatSemantics(), APFloat::rmNearestTiesToEven, &unused); return KeyTy(type, val); } /// Construct a new storage instance. static FloatAttributeStorage *construct(AttributeStorageAllocator &allocator, const KeyTy &key) { const auto &apint = key.second.bitcastToAPInt(); // Here one word's bitwidth equals to that of uint64_t. auto elements = ArrayRef(apint.getRawData(), apint.getNumWords()); auto byteSize = FloatAttributeStorage::totalSizeToAlloc(elements.size()); auto rawMem = allocator.allocate(byteSize, alignof(FloatAttributeStorage)); auto result = ::new (rawMem) FloatAttributeStorage( key.second.getSemantics(), key.first, elements.size()); std::uninitialized_copy(elements.begin(), elements.end(), result->getTrailingObjects()); return result; } /// Returns an APFloat representing the stored value. APFloat getValue() const { auto val = APInt(APFloat::getSizeInBits(semantics), {getTrailingObjects(), numObjects}); return APFloat(semantics, val); } const llvm::fltSemantics &semantics; size_t numObjects; }; /// An attribute representing an integral value. struct IntegerAttributeStorage final : public AttributeStorage, public llvm::TrailingObjects { using KeyTy = std::pair; IntegerAttributeStorage(Type type, size_t numObjects) : AttributeStorage(type), numObjects(numObjects) { assert((type.isIndex() || type.isa()) && "invalid type"); } /// Key equality and hash functions. bool operator==(const KeyTy &key) const { return key == KeyTy(getType(), getValue()); } static unsigned hashKey(const KeyTy &key) { return llvm::hash_combine(key.first, llvm::hash_value(key.second)); } /// Construct a new storage instance. static IntegerAttributeStorage * construct(AttributeStorageAllocator &allocator, const KeyTy &key) { Type type; APInt value; std::tie(type, value) = key; auto elements = ArrayRef(value.getRawData(), value.getNumWords()); auto size = IntegerAttributeStorage::totalSizeToAlloc(elements.size()); auto rawMem = allocator.allocate(size, alignof(IntegerAttributeStorage)); auto result = ::new (rawMem) IntegerAttributeStorage(type, elements.size()); std::uninitialized_copy(elements.begin(), elements.end(), result->getTrailingObjects()); return result; } /// Returns an APInt representing the stored value. APInt getValue() const { if (getType().isIndex()) return APInt(64, {getTrailingObjects(), numObjects}); return APInt(getType().getIntOrFloatBitWidth(), {getTrailingObjects(), numObjects}); } size_t numObjects; }; // An attribute representing a reference to an integer set. struct IntegerSetAttributeStorage : public AttributeStorage { using KeyTy = IntegerSet; IntegerSetAttributeStorage(IntegerSet value) : value(value) {} /// Key equality function. bool operator==(const KeyTy &key) const { return key == value; } /// Construct a new storage instance. static IntegerSetAttributeStorage * construct(AttributeStorageAllocator &allocator, KeyTy key) { return new (allocator.allocate()) IntegerSetAttributeStorage(key); } IntegerSet value; }; /// Opaque Attribute Storage and Uniquing. struct OpaqueAttributeStorage : public AttributeStorage { OpaqueAttributeStorage(Identifier dialectNamespace, StringRef attrData, Type type) : AttributeStorage(type), dialectNamespace(dialectNamespace), attrData(attrData) {} /// The hash key used for uniquing. using KeyTy = std::tuple; bool operator==(const KeyTy &key) const { return key == KeyTy(dialectNamespace, attrData, getType()); } static OpaqueAttributeStorage *construct(AttributeStorageAllocator &allocator, const KeyTy &key) { return new (allocator.allocate()) OpaqueAttributeStorage(std::get<0>(key), allocator.copyInto(std::get<1>(key)), std::get<2>(key)); } // The dialect namespace. Identifier dialectNamespace; // The parser attribute data for this opaque attribute. StringRef attrData; }; /// An attribute representing a string value. struct StringAttributeStorage : public AttributeStorage { using KeyTy = std::pair; StringAttributeStorage(StringRef value, Type type) : AttributeStorage(type), value(value) {} /// Key equality function. bool operator==(const KeyTy &key) const { return key == KeyTy(value, getType()); } /// Construct a new storage instance. static StringAttributeStorage *construct(AttributeStorageAllocator &allocator, const KeyTy &key) { return new (allocator.allocate()) StringAttributeStorage(allocator.copyInto(key.first), key.second); } StringRef value; }; /// An attribute representing a symbol reference. struct SymbolRefAttributeStorage final : public AttributeStorage, public llvm::TrailingObjects { using KeyTy = std::pair>; SymbolRefAttributeStorage(StringRef value, size_t numNestedRefs) : value(value), numNestedRefs(numNestedRefs) {} /// Key equality function. bool operator==(const KeyTy &key) const { return key == KeyTy(value, getNestedRefs()); } /// Construct a new storage instance. static SymbolRefAttributeStorage * construct(AttributeStorageAllocator &allocator, const KeyTy &key) { auto size = SymbolRefAttributeStorage::totalSizeToAlloc( key.second.size()); auto rawMem = allocator.allocate(size, alignof(SymbolRefAttributeStorage)); auto result = ::new (rawMem) SymbolRefAttributeStorage( allocator.copyInto(key.first), key.second.size()); std::uninitialized_copy(key.second.begin(), key.second.end(), result->getTrailingObjects()); return result; } /// Returns the set of nested references. ArrayRef getNestedRefs() const { return {getTrailingObjects(), numNestedRefs}; } StringRef value; size_t numNestedRefs; }; /// An attribute representing a reference to a type. struct TypeAttributeStorage : public AttributeStorage { using KeyTy = Type; TypeAttributeStorage(Type value) : value(value) {} /// Key equality function. bool operator==(const KeyTy &key) const { return key == value; } /// Construct a new storage instance. static TypeAttributeStorage *construct(AttributeStorageAllocator &allocator, KeyTy key) { return new (allocator.allocate()) TypeAttributeStorage(key); } Type value; }; //===----------------------------------------------------------------------===// // Elements Attributes //===----------------------------------------------------------------------===// /// Return the bit width which DenseElementsAttr should use for this type. inline size_t getDenseElementBitWidth(Type eltType) { // Align the width for complex to 8 to make storage and interpretation easier. if (ComplexType comp = eltType.dyn_cast()) return llvm::alignTo<8>(getDenseElementBitWidth(comp.getElementType())) * 2; if (eltType.isIndex()) return IndexType::kInternalStorageBitWidth; return eltType.getIntOrFloatBitWidth(); } /// An attribute representing a reference to a dense vector or tensor object. struct DenseElementsAttributeStorage : public AttributeStorage { public: DenseElementsAttributeStorage(ShapedType ty, bool isSplat) : AttributeStorage(ty), isSplat(isSplat) {} bool isSplat; }; /// An attribute representing a reference to a dense vector or tensor object. struct DenseIntOrFPElementsAttributeStorage : public DenseElementsAttributeStorage { DenseIntOrFPElementsAttributeStorage(ShapedType ty, ArrayRef data, bool isSplat = false) : DenseElementsAttributeStorage(ty, isSplat), data(data) {} struct KeyTy { KeyTy(ShapedType type, ArrayRef data, llvm::hash_code hashCode, bool isSplat = false) : type(type), data(data), hashCode(hashCode), isSplat(isSplat) {} /// The type of the dense elements. ShapedType type; /// The raw buffer for the data storage. ArrayRef data; /// The computed hash code for the storage data. llvm::hash_code hashCode; /// A boolean that indicates if this data is a splat or not. bool isSplat; }; /// Compare this storage instance with the provided key. bool operator==(const KeyTy &key) const { if (key.type != getType()) return false; // For boolean splats we need to explicitly check that the first bit is the // same. Boolean values are packed at the bit level, and even though a splat // is detected the rest of the bits in the first byte may differ from the // splat value. if (key.type.getElementType().isInteger(1)) { if (key.isSplat != isSplat) return false; if (isSplat) return (key.data.front() & 1) == data.front(); } // Otherwise, we can default to just checking the data. return key.data == data; } /// Construct a key from a shaped type, raw data buffer, and a flag that /// signals if the data is already known to be a splat. Callers to this /// function are expected to tag preknown splat values when possible, e.g. one /// element shapes. static KeyTy getKey(ShapedType ty, ArrayRef data, bool isKnownSplat) { // Handle an empty storage instance. if (data.empty()) return KeyTy(ty, data, 0); // If the data is already known to be a splat, the key hash value is // directly the data buffer. if (isKnownSplat) return KeyTy(ty, data, llvm::hash_value(data), isKnownSplat); // Otherwise, we need to check if the data corresponds to a splat or not. // Handle the simple case of only one element. size_t numElements = ty.getNumElements(); assert(numElements != 1 && "splat of 1 element should already be detected"); // Handle boolean values directly as they are packed to 1-bit. if (ty.getElementType().isInteger(1) == 1) return getKeyForBoolData(ty, data, numElements); size_t elementWidth = getDenseElementBitWidth(ty.getElementType()); // Non 1-bit dense elements are padded to 8-bits. size_t storageSize = llvm::divideCeil(elementWidth, CHAR_BIT); assert(((data.size() / storageSize) == numElements) && "data does not hold expected number of elements"); // Create the initial hash value with just the first element. auto firstElt = data.take_front(storageSize); auto hashVal = llvm::hash_value(firstElt); // Check to see if this storage represents a splat. If it doesn't then // combine the hash for the data starting with the first non splat element. for (size_t i = storageSize, e = data.size(); i != e; i += storageSize) if (memcmp(data.data(), &data[i], storageSize)) return KeyTy(ty, data, llvm::hash_combine(hashVal, data.drop_front(i))); // Otherwise, this is a splat so just return the hash of the first element. return KeyTy(ty, firstElt, hashVal, /*isSplat=*/true); } /// Construct a key with a set of boolean data. static KeyTy getKeyForBoolData(ShapedType ty, ArrayRef data, size_t numElements) { ArrayRef splatData = data; bool splatValue = splatData.front() & 1; // Helper functor to generate a KeyTy for a boolean splat value. auto generateSplatKey = [=] { return KeyTy(ty, data.take_front(1), llvm::hash_value(ArrayRef(splatValue ? 1 : 0)), /*isSplat=*/true); }; // Handle the case where the potential splat value is 1 and the number of // elements is non 8-bit aligned. size_t numOddElements = numElements % CHAR_BIT; if (splatValue && numOddElements != 0) { // Check that all bits are set in the last value. char lastElt = splatData.back(); if (lastElt != llvm::maskTrailingOnes(numOddElements)) return KeyTy(ty, data, llvm::hash_value(data)); // If this is the only element, the data is known to be a splat. if (splatData.size() == 1) return generateSplatKey(); splatData = splatData.drop_back(); } // Check that the data buffer corresponds to a splat of the proper mask. char mask = splatValue ? ~0 : 0; return llvm::all_of(splatData, [mask](char c) { return c == mask; }) ? generateSplatKey() : KeyTy(ty, data, llvm::hash_value(data)); } /// Hash the key for the storage. static llvm::hash_code hashKey(const KeyTy &key) { return llvm::hash_combine(key.type, key.hashCode); } /// Construct a new storage instance. static DenseIntOrFPElementsAttributeStorage * construct(AttributeStorageAllocator &allocator, KeyTy key) { // If the data buffer is non-empty, we copy it into the allocator with a // 64-bit alignment. ArrayRef copy, data = key.data; if (!data.empty()) { char *rawData = reinterpret_cast( allocator.allocate(data.size(), alignof(uint64_t))); std::memcpy(rawData, data.data(), data.size()); // If this is a boolean splat, make sure only the first bit is used. if (key.isSplat && key.type.getElementType().isInteger(1)) rawData[0] &= 1; copy = ArrayRef(rawData, data.size()); } return new (allocator.allocate()) DenseIntOrFPElementsAttributeStorage(key.type, copy, key.isSplat); } ArrayRef data; }; /// An attribute representing a reference to a dense vector or tensor object /// containing strings. struct DenseStringElementsAttributeStorage : public DenseElementsAttributeStorage { DenseStringElementsAttributeStorage(ShapedType ty, ArrayRef data, bool isSplat = false) : DenseElementsAttributeStorage(ty, isSplat), data(data) {} struct KeyTy { KeyTy(ShapedType type, ArrayRef data, llvm::hash_code hashCode, bool isSplat = false) : type(type), data(data), hashCode(hashCode), isSplat(isSplat) {} /// The type of the dense elements. ShapedType type; /// The raw buffer for the data storage. ArrayRef data; /// The computed hash code for the storage data. llvm::hash_code hashCode; /// A boolean that indicates if this data is a splat or not. bool isSplat; }; /// Compare this storage instance with the provided key. bool operator==(const KeyTy &key) const { if (key.type != getType()) return false; // Otherwise, we can default to just checking the data. StringRefs compare // by contents. return key.data == data; } /// Construct a key from a shaped type, StringRef data buffer, and a flag that /// signals if the data is already known to be a splat. Callers to this /// function are expected to tag preknown splat values when possible, e.g. one /// element shapes. static KeyTy getKey(ShapedType ty, ArrayRef data, bool isKnownSplat) { // Handle an empty storage instance. if (data.empty()) return KeyTy(ty, data, 0); // If the data is already known to be a splat, the key hash value is // directly the data buffer. if (isKnownSplat) return KeyTy(ty, data, llvm::hash_value(data.front()), isKnownSplat); // Handle the simple case of only one element. assert(ty.getNumElements() != 1 && "splat of 1 element should already be detected"); // Create the initial hash value with just the first element. const auto &firstElt = data.front(); auto hashVal = llvm::hash_value(firstElt); // Check to see if this storage represents a splat. If it doesn't then // combine the hash for the data starting with the first non splat element. for (size_t i = 1, e = data.size(); i != e; i++) if (!firstElt.equals(data[i])) return KeyTy(ty, data, llvm::hash_combine(hashVal, data.drop_front(i))); // Otherwise, this is a splat so just return the hash of the first element. return KeyTy(ty, data.take_front(), hashVal, /*isSplat=*/true); } /// Hash the key for the storage. static llvm::hash_code hashKey(const KeyTy &key) { return llvm::hash_combine(key.type, key.hashCode); } /// Construct a new storage instance. static DenseStringElementsAttributeStorage * construct(AttributeStorageAllocator &allocator, KeyTy key) { // If the data buffer is non-empty, we copy it into the allocator with a // 64-bit alignment. ArrayRef copy, data = key.data; if (data.empty()) { return new (allocator.allocate()) DenseStringElementsAttributeStorage(key.type, copy, key.isSplat); } int numEntries = key.isSplat ? 1 : data.size(); // Compute the amount data needed to store the ArrayRef and StringRef // contents. size_t dataSize = sizeof(StringRef) * numEntries; for (int i = 0; i < numEntries; i++) dataSize += data[i].size(); char *rawData = reinterpret_cast( allocator.allocate(dataSize, alignof(uint64_t))); // Setup a mutable array ref of our string refs so that we can update their // contents. auto mutableCopy = MutableArrayRef( reinterpret_cast(rawData), numEntries); auto stringData = rawData + numEntries * sizeof(StringRef); for (int i = 0; i < numEntries; i++) { memcpy(stringData, data[i].data(), data[i].size()); mutableCopy[i] = StringRef(stringData, data[i].size()); stringData += data[i].size(); } copy = ArrayRef(reinterpret_cast(rawData), numEntries); return new (allocator.allocate()) DenseStringElementsAttributeStorage(key.type, copy, key.isSplat); } ArrayRef data; }; /// An attribute representing a reference to a tensor constant with opaque /// content. struct OpaqueElementsAttributeStorage : public AttributeStorage { using KeyTy = std::tuple; OpaqueElementsAttributeStorage(Type type, Dialect *dialect, StringRef bytes) : AttributeStorage(type), dialect(dialect), bytes(bytes) {} /// Key equality and hash functions. bool operator==(const KeyTy &key) const { return key == std::make_tuple(getType(), dialect, bytes); } static unsigned hashKey(const KeyTy &key) { return llvm::hash_combine(std::get<0>(key), std::get<1>(key), std::get<2>(key)); } /// Construct a new storage instance. static OpaqueElementsAttributeStorage * construct(AttributeStorageAllocator &allocator, KeyTy key) { // TODO: Provide a way to avoid copying content of large opaque // tensors This will likely require a new reference attribute kind. return new (allocator.allocate()) OpaqueElementsAttributeStorage(std::get<0>(key), std::get<1>(key), allocator.copyInto(std::get<2>(key))); } Dialect *dialect; StringRef bytes; }; /// An attribute representing a reference to a sparse vector or tensor object. struct SparseElementsAttributeStorage : public AttributeStorage { using KeyTy = std::tuple; SparseElementsAttributeStorage(Type type, DenseIntElementsAttr indices, DenseElementsAttr values) : AttributeStorage(type), indices(indices), values(values) {} /// Key equality and hash functions. bool operator==(const KeyTy &key) const { return key == std::make_tuple(getType(), indices, values); } static unsigned hashKey(const KeyTy &key) { return llvm::hash_combine(std::get<0>(key), std::get<1>(key), std::get<2>(key)); } /// Construct a new storage instance. static SparseElementsAttributeStorage * construct(AttributeStorageAllocator &allocator, KeyTy key) { return new (allocator.allocate()) SparseElementsAttributeStorage(std::get<0>(key), std::get<1>(key), std::get<2>(key)); } DenseIntElementsAttr indices; DenseElementsAttr values; }; } // namespace detail } // namespace mlir #endif // ATTRIBUTEDETAIL_H_