You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
263 lines
8.9 KiB
263 lines
8.9 KiB
// © 2016 and later: Unicode, Inc. and others.
|
|
// License & terms of use: http://www.unicode.org/copyright.html
|
|
/*
|
|
*******************************************************************************
|
|
* Copyright (C) 2010-2015, International Business Machines
|
|
* Corporation and others. All Rights Reserved.
|
|
*******************************************************************************
|
|
* collationdata.h
|
|
*
|
|
* created on: 2010oct27
|
|
* created by: Markus W. Scherer
|
|
*/
|
|
|
|
#ifndef __COLLATIONDATA_H__
|
|
#define __COLLATIONDATA_H__
|
|
|
|
#include "unicode/utypes.h"
|
|
|
|
#if !UCONFIG_NO_COLLATION
|
|
|
|
#include "unicode/ucol.h"
|
|
#include "unicode/uniset.h"
|
|
#include "collation.h"
|
|
#include "normalizer2impl.h"
|
|
#include "utrie2.h"
|
|
|
|
struct UDataMemory;
|
|
|
|
U_NAMESPACE_BEGIN
|
|
|
|
class UVector32;
|
|
|
|
/**
|
|
* Collation data container.
|
|
* Immutable data created by a CollationDataBuilder, or loaded from a file,
|
|
* or deserialized from API-provided binary data.
|
|
*
|
|
* Includes data for the collation base (root/default), aliased if this is not the base.
|
|
*/
|
|
struct U_I18N_API CollationData : public UMemory {
|
|
// Note: The ucadata.icu loader could discover the reserved ranges by setting an array
|
|
// parallel with the ranges, and resetting ranges that are indexed.
|
|
// The reordering builder code could clone the resulting template array.
|
|
enum {
|
|
REORDER_RESERVED_BEFORE_LATIN = UCOL_REORDER_CODE_FIRST + 14,
|
|
REORDER_RESERVED_AFTER_LATIN
|
|
};
|
|
|
|
enum {
|
|
MAX_NUM_SPECIAL_REORDER_CODES = 8,
|
|
/** C++ only, data reader check scriptStartsLength. */
|
|
MAX_NUM_SCRIPT_RANGES = 256
|
|
};
|
|
|
|
CollationData(const Normalizer2Impl &nfc)
|
|
: trie(NULL),
|
|
ce32s(NULL), ces(NULL), contexts(NULL), base(NULL),
|
|
jamoCE32s(NULL),
|
|
nfcImpl(nfc),
|
|
numericPrimary(0x12000000),
|
|
ce32sLength(0), cesLength(0), contextsLength(0),
|
|
compressibleBytes(NULL),
|
|
unsafeBackwardSet(NULL),
|
|
fastLatinTable(NULL), fastLatinTableLength(0),
|
|
numScripts(0), scriptsIndex(NULL), scriptStarts(NULL), scriptStartsLength(0),
|
|
rootElements(NULL), rootElementsLength(0) {}
|
|
|
|
uint32_t getCE32(UChar32 c) const {
|
|
return UTRIE2_GET32(trie, c);
|
|
}
|
|
|
|
uint32_t getCE32FromSupplementary(UChar32 c) const {
|
|
return UTRIE2_GET32_FROM_SUPP(trie, c);
|
|
}
|
|
|
|
UBool isDigit(UChar32 c) const {
|
|
return c < 0x660 ? c <= 0x39 && 0x30 <= c :
|
|
Collation::hasCE32Tag(getCE32(c), Collation::DIGIT_TAG);
|
|
}
|
|
|
|
UBool isUnsafeBackward(UChar32 c, UBool numeric) const {
|
|
return unsafeBackwardSet->contains(c) || (numeric && isDigit(c));
|
|
}
|
|
|
|
UBool isCompressibleLeadByte(uint32_t b) const {
|
|
return compressibleBytes[b];
|
|
}
|
|
|
|
inline UBool isCompressiblePrimary(uint32_t p) const {
|
|
return isCompressibleLeadByte(p >> 24);
|
|
}
|
|
|
|
/**
|
|
* Returns the CE32 from two contexts words.
|
|
* Access to the defaultCE32 for contraction and prefix matching.
|
|
*/
|
|
static uint32_t readCE32(const UChar *p) {
|
|
return ((uint32_t)p[0] << 16) | p[1];
|
|
}
|
|
|
|
/**
|
|
* Returns the CE32 for an indirect special CE32 (e.g., with DIGIT_TAG).
|
|
* Requires that ce32 is special.
|
|
*/
|
|
uint32_t getIndirectCE32(uint32_t ce32) const;
|
|
/**
|
|
* Returns the CE32 for an indirect special CE32 (e.g., with DIGIT_TAG),
|
|
* if ce32 is special.
|
|
*/
|
|
uint32_t getFinalCE32(uint32_t ce32) const;
|
|
|
|
/**
|
|
* Computes a CE from c's ce32 which has the OFFSET_TAG.
|
|
*/
|
|
int64_t getCEFromOffsetCE32(UChar32 c, uint32_t ce32) const {
|
|
int64_t dataCE = ces[Collation::indexFromCE32(ce32)];
|
|
return Collation::makeCE(Collation::getThreeBytePrimaryForOffsetData(c, dataCE));
|
|
}
|
|
|
|
/**
|
|
* Returns the single CE that c maps to.
|
|
* Sets U_UNSUPPORTED_ERROR if c does not map to a single CE.
|
|
*/
|
|
int64_t getSingleCE(UChar32 c, UErrorCode &errorCode) const;
|
|
|
|
/**
|
|
* Returns the FCD16 value for code point c. c must be >= 0.
|
|
*/
|
|
uint16_t getFCD16(UChar32 c) const {
|
|
return nfcImpl.getFCD16(c);
|
|
}
|
|
|
|
/**
|
|
* Returns the first primary for the script's reordering group.
|
|
* @return the primary with only the first primary lead byte of the group
|
|
* (not necessarily an actual root collator primary weight),
|
|
* or 0 if the script is unknown
|
|
*/
|
|
uint32_t getFirstPrimaryForGroup(int32_t script) const;
|
|
|
|
/**
|
|
* Returns the last primary for the script's reordering group.
|
|
* @return the last primary of the group
|
|
* (not an actual root collator primary weight),
|
|
* or 0 if the script is unknown
|
|
*/
|
|
uint32_t getLastPrimaryForGroup(int32_t script) const;
|
|
|
|
/**
|
|
* Finds the reordering group which contains the primary weight.
|
|
* @return the first script of the group, or -1 if the weight is beyond the last group
|
|
*/
|
|
int32_t getGroupForPrimary(uint32_t p) const;
|
|
|
|
int32_t getEquivalentScripts(int32_t script,
|
|
int32_t dest[], int32_t capacity, UErrorCode &errorCode) const;
|
|
|
|
/**
|
|
* Writes the permutation of primary-weight ranges
|
|
* for the given reordering of scripts and groups.
|
|
* The caller checks for illegal arguments and
|
|
* takes care of [DEFAULT] and memory allocation.
|
|
*
|
|
* Each list element will be a (limit, offset) pair as described
|
|
* for the CollationSettings::reorderRanges.
|
|
* The list will be empty if no ranges are reordered.
|
|
*/
|
|
void makeReorderRanges(const int32_t *reorder, int32_t length,
|
|
UVector32 &ranges, UErrorCode &errorCode) const;
|
|
|
|
/** @see jamoCE32s */
|
|
static const int32_t JAMO_CE32S_LENGTH = 19 + 21 + 27;
|
|
|
|
/** Main lookup trie. */
|
|
const UTrie2 *trie;
|
|
/**
|
|
* Array of CE32 values.
|
|
* At index 0 there must be CE32(U+0000)
|
|
* to support U+0000's special-tag for NUL-termination handling.
|
|
*/
|
|
const uint32_t *ce32s;
|
|
/** Array of CE values for expansions and OFFSET_TAG. */
|
|
const int64_t *ces;
|
|
/** Array of prefix and contraction-suffix matching data. */
|
|
const UChar *contexts;
|
|
/** Base collation data, or NULL if this data itself is a base. */
|
|
const CollationData *base;
|
|
/**
|
|
* Simple array of JAMO_CE32S_LENGTH=19+21+27 CE32s, one per canonical Jamo L/V/T.
|
|
* They are normally simple CE32s, rarely expansions.
|
|
* For fast handling of HANGUL_TAG.
|
|
*/
|
|
const uint32_t *jamoCE32s;
|
|
const Normalizer2Impl &nfcImpl;
|
|
/** The single-byte primary weight (xx000000) for numeric collation. */
|
|
uint32_t numericPrimary;
|
|
|
|
int32_t ce32sLength;
|
|
int32_t cesLength;
|
|
int32_t contextsLength;
|
|
|
|
/** 256 flags for which primary-weight lead bytes are compressible. */
|
|
const UBool *compressibleBytes;
|
|
/**
|
|
* Set of code points that are unsafe for starting string comparison after an identical prefix,
|
|
* or in backwards CE iteration.
|
|
*/
|
|
const UnicodeSet *unsafeBackwardSet;
|
|
|
|
/**
|
|
* Fast Latin table for common-Latin-text string comparisons.
|
|
* Data structure see class CollationFastLatin.
|
|
*/
|
|
const uint16_t *fastLatinTable;
|
|
int32_t fastLatinTableLength;
|
|
|
|
/**
|
|
* Data for scripts and reordering groups.
|
|
* Uses include building a reordering permutation table and
|
|
* providing script boundaries to AlphabeticIndex.
|
|
*/
|
|
int32_t numScripts;
|
|
/**
|
|
* The length of scriptsIndex is numScripts+16.
|
|
* It maps from a UScriptCode or a special reorder code to an entry in scriptStarts.
|
|
* 16 special reorder codes (not all used) are mapped starting at numScripts.
|
|
* Up to MAX_NUM_SPECIAL_REORDER_CODES are codes for special groups like space/punct/digit.
|
|
* There are special codes at the end for reorder-reserved primary ranges.
|
|
*
|
|
* Multiple scripts may share a range and index, for example Hira & Kana.
|
|
*/
|
|
const uint16_t *scriptsIndex;
|
|
/**
|
|
* Start primary weight (top 16 bits only) for a group/script/reserved range
|
|
* indexed by scriptsIndex.
|
|
* The first range (separators & terminators) and the last range (trailing weights)
|
|
* are not reorderable, and no scriptsIndex entry points to them.
|
|
*/
|
|
const uint16_t *scriptStarts;
|
|
int32_t scriptStartsLength;
|
|
|
|
/**
|
|
* Collation elements in the root collator.
|
|
* Used by the CollationRootElements class. The data structure is described there.
|
|
* NULL in a tailoring.
|
|
*/
|
|
const uint32_t *rootElements;
|
|
int32_t rootElementsLength;
|
|
|
|
private:
|
|
int32_t getScriptIndex(int32_t script) const;
|
|
void makeReorderRanges(const int32_t *reorder, int32_t length,
|
|
UBool latinMustMove,
|
|
UVector32 &ranges, UErrorCode &errorCode) const;
|
|
int32_t addLowScriptRange(uint8_t table[], int32_t index, int32_t lowStart) const;
|
|
int32_t addHighScriptRange(uint8_t table[], int32_t index, int32_t highLimit) const;
|
|
};
|
|
|
|
U_NAMESPACE_END
|
|
|
|
#endif // !UCONFIG_NO_COLLATION
|
|
#endif // __COLLATIONDATA_H__
|