You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
1033 lines
41 KiB
1033 lines
41 KiB
// © 2016 and later: Unicode, Inc. and others.
|
|
// License & terms of use: http://www.unicode.org/copyright.html
|
|
/********************************************************************
|
|
* Copyright (c) 2016, International Business Machines Corporation and
|
|
* others. All Rights Reserved.
|
|
********************************************************************/
|
|
|
|
|
|
#include "unicode/utypes.h"
|
|
|
|
#if !UCONFIG_NO_BREAK_ITERATION && !UCONFIG_NO_REGULAR_EXPRESSIONS && !UCONFIG_NO_FORMATTING
|
|
|
|
#include "rbbimonkeytest.h"
|
|
#include "unicode/utypes.h"
|
|
#include "unicode/brkiter.h"
|
|
#include "unicode/utf16.h"
|
|
#include "unicode/uniset.h"
|
|
#include "unicode/unistr.h"
|
|
|
|
#include "charstr.h"
|
|
#include "cmemory.h"
|
|
#include "cstr.h"
|
|
#include "uelement.h"
|
|
#include "uhash.h"
|
|
|
|
#include <iostream>
|
|
#include <stdio.h>
|
|
#include <stdlib.h>
|
|
#include <string>
|
|
|
|
using namespace icu;
|
|
|
|
|
|
void RBBIMonkeyTest::runIndexedTest(int32_t index, UBool exec, const char* &name, char* params) {
|
|
fParams = params; // Work around TESTCASE_AUTO not being able to pass params to test function.
|
|
|
|
TESTCASE_AUTO_BEGIN;
|
|
TESTCASE_AUTO(testMonkey);
|
|
TESTCASE_AUTO_END;
|
|
}
|
|
|
|
//---------------------------------------------------------------------------------------
|
|
//
|
|
// class BreakRule implementation.
|
|
//
|
|
//---------------------------------------------------------------------------------------
|
|
|
|
BreakRule::BreakRule() // : all field default initialized.
|
|
{
|
|
}
|
|
|
|
BreakRule::~BreakRule() {}
|
|
|
|
|
|
//---------------------------------------------------------------------------------------
|
|
//
|
|
// class BreakRules implementation.
|
|
//
|
|
//---------------------------------------------------------------------------------------
|
|
BreakRules::BreakRules(RBBIMonkeyImpl *monkeyImpl, UErrorCode &status) :
|
|
fMonkeyImpl(monkeyImpl), fBreakRules(status), fType(UBRK_COUNT) {
|
|
fCharClasses.adoptInstead(uhash_open(uhash_hashUnicodeString,
|
|
uhash_compareUnicodeString,
|
|
NULL, // value comparator.
|
|
&status));
|
|
if (U_FAILURE(status)) {
|
|
return;
|
|
}
|
|
uhash_setKeyDeleter(fCharClasses.getAlias(), uprv_deleteUObject);
|
|
uhash_setValueDeleter(fCharClasses.getAlias(), uprv_deleteUObject);
|
|
fBreakRules.setDeleter(uprv_deleteUObject);
|
|
|
|
fCharClassList.adoptInstead(new UVector(status));
|
|
|
|
fSetRefsMatcher.adoptInstead(new RegexMatcher(UnicodeString(
|
|
"(?!(?:\\{|=|\\[:)[ \\t]{0,4})" // Negative look behind for '{' or '=' or '[:'
|
|
// (the identifier is a unicode property name or value)
|
|
"(?<ClassName>[A-Za-z_][A-Za-z0-9_]*)"), // The char class name
|
|
0, status));
|
|
|
|
// Match comments and blank lines. Matches will be replaced with "", stripping the comments from the rules.
|
|
fCommentsMatcher.adoptInstead(new RegexMatcher(UnicodeString(
|
|
"(^|(?<=;))" // Start either at start of line, or just after a ';' (look-behind for ';')
|
|
"[ \\t]*+" // Match white space.
|
|
"(#.*)?+" // Optional # plus whatever follows
|
|
"\\R$" // new-line at end of line.
|
|
), 0, status));
|
|
|
|
// Match (initial parse) of a character class definition line.
|
|
fClassDefMatcher.adoptInstead(new RegexMatcher(UnicodeString(
|
|
"[ \\t]*" // leading white space
|
|
"(?<ClassName>[A-Za-z_][A-Za-z0-9_]*)" // The char class name
|
|
"[ \\t]*=[ \\t]*" // =
|
|
"(?<ClassDef>.*?)" // The char class UnicodeSet expression
|
|
"[ \\t]*;$"), // ; <end of line>
|
|
0, status));
|
|
|
|
// Match (initial parse) of a break rule line.
|
|
fRuleDefMatcher.adoptInstead(new RegexMatcher(UnicodeString(
|
|
"[ \\t]*" // leading white space
|
|
"(?<RuleName>[A-Za-z_][A-Za-z0-9_.]*)" // The rule name
|
|
"[ \\t]*:[ \\t]*" // :
|
|
"(?<RuleDef>.*?)" // The rule definition
|
|
"[ \\t]*;$"), // ; <end of line>
|
|
0, status));
|
|
|
|
}
|
|
|
|
|
|
BreakRules::~BreakRules() {}
|
|
|
|
|
|
CharClass *BreakRules::addCharClass(const UnicodeString &name, const UnicodeString &definition, UErrorCode &status) {
|
|
|
|
// Create the expanded definition for this char class,
|
|
// replacing any set references with the corresponding definition.
|
|
|
|
UnicodeString expandedDef;
|
|
UnicodeString emptyString;
|
|
fSetRefsMatcher->reset(definition);
|
|
while (fSetRefsMatcher->find() && U_SUCCESS(status)) {
|
|
const UnicodeString name =
|
|
fSetRefsMatcher->group(fSetRefsMatcher->pattern().groupNumberFromName("ClassName", status), status);
|
|
CharClass *nameClass = static_cast<CharClass *>(uhash_get(fCharClasses.getAlias(), &name));
|
|
const UnicodeString &expansionForName = nameClass ? nameClass->fExpandedDef : name;
|
|
|
|
fSetRefsMatcher->appendReplacement(expandedDef, emptyString, status);
|
|
expandedDef.append(expansionForName);
|
|
}
|
|
fSetRefsMatcher->appendTail(expandedDef);
|
|
|
|
// Verify that the expanded set definition is valid.
|
|
|
|
if (fMonkeyImpl->fDumpExpansions) {
|
|
printf("epandedDef: %s\n", CStr(expandedDef)());
|
|
}
|
|
|
|
LocalPointer<UnicodeSet> s(new UnicodeSet(expandedDef, USET_IGNORE_SPACE, NULL, status), status);
|
|
if (U_FAILURE(status)) {
|
|
IntlTest::gTest->errln("%s:%d: error %s creating UnicodeSet %s\n Expanded set definition: %s",
|
|
__FILE__, __LINE__, u_errorName(status), CStr(name)(), CStr(expandedDef)());
|
|
return nullptr;
|
|
}
|
|
CharClass *cclass = new CharClass(name, definition, expandedDef, s.orphan());
|
|
CharClass *previousClass = static_cast<CharClass *>(uhash_put(fCharClasses.getAlias(),
|
|
new UnicodeString(name), // Key, owned by hash table.
|
|
cclass, // Value, owned by hash table.
|
|
&status));
|
|
|
|
if (previousClass != NULL) {
|
|
// Duplicate class def.
|
|
// These are legitimate, they are adjustments of an existing class.
|
|
// TODO: will need to keep the old around when we handle tailorings.
|
|
IntlTest::gTest->logln("Redefinition of character class %s\n", CStr(cclass->fName)());
|
|
delete previousClass;
|
|
}
|
|
return cclass;
|
|
}
|
|
|
|
|
|
void BreakRules::addRule(const UnicodeString &name, const UnicodeString &definition, UErrorCode &status) {
|
|
LocalPointer<BreakRule> thisRule(new BreakRule);
|
|
thisRule->fName = name;
|
|
thisRule->fRule = definition;
|
|
|
|
// If the rule name contains embedded digits, pad the first numeric field to a fixed length with leading zeroes,
|
|
// This gives a numeric sort order that matches Unicode UAX rule numbering conventions.
|
|
UnicodeString emptyString;
|
|
|
|
// Expand the char class definitions within the rule.
|
|
fSetRefsMatcher->reset(definition);
|
|
while (fSetRefsMatcher->find() && U_SUCCESS(status)) {
|
|
const UnicodeString name =
|
|
fSetRefsMatcher->group(fSetRefsMatcher->pattern().groupNumberFromName("ClassName", status), status);
|
|
CharClass *nameClass = static_cast<CharClass *>(uhash_get(fCharClasses.getAlias(), &name));
|
|
if (!nameClass) {
|
|
IntlTest::gTest->errln("%s:%d char class \"%s\" unrecognized in rule \"%s\"",
|
|
__FILE__, __LINE__, CStr(name)(), CStr(definition)());
|
|
}
|
|
const UnicodeString &expansionForName = nameClass ? nameClass->fExpandedDef : name;
|
|
|
|
fSetRefsMatcher->appendReplacement(thisRule->fExpandedRule, emptyString, status);
|
|
thisRule->fExpandedRule.append(expansionForName);
|
|
}
|
|
fSetRefsMatcher->appendTail(thisRule->fExpandedRule);
|
|
|
|
// If rule begins with a '^' rule chaining is disallowed.
|
|
// Strip off the '^' from the rule expression, and set the flag.
|
|
if (thisRule->fExpandedRule.charAt(0) == u'^') {
|
|
thisRule->fInitialMatchOnly = true;
|
|
thisRule->fExpandedRule.remove(0, 1);
|
|
thisRule->fExpandedRule.trim();
|
|
}
|
|
|
|
// Replace the divide sign (\u00f7) with a regular expression named capture.
|
|
// When running the rules, a match that includes this group means we found a break position.
|
|
|
|
int32_t dividePos = thisRule->fExpandedRule.indexOf((UChar)0x00f7);
|
|
if (dividePos >= 0) {
|
|
thisRule->fExpandedRule.replace(dividePos, 1, UnicodeString("(?<BreakPosition>)"));
|
|
}
|
|
if (thisRule->fExpandedRule.indexOf((UChar)0x00f7) != -1) {
|
|
status = U_ILLEGAL_ARGUMENT_ERROR; // TODO: produce a good error message.
|
|
}
|
|
|
|
// UAX break rule set definitions can be empty, just [].
|
|
// Regular expression set expressions don't accept this. Substitute with [^\u0000-\U0010ffff], which
|
|
// also matches nothing.
|
|
|
|
static const UChar emptySet[] = {(UChar)0x5b, (UChar)0x5d, 0};
|
|
int32_t where = 0;
|
|
while ((where = thisRule->fExpandedRule.indexOf(emptySet, 2, 0)) >= 0) {
|
|
thisRule->fExpandedRule.replace(where, 2, UnicodeString("[^\\u0000-\\U0010ffff]"));
|
|
}
|
|
if (fMonkeyImpl->fDumpExpansions) {
|
|
printf("fExpandedRule: %s\n", CStr(thisRule->fExpandedRule)());
|
|
}
|
|
|
|
// Compile a regular expression for this rule.
|
|
thisRule->fRuleMatcher.adoptInstead(new RegexMatcher(thisRule->fExpandedRule, UREGEX_COMMENTS | UREGEX_DOTALL, status));
|
|
if (U_FAILURE(status)) {
|
|
IntlTest::gTest->errln("%s:%d Error creating regular expression for %s",
|
|
__FILE__, __LINE__, CStr(thisRule->fExpandedRule)());
|
|
return;
|
|
}
|
|
|
|
// Put this new rule into the vector of all Rules.
|
|
fBreakRules.addElement(thisRule.orphan(), status);
|
|
}
|
|
|
|
|
|
bool BreakRules::setKeywordParameter(const UnicodeString &keyword, const UnicodeString &value, UErrorCode &status) {
|
|
if (keyword == UnicodeString("locale")) {
|
|
CharString localeName;
|
|
localeName.append(CStr(value)(), -1, status);
|
|
fLocale = Locale::createFromName(localeName.data());
|
|
return true;
|
|
}
|
|
if (keyword == UnicodeString("type")) {
|
|
if (value == UnicodeString("grapheme")) {
|
|
fType = UBRK_CHARACTER;
|
|
} else if (value == UnicodeString("word")) {
|
|
fType = UBRK_WORD;
|
|
} else if (value == UnicodeString("line")) {
|
|
fType = UBRK_LINE;
|
|
} else if (value == UnicodeString("sentence")) {
|
|
fType = UBRK_SENTENCE;
|
|
} else {
|
|
IntlTest::gTest->errln("%s:%d Unrecognized break type %s", __FILE__, __LINE__, CStr(value)());
|
|
}
|
|
return true;
|
|
}
|
|
// TODO: add tailoring base setting here.
|
|
return false;
|
|
}
|
|
|
|
RuleBasedBreakIterator *BreakRules::createICUBreakIterator(UErrorCode &status) {
|
|
if (U_FAILURE(status)) {
|
|
return NULL;
|
|
}
|
|
RuleBasedBreakIterator *bi = NULL;
|
|
switch(fType) {
|
|
case UBRK_CHARACTER:
|
|
bi = dynamic_cast<RuleBasedBreakIterator *>(BreakIterator::createCharacterInstance(fLocale, status));
|
|
break;
|
|
case UBRK_WORD:
|
|
bi = dynamic_cast<RuleBasedBreakIterator *>(BreakIterator::createWordInstance(fLocale, status));
|
|
break;
|
|
case UBRK_LINE:
|
|
bi = dynamic_cast<RuleBasedBreakIterator *>(BreakIterator::createLineInstance(fLocale, status));
|
|
break;
|
|
case UBRK_SENTENCE:
|
|
bi = dynamic_cast<RuleBasedBreakIterator *>(BreakIterator::createSentenceInstance(fLocale, status));
|
|
break;
|
|
default:
|
|
IntlTest::gTest->errln("%s:%d Bad break iterator type of %d", __FILE__, __LINE__, fType);
|
|
status = U_ILLEGAL_ARGUMENT_ERROR;
|
|
}
|
|
return bi;
|
|
}
|
|
|
|
|
|
void BreakRules::compileRules(UCHARBUF *rules, UErrorCode &status) {
|
|
if (U_FAILURE(status)) {
|
|
return;
|
|
}
|
|
|
|
UnicodeString emptyString;
|
|
for (int32_t lineNumber=0; ;lineNumber++) { // Loop once per input line.
|
|
if (U_FAILURE(status)) {
|
|
return;
|
|
}
|
|
int32_t lineLength = 0;
|
|
const UChar *lineBuf = ucbuf_readline(rules, &lineLength, &status);
|
|
if (lineBuf == NULL) {
|
|
break;
|
|
}
|
|
UnicodeString line(lineBuf, lineLength);
|
|
|
|
// Strip comment lines.
|
|
fCommentsMatcher->reset(line);
|
|
line = fCommentsMatcher->replaceFirst(emptyString, status);
|
|
if (line.isEmpty()) {
|
|
continue;
|
|
}
|
|
|
|
// Recognize character class definition and keyword lines
|
|
fClassDefMatcher->reset(line);
|
|
if (fClassDefMatcher->matches(status)) {
|
|
UnicodeString className = fClassDefMatcher->group(fClassDefMatcher->pattern().groupNumberFromName("ClassName", status), status);
|
|
UnicodeString classDef = fClassDefMatcher->group(fClassDefMatcher->pattern().groupNumberFromName("ClassDef", status), status);
|
|
if (fMonkeyImpl->fDumpExpansions) {
|
|
printf("scanned class: %s = %s\n", CStr(className)(), CStr(classDef)());
|
|
}
|
|
if (setKeywordParameter(className, classDef, status)) {
|
|
// The scanned item was "type = ..." or "locale = ...", etc.
|
|
// which are not actual character classes.
|
|
continue;
|
|
}
|
|
addCharClass(className, classDef, status);
|
|
continue;
|
|
}
|
|
|
|
// Recognize rule lines.
|
|
fRuleDefMatcher->reset(line);
|
|
if (fRuleDefMatcher->matches(status)) {
|
|
UnicodeString ruleName = fRuleDefMatcher->group(fRuleDefMatcher->pattern().groupNumberFromName("RuleName", status), status);
|
|
UnicodeString ruleDef = fRuleDefMatcher->group(fRuleDefMatcher->pattern().groupNumberFromName("RuleDef", status), status);
|
|
if (fMonkeyImpl->fDumpExpansions) {
|
|
printf("scanned rule: %s : %s\n", CStr(ruleName)(), CStr(ruleDef)());
|
|
}
|
|
addRule(ruleName, ruleDef, status);
|
|
continue;
|
|
}
|
|
|
|
IntlTest::gTest->errln("%s:%d: Unrecognized line in rule file %s: \"%s\"\n",
|
|
__FILE__, __LINE__, fMonkeyImpl->fRuleFileName, CStr(line)());
|
|
}
|
|
|
|
// Build the vector of char classes, omitting the dictionary class if there is one.
|
|
// This will be used when constructing the random text to be tested.
|
|
|
|
// Also compute the "other" set, consisting of any characters not included in
|
|
// one or more of the user defined sets.
|
|
|
|
UnicodeSet otherSet((UChar32)0, 0x10ffff);
|
|
int32_t pos = UHASH_FIRST;
|
|
const UHashElement *el = NULL;
|
|
while ((el = uhash_nextElement(fCharClasses.getAlias(), &pos)) != NULL) {
|
|
const UnicodeString *ccName = static_cast<const UnicodeString *>(el->key.pointer);
|
|
CharClass *cclass = static_cast<CharClass *>(el->value.pointer);
|
|
// printf(" Adding %s\n", CStr(*ccName)());
|
|
if (*ccName != cclass->fName) {
|
|
IntlTest::gTest->errln("%s:%d: internal error, set names (%s, %s) inconsistent.\n",
|
|
__FILE__, __LINE__, CStr(*ccName)(), CStr(cclass->fName)());
|
|
}
|
|
const UnicodeSet *set = cclass->fSet.getAlias();
|
|
otherSet.removeAll(*set);
|
|
if (*ccName == UnicodeString("dictionary")) {
|
|
fDictionarySet = *set;
|
|
} else {
|
|
fCharClassList->addElement(cclass, status);
|
|
}
|
|
}
|
|
|
|
if (!otherSet.isEmpty()) {
|
|
// fprintf(stderr, "have an other set.\n");
|
|
UnicodeString pattern;
|
|
CharClass *cclass = addCharClass(UnicodeString("__Others"), otherSet.toPattern(pattern), status);
|
|
fCharClassList->addElement(cclass, status);
|
|
}
|
|
}
|
|
|
|
|
|
const CharClass *BreakRules::getClassForChar(UChar32 c, int32_t *iter) const {
|
|
int32_t localIter = 0;
|
|
int32_t &it = iter? *iter : localIter;
|
|
|
|
while (it < fCharClassList->size()) {
|
|
const CharClass *cc = static_cast<const CharClass *>(fCharClassList->elementAt(it));
|
|
++it;
|
|
if (cc->fSet->contains(c)) {
|
|
return cc;
|
|
}
|
|
}
|
|
return NULL;
|
|
}
|
|
|
|
//---------------------------------------------------------------------------------------
|
|
//
|
|
// class MonkeyTestData implementation.
|
|
//
|
|
//---------------------------------------------------------------------------------------
|
|
|
|
void MonkeyTestData::set(BreakRules *rules, IntlTest::icu_rand &rand, UErrorCode &status) {
|
|
const int32_t dataLength = 1000;
|
|
|
|
// Fill the test string with random characters.
|
|
// First randomly pick a char class, then randomly pick a character from that class.
|
|
// Exclude any characters from the dictionary set.
|
|
|
|
// std::cout << "Populating Test Data" << std::endl;
|
|
fRandomSeed = rand.getSeed(); // Save initial seed for use in error messages,
|
|
// allowing recreation of failing data.
|
|
fBkRules = rules;
|
|
fString.remove();
|
|
for (int32_t n=0; n<dataLength;) {
|
|
int charClassIndex = rand() % rules->fCharClassList->size();
|
|
const CharClass *cclass = static_cast<CharClass *>(rules->fCharClassList->elementAt(charClassIndex));
|
|
if (cclass->fSet->size() == 0) {
|
|
// Some rules or tailorings do end up with empty char classes.
|
|
continue;
|
|
}
|
|
int32_t charIndex = rand() % cclass->fSet->size();
|
|
UChar32 c = cclass->fSet->charAt(charIndex);
|
|
if (U16_IS_TRAIL(c) && fString.length() > 0 && U16_IS_LEAD(fString.charAt(fString.length()-1))) {
|
|
// Character classes may contain unpaired surrogates, e.g. Grapheme_Cluster_Break = Control.
|
|
// Don't let random unpaired surrogates combine in the test data because they might
|
|
// produce an unwanted dictionary character.
|
|
continue;
|
|
}
|
|
|
|
if (!rules->fDictionarySet.contains(c)) {
|
|
fString.append(c);
|
|
++n;
|
|
}
|
|
}
|
|
|
|
// Reset each rule matcher regex with this new string.
|
|
// (Although we are always using the same string object, ICU regular expressions
|
|
// don't like the underlying string data changing without doing a reset).
|
|
|
|
for (int32_t ruleNum=0; ruleNum<rules->fBreakRules.size(); ruleNum++) {
|
|
BreakRule *rule = static_cast<BreakRule *>(rules->fBreakRules.elementAt(ruleNum));
|
|
rule->fRuleMatcher->reset(fString);
|
|
}
|
|
|
|
// Init the expectedBreaks, actualBreaks and ruleForPosition strings (used as arrays).
|
|
// Expected and Actual breaks are one longer than the input string; a non-zero value
|
|
// will indicate a boundary preceding that position.
|
|
|
|
clearActualBreaks();
|
|
fExpectedBreaks = fActualBreaks;
|
|
fRuleForPosition = fActualBreaks;
|
|
f2ndRuleForPos = fActualBreaks;
|
|
|
|
// Apply reference rules to find the expected breaks.
|
|
|
|
fExpectedBreaks.setCharAt(0, (UChar)1); // Force an expected break before the start of the text.
|
|
// ICU always reports a break there.
|
|
// The reference rules do not have a means to do so.
|
|
int32_t strIdx = 0;
|
|
bool initialMatch = true; // True at start of text, and immediately after each boundary,
|
|
// for control over rule chaining.
|
|
while (strIdx < fString.length()) {
|
|
BreakRule *matchingRule = NULL;
|
|
UBool hasBreak = FALSE;
|
|
int32_t ruleNum = 0;
|
|
int32_t matchStart = 0;
|
|
int32_t matchEnd = 0;
|
|
int32_t breakGroup = 0;
|
|
for (ruleNum=0; ruleNum<rules->fBreakRules.size(); ruleNum++) {
|
|
BreakRule *rule = static_cast<BreakRule *>(rules->fBreakRules.elementAt(ruleNum));
|
|
if (rule->fInitialMatchOnly && !initialMatch) {
|
|
// Skip checking this '^' rule. (No rule chaining)
|
|
continue;
|
|
}
|
|
rule->fRuleMatcher->reset();
|
|
if (rule->fRuleMatcher->lookingAt(strIdx, status)) {
|
|
// A candidate rule match, check further to see if we take it or continue to check other rules.
|
|
// Matches of zero or one codepoint count only if they also specify a break.
|
|
matchStart = rule->fRuleMatcher->start(status);
|
|
matchEnd = rule->fRuleMatcher->end(status);
|
|
breakGroup = rule->fRuleMatcher->pattern().groupNumberFromName("BreakPosition", status);
|
|
hasBreak = U_SUCCESS(status);
|
|
if (status == U_REGEX_INVALID_CAPTURE_GROUP_NAME) {
|
|
status = U_ZERO_ERROR;
|
|
}
|
|
if (hasBreak || fString.moveIndex32(matchStart, 1) < matchEnd) {
|
|
matchingRule = rule;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
if (matchingRule == NULL) {
|
|
// No reference rule matched. This is an error in the rules that should never happen.
|
|
IntlTest::gTest->errln("%s:%d Trouble with monkey test reference rules at position %d. ",
|
|
__FILE__, __LINE__, strIdx);
|
|
dump(strIdx);
|
|
status = U_INVALID_FORMAT_ERROR;
|
|
return;
|
|
}
|
|
if (matchingRule->fRuleMatcher->group(status).length() == 0) {
|
|
// Zero length rule match. This is also an error in the rule expressions.
|
|
IntlTest::gTest->errln("%s:%d Zero length rule match.",
|
|
__FILE__, __LINE__);
|
|
status = U_INVALID_FORMAT_ERROR;
|
|
return;
|
|
}
|
|
|
|
// Record which rule matched over the length of the match.
|
|
for (int i = matchStart; i < matchEnd; i++) {
|
|
if (fRuleForPosition.charAt(i) == 0) {
|
|
fRuleForPosition.setCharAt(i, (UChar)ruleNum);
|
|
} else {
|
|
f2ndRuleForPos.setCharAt(i, (UChar)ruleNum);
|
|
}
|
|
}
|
|
|
|
// Break positions appear in rules as a matching named capture of zero length at the break position,
|
|
// the adjusted pattern contains (?<BreakPosition>)
|
|
if (hasBreak) {
|
|
int32_t breakPos = matchingRule->fRuleMatcher->start(breakGroup, status);
|
|
if (U_FAILURE(status) || breakPos < 0) {
|
|
// Rule specified a break, but that break wasn't part of the match, even
|
|
// though the rule as a whole matched.
|
|
// Can't happen with regular expressions derived from (equivalent to) ICU break rules.
|
|
// Shouldn't get here.
|
|
IntlTest::gTest->errln("%s:%d Internal Rule Error.", __FILE__, __LINE__);
|
|
status = U_INVALID_FORMAT_ERROR;
|
|
break;
|
|
}
|
|
fExpectedBreaks.setCharAt(breakPos, (UChar)1);
|
|
// printf("recording break at %d\n", breakPos);
|
|
// For the next iteration, pick up applying rules immediately after the break,
|
|
// which may differ from end of the match. The matching rule may have included
|
|
// context following the boundary that needs to be looked at again.
|
|
strIdx = matchingRule->fRuleMatcher->end(breakGroup, status);
|
|
initialMatch = true;
|
|
} else {
|
|
// Original rule didn't specify a break.
|
|
// Continue applying rules starting on the last code point of this match.
|
|
strIdx = fString.moveIndex32(matchEnd, -1);
|
|
initialMatch = false;
|
|
if (strIdx == matchStart) {
|
|
// Match was only one code point, no progress if we continue.
|
|
// Shouldn't get here, case is filtered out at top of loop.
|
|
CharString ruleName;
|
|
ruleName.appendInvariantChars(matchingRule->fName, status);
|
|
IntlTest::gTest->errln("%s:%d Rule %s internal error",
|
|
__FILE__, __LINE__, ruleName.data());
|
|
status = U_INVALID_FORMAT_ERROR;
|
|
break;
|
|
}
|
|
}
|
|
if (U_FAILURE(status)) {
|
|
IntlTest::gTest->errln("%s:%d status = %s. Unexpected failure, perhaps problem internal to test.",
|
|
__FILE__, __LINE__, u_errorName(status));
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
void MonkeyTestData::clearActualBreaks() {
|
|
fActualBreaks.remove();
|
|
// Actual Breaks length is one longer than the data string length, allowing
|
|
// for breaks before the first and after the last character in the data.
|
|
for (int32_t i=0; i<=fString.length(); i++) {
|
|
fActualBreaks.append((UChar)0);
|
|
}
|
|
}
|
|
|
|
void MonkeyTestData::dump(int32_t around) const {
|
|
printf("\n"
|
|
" char break Rule Character\n"
|
|
" pos code class R I name name\n"
|
|
"---------------------------------------------------------------------------------------------\n");
|
|
|
|
int32_t start;
|
|
int32_t end;
|
|
|
|
if (around == -1) {
|
|
start = 0;
|
|
end = fString.length();
|
|
} else {
|
|
// Display context around a failure.
|
|
start = fString.moveIndex32(around, -30);
|
|
end = fString.moveIndex32(around, +30);
|
|
}
|
|
|
|
for (int charIdx = start; charIdx < end; charIdx=fString.moveIndex32(charIdx, 1)) {
|
|
UErrorCode status = U_ZERO_ERROR;
|
|
UChar32 c = fString.char32At(charIdx);
|
|
const CharClass *cc = fBkRules->getClassForChar(c);
|
|
CharString ccName;
|
|
ccName.appendInvariantChars(cc->fName, status);
|
|
CharString ruleName, secondRuleName;
|
|
const BreakRule *rule = static_cast<BreakRule *>(fBkRules->fBreakRules.elementAt(fRuleForPosition.charAt(charIdx)));
|
|
ruleName.appendInvariantChars(rule->fName, status);
|
|
if (f2ndRuleForPos.charAt(charIdx) > 0) {
|
|
const BreakRule *secondRule = static_cast<BreakRule *>(fBkRules->fBreakRules.elementAt(f2ndRuleForPos.charAt(charIdx)));
|
|
secondRuleName.appendInvariantChars(secondRule->fName, status);
|
|
}
|
|
char cName[200];
|
|
u_charName(c, U_EXTENDED_CHAR_NAME, cName, sizeof(cName), &status);
|
|
|
|
printf(" %4.1d %6.4x %-20s %c %c %-10s %-10s %s\n",
|
|
charIdx, c, ccName.data(),
|
|
fExpectedBreaks.charAt(charIdx) ? '*' : '.',
|
|
fActualBreaks.charAt(charIdx) ? '*' : '.',
|
|
ruleName.data(), secondRuleName.data(), cName
|
|
);
|
|
}
|
|
}
|
|
|
|
|
|
//---------------------------------------------------------------------------------------
|
|
//
|
|
// class RBBIMonkeyImpl
|
|
//
|
|
//---------------------------------------------------------------------------------------
|
|
|
|
RBBIMonkeyImpl::RBBIMonkeyImpl(UErrorCode &status) : fDumpExpansions(FALSE), fThread(this) {
|
|
(void)status; // suppress unused parameter compiler warning.
|
|
}
|
|
|
|
|
|
// RBBIMonkeyImpl setup does all of the setup for a single rule set - compiling the
|
|
// reference rules and creating the icu breakiterator to test,
|
|
// with its type and locale coming from the reference rules.
|
|
|
|
void RBBIMonkeyImpl::setup(const char *ruleFile, UErrorCode &status) {
|
|
fRuleFileName = ruleFile;
|
|
openBreakRules(ruleFile, status);
|
|
if (U_FAILURE(status)) {
|
|
IntlTest::gTest->errln("%s:%d Error %s opening file %s.", __FILE__, __LINE__, u_errorName(status), ruleFile);
|
|
return;
|
|
}
|
|
fRuleSet.adoptInstead(new BreakRules(this, status));
|
|
fRuleSet->compileRules(fRuleCharBuffer.getAlias(), status);
|
|
if (U_FAILURE(status)) {
|
|
IntlTest::gTest->errln("%s:%d Error %s processing file %s.", __FILE__, __LINE__, u_errorName(status), ruleFile);
|
|
return;
|
|
}
|
|
fBI.adoptInstead(fRuleSet->createICUBreakIterator(status));
|
|
fTestData.adoptInstead(new MonkeyTestData());
|
|
}
|
|
|
|
|
|
RBBIMonkeyImpl::~RBBIMonkeyImpl() {
|
|
}
|
|
|
|
|
|
void RBBIMonkeyImpl::openBreakRules(const char *fileName, UErrorCode &status) {
|
|
CharString path;
|
|
path.append(IntlTest::getSourceTestData(status), status);
|
|
path.append("break_rules" U_FILE_SEP_STRING, status);
|
|
path.appendPathPart(fileName, status);
|
|
const char *codePage = "UTF-8";
|
|
fRuleCharBuffer.adoptInstead(ucbuf_open(path.data(), &codePage, TRUE, FALSE, &status));
|
|
}
|
|
|
|
|
|
void RBBIMonkeyImpl::startTest() {
|
|
fThread.start(); // invokes runTest() in a separate thread.
|
|
}
|
|
|
|
void RBBIMonkeyImpl::join() {
|
|
fThread.join();
|
|
}
|
|
|
|
|
|
#define MONKEY_ERROR(msg, index) UPRV_BLOCK_MACRO_BEGIN { \
|
|
IntlTest::gTest->errln("%s:%d %s at index %d. Parameters to reproduce: @rules=%s,seed=%u,loop=1,verbose ", \
|
|
__FILE__, __LINE__, msg, index, fRuleFileName, fTestData->fRandomSeed); \
|
|
if (fVerbose) { fTestData->dump(index); } \
|
|
status = U_INVALID_STATE_ERROR; \
|
|
} UPRV_BLOCK_MACRO_END
|
|
|
|
void RBBIMonkeyImpl::runTest() {
|
|
UErrorCode status = U_ZERO_ERROR;
|
|
int32_t errorCount = 0;
|
|
for (int64_t loopCount = 0; fLoopCount < 0 || loopCount < fLoopCount; loopCount++) {
|
|
status = U_ZERO_ERROR;
|
|
fTestData->set(fRuleSet.getAlias(), fRandomGenerator, status);
|
|
if (fBI.isNull()) {
|
|
IntlTest::gTest->dataerrln("Unable to run test because fBI is null.");
|
|
return;
|
|
}
|
|
// fTestData->dump();
|
|
testForwards(status);
|
|
testPrevious(status);
|
|
testFollowing(status);
|
|
testPreceding(status);
|
|
testIsBoundary(status);
|
|
testIsBoundaryRandom(status);
|
|
|
|
if (fLoopCount < 0 && loopCount % 100 == 0) {
|
|
fprintf(stderr, ".");
|
|
}
|
|
if (U_FAILURE(status)) {
|
|
if (++errorCount > 10) {
|
|
return;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
void RBBIMonkeyImpl::testForwards(UErrorCode &status) {
|
|
if (U_FAILURE(status)) {
|
|
return;
|
|
}
|
|
fTestData->clearActualBreaks();
|
|
fBI->setText(fTestData->fString);
|
|
int32_t previousBreak = -2;
|
|
for (int32_t bk=fBI->first(); bk != BreakIterator::DONE; bk=fBI->next()) {
|
|
if (bk <= previousBreak) {
|
|
MONKEY_ERROR("Break Iterator Stall", bk);
|
|
return;
|
|
}
|
|
if (bk < 0 || bk > fTestData->fString.length()) {
|
|
MONKEY_ERROR("Boundary out of bounds", bk);
|
|
return;
|
|
}
|
|
fTestData->fActualBreaks.setCharAt(bk, 1);
|
|
}
|
|
checkResults("testForwards", FORWARD, status);
|
|
}
|
|
|
|
void RBBIMonkeyImpl::testFollowing(UErrorCode &status) {
|
|
if (U_FAILURE(status)) {
|
|
return;
|
|
}
|
|
fTestData->clearActualBreaks();
|
|
fBI->setText(fTestData->fString);
|
|
int32_t nextBreak = -1;
|
|
for (int32_t i=-1 ; i<fTestData->fString.length(); ++i) {
|
|
int32_t bk = fBI->following(i);
|
|
if (bk == BreakIterator::DONE && i == fTestData->fString.length()) {
|
|
continue;
|
|
}
|
|
if (bk == nextBreak && bk > i) {
|
|
// i is in the gap between two breaks.
|
|
continue;
|
|
}
|
|
if (i == nextBreak && bk > nextBreak) {
|
|
fTestData->fActualBreaks.setCharAt(bk, 1);
|
|
nextBreak = bk;
|
|
continue;
|
|
}
|
|
MONKEY_ERROR("following(i)", i);
|
|
return;
|
|
}
|
|
checkResults("testFollowing", FORWARD, status);
|
|
}
|
|
|
|
|
|
|
|
void RBBIMonkeyImpl::testPrevious(UErrorCode &status) {
|
|
if (U_FAILURE(status)) {return;}
|
|
|
|
fTestData->clearActualBreaks();
|
|
fBI->setText(fTestData->fString);
|
|
int32_t previousBreak = INT32_MAX;
|
|
for (int32_t bk=fBI->last(); bk != BreakIterator::DONE; bk=fBI->previous()) {
|
|
if (bk >= previousBreak) {
|
|
MONKEY_ERROR("Break Iterator Stall", bk);
|
|
return;
|
|
}
|
|
if (bk < 0 || bk > fTestData->fString.length()) {
|
|
MONKEY_ERROR("Boundary out of bounds", bk);
|
|
return;
|
|
}
|
|
fTestData->fActualBreaks.setCharAt(bk, 1);
|
|
}
|
|
checkResults("testPrevius", REVERSE, status);
|
|
}
|
|
|
|
|
|
void RBBIMonkeyImpl::testPreceding(UErrorCode &status) {
|
|
if (U_FAILURE(status)) {
|
|
return;
|
|
}
|
|
fTestData->clearActualBreaks();
|
|
fBI->setText(fTestData->fString);
|
|
int32_t nextBreak = fTestData->fString.length()+1;
|
|
for (int32_t i=fTestData->fString.length()+1 ; i>=0; --i) {
|
|
int32_t bk = fBI->preceding(i);
|
|
// printf("i:%d bk:%d nextBreak:%d\n", i, bk, nextBreak);
|
|
if (bk == BreakIterator::DONE && i == 0) {
|
|
continue;
|
|
}
|
|
if (bk == nextBreak && bk < i) {
|
|
// i is in the gap between two breaks.
|
|
continue;
|
|
}
|
|
if (i<fTestData->fString.length() && fTestData->fString.getChar32Start(i) < i) {
|
|
// i indexes to a trailing surrogate.
|
|
// Break Iterators treat an index to either half as referring to the supplemental code point,
|
|
// with preceding going to some preceding code point.
|
|
if (fBI->preceding(i) != fBI->preceding(fTestData->fString.getChar32Start(i))) {
|
|
MONKEY_ERROR("preceding of trailing surrogate error", i);
|
|
}
|
|
continue;
|
|
}
|
|
if (i == nextBreak && bk < nextBreak) {
|
|
fTestData->fActualBreaks.setCharAt(bk, 1);
|
|
nextBreak = bk;
|
|
continue;
|
|
}
|
|
MONKEY_ERROR("preceding(i)", i);
|
|
return;
|
|
}
|
|
checkResults("testPreceding", REVERSE, status);
|
|
}
|
|
|
|
|
|
void RBBIMonkeyImpl::testIsBoundary(UErrorCode &status) {
|
|
if (U_FAILURE(status)) {
|
|
return;
|
|
}
|
|
fTestData->clearActualBreaks();
|
|
fBI->setText(fTestData->fString);
|
|
for (int i=fTestData->fString.length(); i>=0; --i) {
|
|
if (fBI->isBoundary(i)) {
|
|
fTestData->fActualBreaks.setCharAt(i, 1);
|
|
}
|
|
}
|
|
checkResults("testForwards", FORWARD, status);
|
|
}
|
|
|
|
void RBBIMonkeyImpl::testIsBoundaryRandom(UErrorCode &status) {
|
|
if (U_FAILURE(status)) {
|
|
return;
|
|
}
|
|
fBI->setText(fTestData->fString);
|
|
|
|
int stringLen = fTestData->fString.length();
|
|
for (int i=stringLen; i>=0; --i) {
|
|
int strIdx = fRandomGenerator() % stringLen;
|
|
if (fTestData->fExpectedBreaks.charAt(strIdx) != fBI->isBoundary(strIdx)) {
|
|
IntlTest::gTest->errln("%s:%d testIsBoundaryRandom failure at index %d. Parameters to reproduce: @rules=%s,seed=%u,loop=1,verbose ",
|
|
__FILE__, __LINE__, strIdx, fRuleFileName, fTestData->fRandomSeed);
|
|
if (fVerbose) {
|
|
fTestData->dump(i);
|
|
}
|
|
status = U_INVALID_STATE_ERROR;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
|
|
void RBBIMonkeyImpl::checkResults(const char *msg, CheckDirection direction, UErrorCode &status) {
|
|
if (U_FAILURE(status)) {
|
|
return;
|
|
}
|
|
if (direction == FORWARD) {
|
|
for (int i=0; i<=fTestData->fString.length(); ++i) {
|
|
if (fTestData->fExpectedBreaks.charAt(i) != fTestData->fActualBreaks.charAt(i)) {
|
|
IntlTest::gTest->errln("%s:%d %s failure at index %d. Parameters to reproduce: @rules=%s,seed=%u,loop=1,verbose ",
|
|
__FILE__, __LINE__, msg, i, fRuleFileName, fTestData->fRandomSeed);
|
|
if (fVerbose) {
|
|
fTestData->dump(i);
|
|
}
|
|
status = U_INVALID_STATE_ERROR; // Prevent the test from continuing, which would likely
|
|
break; // produce many redundant errors.
|
|
}
|
|
}
|
|
} else {
|
|
for (int i=fTestData->fString.length(); i>=0; i--) {
|
|
if (fTestData->fExpectedBreaks.charAt(i) != fTestData->fActualBreaks.charAt(i)) {
|
|
IntlTest::gTest->errln("%s:%d %s failure at index %d. Parameters to reproduce: @rules=%s,seed=%u,loop=1,verbose ",
|
|
__FILE__, __LINE__, msg, i, fRuleFileName, fTestData->fRandomSeed);
|
|
if (fVerbose) {
|
|
fTestData->dump(i);
|
|
}
|
|
status = U_INVALID_STATE_ERROR;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
|
|
//---------------------------------------------------------------------------------------
|
|
//
|
|
// class RBBIMonkeyTest implementation.
|
|
//
|
|
//---------------------------------------------------------------------------------------
|
|
RBBIMonkeyTest::RBBIMonkeyTest() {
|
|
}
|
|
|
|
RBBIMonkeyTest::~RBBIMonkeyTest() {
|
|
}
|
|
|
|
|
|
// params, taken from this->fParams.
|
|
// rules=file_name Name of file containing the reference rules.
|
|
// seed=nnnnn Random number starting seed.
|
|
// Setting the seed allows errors to be reproduced.
|
|
// loop=nnn Looping count. Controls running time.
|
|
// -1: run forever.
|
|
// 0 or greater: run length.
|
|
// expansions debug option, show expansions of rules and sets.
|
|
// verbose Display details of the failure.
|
|
//
|
|
// Parameters on the intltest command line follow the test name, and are preceded by '@'.
|
|
// For example,
|
|
// intltest rbbi/RBBIMonkeyTest/testMonkey@rules=line.txt,loop=-1
|
|
//
|
|
void RBBIMonkeyTest::testMonkey() {
|
|
// printf("Test parameters: %s\n", fParams);
|
|
UnicodeString params(fParams);
|
|
UErrorCode status = U_ZERO_ERROR;
|
|
|
|
const char *tests[] = {"grapheme.txt", "word.txt", "line.txt", "line_cj.txt", "sentence.txt", "line_normal.txt",
|
|
"line_normal_cj.txt", "line_loose.txt", "line_loose_cj.txt", "word_POSIX.txt",
|
|
NULL };
|
|
CharString testNameFromParams;
|
|
if (getStringParam("rules", params, testNameFromParams, status)) {
|
|
tests[0] = testNameFromParams.data();
|
|
tests[1] = NULL;
|
|
}
|
|
|
|
int64_t loopCount = quick? 100 : 5000;
|
|
getIntParam("loop", params, loopCount, status);
|
|
|
|
UBool dumpExpansions = FALSE;
|
|
getBoolParam("expansions", params, dumpExpansions, status);
|
|
|
|
UBool verbose = FALSE;
|
|
getBoolParam("verbose", params, verbose, status);
|
|
|
|
int64_t seed = 0;
|
|
getIntParam("seed", params, seed, status);
|
|
|
|
if (params.length() != 0) {
|
|
// Options processing did not consume all of the parameters. Something unrecognized was present.
|
|
CharString unrecognizedParameters;
|
|
unrecognizedParameters.append(CStr(params)(), -1, status);
|
|
errln("%s:%d unrecognized test parameter(s) \"%s\"", __FILE__, __LINE__, unrecognizedParameters.data());
|
|
return;
|
|
}
|
|
|
|
UVector startedTests(status);
|
|
if (U_FAILURE(status)) {
|
|
errln("%s:%d: error %s while setting up test.", __FILE__, __LINE__, u_errorName(status));
|
|
return;
|
|
}
|
|
|
|
// Monkey testing is multi-threaded.
|
|
// Each set of break rules to be tested is run in a separate thread.
|
|
// Each thread/set of rules gets a separate RBBIMonkeyImpl object.
|
|
int32_t i;
|
|
for (i=0; tests[i] != NULL; ++i) {
|
|
logln("beginning testing of %s", tests[i]);
|
|
LocalPointer<RBBIMonkeyImpl> test(new RBBIMonkeyImpl(status));
|
|
if (U_FAILURE(status)) {
|
|
dataerrln("%s:%d: error %s while starting test %s.", __FILE__, __LINE__, u_errorName(status), tests[i]);
|
|
break;
|
|
}
|
|
test->fDumpExpansions = dumpExpansions;
|
|
test->fVerbose = verbose;
|
|
test->fRandomGenerator.seed(static_cast<uint32_t>(seed));
|
|
test->fLoopCount = static_cast<int32_t>(loopCount);
|
|
test->setup(tests[i], status);
|
|
if (U_FAILURE(status)) {
|
|
dataerrln("%s:%d: error %s while starting test %s.", __FILE__, __LINE__, u_errorName(status), tests[i]);
|
|
break;
|
|
}
|
|
test->startTest();
|
|
startedTests.addElement(test.orphan(), status);
|
|
if (U_FAILURE(status)) {
|
|
errln("%s:%d: error %s while starting test %s.", __FILE__, __LINE__, u_errorName(status), tests[i]);
|
|
break;
|
|
}
|
|
}
|
|
|
|
for (i=0; i<startedTests.size(); ++i) {
|
|
RBBIMonkeyImpl *test = static_cast<RBBIMonkeyImpl *>(startedTests.elementAt(i));
|
|
test->join();
|
|
delete test;
|
|
}
|
|
}
|
|
|
|
|
|
UBool RBBIMonkeyTest::getIntParam(UnicodeString name, UnicodeString ¶ms, int64_t &val, UErrorCode &status) {
|
|
name.append(" *= *(-?\\d+) *,? *");
|
|
RegexMatcher m(name, params, 0, status);
|
|
if (m.find()) {
|
|
// The param exists. Convert the string to an int.
|
|
CharString str;
|
|
str.append(CStr(m.group(1, status))(), -1, status);
|
|
val = strtol(str.data(), NULL, 10);
|
|
|
|
// Delete this parameter from the params string.
|
|
m.reset();
|
|
params = m.replaceFirst(UnicodeString(), status);
|
|
return TRUE;
|
|
}
|
|
return FALSE;
|
|
}
|
|
|
|
UBool RBBIMonkeyTest::getStringParam(UnicodeString name, UnicodeString ¶ms, CharString &dest, UErrorCode &status) {
|
|
name.append(" *= *([^ ,]*) *,? *");
|
|
RegexMatcher m(name, params, 0, status);
|
|
if (m.find()) {
|
|
// The param exists.
|
|
dest.append(CStr(m.group(1, status))(), -1, status);
|
|
|
|
// Delete this parameter from the params string.
|
|
m.reset();
|
|
params = m.replaceFirst(UnicodeString(), status);
|
|
return TRUE;
|
|
}
|
|
return FALSE;
|
|
}
|
|
|
|
UBool RBBIMonkeyTest::getBoolParam(UnicodeString name, UnicodeString ¶ms, UBool &dest, UErrorCode &status) {
|
|
name.append("(?: *= *(true|false))? *,? *");
|
|
RegexMatcher m(name, params, UREGEX_CASE_INSENSITIVE, status);
|
|
if (m.find()) {
|
|
if (m.start(1, status) > 0) {
|
|
// user option included a value.
|
|
dest = m.group(1, status).caseCompare(UnicodeString("true"), U_FOLD_CASE_DEFAULT) == 0;
|
|
} else {
|
|
// No explicit user value, implies true.
|
|
dest = TRUE;
|
|
}
|
|
|
|
// Delete this parameter from the params string.
|
|
m.reset();
|
|
params = m.replaceFirst(UnicodeString(), status);
|
|
return TRUE;
|
|
}
|
|
return FALSE;
|
|
}
|
|
|
|
#endif /* !UCONFIG_NO_BREAK_ITERATION && !UCONFIG_NO_REGULAR_EXPRESSIONS && !UCONFIG_NO_FORMATTING */
|