You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
231 lines
9.7 KiB
231 lines
9.7 KiB
// © 2016 and later: Unicode, Inc. and others.
|
|
// License & terms of use: http://www.unicode.org/copyright.html
|
|
/*************************************************************************
|
|
* Copyright (c) 2016, International Business Machines
|
|
* Corporation and others. All Rights Reserved.
|
|
*************************************************************************
|
|
*/
|
|
#ifndef RBBIMONKEYTEST_H
|
|
#define RBBIMONKEYTEST_H
|
|
|
|
#include "unicode/utypes.h"
|
|
|
|
#if !UCONFIG_NO_BREAK_ITERATION && !UCONFIG_NO_REGULAR_EXPRESSIONS && !UCONFIG_NO_FORMATTING
|
|
|
|
#include "intltest.h"
|
|
|
|
#include "unicode/rbbi.h"
|
|
#include "unicode/regex.h"
|
|
#include "unicode/uniset.h"
|
|
#include "unicode/unistr.h"
|
|
#include "unicode/uobject.h"
|
|
|
|
#include "simplethread.h"
|
|
#include "ucbuf.h"
|
|
#include "uhash.h"
|
|
#include "uvector.h"
|
|
|
|
// RBBI Monkey Test. Run break iterators against randomly generated strings, compare results with
|
|
// an independent reference implementation.
|
|
//
|
|
// The monkey test can be run with parameters, e.g.
|
|
// intltest rbbi/RBBIMonkeyTest@loop=-1,rules=word.txt
|
|
// will run word break testing in an infinite loop.
|
|
// Summary of options
|
|
// rules=name Test against the named reference rule file.
|
|
// Files are found in source/test/testdata/break_rules
|
|
// loop=nnn Loop nnn times. -1 for no limit. loop of 1 is useful for debugging.
|
|
// seed=nnnn Random number generator seed. Allows recreation of a failure.
|
|
// Error messages include the necessary seed value.
|
|
// verbose Display details of a failure. Useful for debugging. Use with loop=1.
|
|
// expansions Debug option, show expansions of rules and sets.
|
|
//
|
|
// TODO:
|
|
// Develop a tailoring format.
|
|
// Hook to old tests that use monkey impl to get expected data.
|
|
// Remove old tests.
|
|
|
|
class BreakRules; // Forward declaration
|
|
class RBBIMonkeyImpl;
|
|
|
|
/**
|
|
* Test the RuleBasedBreakIterator class giving different rules
|
|
*/
|
|
class RBBIMonkeyTest: public IntlTest {
|
|
public:
|
|
RBBIMonkeyTest();
|
|
virtual ~RBBIMonkeyTest();
|
|
|
|
void runIndexedTest( int32_t index, UBool exec, const char* &name, char* par = NULL );
|
|
void testMonkey();
|
|
|
|
|
|
private:
|
|
const char *fParams; // Copy of user parameters passed in from IntlTest.
|
|
|
|
|
|
void testRules(const char *ruleFile);
|
|
static UBool getIntParam(UnicodeString name, UnicodeString ¶ms, int64_t &val, UErrorCode &status);
|
|
static UBool getStringParam(UnicodeString name, UnicodeString ¶ms, CharString &dest, UErrorCode &status);
|
|
static UBool getBoolParam(UnicodeString name, UnicodeString ¶ms, UBool &dest, UErrorCode &status);
|
|
|
|
};
|
|
|
|
// The following classes are internal to the RBBI Monkey Test implementation.
|
|
|
|
|
|
|
|
// class CharClass Represents a single character class from the source break rules.
|
|
// Inherits from UObject because instances are adopted by UHashtable, which ultimately
|
|
// deletes them using hash's object deleter function.
|
|
|
|
class CharClass: public UObject {
|
|
public:
|
|
UnicodeString fName;
|
|
UnicodeString fOriginalDef; // set definition as it appeared in user supplied rules.
|
|
UnicodeString fExpandedDef; // set definition with any embedded named sets replaced by their defs, recursively.
|
|
LocalPointer<const UnicodeSet> fSet;
|
|
CharClass(const UnicodeString &name, const UnicodeString &originalDef, const UnicodeString &expandedDef, const UnicodeSet *set) :
|
|
fName(name), fOriginalDef(originalDef), fExpandedDef(expandedDef), fSet(set) {}
|
|
};
|
|
|
|
|
|
// class BreakRule represents a single rule from a set of break rules.
|
|
// Each rule has the set definitions expanded, and
|
|
// is compiled to a regular expression.
|
|
|
|
class BreakRule: public UObject {
|
|
public:
|
|
BreakRule();
|
|
~BreakRule();
|
|
UnicodeString fName; // Name of the rule.
|
|
UnicodeString fRule; // Rule expression, excluding the name, as written in user source.
|
|
UnicodeString fExpandedRule; // Rule expression after expanding the set definitions.
|
|
LocalPointer<RegexMatcher> fRuleMatcher; // Regular expression that matches the rule.
|
|
bool fInitialMatchOnly = false; // True if rule begins with '^', meaning no chaining.
|
|
};
|
|
|
|
|
|
// class BreakRules represents a complete set of break rules, possibly tailored,
|
|
// compiled from testdata break rules.
|
|
|
|
class BreakRules: public UObject {
|
|
public:
|
|
BreakRules(RBBIMonkeyImpl *monkeyImpl, UErrorCode &status);
|
|
~BreakRules();
|
|
|
|
void compileRules(UCHARBUF *rules, UErrorCode &status);
|
|
|
|
const CharClass *getClassForChar(UChar32 c, int32_t *iter=NULL) const;
|
|
|
|
|
|
RBBIMonkeyImpl *fMonkeyImpl; // Pointer back to the owning MonkeyImpl instance.
|
|
icu::UVector fBreakRules; // Contents are of type (BreakRule *).
|
|
|
|
LocalUHashtablePointer fCharClasses; // Key is set name (UnicodeString).
|
|
// Value is (CharClass *)
|
|
LocalPointer<UVector> fCharClassList; // Char Classes, same contents as fCharClasses values,
|
|
// but in a vector so they can be accessed by index.
|
|
UnicodeSet fDictionarySet; // Dictionary set, empty if none is defined.
|
|
Locale fLocale;
|
|
UBreakIteratorType fType;
|
|
|
|
CharClass *addCharClass(const UnicodeString &name, const UnicodeString &def, UErrorCode &status);
|
|
void addRule(const UnicodeString &name, const UnicodeString &def, UErrorCode &status);
|
|
bool setKeywordParameter(const UnicodeString &keyword, const UnicodeString &value, UErrorCode &status);
|
|
RuleBasedBreakIterator *createICUBreakIterator(UErrorCode &status);
|
|
|
|
LocalPointer<RegexMatcher> fSetRefsMatcher;
|
|
LocalPointer<RegexMatcher> fCommentsMatcher;
|
|
LocalPointer<RegexMatcher> fClassDefMatcher;
|
|
LocalPointer<RegexMatcher> fRuleDefMatcher;
|
|
};
|
|
|
|
|
|
// class MonkeyTestData represents a randomly synthesized test data string together
|
|
// with the expected break positions obtained by applying
|
|
// the test break rules.
|
|
|
|
class MonkeyTestData: public UObject {
|
|
public:
|
|
MonkeyTestData() {}
|
|
~MonkeyTestData() {}
|
|
void set(BreakRules *rules, IntlTest::icu_rand &rand, UErrorCode &status);
|
|
void clearActualBreaks();
|
|
void dump(int32_t around = -1) const;
|
|
|
|
uint32_t fRandomSeed; // The initial seed value from the random number genererator.
|
|
const BreakRules *fBkRules; // The break rules used to generate this data.
|
|
UnicodeString fString; // The text.
|
|
UnicodeString fExpectedBreaks; // Breaks as found by the reference rules.
|
|
// Parallel to fString. Non-zero if break preceding.
|
|
UnicodeString fActualBreaks; // Breaks as found by ICU break iterator.
|
|
UnicodeString fRuleForPosition; // Index into BreakRules.fBreakRules of rule that applied at each position.
|
|
// Also parallel to fString.
|
|
UnicodeString f2ndRuleForPos; // As above. A 2nd rule applies when the preceding rule
|
|
// didn't cause a break, and a subsequent rule match starts
|
|
// on the last code point of the preceding match.
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
// class RBBIMonkeyImpl holds (some indirectly) everything associated with running a monkey
|
|
// test for one set of break rules.
|
|
//
|
|
// When running RBBIMonkeyTest with multiple threads, there is a 1:1 correspondence
|
|
// between instances of RBBIMonkeyImpl and threads.
|
|
//
|
|
class RBBIMonkeyImpl: public UObject {
|
|
public:
|
|
RBBIMonkeyImpl(UErrorCode &status);
|
|
~RBBIMonkeyImpl();
|
|
|
|
void setup(const char *ruleFileName, UErrorCode &status);
|
|
|
|
void startTest();
|
|
void runTest();
|
|
void join();
|
|
|
|
LocalUCHARBUFPointer fRuleCharBuffer; // source file contents of the reference rules.
|
|
LocalPointer<BreakRules> fRuleSet;
|
|
LocalPointer<RuleBasedBreakIterator> fBI;
|
|
LocalPointer<MonkeyTestData> fTestData;
|
|
IntlTest::icu_rand fRandomGenerator;
|
|
const char *fRuleFileName;
|
|
UBool fVerbose; // True to do long dump of failing data.
|
|
int32_t fLoopCount;
|
|
|
|
UBool fDumpExpansions; // Debug flag to output epananded form of rules and sets.
|
|
|
|
enum CheckDirection {
|
|
FORWARD = 1,
|
|
REVERSE = 2
|
|
};
|
|
void clearActualBreaks();
|
|
void testForwards(UErrorCode &status);
|
|
void testPrevious(UErrorCode &status);
|
|
void testFollowing(UErrorCode &status);
|
|
void testPreceding(UErrorCode &status);
|
|
void testIsBoundary(UErrorCode &status);
|
|
void testIsBoundaryRandom(UErrorCode &status);
|
|
void checkResults(const char *msg, CheckDirection dir, UErrorCode &status);
|
|
|
|
class RBBIMonkeyThread: public SimpleThread {
|
|
private:
|
|
RBBIMonkeyImpl *fMonkeyImpl;
|
|
public:
|
|
RBBIMonkeyThread(RBBIMonkeyImpl *impl) : fMonkeyImpl(impl) {}
|
|
void run() U_OVERRIDE { fMonkeyImpl->runTest(); }
|
|
};
|
|
private:
|
|
void openBreakRules(const char *fileName, UErrorCode &status);
|
|
RBBIMonkeyThread fThread;
|
|
|
|
};
|
|
|
|
#endif /* !UCONFIG_NO_BREAK_ITERATION && !UCONFIG_NO_REGULAR_EXPRESSIONS && !UCONFIG_NO_FORMATTING */
|
|
|
|
#endif // RBBIMONKEYTEST_H
|