// © 2016 and later: Unicode, Inc. and others. // License & terms of use: http://www.unicode.org/copyright.html /************************************************************************* * Copyright (c) 2016, International Business Machines * Corporation and others. All Rights Reserved. ************************************************************************* */ #ifndef RBBIMONKEYTEST_H #define RBBIMONKEYTEST_H #include "unicode/utypes.h" #if !UCONFIG_NO_BREAK_ITERATION && !UCONFIG_NO_REGULAR_EXPRESSIONS && !UCONFIG_NO_FORMATTING #include "intltest.h" #include "unicode/rbbi.h" #include "unicode/regex.h" #include "unicode/uniset.h" #include "unicode/unistr.h" #include "unicode/uobject.h" #include "simplethread.h" #include "ucbuf.h" #include "uhash.h" #include "uvector.h" // RBBI Monkey Test. Run break iterators against randomly generated strings, compare results with // an independent reference implementation. // // The monkey test can be run with parameters, e.g. // intltest rbbi/RBBIMonkeyTest@loop=-1,rules=word.txt // will run word break testing in an infinite loop. // Summary of options // rules=name Test against the named reference rule file. // Files are found in source/test/testdata/break_rules // loop=nnn Loop nnn times. -1 for no limit. loop of 1 is useful for debugging. // seed=nnnn Random number generator seed. Allows recreation of a failure. // Error messages include the necessary seed value. // verbose Display details of a failure. Useful for debugging. Use with loop=1. // expansions Debug option, show expansions of rules and sets. // // TODO: // Develop a tailoring format. // Hook to old tests that use monkey impl to get expected data. // Remove old tests. class BreakRules; // Forward declaration class RBBIMonkeyImpl; /** * Test the RuleBasedBreakIterator class giving different rules */ class RBBIMonkeyTest: public IntlTest { public: RBBIMonkeyTest(); virtual ~RBBIMonkeyTest(); void runIndexedTest( int32_t index, UBool exec, const char* &name, char* par = NULL ); void testMonkey(); private: const char *fParams; // Copy of user parameters passed in from IntlTest. void testRules(const char *ruleFile); static UBool getIntParam(UnicodeString name, UnicodeString ¶ms, int64_t &val, UErrorCode &status); static UBool getStringParam(UnicodeString name, UnicodeString ¶ms, CharString &dest, UErrorCode &status); static UBool getBoolParam(UnicodeString name, UnicodeString ¶ms, UBool &dest, UErrorCode &status); }; // The following classes are internal to the RBBI Monkey Test implementation. // class CharClass Represents a single character class from the source break rules. // Inherits from UObject because instances are adopted by UHashtable, which ultimately // deletes them using hash's object deleter function. class CharClass: public UObject { public: UnicodeString fName; UnicodeString fOriginalDef; // set definition as it appeared in user supplied rules. UnicodeString fExpandedDef; // set definition with any embedded named sets replaced by their defs, recursively. LocalPointer fSet; CharClass(const UnicodeString &name, const UnicodeString &originalDef, const UnicodeString &expandedDef, const UnicodeSet *set) : fName(name), fOriginalDef(originalDef), fExpandedDef(expandedDef), fSet(set) {} }; // class BreakRule represents a single rule from a set of break rules. // Each rule has the set definitions expanded, and // is compiled to a regular expression. class BreakRule: public UObject { public: BreakRule(); ~BreakRule(); UnicodeString fName; // Name of the rule. UnicodeString fRule; // Rule expression, excluding the name, as written in user source. UnicodeString fExpandedRule; // Rule expression after expanding the set definitions. LocalPointer fRuleMatcher; // Regular expression that matches the rule. bool fInitialMatchOnly = false; // True if rule begins with '^', meaning no chaining. }; // class BreakRules represents a complete set of break rules, possibly tailored, // compiled from testdata break rules. class BreakRules: public UObject { public: BreakRules(RBBIMonkeyImpl *monkeyImpl, UErrorCode &status); ~BreakRules(); void compileRules(UCHARBUF *rules, UErrorCode &status); const CharClass *getClassForChar(UChar32 c, int32_t *iter=NULL) const; RBBIMonkeyImpl *fMonkeyImpl; // Pointer back to the owning MonkeyImpl instance. icu::UVector fBreakRules; // Contents are of type (BreakRule *). LocalUHashtablePointer fCharClasses; // Key is set name (UnicodeString). // Value is (CharClass *) LocalPointer fCharClassList; // Char Classes, same contents as fCharClasses values, // but in a vector so they can be accessed by index. UnicodeSet fDictionarySet; // Dictionary set, empty if none is defined. Locale fLocale; UBreakIteratorType fType; CharClass *addCharClass(const UnicodeString &name, const UnicodeString &def, UErrorCode &status); void addRule(const UnicodeString &name, const UnicodeString &def, UErrorCode &status); bool setKeywordParameter(const UnicodeString &keyword, const UnicodeString &value, UErrorCode &status); RuleBasedBreakIterator *createICUBreakIterator(UErrorCode &status); LocalPointer fSetRefsMatcher; LocalPointer fCommentsMatcher; LocalPointer fClassDefMatcher; LocalPointer fRuleDefMatcher; }; // class MonkeyTestData represents a randomly synthesized test data string together // with the expected break positions obtained by applying // the test break rules. class MonkeyTestData: public UObject { public: MonkeyTestData() {} ~MonkeyTestData() {} void set(BreakRules *rules, IntlTest::icu_rand &rand, UErrorCode &status); void clearActualBreaks(); void dump(int32_t around = -1) const; uint32_t fRandomSeed; // The initial seed value from the random number genererator. const BreakRules *fBkRules; // The break rules used to generate this data. UnicodeString fString; // The text. UnicodeString fExpectedBreaks; // Breaks as found by the reference rules. // Parallel to fString. Non-zero if break preceding. UnicodeString fActualBreaks; // Breaks as found by ICU break iterator. UnicodeString fRuleForPosition; // Index into BreakRules.fBreakRules of rule that applied at each position. // Also parallel to fString. UnicodeString f2ndRuleForPos; // As above. A 2nd rule applies when the preceding rule // didn't cause a break, and a subsequent rule match starts // on the last code point of the preceding match. }; // class RBBIMonkeyImpl holds (some indirectly) everything associated with running a monkey // test for one set of break rules. // // When running RBBIMonkeyTest with multiple threads, there is a 1:1 correspondence // between instances of RBBIMonkeyImpl and threads. // class RBBIMonkeyImpl: public UObject { public: RBBIMonkeyImpl(UErrorCode &status); ~RBBIMonkeyImpl(); void setup(const char *ruleFileName, UErrorCode &status); void startTest(); void runTest(); void join(); LocalUCHARBUFPointer fRuleCharBuffer; // source file contents of the reference rules. LocalPointer fRuleSet; LocalPointer fBI; LocalPointer fTestData; IntlTest::icu_rand fRandomGenerator; const char *fRuleFileName; UBool fVerbose; // True to do long dump of failing data. int32_t fLoopCount; UBool fDumpExpansions; // Debug flag to output epananded form of rules and sets. enum CheckDirection { FORWARD = 1, REVERSE = 2 }; void clearActualBreaks(); void testForwards(UErrorCode &status); void testPrevious(UErrorCode &status); void testFollowing(UErrorCode &status); void testPreceding(UErrorCode &status); void testIsBoundary(UErrorCode &status); void testIsBoundaryRandom(UErrorCode &status); void checkResults(const char *msg, CheckDirection dir, UErrorCode &status); class RBBIMonkeyThread: public SimpleThread { private: RBBIMonkeyImpl *fMonkeyImpl; public: RBBIMonkeyThread(RBBIMonkeyImpl *impl) : fMonkeyImpl(impl) {} void run() U_OVERRIDE { fMonkeyImpl->runTest(); } }; private: void openBreakRules(const char *fileName, UErrorCode &status); RBBIMonkeyThread fThread; }; #endif /* !UCONFIG_NO_BREAK_ITERATION && !UCONFIG_NO_REGULAR_EXPRESSIONS && !UCONFIG_NO_FORMATTING */ #endif // RBBIMONKEYTEST_H