~ubuntu-branches/ubuntu/gutsy/icu/gutsy-updates

static uint32_t testSwitch(tst_strcoll* func, void *collator, int opts, uint32_t strength, const UChar *first, const UChar *second, const char* msg, UBool error) {

963

uint32_t diffs = 0;

964

UCollationResult realResult;

965

uint32_t realStrength;

966

967

uint32_t sLen = u_strlen(first);

968

uint32_t tLen = u_strlen(second);

969

970

realResult = func(collator, opts, first, sLen, second, tLen);

971

realStrength = probeStrength(func, collator, opts, first, sLen, second, tLen, realResult);

972

973

if(strength == UCOL_IDENTICAL && realResult != UCOL_IDENTICAL) {

974

logFailure(msg, "tailoring", first, sLen, second, tLen, realResult, realStrength, UCOL_EQUAL, strength, error);

975

diffs++;

976

} else if(realResult != UCOL_LESS || realStrength != strength) {

977

logFailure(msg, "tailoring", first, sLen, second, tLen, realResult, realStrength, UCOL_LESS, strength, error);

978

diffs++;

979

}

980

return diffs;

981

}

982

983

984

static void testAgainstUCA(UCollator *coll, UCollator *UCA, const char *refName, UBool error, UErrorCode *status) {

985

const UChar *rules = NULL, *current = NULL;

986

int32_t ruleLen = 0;

987

uint32_t strength = 0;

988

uint32_t chOffset = 0; uint32_t chLen = 0;

989

uint32_t exOffset = 0; uint32_t exLen = 0;

990

uint32_t prefixOffset = 0; uint32_t prefixLen = 0;

991

/* uint32_t rExpsLen = 0; */

992

uint32_t firstLen = 0, secondLen = 0;

993

UBool varT = FALSE; UBool top_ = TRUE;

994

uint16_t specs = 0;

995

UBool startOfRules = TRUE;

996

UColTokenParser src;

997

UColOptionSet opts;

998

999

UChar first[256];

1000

UChar second[256];

1001

UChar *rulesCopy = NULL;

1002

1003

uint32_t UCAdiff = 0;

1004

uint32_t Windiff = 1;

1005

UParseError parseError;

1006

1007

src.opts = &opts;

1008

1009

rules = ucol_getRules(coll, &ruleLen);

1010

1011

/*printOutRules(rules);*/

1012

1013

if(U_SUCCESS(*status) && ruleLen > 0) {

1014

rulesCopy = (UChar *)uprv_malloc((ruleLen+UCOL_TOK_EXTRA_RULE_SPACE_SIZE)*sizeof(UChar));

1015

uprv_memcpy(rulesCopy, rules, ruleLen*sizeof(UChar));

1016

src.source = src.current = rulesCopy;

1017

src.end = rulesCopy+ruleLen;

1018

src.extraCurrent = src.end;

1019

src.extraEnd = src.end+UCOL_TOK_EXTRA_RULE_SPACE_SIZE;

1020

*first = *second = 0;

1021

1022

while ((current = ucol_tok_parseNextToken(&src, startOfRules, &parseError,status)) != NULL) {

1023

strength = src.parsedToken.strength;

1024

chOffset = src.parsedToken.charsOffset;

1025

chLen = src.parsedToken.charsLen;

1026

exOffset = src.parsedToken.extensionOffset;

1027

exLen = src.parsedToken.extensionLen;

1028

prefixOffset = src.parsedToken.prefixOffset;

1029

prefixLen = src.parsedToken.prefixLen;

1030

specs = src.parsedToken.flags;

1031

1032

startOfRules = FALSE;

1033

varT = (UBool)((specs & UCOL_TOK_VARIABLE_TOP) != 0);

1034

top_ = (UBool)((specs & UCOL_TOK_TOP) != 0);

1035

1036

u_strncpy(second,rulesCopy+chOffset, chLen);

1037

second[chLen] = 0;

1038

secondLen = chLen;

1039

1040

if(exLen > 0) {

1041

u_strncat(first, rulesCopy+exOffset, exLen);

1042

first[firstLen+exLen] = 0;

1043

firstLen += exLen;

1044

}

1045

1046

if(strength != UCOL_TOK_RESET) {

1047

if((*first<0x3400 || *first>=0xa000) && (*second<0x3400 || *second>=0xa000)) {

1048

UCAdiff += testSwitch(&ucaTest, (void *)UCA, 0, strength, first, second, refName, error);

1049

/*Windiff += testSwitch(&winTest, (void *)lcid, 0, strength, first, second, "Win32");*/

1050

}

1051

}

1052

1053

1054

firstLen = chLen;

1055

u_strcpy(first, second);

1056

1057

}

1058

if(UCAdiff != 0 && Windiff != 0) {

1059

log_verbose("\n");

1060

}

1061

if(UCAdiff == 0) {

1062

log_verbose("No immediate difference with %s!\n", refName);

1063

}

1064

if(Windiff == 0) {

1065

log_verbose("No immediate difference with Win32!\n");

1066

}

1067

uprv_free(rulesCopy);

1068

}

1069

}

1070

1071

static void testCEs(UCollator *coll, UErrorCode *status) {

1072

1073

const UChar *rules = NULL, *current = NULL;

1074

int32_t ruleLen = 0;

1075

1076

uint32_t strength = 0;

1077

uint32_t maxStrength = UCOL_IDENTICAL;

1078

uint32_t baseCE, baseContCE, nextCE, nextContCE, currCE, currContCE;

1079

uint32_t lastCE;

1080

uint32_t lastContCE;

1081

1082

int32_t result = 0;

1083

uint32_t chOffset = 0; uint32_t chLen = 0;

1084

uint32_t exOffset = 0; uint32_t exLen = 0;

1085

uint32_t prefixOffset = 0; uint32_t prefixLen = 0;

1086

uint32_t oldOffset = 0;

1087

1088

/* uint32_t rExpsLen = 0; */

1089

/* uint32_t firstLen = 0; */

1090

uint16_t specs = 0;

1091

UBool varT = FALSE; UBool top_ = TRUE;

1092

UBool startOfRules = TRUE;

1093

UColTokenParser src;

1094

UColOptionSet opts;

1095

UParseError parseError;

1096

UChar *rulesCopy = NULL;

1097

collIterate c;

1098

1099

baseCE=baseContCE=nextCE=nextContCE=currCE=currContCE=lastCE=lastContCE = UCOL_NOT_FOUND;

1100

1101

src.opts = &opts;

1102

1103

rules = ucol_getRules(coll, &ruleLen);

1104

1105

ucol_initInverseUCA(status);

1106

1107

if(U_SUCCESS(*status) && ruleLen > 0) {

1108

rulesCopy = (UChar *)uprv_malloc((ruleLen+UCOL_TOK_EXTRA_RULE_SPACE_SIZE)*sizeof(UChar));

1109

uprv_memcpy(rulesCopy, rules, ruleLen*sizeof(UChar));

1110

src.source = src.current = rulesCopy;

1111

src.end = rulesCopy+ruleLen;

1112

src.extraCurrent = src.end;

1113

src.extraEnd = src.end+UCOL_TOK_EXTRA_RULE_SPACE_SIZE;

1114

1115

while ((current = ucol_tok_parseNextToken(&src, startOfRules, &parseError,status)) != NULL) {

1116

strength = src.parsedToken.strength;

1117

chOffset = src.parsedToken.charsOffset;

1118

chLen = src.parsedToken.charsLen;

1119

exOffset = src.parsedToken.extensionOffset;

1120

exLen = src.parsedToken.extensionLen;

1121

prefixOffset = src.parsedToken.prefixOffset;

1122

prefixLen = src.parsedToken.prefixLen;

1123

specs = src.parsedToken.flags;

1124

1125

startOfRules = FALSE;

1126

varT = (UBool)((specs & UCOL_TOK_VARIABLE_TOP) != 0);

1127

top_ = (UBool)((specs & UCOL_TOK_TOP) != 0);

1128

1129

init_collIterate(coll, rulesCopy+chOffset, chLen, &c);

1130

1131

currCE = ucol_getNextCE(coll, &c, status);

1132

if(currCE == 0 && UCOL_ISTHAIPREVOWEL(*(rulesCopy+chOffset))) {

1133

log_verbose("Thai prevowel detected. Will pick next CE\n");

1134

currCE = ucol_getNextCE(coll, &c, status);

1135

}

1136

1137

currContCE = ucol_getNextCE(coll, &c, status);

1138

if(!isContinuation(currContCE)) {

1139

currContCE = 0;

1140

}

1141

1142

if(strength == UCOL_TOK_RESET) {

1143

if(top_ == TRUE) {

1144

nextCE = baseCE = currCE = UCOL_RESET_TOP_VALUE;

1145

nextContCE = baseContCE = currContCE = 0;

1146

} else {

1147

nextCE = baseCE = currCE;

1148

nextContCE = baseContCE = currContCE;

1149

}

1150

maxStrength = UCOL_IDENTICAL;

1151

} else {

1152

if(strength < maxStrength) {

1153

maxStrength = strength;

1154

if(baseCE == UCOL_RESET_TOP_VALUE) {

1155

log_verbose("Resetting to [top]\n");

1156

nextCE = UCOL_NEXT_TOP_VALUE;

1157

nextContCE = 0;

1158

} else {

1159

result = ucol_inv_getNextCE(baseCE & 0xFFFFFF3F, baseContCE, &nextCE, &nextContCE, maxStrength);

1160

}

1161

if(result < 0) {

1162

if(isTailored(coll, *(rulesCopy+oldOffset), status)) {

1163

log_verbose("Reset is tailored codepoint %04X, don't know how to continue, taking next test\n", *(rulesCopy+oldOffset));

1164

return;

1165

} else {

1166

log_err("couldn't find the CE\n");

1167

return;

1168

}

1169

}

1170

}

1171

1172

currCE &= 0xFFFFFF3F;

1173

currContCE &= 0xFFFFFFBF;

1174

1175

if(maxStrength == UCOL_IDENTICAL) {

1176

if(baseCE != currCE || baseContCE != currContCE) {

1177

log_err("current CE (initial strength UCOL_EQUAL)\n");

1178

}

1179

} else {

1180

if(strength == UCOL_IDENTICAL) {

1181

if(lastCE != currCE || lastContCE != currContCE) {

1182

log_err("current CE (initial strength UCOL_EQUAL)\n");

1183

}

1184

} else {

1185

if(currCE > nextCE || (currCE == nextCE && currContCE >= nextContCE)) {

1186

log_err("current CE is not less than base CE\n");

1187

}

1188

if(currCE < lastCE || (currCE == lastCE && currContCE <= lastContCE)) {

1189

log_err("sequence of generated CEs is broken\n");

1190

}

1191

}

1192

}

1193

1194

}

1195

1196

oldOffset = chOffset;

1197

lastCE = currCE & 0xFFFFFF3F;

1198

lastContCE = currContCE & 0xFFFFFFBF;

1199

}

1200

uprv_free(rulesCopy);

1201

}

1202

}

1203

1204

#if 0

1205

/* these locales are now picked from index RB */

1206

static const char* localesToTest[] = {

1207

"ar", "bg", "ca", "cs", "da",

1208

"el", "en_BE", "en_US_POSIX",

1209

"es", "et", "fi", "fr", "hi",

1210

"hr", "hu", "is", "iw", "ja",

1211

"ko", "lt", "lv", "mk", "mt",

1212

"nb", "nn", "nn_NO", "pl", "ro",

1213

"ru", "sh", "sk", "sl", "sq",

1214

"sr", "sv", "th", "tr", "uk",

1215

"vi", "zh", "zh_TW"

1216

};

1217

#endif

1218

1219

static const char* rulesToTest[] = {

1220

/*"& Z < p, P",*/

1221

/* Cui Mins rules */

1222

"<o,O<p,P<q,Q<'?'/u<r,R<u,U", /*"<o,O<p,P<q,Q<r,R<u,U & Qu<'?'",*/

1223

"<o,O<p,P<q,Q;'?'/u<r,R<u,U", /*"<o,O<p,P<q,Q<r,R<u,U & Qu;'?'",*/

1224

"<o,O<p,P<q,Q,'?'/u<r,R<u,U", /*"<o,O<p,P<q,Q<r,R<u,U&'Qu','?'",*/

1225

"<3<4<5<c,C<f,F<m,M<o,O<p,P<q,Q;'?'/u<r,R<u,U", /*"<'?'<3<4<5<a,A<f,F<m,M<o,O<p,P<q,Q<r,R<u,U & Qu;'?'",*/

1226

"<'?';Qu<3<4<5<c,C<f,F<m,M<o,O<p,P<q,Q<r,R<u,U", /*"<'?'<3<4<5<a,A<f,F<m,M<o,O<p,P<q,Q<r,R<u,U & '?';Qu",*/

1227

"<3<4<5<c,C<f,F<m,M<o,O<p,P<q,Q;'?'/um<r,R<u,U", /*"<'?'<3<4<5<a,A<f,F<m,M<o,O<p,P<q,Q<r,R<u,U & Qum;'?'",*/

1228

"<'?';Qum<3<4<5<c,C<f,F<m,M<o,O<p,P<q,Q<r,R<u,U" /*"<'?'<3<4<5<a,A<f,F<m,M<o,O<p,P<q,Q<r,R<u,U & '?';Qum"*/

1229

};

1230

1231

static UBool hasCollationElements(const char *locName) {

1232

1233

UErrorCode status = U_ZERO_ERROR;

1234

UResourceBundle *ColEl = NULL;

1235

1236

UResourceBundle *loc = ures_open(NULL, locName, &status);;

1237

1238

if(U_SUCCESS(status)) {

1239

status = U_ZERO_ERROR;

1240

ColEl = ures_getByKey(loc, "CollationElements", ColEl, &status);

1241

if(status == U_ZERO_ERROR) { /* do the test - there are real elements */

1242

ures_close(ColEl);

1243

ures_close(loc);

1244

return TRUE;

1245

}

1246

ures_close(ColEl);

1247

ures_close(loc);

1248

}

1249

return FALSE;

1250

}

1251

1252

1253

static void TestCollations(void) {

1254

int32_t noOfLoc = uloc_countAvailable();

1255

int32_t i = 0, j = 0;

1256

1257

UErrorCode status = U_ZERO_ERROR;

1258

char cName[256];

1259

UChar name[256];

1260

int32_t nameSize;

1261

1262

1263

const char *locName = NULL;

1264

UCollator *coll = NULL;

1265

UCollator *UCA = ucol_open("", &status);

1266

UColAttributeValue oldStrength = ucol_getAttribute(UCA, UCOL_STRENGTH, &status);

1267

ucol_setAttribute(UCA, UCOL_STRENGTH, UCOL_QUATERNARY, &status);

1268

1269

for(i = 0; i<noOfLoc; i++) {

1270

status = U_ZERO_ERROR;

1271

locName = uloc_getAvailable(i);

1272

if(uprv_strcmp("ja", locName) == 0) {

1273

log_verbose("Don't know how to test prefixes\n");

1274

continue;

1275

}

1276

if(hasCollationElements(locName)) {

1277

nameSize = uloc_getDisplayName(locName, NULL, name, 256, &status);

1278

for(j = 0; j<nameSize; j++) {

1279

cName[j] = (char)name[j];

1280

}

1281

cName[nameSize] = 0;

1282

log_verbose("\nTesting locale %s (%s)\n", locName, cName);

1283

coll = ucol_open(locName, &status);

1284

testAgainstUCA(coll, UCA, "UCA", FALSE, &status);

1285

ucol_close(coll);

1286

}

1287

}

1288

ucol_setAttribute(UCA, UCOL_STRENGTH, oldStrength, &status);

1289

ucol_close(UCA);

1290

}

1291

1292

static void RamsRulesTest(void) {

1293

UErrorCode status = U_ZERO_ERROR;

1294

int32_t i = 0;

1295

UCollator *coll = NULL;

1296

/* UCollator *UCA = ucol_open("", &status); */

1297

UChar rule[2048];

1298

uint32_t ruleLen;

1299

int32_t noOfLoc = uloc_countAvailable();

1300

const char *locName = NULL;

1301

1302

log_verbose("RamsRulesTest\n");

1303

1304

for(i = 0; i<noOfLoc; i++) {

1305

status = U_ZERO_ERROR;

1306

locName = uloc_getAvailable(i);

1307

if(hasCollationElements(locName)) {

1308

if (uprv_strcmp("ja", locName)==0) {

1309

log_verbose("Don't know how to test Japanese because of prefixes\n");

1310

continue;

1311

}

1312

log_verbose("Testing locale %s\n", locName);

1313

coll = ucol_open(locName, &status);

1314

if(U_SUCCESS(status)) {

1315

if(coll->image->jamoSpecial == TRUE) {

1316

log_err("%s has special JAMOs\n", locName);

1317

}

1318

ucol_setAttribute(coll, UCOL_CASE_FIRST, UCOL_OFF, &status);

1319

testCollator(coll, &status);

1320

testCEs(coll, &status);

1321

ucol_close(coll);

1322

}

1323

}

1324

}

1325

1326

for(i = 0; i<sizeof(rulesToTest)/sizeof(rulesToTest[0]); i++) {

1327

log_verbose("Testing rule: %s\n", rulesToTest[i]);

1328

u_uastrcpy(rule, rulesToTest[i]);

1329

ruleLen = u_strlen(rule);

1330

coll = ucol_openRules(rule, ruleLen, UCOL_OFF, UCOL_TERTIARY, NULL,&status);

1331

if(U_SUCCESS(status)) {

1332

testCollator(coll, &status);

1333

testCEs(coll, &status);

1334

ucol_close(coll);

1335

}

1336

}

1337

1338

}

1339

1340

static void IsTailoredTest(void) {

1341

UErrorCode status = U_ZERO_ERROR;

1342

uint32_t i = 0;

1343

UCollator *coll = NULL;

1344

UChar rule[2048];

1345

UChar tailored[2048];

1346

UChar notTailored[2048];

1347

uint32_t ruleLen, tailoredLen, notTailoredLen;

1348

1349

log_verbose("IsTailoredTest\n");

1350

1351

u_uastrcpy(rule, "&Z < A, B, C;c < d");

1352

ruleLen = u_strlen(rule);

1353

1354

u_uastrcpy(tailored, "ABCcd");

1355

tailoredLen = u_strlen(tailored);

1356

1357

u_uastrcpy(notTailored, "ZabD");

1358

notTailoredLen = u_strlen(notTailored);

1359

1360

coll = ucol_openRules(rule, ruleLen, UCOL_OFF, UCOL_TERTIARY, NULL,&status);

1361

if(U_SUCCESS(status)) {

1362

for(i = 0; i<tailoredLen; i++) {

1363

if(!isTailored(coll, tailored[i], &status)) {

1364

log_err("%i: %04X should be tailored - it is reported as not\n", i, tailored[i]);

1365

}

1366

}

1367

for(i = 0; i<notTailoredLen; i++) {

1368

if(isTailored(coll, notTailored[i], &status)) {

1369

log_err("%i: %04X should not be tailored - it is reported as it is\n", i, notTailored[i]);

1370

}

1371

}

1372

ucol_close(coll);

1373

}

1374

}

1375

1376

static void genericOrderingTestWithResult(UCollator *coll, const char *s[], uint32_t size, UCollationResult result) {

1377

UChar t1[2048] = {0};

1378

UChar t2[2048] = {0};

1379

UCollationElements *iter;

1380

UErrorCode status = U_ZERO_ERROR;

1381

1382

uint32_t i = 0, j = 0;

1383

log_verbose("testing sequence:\n");

1384

for(i = 0; i < size; i++) {

1385

log_verbose("%s\n", s[i]);

1386

}

1387

1388

iter = ucol_openElements(coll, t1, u_strlen(t1), &status);

1389

if (U_FAILURE(status)) {

1390

log_err("Creation of iterator failed\n");

1391

}

1392

for(i = 0; i < size-1; i++) {

1393

for(j = i+1; j < size; j++) {

1394

u_unescape(s[i], t1, 2048);

1395

u_unescape(s[j], t2, 2048);

1396

doTest(coll, t1, t2, result);

1397

/* synwee : added collation element iterator test */

1398

ucol_setText(iter, t1, u_strlen(t1), &status);

1399

backAndForth(iter);

1400

ucol_setText(iter, t2, u_strlen(t2), &status);

1401

backAndForth(iter);

1402

}

1403

}

1404

ucol_closeElements(iter);

1405

}

1406

1407

static void genericOrderingTest(UCollator *coll, const char *s[], uint32_t size) {

1408

genericOrderingTestWithResult(coll, s, size, UCOL_LESS);

1409

}

1410

1411

static void genericLocaleStarter(const char *locale, const char *s[], uint32_t size) {

1412

UErrorCode status = U_ZERO_ERROR;

1413

UCollator *coll = ucol_open(locale, &status);

1414

1415

log_verbose("Locale starter for %s\n", locale);

1416

1417

if(U_SUCCESS(status)) {

1418

genericOrderingTest(coll, s, size);

1419

} else {

1420

log_err("Unable to open collator for locale %s\n", locale);

1421

}

1422

ucol_close(coll);

1423

}

1424

1425

#if 0

1426

/* currently not used with options */

1427

static void genericRulesStarterWithOptions(const char *rules, const char *s[], uint32_t size, const UColAttribute *attrs, const UColAttributeValue *values, uint32_t attsize) {

1428

UErrorCode status = U_ZERO_ERROR;

1429

UChar rlz[RULE_BUFFER_LEN] = { 0 };

1430

uint32_t rlen = u_unescape(rules, rlz, RULE_BUFFER_LEN);

1431

uint32_t i;

1432

1433

UCollator *coll = ucol_openRules(rlz, rlen, UCOL_DEFAULT, UCOL_DEFAULT,NULL, &status);

1434

1435

log_verbose("Rules starter for %s\n", rules);

1436

1437

if(U_SUCCESS(status)) {

1438

log_verbose("Setting attributes\n");

1439

for(i = 0; i < attsize; i++) {

1440

ucol_setAttribute(coll, attrs[i], values[i], &status);

1441

}

1442

1443

genericOrderingTest(coll, s, size);

1444

} else {

1445

log_err("Unable to open collator with rules %s\n", rules);

1446

}

1447

ucol_close(coll);

1448

}

1449

#endif

1450

1451

static void genericLocaleStarterWithOptions(const char *locale, const char *s[], uint32_t size, const UColAttribute *attrs, const UColAttributeValue *values, uint32_t attsize) {

1452

UErrorCode status = U_ZERO_ERROR;

1453

uint32_t i;

1454

1455

UCollator *coll = ucol_open(locale, &status);

1456

1457

log_verbose("Locale starter for %s\n", locale);

1458

1459

if(U_SUCCESS(status)) {

1460

1461

log_verbose("Setting attributes\n");

1462

for(i = 0; i < attsize; i++) {

1463

ucol_setAttribute(coll, attrs[i], values[i], &status);

1464

}

1465

1466

genericOrderingTest(coll, s, size);

1467

} else {

1468

log_err("Unable to open collator for locale %s\n", locale);

1469

}

1470

ucol_close(coll);

1471

}

1472

1473

static void genericRulesTestWithResult(const char *rules, const char *s[], uint32_t size, UCollationResult result) {

1474

UErrorCode status = U_ZERO_ERROR;

1475

UChar rlz[RULE_BUFFER_LEN] = { 0 };

1476

uint32_t rlen = u_unescape(rules, rlz, RULE_BUFFER_LEN);

1477

1478

UCollator *coll = NULL;

1479

coll = ucol_openRules(rlz, rlen, UCOL_DEFAULT, UCOL_DEFAULT,NULL, &status);

1480

log_verbose("Rules starter for %s\n", rules);

1481

1482

if(U_SUCCESS(status)) {

1483

genericOrderingTestWithResult(coll, s, size, result);

1484

ucol_close(coll);

1485

} else {

1486

log_err("Unable to open collator with rules %s\n", rules);

1487

}

1488

}

1489

1490

static void genericRulesStarter(const char *rules, const char *s[], uint32_t size) {

1491

genericRulesTestWithResult(rules, s, size, UCOL_LESS);

1492

}

1493

1494

const static char chTest[][20] = {

1495

"c",

1496

"C",

1497

"ca", "cb", "cx", "cy", "CZ",

1498

"c\\u030C", "C\\u030C",

1499

"h",

1500

"H",

1501

"ha", "Ha", "harly", "hb", "HB", "hx", "HX", "hy", "HY",

1502

"ch", "cH", "Ch", "CH",

1503

"cha", "charly", "che", "chh", "chch", "chr",

1504

"i", "I", "iarly",

1505

"r", "R",

1506

"r\\u030C", "R\\u030C",

1507

"s",

1508

"S",

1509

"s\\u030C", "S\\u030C",

1510

"z", "Z",

1511

"z\\u030C", "Z\\u030C"

1512

};

1513

1514

static void TestChMove(void) {

1515

UChar t1[256] = {0};

1516

UChar t2[256] = {0};

1517

1518

uint32_t i = 0, j = 0;

1519

uint32_t size = 0;

1520

UErrorCode status = U_ZERO_ERROR;

1521

1522

UCollator *coll = ucol_open("cs", &status);

1523

1524

if(U_SUCCESS(status)) {

1525

size = sizeof(chTest)/sizeof(chTest[0]);

1526

for(i = 0; i < size-1; i++) {

1527

for(j = i+1; j < size; j++) {

1528

u_unescape(chTest[i], t1, 256);

1529

u_unescape(chTest[j], t2, 256);

1530

doTest(coll, t1, t2, UCOL_LESS);

1531

}

1532

}

1533

}

1534

else {

1535

log_err("Can't open collator");

1536

}

1537

ucol_close(coll);

1538

}

1539

1540

const static char impTest[][20] = {

1541

"\\u4e00",

1542

"a",

1543

"A",

1544

"b",

1545

"B",

1546

"\\u4e01"

1547

};

1548

1549

1550

static void TestImplicitTailoring(void) {

1551

UChar t1[256] = {0};

1552

UChar t2[256] = {0};

1553

1554

const char *rule = "&\\u4e00 < a <<< A < b <<< B";

1555

1556

uint32_t i = 0, j = 0;

1557

uint32_t size = 0;

1558

uint32_t ruleLen = 0;

1559

UErrorCode status = U_ZERO_ERROR;

1560

UCollator *coll = NULL;

1561

ruleLen = u_unescape(rule, t1, 256);

1562

1563

coll = ucol_openRules(t1, ruleLen, UCOL_OFF, UCOL_TERTIARY,NULL, &status);

1564

1565

if(U_SUCCESS(status)) {

1566

size = sizeof(impTest)/sizeof(impTest[0]);

1567

for(i = 0; i < size-1; i++) {

1568

for(j = i+1; j < size; j++) {

1569

u_unescape(impTest[i], t1, 256);

1570

u_unescape(impTest[j], t2, 256);

1571

doTest(coll, t1, t2, UCOL_LESS);

1572

}

1573

}

1574

}

1575

else {

1576

log_err("Can't open collator");

1577

}

1578

ucol_close(coll);

1579

}

1580

1581

static void TestFCDProblem(void) {

1582

UChar t1[256] = {0};

1583

UChar t2[256] = {0};

1584

1585

const char *s1 = "\\u0430\\u0306\\u0325";

1586

const char *s2 = "\\u04D1\\u0325";

1587

1588

UErrorCode status = U_ZERO_ERROR;

1589

UCollator *coll = ucol_open("", &status);

1590

u_unescape(s1, t1, 256);

1591

u_unescape(s2, t2, 256);

1592

1593

ucol_setAttribute(coll, UCOL_NORMALIZATION_MODE, UCOL_OFF, &status);

1594

doTest(coll, t1, t2, UCOL_EQUAL);

1595

1596

ucol_setAttribute(coll, UCOL_NORMALIZATION_MODE, UCOL_ON, &status);

1597

doTest(coll, t1, t2, UCOL_EQUAL);

1598

1599

ucol_close(coll);

1600

}

1601

1602

#define NORM_BUFFER_TEST_LEN 32

1603

typedef struct {

1604

UChar32 u;

1605

UChar NFC[NORM_BUFFER_TEST_LEN];

1606

UChar NFD[NORM_BUFFER_TEST_LEN];

1607

} tester;

1608

1609

static void TestComposeDecompose(void) {

1610

int32_t noOfLoc = uloc_countAvailable();

1611

int32_t i = 0, j = 0;

1612

1613

UErrorCode status = U_ZERO_ERROR;

1614

1615

const char *locName = NULL;

1616

1617

uint32_t nfcSize;

1618

uint32_t nfdSize;

1619

tester **t = uprv_malloc(0x30000 * sizeof(tester *));

1620

uint32_t noCases = 0;

1621

UCollator *coll = NULL;

1622

UChar32 u = 0;

1623

UChar comp[NORM_BUFFER_TEST_LEN];

1624

uint32_t len = 0;

1625

1626

t[0] = (tester *)uprv_malloc(sizeof(tester));

1627

1628

for(u = 0; u < 0x30000; u++) {

1629

len = 0;

1630

UTF_APPEND_CHAR_UNSAFE(comp, len, u);

1631

nfcSize = unorm_normalize(comp, len, UNORM_NFC, 0, t[noCases]->NFC, NORM_BUFFER_TEST_LEN, &status);

1632

nfdSize = unorm_normalize(comp, len, UNORM_NFD, 0, t[noCases]->NFD, NORM_BUFFER_TEST_LEN, &status);

1633

1634

if(nfcSize != nfdSize || (uprv_memcmp(t[noCases]->NFC, t[noCases]->NFD, nfcSize * sizeof(UChar)) != 0)

1635

|| (len != nfdSize || (uprv_memcmp(comp, t[noCases]->NFD, nfdSize * sizeof(UChar)) != 0))) {

1636

t[noCases]->u = u;

1637

if(len != nfdSize || (uprv_memcmp(comp, t[noCases]->NFD, nfdSize * sizeof(UChar)) != 0)) {

1638

u_strncpy(t[noCases]->NFC, comp, len);

1639

t[noCases]->NFC[len] = 0;

1640

}

1641

noCases++;

1642

t[noCases] = (tester *)uprv_malloc(sizeof(tester));

1643

uprv_memset(t[noCases], 0, sizeof(tester));

1644

}

1645

}

1646

1647

log_verbose("Testing UCA extensively\n");

1648

coll = ucol_open("", &status);

1649

for(u=0; u<noCases; u++) {

1650

if(!ucol_equal(coll, t[u]->NFC, -1, t[u]->NFD, -1)) {

1651

log_err("Failure: codePoint %05X fails TestComposeDecompose in the UCA\n", t[u]->u);

1652

doTest(coll, t[u]->NFC, t[u]->NFD, UCOL_EQUAL);

1653

}

1654

}

1655

1656

for(u = 0; u < 0x30000; u++) {

1657

if(!(u&0xFFFF)) {

1658

log_verbose("%08X ", u);

1659

}

1660

uprv_memset(t[noCases], 0, sizeof(tester));

1661

t[noCases]->u = u;

1662

len = 0;

1663

UTF_APPEND_CHAR_UNSAFE(comp, len, u);

1664

comp[len] = 0;

1665

nfcSize = unorm_normalize(comp, len, UNORM_NFC, 0, t[noCases]->NFC, NORM_BUFFER_TEST_LEN, &status);

1666

nfdSize = unorm_normalize(comp, len, UNORM_NFD, 0, t[noCases]->NFD, NORM_BUFFER_TEST_LEN, &status);

1667

doTest(coll, comp, t[noCases]->NFD, UCOL_EQUAL);

1668

doTest(coll, comp, t[noCases]->NFC, UCOL_EQUAL);

1669

}

1670

1671

1672

ucol_close(coll);

1673

1674

log_verbose("Testing locales, number of cases = %i\n", noCases);

1675

for(i = 0; i<noOfLoc; i++) {

1676

status = U_ZERO_ERROR;

1677

locName = uloc_getAvailable(i);

1678

if(hasCollationElements(locName)) {

1679

char cName[256];

1680

UChar name[256];

1681

int32_t nameSize = uloc_getDisplayName(locName, NULL, name, sizeof(cName), &status);

1682

1683

for(j = 0; j<nameSize; j++) {

1684

cName[j] = (char)name[j];

1685

}

1686

cName[nameSize] = 0;

1687

log_verbose("\nTesting locale %s (%s)\n", locName, cName);

1688

1689

coll = ucol_open(locName, &status);

1690

ucol_setStrength(coll, UCOL_IDENTICAL);

1691

1692

for(u=0; u<noCases; u++) {

1693

if(!ucol_equal(coll, t[u]->NFC, -1, t[u]->NFD, -1)) {

1694

log_err("Failure: codePoint %05X fails TestComposeDecompose for locale %s\n", t[u]->u, cName);

1695

doTest(coll, t[u]->NFC, t[u]->NFD, UCOL_EQUAL);

1696

}

1697

}

1698

ucol_close(coll);

1699

}

1700

}

1701

for(u = 0; u <= noCases; u++) {

1702

uprv_free(t[u]);

1703

}

1704

uprv_free(t);

1705

}

1706

1707

static void TestEmptyRule(void) {

1708

UErrorCode status = U_ZERO_ERROR;

1709

UChar rulez[] = { 0 };

1710

UCollator *coll = ucol_openRules(rulez, 0, UCOL_OFF, UCOL_TERTIARY,NULL, &status);

1711

1712

ucol_close(coll);

1713

}

1714

1715

static void TestUCARules(void) {

1716

UErrorCode status = U_ZERO_ERROR;

1717

UChar b[256];

1718

UChar *rules = b;

1719

UCollator *UCAfromRules = NULL;

1720

UCollator *coll = ucol_open("", &status);

1721

uint32_t ruleLen = ucol_getRulesEx(coll, UCOL_FULL_RULES, rules, 256);

1722

1723

log_verbose("TestUCARules\n");

1724

if(ruleLen > 256) {

1725

rules = (UChar *)malloc((ruleLen+1)*sizeof(UChar));

1726

ruleLen = ucol_getRulesEx(coll, UCOL_FULL_RULES, rules, ruleLen);

1727

}

1728

log_verbose("Rules length is %d\n", ruleLen);

1729

UCAfromRules = ucol_openRules(rules, ruleLen, UCOL_OFF, UCOL_TERTIARY, NULL,&status);

1730

if(U_SUCCESS(status)) {

1731

ucol_close(UCAfromRules);

1732

} else {

1733

log_verbose("Unable to create a collator from UCARules!\n");

1734

}

1735

1736

u_unescape(blah, b, 256);

1737

ucol_getSortKey(coll, b, 1, res, 256);

1738

1739

ucol_close(coll);

1740

if(rules != b) {

1741

free(rules);

1742

}

1743

}

1744

1745

1746

/* Pinyin tonal order */

1747

1748

A < .. (\u0101) < .. (\u00e1) < .. (\u01ce) < .. (\u00e0)

1749

(w/macron)< (w/acute)< (w/caron)< (w/grave)

1750

E < .. (\u0113) < .. (\u00e9) < .. (\u011b) < .. (\u00e8)

1751

I < .. (\u012b) < .. (\u00ed) < .. (\u01d0) < .. (\u00ec)

1752

O < .. (\u014d) < .. (\u00f3) < .. (\u01d2) < .. (\u00f2)

1753

U < .. (\u016b) < .. (\u00fa) < .. (\u01d4) < .. (\u00f9)

1754

< .. (\u01d6) < .. (\u01d8) < .. (\u01da) < .. (\u01dc) <

1755

.. (\u00fc)

1756

1757

However, in testing we got the following order:

1758

A < .. (\u00e1) < .. (\u00e0) < .. (\u01ce) < .. (\u0101)

1759

(w/acute)< (w/grave)< (w/caron)< (w/macron)

1760

E < .. (\u00e9) < .. (\u00e8) < .. (\u00ea) < .. (\u011b) <

1761

.. (\u0113)

1762

I < .. (\u00ed) < .. (\u00ec) < .. (\u01d0) < .. (\u012b)

1763

O < .. (\u00f3) < .. (\u00f2) < .. (\u01d2) < .. (\u014d)

1764

U < .. (\u00fa) < .. (\u00f9) < .. (\u01d4) < .. (\u00fc) <

1765

.. (\u01d8)

1766

< .. (\u01dc) < .. (\u01da) < .. (\u01d6) < .. (\u016b)

1767

1768

1769

static void TestBefore(void) {

1770

const static char *data[] = {

1771

"\\u0101", "\\u00e1", "\\u01ce", "\\u00e0", "A",

1772

"\\u0113", "\\u00e9", "\\u011b", "\\u00e8", "E",

1773

"\\u012b", "\\u00ed", "\\u01d0", "\\u00ec", "I",

1774

"\\u014d", "\\u00f3", "\\u01d2", "\\u00f2", "O",

1775

"\\u016b", "\\u00fa", "\\u01d4", "\\u00f9", "U",

1776

"\\u01d6", "\\u01d8", "\\u01da", "\\u01dc", "\\u00fc"

1777

};

1778

genericRulesStarter(

1779

"&[before 1]a<\\u0101<\\u00e1<\\u01ce<\\u00e0"

1780

"&[before 1]e<\\u0113<\\u00e9<\\u011b<\\u00e8"

1781

"&[before 1]i<\\u012b<\\u00ed<\\u01d0<\\u00ec"

1782

"&[before 1]o<\\u014d<\\u00f3<\\u01d2<\\u00f2"

1783

"&[before 1]u<\\u016b<\\u00fa<\\u01d4<\\u00f9"

1784

"&u<\\u01d6<\\u01d8<\\u01da<\\u01dc<\\u00fc",

1785

data, sizeof(data)/sizeof(data[0]));

1786

}

1787

1788

static void TestJ784(void) {

1789

const static char *data[] = {

1790

"A", "\\u0101", "\\u00e1", "\\u01ce", "\\u00e0",

1791

"E", "\\u0113", "\\u00e9", "\\u011b", "\\u00e8",

1792

"I", "\\u012b", "\\u00ed", "\\u01d0", "\\u00ec",

1793

"O", "\\u014d", "\\u00f3", "\\u01d2", "\\u00f2",

1794

"U", "\\u016b", "\\u00fa", "\\u01d4", "\\u00f9",

1795

"\\u00fc",

1796

"\\u01d6", "\\u01d8", "\\u01da", "\\u01dc"

1797

};

1798

genericLocaleStarter("zh", data, sizeof(data)/sizeof(data[0]));

1799

}

1800

1801

1802

static void TestJ831(void) {

1803

const static char *data[] = {

1804

"I",

1805

"i",

1806

"Y",

1807

"y"

1808

};

1809

genericLocaleStarter("lv", data, sizeof(data)/sizeof(data[0]));

1810

}

1811

1812

static void TestJ815(void) {

1813

const static char *data[] = {

1814

"aa",

1815

"Aa",

1816

"ab",

1817

"Ab",

1818

"ad",

1819

"Ad",

1820

"ae",

1821

"Ae",

1822

"\\u00e6",

1823

"\\u00c6",

1824

"af",

1825

"Af",

1826

"b",

1827

"B"

1828

};

1829

genericLocaleStarter("fr", data, sizeof(data)/sizeof(data[0]));

1830

genericRulesStarter("[backwards 2]&A<<\\u00e6/e<<<\\u00c6/E", data, sizeof(data)/sizeof(data[0]));

1831

}

1832

1833

1834

1835

"& a < b < c < d& r < c", "& a < b < d& r < c",

1836

"& a < b < c < d& c < m", "& a < b < c < m < d",

1837

"& a < b < c < d& a < m", "& a < m < b < c < d",

1838

"& a <<< b << c < d& a < m", "& a <<< b << c < m < d",

1839

"& a < b < c < d& [before 1] c < m", "& a < b < m < c < d",

1840

"& a < b <<< c << d <<< e& [before 3] e <<< x", "& a < b <<< c << d <<< x <<< e",

1841

"& a < b <<< c << d <<< e& [before 2] e <<< x", "& a < b <<< c <<< x << d <<< e",

1842

"& a < b <<< c << d <<< e& [before 1] e <<< x", "& a <<< x < b <<< c << d <<< e",

1843

"& a < b <<< c << d <<< e <<< f < g& [before 1] g < x", "& a < b <<< c << d <<< e <<< f < x < g",

1844

1845

static void TestRedundantRules(void) {

1846

int32_t i;

1847

1848

const static char *rules[] = {

1849

"& a <<< b <<< c << d <<< e& [before 1] e <<< x",

1850

"& a < b <<< c << d <<< e& [before 1] e <<< x",

1851

"& a < b < c < d& [before 1] c < m",

1852

"& a < b <<< c << d <<< e& [before 3] e <<< x",

1853

"& a < b <<< c << d <<< e& [before 2] e <<< x",

1854

"& a < b <<< c << d <<< e <<< f < g& [before 1] g < x",

1855

"& a <<< b << c < d& a < m",

1856

"&a<b<<b\\u0301 &z<b",

1857

"&z<m<<<q<<<m",

1858

"&z<<<m<q<<<m",

1859

"& a < b < c < d& r < c",

1860

"& a < b < c < d& r < c",

1861

"& a < b < c < d& c < m",

1862

"& a < b < c < d& a < m"

1863

};

1864

1865

const static char *expectedRules[] = {

1866

"&\\u3029<<<x",

1867

"& a <<< x < b <<< c << d <<< e",

1868

"& a < b < m < c < d",

1869

"& a < b <<< c << d <<< x <<< e",

1870

"& a < b <<< c <<< x << d <<< e",

1871

"& a < b <<< c << d <<< e <<< f < x < g",

1872

"& a <<< b << c < m < d",

1873

"&a<b\\u0301 &z<b",

1874

"&z<q<<<m",

1875

"&z<q<<<m",

1876

"& a < b < d& r < c",

1877

"& a < b < d& r < c",

1878

"& a < b < c < m < d",

1879

"& a < m < b < c < d"

1880

};

1881

1882

const static char *testdata[][8] = {

1883

{"\\u3029", "x"},

1884

{"a", "x", "b", "c", "d", "e"},

1885

{"a", "b", "m", "c", "d"},

1886

{"a", "b", "c", "d", "x", "e"},

1887

{"a", "b", "c", "x", "d", "e"},

1888

{"a", "b", "c", "d", "e", "f", "x", "g"},

1889

{"a", "b", "c", "m", "d"},

1890

{"a", "b\\u0301", "z", "b"},

1891

{"z", "q", "m"},

1892

{"z", "q", "m"},

1893

{"a", "b", "d"},

1894

{"r", "c"},

1895

{"a", "b", "c", "m", "d"},

1896

{"a", "m", "b", "c", "d"}

1897

};

1898

1899

const static uint32_t testdatalen[] = {

1900

1901

1902

1903

1904

1905

1906

1907

1908

1909

1910

1911

1912

1913

1914

};

1915

1916

1917

1918

UCollator *credundant = NULL;

1919

UCollator *cresulting = NULL;

1920

UErrorCode status = U_ZERO_ERROR;

1921

UChar rlz[2048] = { 0 };

1922

uint32_t rlen = 0;

1923

1924

for(i = 0; i<sizeof(rules)/sizeof(rules[0]); i++) {

1925

log_verbose("testing rule %s, expected to be %s\n", rules[i], expectedRules[i]);

1926

rlen = u_unescape(rules[i], rlz, 2048);

1927

1928

credundant = ucol_openRules(rlz, rlen, UCOL_DEFAULT, UCOL_DEFAULT, NULL,&status);

1929

rlen = u_unescape(expectedRules[i], rlz, 2048);

1930

cresulting = ucol_openRules(rlz, rlen, UCOL_DEFAULT, UCOL_DEFAULT, NULL,&status);

1931

1932

testAgainstUCA(cresulting, credundant, "expected", TRUE, &status);

1933

1934

ucol_close(credundant);

1935

ucol_close(cresulting);

1936

1937

log_verbose("testing using data\n");

1938

1939

genericRulesStarter(rules[i], testdata[i], testdatalen[i]);

1940

}

1941

1942

}

1943

1944

static void TestExpansionSyntax(void) {

1945

int32_t i;

1946

1947

const static char *rules[] = {

1948

"&AE <<< a << b <<< c &d <<< f",

1949

"&AE <<< a <<< b << c << d < e < f <<< g",

1950

"&AE <<< B <<< C / D <<< F"

1951

};

1952

1953

const static char *expectedRules[] = {

1954

"&A <<< a / E << b / E <<< c /E &d <<< f",

1955

"&A <<< a / E <<< b / E << c / E << d / E < e < f <<< g",

1956

"&A <<< B / E <<< C / ED <<< F / E"

1957

};

1958

1959

const static char *testdata[][8] = {

1960

{"AE", "a", "b", "c"},

1961

{"AE", "a", "b", "c", "d", "e", "f", "g"},

1962

{"AE", "B", "C"} /* / ED <<< F / E"},*/

1963

};

1964

1965

const static uint32_t testdatalen[] = {

1966

1967

1968

1969

};

1970

1971

1972

1973

UCollator *credundant = NULL;

1974

UCollator *cresulting = NULL;

1975

UErrorCode status = U_ZERO_ERROR;

1976

UChar rlz[2048] = { 0 };

1977

uint32_t rlen = 0;

1978

1979

for(i = 0; i<sizeof(rules)/sizeof(rules[0]); i++) {

1980

log_verbose("testing rule %s, expected to be %s\n", rules[i], expectedRules[i]);

1981

rlen = u_unescape(rules[i], rlz, 2048);

1982

1983

credundant = ucol_openRules(rlz, rlen, UCOL_DEFAULT, UCOL_DEFAULT, NULL, &status);

1984

rlen = u_unescape(expectedRules[i], rlz, 2048);

1985

cresulting = ucol_openRules(rlz, rlen, UCOL_DEFAULT, UCOL_DEFAULT, NULL,&status);

1986

1987

/* testAgainstUCA still doesn't handle expansions correctly, so this is not run */

1988

/* as a hard error test, but only in information mode */

1989

testAgainstUCA(cresulting, credundant, "expected", FALSE, &status);

1990

1991

ucol_close(credundant);

1992

ucol_close(cresulting);

1993

1994

log_verbose("testing using data\n");

1995

1996

genericRulesStarter(rules[i], testdata[i], testdatalen[i]);

1997

}

1998

}

1999

2000

static void TestCase(void)

2001

{

2002

const static UChar gRules[MAX_TOKEN_LEN] =

2003

/*" & 0 < 1,\u2461<a,A"*/

2004

{ 0x0026, 0x0030, 0x003C, 0x0031, 0x002C, 0x2460, 0x003C, 0x0061, 0x002C, 0x0041, 0x0000 };

2005

2006

const static UChar testCase[][MAX_TOKEN_LEN] =

2007

{

2008

/*0*/ {0x0031 /*'1'*/, 0x0061/*'a'*/, 0x0000},

2009

/*1*/ {0x0031 /*'1'*/, 0x0041/*'A'*/, 0x0000},

2010

/*2*/ {0x2460 /*circ'1'*/, 0x0061/*'a'*/, 0x0000},

2011

/*3*/ {0x2460 /*circ'1'*/, 0x0041/*'A'*/, 0x0000}

2012

};

2013

2014

const static UCollationResult caseTestResults[][9] =

2015

{

2016

{ UCOL_LESS, UCOL_LESS, UCOL_LESS, 0, UCOL_LESS, UCOL_LESS, 0, 0, UCOL_LESS },

2017

{ UCOL_GREATER, UCOL_LESS, UCOL_LESS, 0, UCOL_LESS, UCOL_LESS, 0, 0, UCOL_GREATER },

2018

{ UCOL_LESS, UCOL_LESS, UCOL_LESS, 0, UCOL_GREATER, UCOL_LESS, 0, 0, UCOL_LESS },

2019

{ UCOL_GREATER, UCOL_LESS, UCOL_GREATER, 0, UCOL_LESS, UCOL_LESS, 0, 0, UCOL_GREATER }

2020

2021

};

2022

2023

const static UColAttributeValue caseTestAttributes[][2] =

2024

{

2025

{ UCOL_LOWER_FIRST, UCOL_OFF},

2026

{ UCOL_UPPER_FIRST, UCOL_OFF},

2027

{ UCOL_LOWER_FIRST, UCOL_ON},

2028

{ UCOL_UPPER_FIRST, UCOL_ON}

2029

2030

};

2031

int32_t i,j,k;

2032

UErrorCode status = U_ZERO_ERROR;

2033

UCollator *myCollation;

2034

myCollation = ucol_open("en_US", &status);

2035

if(U_FAILURE(status)){

2036

log_err("ERROR: in creation of rule based collator: %s\n", myErrorName(status));

2037

return;

2038

}

2039

log_verbose("Testing different case settings\n");

2040

ucol_setStrength(myCollation, UCOL_TERTIARY);

2041

2042

for(k = 0; k<4; k++) {

2043

ucol_setAttribute(myCollation, UCOL_CASE_FIRST, caseTestAttributes[k][0], &status);

2044

ucol_setAttribute(myCollation, UCOL_CASE_LEVEL, caseTestAttributes[k][1], &status);

2045

log_verbose("Case first = %d, Case level = %d\n", caseTestAttributes[k][0], caseTestAttributes[k][1]);

2046

for (i = 0; i < 3 ; i++) {

2047

for(j = i+1; j<4; j++) {

2048

doTest(myCollation, testCase[i], testCase[j], caseTestResults[k][3*i+j-1]);

2049

}

2050

}

2051

}

2052

ucol_close(myCollation);

2053

2054

myCollation = ucol_openRules(gRules, u_strlen(gRules), UCOL_OFF, UCOL_TERTIARY,NULL, &status);

2055

if(U_FAILURE(status)){

2056

log_err("ERROR: in creation of rule based collator: %s\n", myErrorName(status));

2057

return;

2058

}

2059

log_verbose("Testing different case settings with custom rules\n");

2060

ucol_setStrength(myCollation, UCOL_TERTIARY);

2061

2062

for(k = 0; k<4; k++) {

2063

ucol_setAttribute(myCollation, UCOL_CASE_FIRST, caseTestAttributes[k][0], &status);

2064

ucol_setAttribute(myCollation, UCOL_CASE_LEVEL, caseTestAttributes[k][1], &status);

2065

for (i = 0; i < 3 ; i++) {

2066

for(j = i+1; j<4; j++) {

2067

log_verbose("k:%d, i:%d, j:%d\n", k, i, j);

2068

doTest(myCollation, testCase[i], testCase[j], caseTestResults[k][3*i+j-1]);

2069

}

2070

}

2071

}

2072

ucol_close(myCollation);

2073

{

2074

const static char *lowerFirst[] = {

2075

"h",

2076

"H",

2077

"ch",

2078

"Ch",

2079

"CH",

2080

"cha",

2081

"chA",

2082

"Cha",

2083

"ChA",

2084

"CHa",

2085

"CHA",

2086

"i",

2087

"I"

2088

};

2089

2090

const static char *upperFirst[] = {

2091

"H",

2092

"h",

2093

"CH",

2094

"Ch",

2095

"ch",

2096

"CHA",

2097

"CHa",

2098

"ChA",

2099

"Cha",

2100

"chA",

2101

"cha",

2102

"I",

2103

"i"

2104

};

2105

log_verbose("mixed case test\n");

2106

log_verbose("lower first, case level off\n");

2107

genericRulesStarter("[casefirst lower]&H<ch<<<Ch<<<CH", lowerFirst, sizeof(lowerFirst)/sizeof(lowerFirst[0]));

2108

log_verbose("upper first, case level off\n");

2109

genericRulesStarter("[casefirst upper]&H<ch<<<Ch<<<CH", upperFirst, sizeof(upperFirst)/sizeof(upperFirst[0]));

2110

log_verbose("lower first, case level on\n");

2111

genericRulesStarter("[casefirst lower][caselevel on]&H<ch<<<Ch<<<CH", lowerFirst, sizeof(lowerFirst)/sizeof(lowerFirst[0]));

2112

log_verbose("upper first, case level on\n");

2113

genericRulesStarter("[casefirst upper][caselevel on]&H<ch<<<Ch<<<CH", upperFirst, sizeof(upperFirst)/sizeof(upperFirst[0]));

2114

}

2115

2116

}

2117

2118

static void TestIncrementalNormalize(void) {

2119

2120

UChar baseA =0x41;

2121

/* UChar baseB = 0x42;*/

2122

UChar ccMix[] = {0x316, 0x321, 0x300};

2123

2124

0x316 is combining grave accent below, cc=220

2125

0x321 is combining palatalized hook below, cc=202

2126

0x300 is combining grave accent, cc=230

2127

2128

2129

int maxSLen = 2000;

2130

int sLen;

2131

int i;

2132

2133

UCollator *coll;

2134

UErrorCode status = U_ZERO_ERROR;

2135

UCollationResult result;

2136

2137

{

2138

/* Test 1. Run very long unnormalized strings, to force overflow of*/

2139

/* most buffers along the way.*/

2140

UChar *strA;

2141

UChar *strB;

2142

2143

strA = uprv_malloc((maxSLen+1) * sizeof(UChar));

2144

strB = uprv_malloc((maxSLen+1) * sizeof(UChar));

2145

2146

coll = ucol_open("en_US", &status);

2147

ucol_setNormalization(coll, UNORM_NFD);

2148

2149

/* for (sLen = 4; sLen<maxSLen; sLen++) { */

2150

for (sLen = 1000; sLen<1001; sLen++) {

2151

strA[0] = baseA;

2152

strB[0] = baseA;

2153

for (i=1; i<=sLen-1; i++) {

2154

strA[i] = ccMix[i % 3];

2155

strB[sLen-i] = ccMix[i % 3];

2156

}

2157

strA[sLen] = 0;

2158

strB[sLen] = 0;

2159

2160

ucol_setStrength(coll, UCOL_TERTIARY); /* Do test with default strength, which runs*/

2161

doTest(coll, strA, strB, UCOL_EQUAL); /* optimized functions in the impl*/

2162

ucol_setStrength(coll, UCOL_IDENTICAL); /* Do again with the slow, general impl.*/

2163

doTest(coll, strA, strB, UCOL_EQUAL);

2164

}

2165

uprv_free(strA);

2166

uprv_free(strB);

2167

}

2168

2169

2170

/* Test 2: Non-normal sequence in a string that extends to the last character*/

2171

/* of the string. Checks a couple of edge cases.*/

2172

2173

{

2174

UChar strA[] = {0x41, 0x41, 0x300, 0x316, 0};

2175

UChar strB[] = {0x41, 0xc0, 0x316, 0};

2176

ucol_setStrength(coll, UCOL_TERTIARY);

2177

doTest(coll, strA, strB, UCOL_EQUAL);

2178

}

2179

2180

/* Test 3: Non-normal sequence is terminated by a surrogate pair.*/

2181

2182

{

2183

UChar strA[] = {0x41, 0x41, 0x300, 0x316, 0xD801, 0xDC00, 0};

2184

UChar strB[] = {0x41, 0xc0, 0x316, 0xD800, 0xDC00, 0};

2185

ucol_setStrength(coll, UCOL_TERTIARY);

2186

doTest(coll, strA, strB, UCOL_GREATER);

2187

}

2188

2189

/* Test 4: Imbedded nulls do not terminate a string when length is specified.*/

2190

2191

{

2192

UChar strA[] = {0x41, 0x00, 0x42, 0x00};

2193

UChar strB[] = {0x41, 0x00, 0x00, 0x00};

2194

char sortKeyA[50];

2195

char sortKeyAz[50];

2196

char sortKeyB[50];

2197

char sortKeyBz[50];

2198

int r;

2199

2200

result = ucol_strcoll(coll, strA, -3, strB, -3);

2201

if (result != UCOL_GREATER) {

2202

log_err("ERROR 1 in test 4\n");

2203

}

2204

result = ucol_strcoll(coll, strA, -1, strB, -1);

2205

if (result != UCOL_EQUAL) {

2206

log_err("ERROR 2 in test 4\n");

2207

}

2208

2209

ucol_getSortKey(coll, strA, 3, (uint8_t *)sortKeyA, sizeof(sortKeyA));

2210

ucol_getSortKey(coll, strA, -1, (uint8_t *)sortKeyAz, sizeof(sortKeyAz));

2211

ucol_getSortKey(coll, strB, 3, (uint8_t *)sortKeyB, sizeof(sortKeyB));

2212

ucol_getSortKey(coll, strB, -1, (uint8_t *)sortKeyBz, sizeof(sortKeyBz));

2213

2214

r = strcmp(sortKeyA, sortKeyAz);

2215

if (r <= 0) {

2216

log_err("Error 3 in test 4\n");

2217

}

2218

r = strcmp(sortKeyA, sortKeyB);

2219

if (r <= 0) {

2220

log_err("Error 4 in test 4\n");

2221

}

2222

r = strcmp(sortKeyAz, sortKeyBz);

2223

if (r != 0) {

2224

log_err("Error 5 in test 4\n");

2225

}

2226

2227

ucol_setStrength(coll, UCOL_IDENTICAL);

2228

ucol_getSortKey(coll, strA, 3, (uint8_t *)sortKeyA, sizeof(sortKeyA));

2229

ucol_getSortKey(coll, strA, -1, (uint8_t *)sortKeyAz, sizeof(sortKeyAz));

2230

ucol_getSortKey(coll, strB, 3, (uint8_t *)sortKeyB, sizeof(sortKeyB));

2231

ucol_getSortKey(coll, strB, -1, (uint8_t *)sortKeyBz, sizeof(sortKeyBz));

2232

2233

r = strcmp(sortKeyA, sortKeyAz);

2234

if (r <= 0) {

2235

log_err("Error 6 in test 4\n");

2236

}

2237

r = strcmp(sortKeyA, sortKeyB);

2238

if (r <= 0) {

2239

log_err("Error 7 in test 4\n");

2240

}

2241

r = strcmp(sortKeyAz, sortKeyBz);

2242

if (r != 0) {

2243

log_err("Error 8 in test 4\n");

2244

}

2245

ucol_setStrength(coll, UCOL_TERTIARY);

2246

}

2247

2248

2249

/* Test 5: Null characters in non-normal source strings.*/

2250

2251

{

2252

UChar strA[] = {0x41, 0x41, 0x300, 0x316, 0x00, 0x42, 0x00};

2253

UChar strB[] = {0x41, 0x41, 0x300, 0x316, 0x00, 0x00, 0x00};

2254

char sortKeyA[50];

2255

char sortKeyAz[50];

2256

char sortKeyB[50];

2257

char sortKeyBz[50];

2258

int r;

2259

2260

result = ucol_strcoll(coll, strA, 6, strB, 6);

2261

if (result != UCOL_GREATER) {

2262

log_err("ERROR 1 in test 5\n");

2263

}

2264

result = ucol_strcoll(coll, strA, -1, strB, -1);

2265

if (result != UCOL_EQUAL) {

2266

log_err("ERROR 2 in test 5\n");

2267

}

2268

2269

ucol_getSortKey(coll, strA, 6, (uint8_t *)sortKeyA, sizeof(sortKeyA));

2270

ucol_getSortKey(coll, strA, -1, (uint8_t *)sortKeyAz, sizeof(sortKeyAz));

2271

ucol_getSortKey(coll, strB, 6, (uint8_t *)sortKeyB, sizeof(sortKeyB));

2272

ucol_getSortKey(coll, strB, -1, (uint8_t *)sortKeyBz, sizeof(sortKeyBz));

2273

2274

r = strcmp(sortKeyA, sortKeyAz);

2275

if (r <= 0) {

2276

log_err("Error 3 in test 5\n");

2277

}

2278

r = strcmp(sortKeyA, sortKeyB);

2279

if (r <= 0) {

2280

log_err("Error 4 in test 5\n");

2281

}

2282

r = strcmp(sortKeyAz, sortKeyBz);

2283

if (r != 0) {

2284

log_err("Error 5 in test 5\n");

2285

}

2286

2287

ucol_setStrength(coll, UCOL_IDENTICAL);

2288

ucol_getSortKey(coll, strA, 6, (uint8_t *)sortKeyA, sizeof(sortKeyA));

2289

ucol_getSortKey(coll, strA, -1, (uint8_t *)sortKeyAz, sizeof(sortKeyAz));

2290

ucol_getSortKey(coll, strB, 6, (uint8_t *)sortKeyB, sizeof(sortKeyB));

2291

ucol_getSortKey(coll, strB, -1, (uint8_t *)sortKeyBz, sizeof(sortKeyBz));

2292

2293

r = strcmp(sortKeyA, sortKeyAz);

2294

if (r <= 0) {

2295

log_err("Error 6 in test 5\n");

2296

}

2297

r = strcmp(sortKeyA, sortKeyB);

2298

if (r <= 0) {

2299

log_err("Error 7 in test 5\n");

2300

}

2301

r = strcmp(sortKeyAz, sortKeyBz);

2302

if (r != 0) {

2303

log_err("Error 8 in test 5\n");

2304

}

2305

ucol_setStrength(coll, UCOL_TERTIARY);

2306

}

2307

2308

2309

/* Test 6: Null character as base of a non-normal combining sequence.*/

2310

2311

{

2312

UChar strA[] = {0x41, 0x0, 0x300, 0x316, 0x41, 0x302, 0x00};

2313

UChar strB[] = {0x41, 0x0, 0x302, 0x316, 0x41, 0x300, 0x00};

2314

2315

result = ucol_strcoll(coll, strA, 5, strB, 5);

2316

if (result != UCOL_LESS) {

2317

log_err("Error 1 in test 6\n");

2318

}

2319

result = ucol_strcoll(coll, strA, -1, strB, -1);

2320

if (result != UCOL_EQUAL) {

2321

log_err("Error 2 in test 6\n");

2322

}

2323

}

2324

2325

ucol_close(coll);

2326

}

2327

2328

2329

2330

#if 0

2331

static void TestGetCaseBit(void) {

2332

static const char *caseBitData[] = {

2333

"a", "A", "ch", "Ch", "CH",

2334

"\\uFF9E", "\\u0009"

2335

};

2336

2337

static const uint8_t results[] = {

2338

UCOL_LOWER_CASE, UCOL_UPPER_CASE, UCOL_LOWER_CASE, UCOL_MIXED_CASE, UCOL_UPPER_CASE,

2339

UCOL_UPPER_CASE, UCOL_LOWER_CASE

2340

};

2341

2342

uint32_t i, blen = 0;

2343

UChar b[256] = {0};

2344

UErrorCode status = U_ZERO_ERROR;

2345

UCollator *UCA = ucol_open("", &status);

2346

uint8_t res = 0;

2347

2348

for(i = 0; i<sizeof(results)/sizeof(results[0]); i++) {

2349

blen = u_unescape(caseBitData[i], b, 256);

2350

res = ucol_uprv_getCaseBits(UCA, b, blen, &status);

2351

if(results[i] != res) {

2352

log_err("Expected case = %02X, got %02X for %04X\n", results[i], res, b[0]);

2353

}

2354

}

2355

}

2356

#endif

2357

2358

static void TestHangulTailoring(void) {

2359

static const char *koreanData[] = {

2360

"\\uac00", "\\u4f3d", "\\u4f73", "\\u5047", "\\u50f9", "\\u52a0", "\\u53ef", "\\u5475",

2361

"\\u54e5", "\\u5609", "\\u5ac1", "\\u5bb6", "\\u6687", "\\u67b6", "\\u67b7", "\\u67ef",

2362

"\\u6b4c", "\\u73c2", "\\u75c2", "\\u7a3c", "\\u82db", "\\u8304", "\\u8857", "\\u8888",

2363

"\\u8a36", "\\u8cc8", "\\u8dcf", "\\u8efb", "\\u8fe6", "\\u99d5",

2364

"\\u4EEE", "\\u50A2", "\\u5496", "\\u54FF", "\\u5777", "\\u5B8A", "\\u659D", "\\u698E",

2365

"\\u6A9F", "\\u73C8", "\\u7B33", "\\u801E", "\\u8238", "\\u846D", "\\u8B0C"

2366

};

2367

2368

const char *rules =

2369

"&\\uac00 <<< \\u4f3d <<< \\u4f73 <<< \\u5047 <<< \\u50f9 <<< \\u52a0 <<< \\u53ef <<< \\u5475 "

2370

"<<< \\u54e5 <<< \\u5609 <<< \\u5ac1 <<< \\u5bb6 <<< \\u6687 <<< \\u67b6 <<< \\u67b7 <<< \\u67ef "

2371

"<<< \\u6b4c <<< \\u73c2 <<< \\u75c2 <<< \\u7a3c <<< \\u82db <<< \\u8304 <<< \\u8857 <<< \\u8888 "

2372

"<<< \\u8a36 <<< \\u8cc8 <<< \\u8dcf <<< \\u8efb <<< \\u8fe6 <<< \\u99d5 "

2373

"<<< \\u4EEE <<< \\u50A2 <<< \\u5496 <<< \\u54FF <<< \\u5777 <<< \\u5B8A <<< \\u659D <<< \\u698E "

2374

"<<< \\u6A9F <<< \\u73C8 <<< \\u7B33 <<< \\u801E <<< \\u8238 <<< \\u846D <<< \\u8B0C";

2375

2376

2377

UErrorCode status = U_ZERO_ERROR;

2378

UChar rlz[2048] = { 0 };

2379

uint32_t rlen = u_unescape(rules, rlz, 2048);

2380

2381

UCollator *coll = ucol_openRules(rlz, rlen, UCOL_DEFAULT, UCOL_DEFAULT,NULL, &status);

2382

2383

log_verbose("Using start of korean rules\n");

2384

2385

if(U_SUCCESS(status)) {

2386

genericOrderingTest(coll, koreanData, sizeof(koreanData)/sizeof(koreanData[0]));

2387

} else {

2388

log_err("Unable to open collator with rules %s\n", rules);

2389

}

2390

2391

log_verbose("Setting jamoSpecial to TRUE and testing once more\n");

2392

((UCATableHeader *)coll->image)->jamoSpecial = TRUE; /* don't try this at home */

2393

genericOrderingTest(coll, koreanData, sizeof(koreanData)/sizeof(koreanData[0]));

2394

2395

ucol_close(coll);

2396

2397

log_verbose("Using ko__LOTUS locale\n");

2398

genericLocaleStarter("ko__LOTUS", koreanData, sizeof(koreanData)/sizeof(koreanData[0]));

2399

}

2400

2401

static void TestCompressOverlap(void) {

2402

UChar secstr[150];

2403

UChar tertstr[150];

2404

UErrorCode status = U_ZERO_ERROR;

2405

UCollator *coll;

2406

char result[200];

2407

uint32_t resultlen;

2408

int count = 0;

2409

char *tempptr;

2410

2411

coll = ucol_open("", &status);

2412

2413

if (U_FAILURE(status)) {

2414

log_err("Collator can't be created\n");

2415

return;

2416

}

2417

while (count < 149) {

2418

secstr[count] = 0x0020; /* [06, 05, 05] */

2419

tertstr[count] = 0x0020;

2420

count ++;

2421

}

2422

2423

/* top down compression ----------------------------------- */

2424

secstr[count] = 0x0332; /* [, 87, 05] */

2425

tertstr[count] = 0x3000; /* [06, 05, 07] */

2426

2427

/* no compression secstr should have 150 secondary bytes, tertstr should

2428

have 150 tertiary bytes.

2429

with correct overlapping compression, secstr should have 4 secondary

2430

bytes, tertstr should have > 2 tertiary bytes */

2431

resultlen = ucol_getSortKey(coll, secstr, 150, (uint8_t *)result, 250);

2432

tempptr = uprv_strchr(result, 1) + 1;

2433

while (*(tempptr + 1) != 1) {

2434

/* the last secondary collation element is not checked since it is not

2435

part of the compression */

2436

if (*tempptr < UCOL_COMMON_TOP2 - UCOL_TOP_COUNT2) {

2437

log_err("Secondary compression overlapped\n");

2438

}

2439

tempptr ++;

2440

}

2441

2442

/* tertiary top/bottom/common for en_US is similar to the secondary

2443

top/bottom/common */

2444

resultlen = ucol_getSortKey(coll, tertstr, 150, (uint8_t *)result, 250);

2445

tempptr = uprv_strrchr(result, 1) + 1;

2446

while (*(tempptr + 1) != 0) {

2447

/* the last secondary collation element is not checked since it is not

2448

part of the compression */

2449

if (*tempptr < coll->tertiaryTop - coll->tertiaryTopCount) {

2450

log_err("Tertiary compression overlapped\n");

2451

}

2452

tempptr ++;

2453

}

2454

2455

/* bottom up compression ------------------------------------- */

2456

secstr[count] = 0;

2457

tertstr[count] = 0;

2458

resultlen = ucol_getSortKey(coll, secstr, 150, (uint8_t *)result, 250);

2459

tempptr = uprv_strchr(result, 1) + 1;

2460

while (*(tempptr + 1) != 1) {

2461

/* the last secondary collation element is not checked since it is not

2462

part of the compression */

2463

if (*tempptr > UCOL_COMMON_BOT2 + UCOL_BOT_COUNT2) {

2464

log_err("Secondary compression overlapped\n");

2465

}

2466

tempptr ++;

2467

}

2468

2469

/* tertiary top/bottom/common for en_US is similar to the secondary

2470

top/bottom/common */

2471

resultlen = ucol_getSortKey(coll, tertstr, 150, (uint8_t *)result, 250);

2472

tempptr = uprv_strrchr(result, 1) + 1;

2473

while (*(tempptr + 1) != 0) {

2474

/* the last secondary collation element is not checked since it is not

2475

part of the compression */

2476

if (*tempptr > coll->tertiaryBottom + coll->tertiaryBottomCount) {

2477

log_err("Tertiary compression overlapped\n");

2478

}

2479

tempptr ++;

2480

}

2481

2482

ucol_close(coll);

2483

}

2484

2485

static void TestCyrillicTailoring(void) {

2486

static const char *test[] = {

2487

"\\u0410b",

2488

"\\u0410\\u0306a",

2489

"\\u04d0A"

2490

};

2491

genericLocaleStarter("ru", test, 3);

2492

genericRulesStarter("&\\u0410 = \\u0410", test, 3);

2493

genericRulesStarter("&Z < \\u0410", test, 3);

2494

genericRulesStarter("&\\u0410 = \\u0410 < \\u04d0", test, 3);

2495

genericRulesStarter("&Z < \\u0410 < \\u04d0", test, 3);

2496

genericRulesStarter("&\\u0410 = \\u0410 < \\u0410\\u0301", test, 3);

2497

genericRulesStarter("&Z < \\u0410 < \\u0410\\u0301", test, 3);

2498

}

2499

2500

static void TestContraction(void) {

2501

const static char *testrules[] = {

2502

"&A = AB / B",

2503

"&A = A\\u0306/\\u0306",

2504

"&c = ch / h"

2505

};

2506

const static UChar testdata[][2] = {

2507

{0x0041 /* 'A' */, 0x0042 /* 'B' */},

2508

{0x0041 /* 'A' */, 0x0306 /* combining breve */},

2509

{0x0063 /* 'c' */, 0x0068 /* 'h' */}

2510

};

2511

const static UChar testdata2[][2] = {

2512

{0x0063 /* 'c' */, 0x0067 /* 'g' */},

2513

{0x0063 /* 'c' */, 0x0068 /* 'h' */},

2514

{0x0063 /* 'c' */, 0x006C /* 'l' */}

2515

};

2516

const static char *testrules3[] = {

2517

"&z < xyz &xyzw << B",

2518

"&z < xyz &xyz << B / w",

2519

"&z < ch &achm << B",

2520

"&z < ch &a << B / chm",

2521

"&\\ud800\\udc00w << B",

2522

"&\\ud800\\udc00 << B / w",

2523

"&a\\ud800\\udc00m << B",

2524

"&a << B / \\ud800\\udc00m",

2525

};

2526

2527

UErrorCode status = U_ZERO_ERROR;

2528

UCollator *coll;

2529

UChar rule[256] = {0};

2530

uint32_t rlen = 0;

2531

int i;

2532

2533

for (i = 0; i < sizeof(testrules) / sizeof(testrules[0]); i ++) {

2534

UCollationElements *iter1;

2535

int j = 0;

2536

log_verbose("Rule %s for testing\n", testrules[i]);

2537

rlen = u_unescape(testrules[i], rule, 32);

2538

coll = ucol_openRules(rule, rlen, UCOL_ON, UCOL_TERTIARY,NULL, &status);

2539

if (U_FAILURE(status)) {

2540

log_err("Collator creation failed %s\n", testrules[i]);

2541

return;

2542

}

2543

iter1 = ucol_openElements(coll, testdata[i], 2, &status);

2544

if (U_FAILURE(status)) {

2545

log_err("Collation iterator creation failed\n");

2546

return;

2547

}

2548

while (j < 2) {

2549

UCollationElements *iter2 = ucol_openElements(coll,

2550

&(testdata[i][j]),

2551

1, &status);

2552

uint32_t ce;

2553

if (U_FAILURE(status)) {

2554

log_err("Collation iterator creation failed\n");

2555

return;

2556

}

2557

ce = ucol_next(iter2, &status);

2558

while (ce != UCOL_NULLORDER) {

2559

if ((uint32_t)ucol_next(iter1, &status) != ce) {

2560

log_err("Collation elements in contraction split does not match\n");

2561

return;

2562

}

2563

ce = ucol_next(iter2, &status);

2564

}

2565

j ++;

2566

ucol_closeElements(iter2);

2567

}

2568

if (ucol_next(iter1, &status) != UCOL_NULLORDER) {

2569

log_err("Collation elements not exhausted\n");

2570

return;

2571

}

2572

ucol_closeElements(iter1);

2573

ucol_close(coll);

2574

}

2575

2576

rlen = u_unescape("& a < b < c < ch < d & c = ch / h", rule, 256);

2577

coll = ucol_openRules(rule, rlen, UCOL_ON, UCOL_TERTIARY,NULL, &status);

2578

if (ucol_strcoll(coll, testdata2[0], 2, testdata2[1], 2) != UCOL_LESS) {

2579

log_err("Expected \\u%04x\\u%04x < \\u%04x\\u%04x\n",

2580

testdata2[0][0], testdata2[0][1], testdata2[1][0],

2581

testdata2[1][1]);

2582

return;

2583

}

2584

if (ucol_strcoll(coll, testdata2[1], 2, testdata2[2], 2) != UCOL_LESS) {

2585

log_err("Expected \\u%04x\\u%04x < \\u%04x\\u%04x\n",

2586

testdata2[1][0], testdata2[1][1], testdata2[2][0],

2587

testdata2[2][1]);

2588

return;

2589

}

2590

ucol_close(coll);

2591

2592

for (i = 0; i < sizeof(testrules3) / sizeof(testrules3[0]); i += 2) {

2593

UCollator *coll1,

2594

*coll2;

2595

UCollationElements *iter1,

2596

*iter2;

2597

UChar ch = 0x0042 /* 'B' */;

2598

uint32_t ce;

2599

rlen = u_unescape(testrules3[i], rule, 32);

2600

coll1 = ucol_openRules(rule, rlen, UCOL_ON, UCOL_TERTIARY,NULL, &status);

2601

rlen = u_unescape(testrules3[i + 1], rule, 32);

2602

coll2 = ucol_openRules(rule, rlen, UCOL_ON, UCOL_TERTIARY,NULL, &status);

2603

if (U_FAILURE(status)) {

2604

log_err("Collator creation failed %s\n", testrules[i]);

2605

return;

2606

}

2607

iter1 = ucol_openElements(coll1, &ch, 1, &status);

2608

iter2 = ucol_openElements(coll2, &ch, 1, &status);

2609

if (U_FAILURE(status)) {

2610

log_err("Collation iterator creation failed\n");

2611

return;

2612

}

2613

ce = ucol_next(iter1, &status);

2614

if (U_FAILURE(status)) {

2615

log_err("Retrieving ces failed\n");

2616

return;

2617

}

2618

while (ce != UCOL_NULLORDER) {

2619

if (ce != (uint32_t)ucol_next(iter2, &status)) {

2620

log_err("CEs does not match\n");

2621

return;

2622

}

2623

ce = ucol_next(iter1, &status);

2624

if (U_FAILURE(status)) {

2625

log_err("Retrieving ces failed\n");

2626

return;

2627

}

2628

}

2629

if (ucol_next(iter2, &status) != UCOL_NULLORDER) {

2630

log_err("CEs not exhausted\n");

2631

return;

2632

}

2633

ucol_closeElements(iter1);

2634

ucol_closeElements(iter2);

2635

ucol_close(coll1);

2636

ucol_close(coll2);

2637

}

2638

}

2639

2640

static void TestExpansion(void) {

2641

const static char *testrules[] = {

2642

"&J << K / B & K << M",

2643

"&J << K / B << M"

2644

};

2645

const static UChar testdata[][3] = {

2646

{0x004A /*'J'*/, 0x0041 /*'A'*/, 0},

2647

{0x004D /*'M'*/, 0x0041 /*'A'*/, 0},

2648

{0x004B /*'K'*/, 0x0041 /*'A'*/, 0},

2649

{0x004B /*'K'*/, 0x0043 /*'C'*/, 0},

2650

{0x004A /*'J'*/, 0x0043 /*'C'*/, 0},

2651

{0x004D /*'M'*/, 0x0043 /*'C'*/, 0}

2652

};

2653

2654

UErrorCode status = U_ZERO_ERROR;

2655

UCollator *coll;

2656

UChar rule[256] = {0};

2657

uint32_t rlen = 0;

2658

int i;

2659

2660

for (i = 0; i < sizeof(testrules) / sizeof(testrules[0]); i ++) {

2661

int j = 0;

2662

log_verbose("Rule %s for testing\n", testrules[i]);

2663

rlen = u_unescape(testrules[i], rule, 32);

2664

coll = ucol_openRules(rule, rlen, UCOL_ON, UCOL_TERTIARY,NULL, &status);

2665

if (U_FAILURE(status)) {

2666

log_err("Collator creation failed %s\n", testrules[i]);

2667

return;

2668

}

2669

2670

for (j = 0; j < 5; j ++) {

2671

doTest(coll, testdata[j], testdata[j + 1], UCOL_LESS);

2672

}

2673

ucol_close(coll);

2674

}

2675

}

2676

2677

#if 0

2678

/* this test tests the current limitations of the engine */

2679

/* it always fail, so it is disabled by default */

2680

static void TestLimitations(void) {

2681

/* recursive expansions */

2682

{

2683

static const char *rule = "&a=b/c&d=c/e";

2684

static const char *tlimit01[] = {"add","b","adf"};

2685

static const char *tlimit02[] = {"aa","b","af"};

2686

log_verbose("recursive expansions\n");

2687

genericRulesStarter(rule, tlimit01, sizeof(tlimit01)/sizeof(tlimit01[0]));

2688

genericRulesStarter(rule, tlimit02, sizeof(tlimit02)/sizeof(tlimit02[0]));

2689

}

2690

/* contractions spanning expansions */

2691

{

2692

static const char *rule = "&a<<<c/e&g<<<eh";

2693

static const char *tlimit01[] = {"ad","c","af","f","ch","h"};

2694

static const char *tlimit02[] = {"ad","c","ch","af","f","h"};

2695

log_verbose("contractions spanning expansions\n");

2696

genericRulesStarter(rule, tlimit01, sizeof(tlimit01)/sizeof(tlimit01[0]));

2697

genericRulesStarter(rule, tlimit02, sizeof(tlimit02)/sizeof(tlimit02[0]));

2698

}

2699

/* normalization: nulls in contractions */

2700

{

2701

static const char *rule = "&a<<<\\u0000\\u0302";

2702

static const char *tlimit01[] = {"a","\\u0000\\u0302\\u0327"};

2703

static const char *tlimit02[] = {"\\u0000\\u0302\\u0327","a"};

2704

static const UColAttribute att[] = { UCOL_DECOMPOSITION_MODE };

2705

static const UColAttributeValue valOn[] = { UCOL_ON };

2706

static const UColAttributeValue valOff[] = { UCOL_OFF };

2707

2708

log_verbose("NULL in contractions\n");

2709

genericRulesStarterWithOptions(rule, tlimit01, 2, att, valOn, 1);

2710

genericRulesStarterWithOptions(rule, tlimit02, 2, att, valOn, 1);

2711

genericRulesStarterWithOptions(rule, tlimit01, 2, att, valOff, 1);

2712

genericRulesStarterWithOptions(rule, tlimit02, 2, att, valOff, 1);

2713

2714

}

2715

/* normalization: contractions spanning normalization */

2716

{

2717

static const char *rule = "&a<<<\\u0000\\u0302";

2718

static const char *tlimit01[] = {"a","\\u0000\\u0302\\u0327"};

2719

static const char *tlimit02[] = {"\\u0000\\u0302\\u0327","a"};

2720

static const UColAttribute att[] = { UCOL_DECOMPOSITION_MODE };

2721

static const UColAttributeValue valOn[] = { UCOL_ON };

2722

static const UColAttributeValue valOff[] = { UCOL_OFF };

2723

2724

log_verbose("contractions spanning normalization\n");

2725

genericRulesStarterWithOptions(rule, tlimit01, 2, att, valOn, 1);

2726

genericRulesStarterWithOptions(rule, tlimit02, 2, att, valOn, 1);

2727

genericRulesStarterWithOptions(rule, tlimit01, 2, att, valOff, 1);

2728

genericRulesStarterWithOptions(rule, tlimit02, 2, att, valOff, 1);

2729

2730

}

2731

/* variable top: */

2732

{

2733

/*static const char *rule2 = "&\\u2010<x=[variable top]<z";*/

2734

static const char *rule = "&\\u2010<x<[variable top]=z";

2735

/*static const char *rule3 = "&' '<x<[variable top]=z";*/

2736

static const char *tlimit01[] = {" ", "z", "zb", "a", " b", "xb", "b", "c" };

2737

static const char *tlimit02[] = {"-", "-x", "x","xb", "-z", "z", "zb", "-a", "a", "-b", "b", "c"};

2738

static const char *tlimit03[] = {" ", "xb", "z", "zb", "a", " b", "b", "c" };

2739

static const UColAttribute att[] = { UCOL_ALTERNATE_HANDLING, UCOL_STRENGTH };

2740

static const UColAttributeValue valOn[] = { UCOL_SHIFTED, UCOL_QUATERNARY };

2741

static const UColAttributeValue valOff[] = { UCOL_NON_IGNORABLE, UCOL_TERTIARY };

2742

2743

log_verbose("variable top\n");

2744

genericRulesStarterWithOptions(rule, tlimit03, sizeof(tlimit03)/sizeof(tlimit03[0]), att, valOn, sizeof(att)/sizeof(att[0]));

2745

genericRulesStarterWithOptions(rule, tlimit01, sizeof(tlimit01)/sizeof(tlimit01[0]), att, valOn, sizeof(att)/sizeof(att[0]));

2746

genericRulesStarterWithOptions(rule, tlimit02, sizeof(tlimit02)/sizeof(tlimit02[0]), att, valOn, sizeof(att)/sizeof(att[0]));

2747

genericRulesStarterWithOptions(rule, tlimit01, sizeof(tlimit01)/sizeof(tlimit01[0]), att, valOff, sizeof(att)/sizeof(att[0]));

2748

genericRulesStarterWithOptions(rule, tlimit02, sizeof(tlimit02)/sizeof(tlimit02[0]), att, valOff, sizeof(att)/sizeof(att[0]));

2749

2750

}

2751

/* case level */

2752

{

2753

static const char *rule = "&c<ch<<<cH<<<Ch<<<CH";

2754

static const char *tlimit01[] = {"c","CH","Ch","cH","ch"};

2755

static const char *tlimit02[] = {"c","CH","cH","Ch","ch"};

2756

static const UColAttribute att[] = { UCOL_CASE_FIRST};

2757

static const UColAttributeValue valOn[] = { UCOL_UPPER_FIRST};

2758

/*static const UColAttributeValue valOff[] = { UCOL_OFF};*/

2759

log_verbose("case level\n");

2760

genericRulesStarterWithOptions(rule, tlimit01, sizeof(tlimit01)/sizeof(tlimit01[0]), att, valOn, sizeof(att)/sizeof(att[0]));

2761

genericRulesStarterWithOptions(rule, tlimit02, sizeof(tlimit02)/sizeof(tlimit02[0]), att, valOn, sizeof(att)/sizeof(att[0]));

2762

/*genericRulesStarterWithOptions(rule, tlimit01, sizeof(tlimit01)/sizeof(tlimit01[0]), att, valOff, sizeof(att)/sizeof(att[0]));*/

2763

/*genericRulesStarterWithOptions(rule, tlimit02, sizeof(tlimit02)/sizeof(tlimit02[0]), att, valOff, sizeof(att)/sizeof(att[0]));*/

2764

}

2765

2766

}

2767

#endif

2768

2769

static void TestBocsuCoverage(void) {

2770

UErrorCode status = U_ZERO_ERROR;

2771

const char *testString = "\\u0041\\u0441\\u4441\\U00044441\\u4441\\u0441\\u0041";

2772

UChar test[256] = {0};

2773

uint32_t tlen = u_unescape(testString, test, 32);

2774

uint8_t key[256] = {0};

2775

uint32_t klen = 0;

2776

2777

UCollator *coll = ucol_open("", &status);

2778

ucol_setAttribute(coll, UCOL_STRENGTH, UCOL_IDENTICAL, &status);

2779

2780

klen = ucol_getSortKey(coll, test, tlen, key, 256);

2781

2782

ucol_close(coll);

2783

}

2784

2785

static void TestVariableTopSetting(void) {

2786

UErrorCode status = U_ZERO_ERROR;

2787

const UChar *current = NULL;

2788

uint32_t varTopOriginal = 0, varTop1, varTop2;

2789

UCollator *coll = ucol_open("", &status);

2790

2791

uint32_t strength = 0;

2792

uint16_t specs = 0;

2793

uint32_t chOffset = 0;

2794

uint32_t chLen = 0;

2795

uint32_t exOffset = 0;

2796

uint32_t exLen = 0;

2797

uint32_t oldChOffset = 0;

2798

uint32_t oldChLen = 0;

2799

uint32_t oldExOffset = 0;

2800

uint32_t oldExLen = 0;

2801

uint32_t prefixOffset = 0;

2802

uint32_t prefixLen = 0;

2803

2804

UBool startOfRules = TRUE;

2805

UColTokenParser src;

2806

UColOptionSet opts;

2807

2808

UChar *rulesCopy = NULL;

2809

uint32_t rulesLen;

2810

2811

UCollationResult result;

2812

2813

UChar first[256] = { 0 };

2814

UChar second[256] = { 0 };

2815

UParseError parseError;

2816

src.opts = &opts;

2817

2818

log_verbose("Slide variable top over UCARules\n");

2819

rulesLen = ucol_getRulesEx(coll, UCOL_FULL_RULES, rulesCopy, 0);

2820

rulesCopy = (UChar *)uprv_malloc((rulesLen+UCOL_TOK_EXTRA_RULE_SPACE_SIZE)*sizeof(UChar));

2821

rulesLen = ucol_getRulesEx(coll, UCOL_FULL_RULES, rulesCopy, rulesLen+UCOL_TOK_EXTRA_RULE_SPACE_SIZE);

2822

2823

if(U_SUCCESS(status) && rulesLen > 0) {

2824

ucol_setAttribute(coll, UCOL_ALTERNATE_HANDLING, UCOL_SHIFTED, &status);

2825

src.source = src.current = rulesCopy;

2826

src.end = rulesCopy+rulesLen;

2827

src.extraCurrent = src.end;

2828

src.extraEnd = src.end+UCOL_TOK_EXTRA_RULE_SPACE_SIZE;

2829

2830

while ((current = ucol_tok_parseNextToken(&src, startOfRules, &parseError,&status)) != NULL) {

2831

strength = src.parsedToken.strength;

2832

chOffset = src.parsedToken.charsOffset;

2833

chLen = src.parsedToken.charsLen;

2834

exOffset = src.parsedToken.extensionOffset;

2835

exLen = src.parsedToken.extensionLen;

2836

prefixOffset = src.parsedToken.prefixOffset;

2837

prefixLen = src.parsedToken.prefixLen;

2838

specs = src.parsedToken.flags;

2839

2840

startOfRules = FALSE;

2841

if(0) {

2842

log_verbose("%04X %d ", *(rulesCopy+chOffset), chLen);

2843

}

2844

if(strength == UCOL_PRIMARY) {

2845

status = U_ZERO_ERROR;

2846

varTopOriginal = ucol_getVariableTop(coll, &status);

2847

varTop1 = ucol_setVariableTop(coll, rulesCopy+oldChOffset, oldChLen, &status);

2848

if(U_FAILURE(status)) {

2849

if(status == U_PRIMARY_TOO_LONG_ERROR) {

2850

log_verbose("= Expected failure for %04X =", *(rulesCopy+oldChOffset));

2851

} else {

2852

log_err("Unexpected failure setting variable top for %04X at offset %d. Error %s\n", *(rulesCopy+oldChOffset), oldChOffset, u_errorName(status));

2853

}

2854

continue;

2855

}

2856

varTop2 = ucol_getVariableTop(coll, &status);

2857

if((varTop1 & 0xFFFF0000) != (varTop2 & 0xFFFF0000)) {

2858

log_err("cannot retrieve set varTop value!\n");

2859

continue;

2860

}

2861

2862

if((varTop1 & 0xFFFF0000) > 0 && oldExLen == 0) {

2863

2864

u_strncpy(first, rulesCopy+oldChOffset, oldChLen);

2865

u_strncpy(first+oldChLen, rulesCopy+chOffset, chLen);

2866

u_strncpy(first+oldChLen+chLen, rulesCopy+oldChOffset, oldChLen);

2867

first[2*oldChLen+chLen] = 0;

2868

2869

if(oldExLen == 0) {

2870

u_strncpy(second, rulesCopy+chOffset, chLen);

2871

second[chLen] = 0;

2872

} else { /* This is skipped momentarily, but should work once UCARules are fully UCA conformant */

2873

u_strncpy(second, rulesCopy+oldExOffset, oldExLen);

2874

u_strncpy(second+oldChLen, rulesCopy+chOffset, chLen);

2875

u_strncpy(second+oldChLen+chLen, rulesCopy+oldExOffset, oldExLen);

2876

second[2*oldExLen+chLen] = 0;

2877

}

2878

result = ucol_strcoll(coll, first, -1, second, -1);

2879

if(result == UCOL_EQUAL) {

2880

doTest(coll, first, second, UCOL_EQUAL);

2881

} else {

2882

log_verbose("Suspicious strcoll result for %04X and %04X\n", *(rulesCopy+oldChOffset), *(rulesCopy+chOffset));

2883

}

2884

}

2885

}

2886

if(strength != UCOL_TOK_RESET) {

2887

oldChOffset = chOffset;

2888

oldChLen = chLen;

2889

oldExOffset = exOffset;

2890

oldExLen = exLen;

2891

}

2892

}

2893

}

2894

else {

2895

log_err("Unexpected failure getting rules %s\n", u_errorName(status));

2896

return;

2897

}

2898

if (U_FAILURE(status)) {

2899

log_err("Error parsing rules %s\n", u_errorName(status));

2900

return;

2901

}

2902

status = U_ZERO_ERROR;

2903

2904

log_verbose("Testing setting variable top to contractions\n");

2905

{

2906

/* uint32_t tailoredCE = UCOL_NOT_FOUND; */

2907

UChar *conts = (UChar *)((uint8_t *)coll->image + coll->image->contractionUCACombos);

2908

while(*conts != 0) {

2909

if(*(conts+2) == 0) {

2910

varTop1 = ucol_setVariableTop(coll, conts, -1, &status);

2911

} else {

2912

varTop1 = ucol_setVariableTop(coll, conts, 3, &status);

2913

}

2914

if(U_FAILURE(status)) {

2915

log_err("Couldn't set variable top to a contraction\n");

2916

}

2917

conts+=3;

2918

}

2919

2920

status = U_ZERO_ERROR;

2921

2922

first[0] = 0x0040;

2923

first[1] = 0x0050;

2924

first[2] = 0x0000;

2925

2926

ucol_setVariableTop(coll, first, -1, &status);

2927

2928

if(U_SUCCESS(status)) {

2929

log_err("Invalid contraction succeded in setting variable top!\n");

2930

}

2931

2932

}

2933

2934

log_verbose("Test restoring variable top\n");

2935

2936

status = U_ZERO_ERROR;

2937

ucol_restoreVariableTop(coll, varTopOriginal, &status);

2938

if(varTopOriginal != ucol_getVariableTop(coll, &status)) {

2939

log_err("Couldn't restore old variable top\n");

2940

}

2941

2942

log_verbose("Testing calling with error set\n");

2943

2944

status = U_INTERNAL_PROGRAM_ERROR;

2945

varTop1 = ucol_setVariableTop(coll, first, 1, &status);

2946

varTop2 = ucol_getVariableTop(coll, &status);

2947

ucol_restoreVariableTop(coll, varTop2, &status);

2948

varTop1 = ucol_setVariableTop(NULL, first, 1, &status);

2949

varTop2 = ucol_getVariableTop(NULL, &status);

2950

ucol_restoreVariableTop(NULL, varTop2, &status);

2951

if(status != U_INTERNAL_PROGRAM_ERROR) {

2952

log_err("Bad reaction to passed error!\n");

2953

}

2954

uprv_free(rulesCopy);

2955

ucol_close(coll);

2956

}

2957

2958

static void TestNonChars(void) {

2959

static const char *test[] = {

2960

"\\u0000",

2961

"\\uFFFE", "\\uFFFF",

2962

"\\U0001FFFE", "\\U0001FFFF",

2963

"\\U0002FFFE", "\\U0002FFFF",

2964

"\\U0003FFFE", "\\U0003FFFF",

2965

"\\U0004FFFE", "\\U0004FFFF",

2966

"\\U0005FFFE", "\\U0005FFFF",

2967

"\\U0006FFFE", "\\U0006FFFF",

2968

"\\U0007FFFE", "\\U0007FFFF",

2969

"\\U0008FFFE", "\\U0008FFFF",

2970

"\\U0009FFFE", "\\U0009FFFF",

2971

"\\U000AFFFE", "\\U000AFFFF",

2972

"\\U000BFFFE", "\\U000BFFFF",

2973

"\\U000CFFFE", "\\U000CFFFF",

2974

"\\U000DFFFE", "\\U000DFFFF",

2975

"\\U000EFFFE", "\\U000EFFFF",

2976

"\\U000FFFFE", "\\U000FFFFF",

2977

"\\U0010FFFE", "\\U0010FFFF"

2978

};

2979

UErrorCode status = U_ZERO_ERROR;

2980

UCollator *coll = ucol_open("en_US", &status);

2981

2982

log_verbose("Test non characters\n");

2983

2984

if(U_SUCCESS(status)) {

2985

genericOrderingTestWithResult(coll, test, 35, UCOL_EQUAL);

2986

} else {

2987

log_err("Unable to open collator\n");

2988

}

2989

2990

ucol_close(coll);

2991

}

2992

2993

static void TestExtremeCompression(void) {

2994

static char *test[4];

2995

int32_t i = 0;

2996

2997

for(i = 0; i<4; i++) {

2998

test[i] = (char *)uprv_malloc(2048*sizeof(char));

2999

uprv_memset(test[i], 'a', 2046*sizeof(char));

3000

test[i][2046] = (char)('a'+i);

3001

test[i][2047] = 0;

3002

}

3003

3004

genericLocaleStarter("en_US", (const char **)test, 4);

3005

3006

for(i = 0; i<4; i++) {

3007

uprv_free(test[i]);

3008

}

3009

}

3010

3011

static void TestSurrogates(void) {

3012

static const char *test[] = {

3013

"z","\\ud900\\udc25", "\\ud805\\udc50",

3014

"\\ud800\\udc00y", "\\ud800\\udc00r",

3015

"\\ud800\\udc00f", "\\ud800\\udc00",

3016

"\\ud800\\udc00c", "\\ud800\\udc00b",

3017

"\\ud800\\udc00fa", "\\ud800\\udc00fb",

3018

"\\ud800\\udc00a",

3019

"c", "b"

3020

};

3021

3022

static const char *rule =

3023

"&z < \\ud900\\udc25 < \\ud805\\udc50"

3024

"< \\ud800\\udc00y < \\ud800\\udc00r"

3025

"< \\ud800\\udc00f << \\ud800\\udc00"

3026

"< \\ud800\\udc00fa << \\ud800\\udc00fb"

3027

"< \\ud800\\udc00a < c < b" ;

3028

3029

genericRulesStarter(rule, test, 14);

3030

}

3031

3032

/* This is a test for prefix implementation, used by JIS X 4061 collation rules */

3033

static void TestPrefix(void) {

3034

uint32_t i;

3035

3036

static struct {

3037

const char *rules;

3038

const char *data[50];

3039

const uint32_t len;

3040

} tests[] = {

3041

{ "&z <<< z|a",

3042

{"zz", "za"}, 2 },

3043

{ "[strength I]"

3044

"&a=\\ud900\\udc25"

3045

"&z<<<\\ud900\\udc25|a",

3046

{"aa", "az", "\\ud900\\udc25z", "\\ud900\\udc25a", "zz"}, 4 },

3047

};

3048

3049

3050

for(i = 0; i<(sizeof(tests)/sizeof(tests[0])); i++) {

3051

genericRulesStarter(tests[i].rules, tests[i].data, tests[i].len);

3052

}

3053

}

3054

3055

/* This test uses data suplied by Masashiko Maedera to test the implementation */

3056

/* JIS X 4061 collation order implementation */

3057

static void TestNewJapanese(void) {

3058

3059

static const char *test1[] = {

3060

"\\u30b7\\u30e3\\u30fc\\u30ec",

3061

"\\u30b7\\u30e3\\u30a4",

3062

"\\u30b7\\u30e4\\u30a3",

3063

"\\u30b7\\u30e3\\u30ec",

3064

"\\u3061\\u3087\\u3053",

3065

"\\u3061\\u3088\\u3053",

3066

"\\u30c1\\u30e7\\u30b3\\u30ec\\u30fc\\u30c8",

3067

"\\u3066\\u30fc\\u305f",

3068

"\\u30c6\\u30fc\\u30bf",

3069

"\\u30c6\\u30a7\\u30bf",

3070

"\\u3066\\u3048\\u305f",

3071

"\\u3067\\u30fc\\u305f",

3072

"\\u30c7\\u30fc\\u30bf",

3073

"\\u30c7\\u30a7\\u30bf",

3074

"\\u3067\\u3048\\u305f",

3075

"\\u3066\\u30fc\\u305f\\u30fc",

3076

"\\u30c6\\u30fc\\u30bf\\u30a1",

3077

"\\u30c6\\u30a7\\u30bf\\u30fc",

3078

"\\u3066\\u3047\\u305f\\u3041",

3079

"\\u3066\\u3048\\u305f\\u30fc",

3080

"\\u3067\\u30fc\\u305f\\u30fc",

3081

"\\u30c7\\u30fc\\u30bf\\u30a1",

3082

"\\u3067\\u30a7\\u305f\\u30a1",

3083

"\\u30c7\\u3047\\u30bf\\u3041",

3084

"\\u30c7\\u30a8\\u30bf\\u30a2",

3085

"\\u3072\\u3086",

3086

"\\u3073\\u3085\\u3042",

3087

"\\u3074\\u3085\\u3042",

3088

"\\u3073\\u3085\\u3042\\u30fc",

3089

"\\u30d3\\u30e5\\u30a2\\u30fc",

3090

"\\u3074\\u3085\\u3042\\u30fc",

3091

"\\u30d4\\u30e5\\u30a2\\u30fc",

3092

"\\u30d2\\u30e5\\u30a6",

3093

"\\u30d2\\u30e6\\u30a6",

3094

"\\u30d4\\u30e5\\u30a6\\u30a2",

3095

"\\u3073\\u3085\\u30fc\\u3042\\u30fc",

3096

"\\u30d3\\u30e5\\u30fc\\u30a2\\u30fc",

3097

"\\u30d3\\u30e5\\u30a6\\u30a2\\u30fc",

3098

"\\u3072\\u3085\\u3093",

3099

"\\u3074\\u3085\\u3093",

3100

"\\u3075\\u30fc\\u308a",

3101

"\\u30d5\\u30fc\\u30ea",

3102

"\\u3075\\u3045\\u308a",

3103

"\\u3075\\u30a5\\u308a",

3104

"\\u3075\\u30a5\\u30ea",

3105

"\\u30d5\\u30a6\\u30ea",

3106

"\\u3076\\u30fc\\u308a",

3107

"\\u30d6\\u30fc\\u30ea",

3108

"\\u3076\\u3045\\u308a",

3109

"\\u30d6\\u30a5\\u308a",

3110

"\\u3077\\u3046\\u308a",

3111

"\\u30d7\\u30a6\\u30ea",

3112

"\\u3075\\u30fc\\u308a\\u30fc",

3113

"\\u30d5\\u30a5\\u30ea\\u30fc",

3114

"\\u3075\\u30a5\\u308a\\u30a3",

3115

"\\u30d5\\u3045\\u308a\\u3043",

3116

"\\u30d5\\u30a6\\u30ea\\u30fc",

3117

"\\u3075\\u3046\\u308a\\u3043",

3118

"\\u30d6\\u30a6\\u30ea\\u30a4",

3119

"\\u3077\\u30fc\\u308a\\u30fc",

3120

"\\u3077\\u30a5\\u308a\\u30a4",

3121

"\\u3077\\u3046\\u308a\\u30fc",

3122

"\\u30d7\\u30a6\\u30ea\\u30a4",

3123

"\\u30d5\\u30fd",

3124

"\\u3075\\u309e",

3125

"\\u3076\\u309d",

3126

"\\u3076\\u3075",

3127

"\\u3076\\u30d5",

3128

"\\u30d6\\u3075",

3129

"\\u30d6\\u30d5",

3130

"\\u3076\\u309e",

3131

"\\u3076\\u3077",

3132

"\\u30d6\\u3077",

3133

"\\u3077\\u309d",

3134

"\\u30d7\\u30fd",

3135

"\\u3077\\u3075",

3136

};

3137

3138

static const char *test2[] = {

3139

"\\u306f\\u309d", /* H\\u309d */

3140

"\\u30cf\\u30fd", /* K\\u30fd */

3141

"\\u306f\\u306f", /* HH */

3142

"\\u306f\\u30cf", /* HK */

3143

"\\u30cf\\u30cf", /* KK */

3144

"\\u306f\\u309e", /* H\\u309e */

3145

"\\u30cf\\u30fe", /* K\\u30fe */

3146

"\\u306f\\u3070", /* HH\\u309b */

3147

"\\u30cf\\u30d0", /* KK\\u309b */

3148

"\\u306f\\u3071", /* HH\\u309c */

3149

"\\u30cf\\u3071", /* KH\\u309c */

3150

"\\u30cf\\u30d1", /* KK\\u309c */

3151

"\\u3070\\u309d", /* H\\u309b\\u309d */

3152

"\\u30d0\\u30fd", /* K\\u309b\\u30fd */

3153

"\\u3070\\u306f", /* H\\u309bH */

3154

"\\u30d0\\u30cf", /* K\\u309bK */

3155

"\\u3070\\u309e", /* H\\u309b\\u309e */

3156

"\\u30d0\\u30fe", /* K\\u309b\\u30fe */

3157

"\\u3070\\u3070", /* H\\u309bH\\u309b */

3158

"\\u30d0\\u3070", /* K\\u309bH\\u309b */

3159

"\\u30d0\\u30d0", /* K\\u309bK\\u309b */

3160

"\\u3070\\u3071", /* H\\u309bH\\u309c */

3161

"\\u30d0\\u30d1", /* K\\u309bK\\u309c */

3162

"\\u3071\\u309d", /* H\\u309c\\u309d */

3163

"\\u30d1\\u30fd", /* K\\u309c\\u30fd */

3164

"\\u3071\\u306f", /* H\\u309cH */

3165

"\\u30d1\\u30cf", /* K\\u309cK */

3166

"\\u3071\\u3070", /* H\\u309cH\\u309b */

3167

"\\u3071\\u30d0", /* H\\u309cK\\u309b */

3168

"\\u30d1\\u30d0", /* K\\u309cK\\u309b */

3169

"\\u3071\\u3071", /* H\\u309cH\\u309c */

3170

"\\u30d1\\u30d1", /* K\\u309cK\\u309c */

3171

};

3172

3173

static const char *test3[] = {

3174

"\\u221er\\u221e",

3175

"\\u221eR#",

3176

"\\u221et\\u221e",

3177

"#r\\u221e",

3178

"#R#",

3179

"#t%",

3180

"#T%",

3181

"8t\\u221e",

3182

"8T\\u221e",

3183

"8t#",

3184

"8T#",

3185

"8t%",

3186

"8T%",

3187

"8t8",

3188

"8T8",

3189

"\\u03c9r\\u221e",

3190

"\\u03a9R%",

3191

"rr\\u221e",

3192

"rR\\u221e",

3193

"Rr\\u221e",

3194

"RR\\u221e",

3195

"RT%",

3196

"rt8",

3197

"tr\\u221e",

3198

"tr8",

3199

"TR8",

3200

"tt8",

3201

"\\u30b7\\u30e3\\u30fc\\u30ec",

3202

};

3203

3204

static const UColAttribute att[] = { UCOL_ALTERNATE_HANDLING};

3205

static const UColAttributeValue valShifted[] = { UCOL_SHIFTED };

3206

3207

genericLocaleStarter("ja", test1, sizeof(test1)/sizeof(test1[0]));

3208

genericLocaleStarter("ja", test2, sizeof(test2)/sizeof(test2[0]));

3209

/*genericLocaleStarter("ja", test3, sizeof(test3)/sizeof(test3[0]));*/

3210

genericLocaleStarterWithOptions("ja", test1, sizeof(test1)/sizeof(test1[0]), att, valShifted, 1);

3211

genericLocaleStarterWithOptions("ja", test2, sizeof(test2)/sizeof(test2[0]), att, valShifted, 1);

3212

}

3213

3214

static void TestStrCollIdenticalPrefix(void) {

3215

const char* rule = "&\\ud9b0\\udc70=\\ud9b0\\udc71";

3216

const char* test[] = {

3217

"ab\\ud9b0\\udc70",

3218

"ab\\ud9b0\\udc71"

3219

};

3220

genericRulesTestWithResult(rule, test, sizeof(test)/sizeof(test[0]), UCOL_EQUAL);

3221

}

3222

/* Contractions should have all their canonically equivalent */

3223

/* strings included */

3224

static void TestContractionClosure(void) {

3225

static struct {

3226

const char *rules;

3227

const char *data[50];

3228

const uint32_t len;

3229

} tests[] = {

3230

{ "&b=\\u00e4\\u00e4",

3231

{ "b", "\\u00e4\\u00e4", "a\\u0308a\\u0308", "\\u00e4a\\u0308", "a\\u0308\\u00e4" }, 5},

3232

{ "&b=\\u00C5",

3233

{ "b", "\\u00C5", "A\\u030A", "\\u212B" }, 4},

3234

};

3235

uint32_t i;

3236

3237

3238

for(i = 0; i<(sizeof(tests)/sizeof(tests[0])); i++) {

3239

genericRulesTestWithResult(tests[i].rules, tests[i].data, tests[i].len, UCOL_EQUAL);

3240

}

3241

}

3242

3243

/* This tests also fails*/

3244

static void TestBeforePrefixFailure(void) {

3245

static struct {

3246

const char *rules;

3247

const char *data[50];

3248

const uint32_t len;

3249

} tests[] = {

3250

{ "&g <<< a"

3251

"&[before 3]\\uff41 <<< x",

3252

{"x", "\\uff41"}, 2 },

3253

{ "&\\u30A7=\\u30A7=\\u3047=\\uff6a"

3254

"&\\u30A8=\\u30A8=\\u3048=\\uff74"

3255

"&[before 3]\\u30a7<<<\\u30a9",

3256

{"\\u30a9", "\\u30a7"}, 2 },

3257

{ "&[before 3]\\u30a7<<<\\u30a9"

3258

"&\\u30A7=\\u30A7=\\u3047=\\uff6a"

3259

"&\\u30A8=\\u30A8=\\u3048=\\uff74",

3260

{"\\u30a9", "\\u30a7"}, 2 },

3261

};

3262

uint32_t i;

3263

3264

3265

for(i = 0; i<(sizeof(tests)/sizeof(tests[0])); i++) {

3266

genericRulesStarter(tests[i].rules, tests[i].data, tests[i].len);

3267

}

3268

3269

#if 0

3270

const char* rule1 =

3271

"&\\u30A7=\\u30A7=\\u3047=\\uff6a"

3272

"&\\u30A8=\\u30A8=\\u3048=\\uff74"

3273

"&[before 3]\\u30a7<<<\\u30c6|\\u30fc";

3274

const char* rule2 =

3275

"&[before 3]\\u30a7<<<\\u30c6|\\u30fc"

3276

"&\\u30A7=\\u30A7=\\u3047=\\uff6a"

3277

"&\\u30A8=\\u30A8=\\u3048=\\uff74";

3278

const char* test[] = {

3279

"\\u30c6\\u30fc\\u30bf",

3280

"\\u30c6\\u30a7\\u30bf",

3281

};

3282

genericRulesStarter(rule1, test, sizeof(test)/sizeof(test[0]));

3283

genericRulesStarter(rule2, test, sizeof(test)/sizeof(test[0]));

3284

/* this piece of code should be in some sort of verbose mode */

3285

/* it gets the collation elements for elements and prints them */

3286

/* This is useful when trying to see whether the problem is */

3287

{

3288

UErrorCode status = U_ZERO_ERROR;

3289

uint32_t i = 0;

3290

UCollationElements *it = NULL;

3291

uint32_t CE;

3292

UChar string[256];

3293

uint32_t uStringLen;

3294

UCollator *coll = NULL;

3295

3296

uStringLen = u_unescape(rule1, string, 256);

3297

3298

coll = ucol_openRules(string, uStringLen, UCOL_DEFAULT, UCOL_DEFAULT, NULL, &status);

3299

3300

/*coll = ucol_open("ja_JP_JIS", &status);*/

3301

it = ucol_openElements(coll, string, 0, &status);

3302

3303

for(i = 0; i < sizeof(test)/sizeof(test[0]); i++) {

3304

log_verbose("%s\n", test[i]);

3305

uStringLen = u_unescape(test[i], string, 256);

3306

ucol_setText(it, string, uStringLen, &status);

3307

3308

while((CE=ucol_next(it, &status)) != UCOL_NULLORDER) {

3309

log_verbose("%08X\n", CE);

3310

}

3311

log_verbose("\n");

3312

3313

}

3314

3315

ucol_closeElements(it);

3316

ucol_close(coll);

3317

}

3318

#endif

3319

}

3320

3321

static void TestPrefixCompose(void) {

3322

const char* rule1 =

3323

"&\\u30a7<<<\\u30ab|\\u30fc=\\u30ac|\\u30fc";

3324

3325

const char* test[] = {

3326

"\\u30c6\\u30fc\\u30bf",

3327

"\\u30c6\\u30a7\\u30bf",

3328

};

3329

3330

{

3331

UErrorCode status = U_ZERO_ERROR;

3332

/*uint32_t i = 0;*/

3333

/*UCollationElements *it = NULL;*/

3334

/* uint32_t CE;*/

3335

UChar string[256];

3336

uint32_t uStringLen;

3337

UCollator *coll = NULL;

3338

3339

uStringLen = u_unescape(rule1, string, 256);

3340

3341

coll = ucol_openRules(string, uStringLen, UCOL_DEFAULT, UCOL_DEFAULT, NULL, &status);

3342

ucol_close(coll);

3343

}

3344

3345

3346

}

3347

3348

static int tMemCmp(const uint8_t *first, const uint8_t *second) {

3349

int32_t firstLen = uprv_strlen((const char *)first);

3350

int32_t secondLen = uprv_strlen((const char *)second);

3351

return uprv_memcmp(first, second, uprv_min(firstLen, secondLen));

3352

}

3353

3354

3355

3356

static void TestMergeSortKeys(void) {

3357

UErrorCode status = U_ZERO_ERROR;

3358

3359

const char* cases[] = {

3360

"abc",

3361

"abcd",

3362

"abcde"

3363

};

3364

uint32_t casesSize = sizeof(cases)/sizeof(cases[0]);

3365

const char* prefix = "foo";

3366

const char* suffix = "egg";

3367

char outBuff1[256], outBuff2[256];

3368

3369

uint8_t **sortkeys = (uint8_t **)uprv_malloc(casesSize*sizeof(uint8_t *));

3370

uint8_t **mergedPrefixkeys = (uint8_t **)uprv_malloc(casesSize*sizeof(uint8_t *));

3371

uint8_t **mergedSuffixkeys = (uint8_t **)uprv_malloc(casesSize*sizeof(uint8_t *));

3372

uint32_t *sortKeysLen = (uint32_t *)uprv_malloc(casesSize*sizeof(uint32_t));

3373

uint8_t prefixKey[256], suffixKey[256];

3374

uint32_t prefixKeyLen = 0, suffixKeyLen = 0, i = 0;

3375

UChar buffer[256];

3376

uint32_t unescapedLen = 0, l1 = 0, l2 = 0;

3377

UColAttributeValue strength;

3378

3379

UCollator *coll = ucol_open("en", &status);

3380

log_verbose("ucol_mergeSortkeys test\n");

3381

log_verbose("Testing order of the test cases\n");

3382

genericLocaleStarter("en", cases, casesSize);

3383

3384

for(i = 0; i<casesSize; i++) {

3385

sortkeys[i] = (uint8_t *)uprv_malloc(256*sizeof(uint8_t));

3386

mergedPrefixkeys[i] = (uint8_t *)uprv_malloc(256*sizeof(uint8_t));

3387

mergedSuffixkeys[i] = (uint8_t *)uprv_malloc(256*sizeof(uint8_t));

3388

}

3389

3390

unescapedLen = u_unescape(prefix, buffer, 256);

3391

prefixKeyLen = ucol_getSortKey(coll, buffer, unescapedLen, prefixKey, 256);

3392

3393

unescapedLen = u_unescape(suffix, buffer, 256);

3394

suffixKeyLen = ucol_getSortKey(coll, buffer, unescapedLen, suffixKey, 256);

3395

3396

log_verbose("Massaging data with prefixes and different strengths\n");

3397

strength = UCOL_PRIMARY;

3398

while(strength <= UCOL_IDENTICAL) {

3399

log_verbose("Strength %s\n", strengthsC[strength<=UCOL_QUATERNARY?strength:4]);

3400

ucol_setAttribute(coll, UCOL_STRENGTH, strength, &status);

3401

for(i = 0; i<casesSize; i++) {

3402

unescapedLen = u_unescape(cases[i], buffer, 256);

3403

sortKeysLen[i] = ucol_getSortKey(coll, buffer, unescapedLen, sortkeys[i], 256);

3404

ucol_mergeSortkeys(prefixKey, prefixKeyLen, sortkeys[i], sortKeysLen[i], mergedPrefixkeys[i], 256);

3405

ucol_mergeSortkeys(sortkeys[i], sortKeysLen[i], suffixKey, suffixKeyLen, mergedSuffixkeys[i], 256);

3406

if(i>0) {

3407

if(tMemCmp(mergedPrefixkeys[i-1], mergedPrefixkeys[i]) >= 0) {

3408

log_err("Error while comparing prefixed keys @ strength %s:\n", strengthsC[strength<=UCOL_QUATERNARY?strength:4]);

3409

log_err("%s\n%s\n",

3410

ucol_sortKeyToString(coll, mergedPrefixkeys[i-1], outBuff1, &l1),

3411

ucol_sortKeyToString(coll, mergedPrefixkeys[i], outBuff2, &l2));

3412

}

3413

if(tMemCmp(mergedSuffixkeys[i-1], mergedSuffixkeys[i]) >= 0) {

3414

log_err("Error while comparing suffixed keys @ strength %s:\n", strengthsC[strength<=UCOL_QUATERNARY?strength:4]);

3415

log_err("%s\n%s\n",

3416

ucol_sortKeyToString(coll, mergedSuffixkeys[i-1], outBuff1, &l1),

3417

ucol_sortKeyToString(coll, mergedSuffixkeys[i], outBuff2, &l2));

3418

}

3419

}

3420

}

3421

if(strength == UCOL_QUATERNARY) {

3422

strength = UCOL_IDENTICAL;

3423

} else {

3424

strength++;

3425

}

3426

}

3427

3428

{

3429

uint8_t smallBuf[3];

3430

uint32_t reqLen = 0;

3431

log_verbose("testing buffer overflow\n");

3432

reqLen = ucol_mergeSortkeys(prefixKey, prefixKeyLen, suffixKey, suffixKeyLen, smallBuf, 3);

3433

if(reqLen != (prefixKeyLen+suffixKeyLen-1)) {

3434

log_err("Wrong preflight size for merged sortkey\n");

3435

}

3436

}

3437

3438

3439

for(i = 0; i<casesSize; i++) {

3440

uprv_free(sortkeys[i]);

3441

uprv_free(mergedPrefixkeys[i]);

3442

uprv_free(mergedSuffixkeys[i]);

3443

}

3444

uprv_free(sortkeys);

3445

uprv_free(mergedPrefixkeys);

3446

uprv_free(mergedSuffixkeys);

3447

uprv_free(sortKeysLen);

3448

ucol_close(coll);

3449

/* need to finish this up */

3450

}

3451

3452

3453

[last variable] last variable value

3454

[last primary ignorable] largest CE for primary ignorable

3455

[last secondary ignorable] largest CE for secondary ignorable

3456

[last tertiary ignorable] largest CE for tertiary ignorable

3457

[top] guaranteed to be above all implicit CEs, for now and in the future (in 1.8)

3458

3459

3460

static void TestRuleOptions(void) {

3461

static struct {

3462

const char *rules;

3463

const char *data[50];

3464

const uint32_t len;

3465

} tests[] = {

3466

{ "&[last variable]<z"

3467

"&[last primary ignorable]<x"

3468

"&[last secondary ignorable]<<y"

3469

"&[last tertiary ignorable]<<<w"

3470

"&[top]<u",

3471

{"\\ufffb", "w", "y", "\\u20e3", "x", "\\u137c", "z", "u"}, 7 },

3472

{ "&[before 1][first tertiary ignorable]<<<k",

3473

{ "\\u0000", "k"}, 2}, /* you cannot go before first tertiary ignorable */

3474

/* - all befores here amount to zero */

3475

{ "&[before 3][last primary ignorable]<<<k",

3476

{ "k", "\\u20e3"}, 2},

3477

};

3478

uint32_t i;

3479

3480

3481

for(i = 0; i<(sizeof(tests)/sizeof(tests[0])); i++) {

3482

genericRulesStarter(tests[i].rules, tests[i].data, tests[i].len);

3483

}

3484

}

3485

3486

void addMiscCollTest(TestNode** root)

3487

{

3488

3489

addTest(root, &TestRuleOptions, "tscoll/cmsccoll/TestRuleOptions");

3490

addTest(root, &TestBeforePrefixFailure, "tscoll/cmsccoll/TestBeforePrefixFailure");

3491

addTest(root, &TestContractionClosure, "tscoll/cmsccoll/TestContractionClosure");

3492

addTest(root, &TestMergeSortKeys, "tscoll/cmsccoll/TestMergeSortKeys");

3493

addTest(root, &TestPrefixCompose, "tscoll/cmsccoll/TestPrefixCompose");

3494

addTest(root, &TestStrCollIdenticalPrefix, "tscoll/cmsccoll/TestStrCollIdenticalPrefix");

3495

addTest(root, &TestPrefix, "tscoll/cmsccoll/TestPrefix");

3496

addTest(root, &TestNewJapanese, "tscoll/cmsccoll/TestNewJapanese");

3497

/*addTest(root, &TestLimitations, "tscoll/cmsccoll/TestLimitations");*/

3498

addTest(root, &TestNonChars, "tscoll/cmsccoll/TestNonChars");

3499

addTest(root, &TestExtremeCompression, "tscoll/cmsccoll/TestExtremeCompression");

3500

addTest(root, &TestSurrogates, "tscoll/cmsccoll/TestSurrogates");

3501

addTest(root, &TestVariableTopSetting, "tscoll/cmsccoll/TestVariableTopSetting");

3502

addTest(root, &TestBocsuCoverage, "tscoll/cmsccoll/TestBocsuCoverage");

3503

addTest(root, &TestCyrillicTailoring, "tscoll/cmsccoll/TestCyrillicTailoring");

3504

addTest(root, &TestCase, "tscoll/cmsccoll/TestCase");

3505

addTest(root, &IncompleteCntTest, "tscoll/cmsccoll/IncompleteCntTest");

3506

addTest(root, &BlackBirdTest, "tscoll/cmsccoll/BlackBirdTest");

3507

addTest(root, &FunkyATest, "tscoll/cmsccoll/FunkyATest");

3508

addTest(root, &BillFairmanTest, "tscoll/cmsccoll/BillFairmanTest");

3509

addTest(root, &RamsRulesTest, "tscoll/cmsccoll/RamsRulesTest");

3510

addTest(root, &IsTailoredTest, "tscoll/cmsccoll/IsTailoredTest");

3511

addTest(root, &TestCollations, "tscoll/cmsccoll/TestCollations");

3512

addTest(root, &TestChMove, "tscoll/cmsccoll/TestChMove");

3513

addTest(root, &TestImplicitTailoring, "tscoll/cmsccoll/TestImplicitTailoring");

3514

addTest(root, &TestFCDProblem, "tscoll/cmsccoll/TestFCDProblem");

3515

addTest(root, &TestEmptyRule, "tscoll/cmsccoll/TestEmptyRule");

3516

addTest(root, &TestJ784, "tscoll/cmsccoll/TestJ784");

3517

addTest(root, &TestJ815, "tscoll/cmsccoll/TestJ815");

3518

addTest(root, &TestJ831, "tscoll/cmsccoll/TestJ831");

3519

addTest(root, &TestBefore, "tscoll/cmsccoll/TestBefore");

3520

addTest(root, &TestRedundantRules, "tscoll/cmsccoll/TestRedundantRules");

3521

addTest(root, &TestExpansionSyntax, "tscoll/cmsccoll/TestExpansionSyntax");

3522

addTest(root, &TestHangulTailoring, "tscoll/cmsccoll/TestHangulTailoring");

3523

addTest(root, &TestUCARules, "tscoll/cmsccoll/TestUCARules");

3524

addTest(root, &TestIncrementalNormalize, "tscoll/cmsccoll/TestIncrementalNormalize");

3525

addTest(root, &TestComposeDecompose, "tscoll/cmsccoll/TestComposeDecompose");

3526

addTest(root, &TestCompressOverlap, "tscoll/cmsccoll/TestCompressOverlap");

3527

addTest(root, &TestContraction, "tscoll/cmsccoll/TestContraction");

3528

addTest(root, &TestExpansion, "tscoll/cmsccoll/TestExpansion");

3529

/*addTest(root, &PrintMarkDavis, "tscoll/cmsccoll/PrintMarkDavis");*/ /* this test doesn't test - just prints sortkeys */

3530

/*addTest(root, &TestGetCaseBit, "tscoll/cmsccoll/TestGetCaseBit");*/ /*this one requires internal things to be exported */

3531

}

Older »