~ubuntu-branches/ubuntu/gutsy/icu/gutsy-updates

fprintf(stderr, "Continuation error at %i: %08X %08X & %08X %08X\n", i, inverseTable[i-1][0], inverseTable[i-1][1], inverseTable[i][0], inverseTable[i][1]);

271

}

272

}

273

274

result->tableSize = inversePos;

275

result->contsSize = sContPos;

276

277

result->table = headerByteSize;

278

result->conts = headerByteSize + inverseTableByteSize;

279

280

memcpy((uint8_t *)result + result->table, inverseTable, inverseTableByteSize);

281

memcpy((uint8_t *)result + result->conts, stringContinue, contsByteSize);

282

283

} else {

284

*status = U_MEMORY_ALLOCATION_ERROR;

285

return NULL;

286

}

287

288

return result;

289

}

290

291

292

static void writeOutInverseData(InverseTableHeader *data,

293

const char *outputDir,

294

const char *copyright,

295

UErrorCode *status)

296

{

297

UNewDataMemory *pData;

298

299

long dataLength;

300

301

pData=udata_create(outputDir, INVC_DATA_TYPE, INVC_DATA_NAME, &invUcaDataInfo,

302

303

304

if(U_FAILURE(*status)) {

305

fprintf(stderr, "Error: unable to create data memory, error %d\n", *status);

306

return;

307

}

308

309

/* write the data to the file */

310

if (VERBOSE) {

311

fprintf(stdout, "Writing out inverse UCA table: %s%c%s.%s\n", outputDir, U_FILE_SEP_CHAR,

312

INVC_DATA_NAME,

313

INVC_DATA_TYPE);

314

}

315

udata_writeBlock(pData, data, data->byteSize);

316

317

/* finish up */

318

dataLength=udata_finish(pData, status);

319

if(U_FAILURE(*status)) {

320

fprintf(stderr, "Error: error %d writing the output file\n", *status);

321

return;

322

}

323

}

324

325

326

327

static int32_t hex2num(char hex) {

328

if(hex>='0' && hex <='9') {

329

return hex-'0';

330

} else if(hex>='a' && hex<='f') {

331

return hex-'a'+10;

332

} else if(hex>='A' && hex<='F') {

333

return hex-'A'+10;

334

} else {

335

return 0;

336

}

337

}

338

339

UCAElements *readAnElement(FILE *data, UErrorCode *status) {

340

char buffer[2048], primary[100], secondary[100], tertiary[100];

341

UBool detectedContraction;

342

int32_t i = 0;

343

unsigned int theValue;

344

char *pointer = NULL;

345

char *commentStart = NULL;

346

char *startCodePoint = NULL;

347

char *endCodePoint = NULL;

348

char *spacePointer = NULL;

349

char *result = fgets(buffer, 2048, data);

350

if(U_FAILURE(*status)) {

351

return 0;

352

}

353

*primary = *secondary = *tertiary = '\0';

354

if(result == NULL) {

355

if(feof(data)) {

356

return NULL;

357

} else {

358

fprintf(stderr, "empty line but no EOF!\n");

359

*status = U_INVALID_FORMAT_ERROR;

360

return NULL;

361

}

362

}

363

if(buffer[0] == '#' || buffer[0] == '\n') {

364

return NULL; // just a comment, skip whole line

365

}

366

367

UCAElements *element = ≤ //(UCAElements *)malloc(sizeof(UCAElements));

368

369

if(buffer[0] == '[') {

370

const char *vt = "[variable top = ";

371

uint32_t vtLen = uprv_strlen(vt);

372

if(uprv_strncmp(buffer, vt, vtLen) == 0) {

373

element->variableTop = TRUE;

374

if(sscanf(buffer+vtLen, "%4x", &theValue) != 1) /* read first code point */

375

{

376

fprintf(stderr, " scanf(hex) failed!\n ");

377

}

378

element->cPoints[0] = (UChar)theValue;

379

return element; // just a comment, skip whole line

380

} else {

381

*status = U_INVALID_FORMAT_ERROR;

382

return NULL;

383

}

384

}

385

element->variableTop = FALSE;

386

387

startCodePoint = buffer;

388

endCodePoint = strchr(startCodePoint, ';');

389

390

if(endCodePoint == 0) {

391

fprintf(stderr, "error - line with no code point!\n");

392

*status = U_INVALID_FORMAT_ERROR; /* No code point - could be an error, but probably only an empty line */

393

return NULL;

394

} else {

395

*(endCodePoint) = 0;

396

}

397

398

if(element != NULL) {

399

memset(element, 0, sizeof(*element));

400

} else {

401

*status = U_MEMORY_ALLOCATION_ERROR;

402

return NULL;

403

}

404

405

element->cPoints = element->uchars;

406

407

spacePointer = strchr(buffer, ' ');

408

if(sscanf(buffer, "%4x", &theValue) != 1) /* read first code point */

409

{

410

fprintf(stderr, " scanf(hex) failed!\n ");

411

}

412

element->cPoints[0] = (UChar)theValue;

413

414

if(spacePointer == 0) {

415

detectedContraction = FALSE;

416

element->cSize = 1;

417

} else {

418

i = 1;

419

detectedContraction = TRUE;

420

while(spacePointer != NULL) {

421

sscanf(spacePointer+1, "%4x", &theValue);

422

element->cPoints[i++] = (UChar)theValue;

423

spacePointer = strchr(spacePointer+1, ' ');

424

}

425

426

element->cSize = i;

427

428

//fprintf(stderr, "Number of codepoints in contraction: %i\n", i);

429

}

430

431

startCodePoint = endCodePoint+1;

432

433

commentStart = strchr(startCodePoint, '#');

434

if(commentStart == NULL) {

435

commentStart = strlen(startCodePoint) + startCodePoint - 1;

436

}

437

438

i = 0;

439

uint32_t CEindex = 0;

440

element->noOfCEs = 0;

441

for(;;) {

442

endCodePoint = strchr(startCodePoint, ']');

443

if(endCodePoint == NULL || endCodePoint >= commentStart) {

444

break;

445

}

446

pointer = strchr(startCodePoint, '[');

447

pointer++;

448

449

element->sizePrim[i]=readElement(&pointer, primary, ',', status);

450

element->sizeSec[i]=readElement(&pointer, secondary, ',', status);

451

element->sizeTer[i]=readElement(&pointer, tertiary, ']', status);

452

453

454

/* I want to get the CEs entered right here, including continuation */

455

element->CEs[CEindex++] = getSingleCEValue(primary, secondary, tertiary, status);

456

457

uint32_t CEi = 1;

458

while(2*CEi<element->sizePrim[i] || CEi<element->sizeSec[i] || CEi<element->sizeTer[i]) {

459

uint32_t value = UCOL_CONTINUATION_MARKER; /* Continuation marker */

460

if(2*CEi<element->sizePrim[i]) {

461

value |= ((hex2num(*(primary+4*CEi))&0xF)<<28);

462

value |= ((hex2num(*(primary+4*CEi+1))&0xF)<<24);

463

}

464

465

if(2*CEi+1<element->sizePrim[i]) {

466

value |= ((hex2num(*(primary+4*CEi+2))&0xF)<<20);

467

value |= ((hex2num(*(primary+4*CEi+3))&0xF)<<16);

468

}

469

470

if(CEi<element->sizeSec[i]) {

471

value |= ((hex2num(*(secondary+2*CEi))&0xF)<<12);

472

value |= ((hex2num(*(secondary+2*CEi+1))&0xF)<<8);

473

}

474

475

if(CEi<element->sizeTer[i]) {

476

value |= ((hex2num(*(tertiary+2*CEi))&0x3)<<4);

477

value |= (hex2num(*(tertiary+2*CEi+1))&0xF);

478

}

479

480

CEi++;

481

482

element->CEs[CEindex++] = value;

483

}

484

485

startCodePoint = endCodePoint+1;

486

i++;

487

}

488

element->noOfCEs = CEindex;

489

490

element->isThai = UCOL_ISTHAIPREVOWEL(element->cPoints[0]);

491

492

// we don't want any strange stuff after useful data!

493

while(pointer < commentStart) {

494

if(*pointer != ' ') {

495

*status=U_INVALID_FORMAT_ERROR;

496

break;

497

}

498

pointer++;

499

}

500

501

if(U_FAILURE(*status)) {

502

fprintf(stderr, "problem putting stuff in hash table\n");

503

*status = U_INTERNAL_PROGRAM_ERROR;

504

return NULL;

505

}

506

507

return element;

508

}

509

510

511

void writeOutData(UCATableHeader *data,

512

UChar contractions[][3],

513

uint32_t noOfcontractions,

514

const char *outputDir,

515

const char *copyright,

516

UErrorCode *status)

517

{

518

if(U_FAILURE(*status)) {

519

return;

520

}

521

522

uint32_t size = data->size;

523

524

if(noOfcontractions != 0) {

525

contractions[noOfcontractions][0] = 0;

526

contractions[noOfcontractions][1] = 0;

527

contractions[noOfcontractions][2] = 0;

528

noOfcontractions++;

529

530

531

data->contractionUCACombos = size;

532

data->size += paddedsize((noOfcontractions*3*sizeof(UChar)));

533

}

534

535

UNewDataMemory *pData;

536

537

long dataLength;

538

539

pData=udata_create(outputDir, UCA_DATA_TYPE, UCA_DATA_NAME, &ucaDataInfo,

540

541

542

if(U_FAILURE(*status)) {

543

fprintf(stderr, "Error: unable to create data memory, error %d\n", *status);

544

return;

545

}

546

547

/* write the data to the file */

548

if (VERBOSE) {

549

fprintf(stdout, "Writing out UCA table: %s%c%s.%s\n", outputDir,

550

U_FILE_SEP_CHAR,

551

UCA_DATA_NAME,

552

UCA_DATA_TYPE);

553

}

554

udata_writeBlock(pData, data, size);

555

556

if(noOfcontractions != 0) {

557

udata_writeBlock(pData, contractions, noOfcontractions*3*sizeof(UChar));

558

udata_writePadding(pData, paddedsize((noOfcontractions*3*sizeof(UChar))) - noOfcontractions*3*sizeof(uint16_t));

559

}

560

561

/* finish up */

562

dataLength=udata_finish(pData, status);

563

if(U_FAILURE(*status)) {

564

fprintf(stderr, "Error: error %d writing the output file\n", *status);

565

return;

566

}

567

}

568

569

static int32_t

570

write_uca_table(const char *filename,

571

const char *outputDir,

572

const char *copyright,

573

UErrorCode *status)

574

{

575

FILE *data = fopen(filename, "r");

576

uint32_t line = 0;

577

UCAElements *element = NULL;

578

UChar variableTopValue = 0;

579

UCATableHeader *myD = (UCATableHeader *)uprv_malloc(sizeof(UCATableHeader));

580

UColOptionSet *opts = (UColOptionSet *)uprv_malloc(sizeof(UColOptionSet));

581

UChar contractionCEs[256][3];

582

uint32_t noOfContractions = 0;

583

584

585

if(data == NULL) {

586

fprintf(stderr, "Couldn't open file: %s\n", filename);

587

return -1;

588

}

589

590

memset(inverseTable, 0xDA, sizeof(int32_t)*3*0xFFFF);

591

592

opts->variableTopValue = variableTopValue;

593

opts->strength = UCOL_TERTIARY;

594

opts->frenchCollation = UCOL_OFF;

595

opts->alternateHandling = UCOL_NON_IGNORABLE; /* attribute for handling variable elements*/

596

opts->caseFirst = UCOL_OFF; /* who goes first, lower case or uppercase */

597

opts->caseLevel = UCOL_OFF; /* do we have an extra case level */

598

opts->normalizationMode = UCOL_OFF; /* attribute for normalization */

599

opts->hiraganaQ = UCOL_OFF; /* attribute for JIS X 4061, used only in Japanese */

600

/* populate the version info struct with version info*/

601

myD->version[0] = UCOL_BUILDER_VERSION;

602

/*TODO:The fractional rules version should be taken from FractionalUCA.txt*/

603

myD->version[1] = UCOL_FRACTIONAL_UCA_VERSION;

604

myD->jamoSpecial = FALSE;

605

606

tempUCATable *t = uprv_uca_initTempTable(myD, opts, NULL, IMPLICIT_TAG, status);

607

608

#if 0

609

IMPLICIT_TAG = 9,

610

611

*****************************************************************************************

612

* NON_CHARACTER FDD0 - FDEF, FFFE, FFFF, 1FFFE, 1FFFF, 2FFFE, 2FFFF,...e.g. **FFFE, **FFFF

613

******************************************************************************************

614

615

#endif

616

617

// * set to zero

618

struct {

619

UChar32 start;

620

UChar32 end;

621

int32_t value;

622

} ranges[] =

623

{

624

#if 0

625

{0xAC00, 0xD7AF, UCOL_SPECIAL_FLAG | (HANGUL_SYLLABLE_TAG << 24) }, //0 HANGUL_SYLLABLE_TAG,/* AC00-D7AF*/

626

{0xD800, 0xDBFF, UCOL_SPECIAL_FLAG | (LEAD_SURROGATE_TAG << 24) }, //1 LEAD_SURROGATE_TAG, /* D800-DBFF*/

627

{0xDC00, 0xDFFF, UCOL_SPECIAL_FLAG | (TRAIL_SURROGATE_TAG << 24) }, //2 TRAIL_SURROGATE DC00-DFFF

628

{0x3400, 0x4DB5, UCOL_SPECIAL_FLAG | (CJK_IMPLICIT_TAG << 24) }, //3 CJK_IMPLICIT_TAG, /* 0x3400-0x4DB5*/

629

{0x4E00, 0x9FA5, UCOL_SPECIAL_FLAG | (CJK_IMPLICIT_TAG << 24) }, //4 CJK_IMPLICIT_TAG, /* 0x4E00-0x9FA5*/

630

{0xF900, 0xFA2D, UCOL_SPECIAL_FLAG | (CJK_IMPLICIT_TAG << 24) }, //5 CJK_IMPLICIT_TAG, /* 0xF900-0xFA2D*/

631

{0x20000, 0x2A6D6, UCOL_SPECIAL_FLAG | (CJK_IMPLICIT_TAG << 24) }, //6 CJK_IMPLICIT_TAG, /* 0x20000-0x2A6D6*/

632

{0x2F800, 0x2FA1D, UCOL_SPECIAL_FLAG | (CJK_IMPLICIT_TAG << 24) }, //7 CJK_IMPLICIT_TAG, /* 0x2F800-0x2FA1D*/

633

#endif

634

{0xAC00, 0xD7B0, UCOL_SPECIAL_FLAG | (HANGUL_SYLLABLE_TAG << 24) }, //0 HANGUL_SYLLABLE_TAG,/* AC00-D7AF*/

635

{0xD800, 0xDC00, UCOL_SPECIAL_FLAG | (LEAD_SURROGATE_TAG << 24) }, //1 LEAD_SURROGATE_TAG, /* D800-DBFF*/

636

{0xDC00, 0xE000, UCOL_SPECIAL_FLAG | (TRAIL_SURROGATE_TAG << 24) }, //2 TRAIL_SURROGATE DC00-DFFF

637

{0x3400, 0x4DB6, UCOL_SPECIAL_FLAG | (CJK_IMPLICIT_TAG << 24) }, //3 CJK_IMPLICIT_TAG, /* 0x3400-0x4DB5*/

638

{0x4E00, 0x9FA6, UCOL_SPECIAL_FLAG | (CJK_IMPLICIT_TAG << 24) }, //4 CJK_IMPLICIT_TAG, /* 0x4E00-0x9FA5*/

639

{0xF900, 0xFA2E, UCOL_SPECIAL_FLAG | (CJK_IMPLICIT_TAG << 24) }, //5 CJK_IMPLICIT_TAG, /* 0xF900-0xFA2D*/

640

//{0x20000, 0x2A6D7, UCOL_SPECIAL_FLAG | (CJK_IMPLICIT_TAG << 24) }, //6 CJK_IMPLICIT_TAG, /* 0x20000-0x2A6D6*/

641

//{0x2F800, 0x2FA1E, UCOL_SPECIAL_FLAG | (CJK_IMPLICIT_TAG << 24) }, //7 CJK_IMPLICIT_TAG, /* 0x2F800-0x2FA1D*/

642

};

643

uint32_t i = 0;

644

645

for(i = 0; i<sizeof(ranges)/sizeof(ranges[0]); i++) {

646

/*ucmpe32_setRange32(t->mapping, ranges[i].start, ranges[i].end, ranges[i].value); */

647

utrie_setRange32(t->mapping, ranges[i].start, ranges[i].end, ranges[i].value, TRUE);

648

}

649

650

651

int32_t surrogateCount = 0;

652

while(!feof(data)) {

653

if(U_FAILURE(*status)) {

654

fprintf(stderr, "Something returned an error %i (%s) while processing line %i of %s. Exiting...\n",

655

*status, u_errorName(*status), line, filename);

656

exit(*status);

657

}

658

659

element = readAnElement(data, status);

660

line++;

661

if(element != NULL) {

662

// we have read the line, now do something sensible with the read data!

663

if(element->variableTop == TRUE && variableTopValue == 0) {

664

t->options->variableTopValue = element->cPoints[0];

665

}

666

667

// if element is a contraction, we want to add it to contractions

668

if(element->cSize > 1) { // this is a contraction

669

if(UTF_IS_LEAD(element->cPoints[0]) && UTF_IS_TRAIL(element->cPoints[1]) && element->cSize == 2) {

670

surrogateCount++;

671

} else {

672

contractionCEs[noOfContractions][0] = element->cPoints[0];

673

contractionCEs[noOfContractions][1] = element->cPoints[1];

674

if(element->cSize > 2) { // the third one

675

contractionCEs[noOfContractions][2] = element->cPoints[2];

676

} else {

677

contractionCEs[noOfContractions][2] = 0;

678

}

679

noOfContractions++;

680

}

681

}

682

683

/* we're first adding to inverse, because addAnElement will reverse the order */

684

/* of code points and stuff... we don't want that to happen */

685

addToInverse(element, status);

686

uprv_uca_addAnElement(t, element, status);

687

}

688

}

689

690

691

if (VERBOSE) {

692

fprintf(stdout, "\nLines read: %i\n", line);

693

fprintf(stdout, "Surrogate count: %i\n", surrogateCount);

694

fprintf(stdout, "Raw data breakdown:\n");

695

/*fprintf(stdout, "Compact array stage1 top: %i, stage2 top: %i\n", t->mapping->stage1Top, t->mapping->stage2Top);*/

696

fprintf(stdout, "Number of contractions: %i\n", noOfContractions);

697

fprintf(stdout, "Contraction image size: %i\n", t->image->contractionSize);

698

fprintf(stdout, "Expansions size: %i\n", t->expansions->position);

699

}

700

701

/* test */

702

UCATableHeader *myData = uprv_uca_assembleTable(t, status);

703

704

if (VERBOSE) {

705

fprintf(stdout, "Compacted data breakdown:\n");

706

/*fprintf(stdout, "Compact array stage1 top: %i, stage2 top: %i\n", t->mapping->stage1Top, t->mapping->stage2Top);*/

707

fprintf(stdout, "Number of contractions: %i\n", noOfContractions);

708

fprintf(stdout, "Contraction image size: %i\n", t->image->contractionSize);

709

fprintf(stdout, "Expansions size: %i\n", t->expansions->position);

710

}

711

712

writeOutData(myData, contractionCEs, noOfContractions, outputDir, copyright, status);

713

714

InverseTableHeader *inverse = assembleInverseTable(status);

715

writeOutInverseData(inverse, outputDir, copyright, status);

716

717

uprv_uca_closeTempTable(t);

718

uprv_free(myD);

719

uprv_free(opts);

720

721

722

uprv_free(myData);

723

uprv_free(inverse);

724

fclose(data);

725

726

return 0;

727

}

728

729

static UOption options[]={

730

UOPTION_HELP_H, /* 0 Numbers for those who*/

731

UOPTION_HELP_QUESTION_MARK, /* 1 can't count. */

732

UOPTION_COPYRIGHT, /* 2 */

733

UOPTION_VERSION, /* 3 */

734

UOPTION_DESTDIR, /* 4 */

735

UOPTION_SOURCEDIR, /* 5 */

736

UOPTION_VERBOSE, /* 6 */

737

UOPTION_ICUDATADIR /* 7 */

738

/* weiv can't count :))))) */

739

};

740

741

int main(int argc, char* argv[]) {

742

UErrorCode status = U_ZERO_ERROR;

743

const char* destdir = NULL;

744

const char* srcDir = NULL;

745

char filename[300];

746

char *basename = NULL;

747

const char *copyright = NULL;

748

749

U_MAIN_INIT_ARGS(argc, argv);

750

751

/* preset then read command line options */

752

options[4].value=u_getDataDirectory();

753

options[5].value="";

754

argc=u_parseArgs(argc, argv, sizeof(options)/sizeof(options[0]), options);

755

756

/* error handling, printing usage message */

757

if(argc<0) {

758

fprintf(stderr,

759

"error in command line argument \"%s\"\n",

760

argv[-argc]);

761

} else if(argc<2) {

762

argc=-1;

763

}

764

if(options[0].doesOccur || options[1].doesOccur) {

765

fprintf(stderr,

766

"usage: %s [-options] file\n"

767

"\tRead in UCA collation text data and write out the binary collation data\n"

768

"options:\n"

769

"\t-h or -? or --help this usage text\n"

770

"\t-V or --version show a version message\n"

771

"\t-c or --copyright include a copyright notice\n"

772

"\t-d or --destdir destination directory, followed by the path\n"

773

"\t-s or --sourcedir source directory, followed by the path\n"

774

"\t-v or --verbose turn on verbose output\n"

775

"\t-i or --icudatadir directory for locating any needed intermediate data files,\n"

776

"\t followed by path, defaults to %s\n",

777

argv[0], u_getDataDirectory());

778

return argc<0 ? U_ILLEGAL_ARGUMENT_ERROR : U_ZERO_ERROR;

779

}

780

781

if(options[3].doesOccur) {

782

fprintf(stdout, "genuca version %hu.%hu, ICU tool to read UCA text data and create UCA data tables for collation.\n",

783

ucaDataInfo.formatVersion[0], ucaDataInfo.formatVersion[1]);

784

785

786

exit(0);

787

}

788

789

/* get the options values */

790

destdir = options[4].value;

791

srcDir = options[5].value;

792

VERBOSE = options[6].doesOccur;

793

794

if (options[2].doesOccur) {

795

796

}

797

798

if (options[7].doesOccur) {

799

u_setDataDirectory(options[7].value);

800

}

801

802

/* prepare the filename beginning with the source dir */

803

uprv_strcpy(filename, srcDir);

804

basename=filename+uprv_strlen(filename);

805

806

if(basename>filename && *(basename-1)!=U_FILE_SEP_CHAR) {

807

*basename++ = U_FILE_SEP_CHAR;

808

}

809

810

if(argc < 0) {

811

uprv_strcpy(basename, "FractionalUCA.txt");

812

} else {

813

argv++;

814

uprv_strcpy(basename, getLongPathname(*argv));

815

}

816

817

return write_uca_table(filename, destdir, copyright, &status);

818

}

819

820

821

* Hey, Emacs, please set the following:

822

823

* Local Variables:

824

* indent-tabs-mode: nil

825

* End:

826

827

Older »