~michael-rynn-500/d2-xml/d2-xml-dev : revision 91

1

module xml.parse.dxml;

2

3

import alt.buffer, std.stdint;

4

import xml.xmlchar, std.variant, std.string, std.conv, std.utf;

4

import xml.xmlchar, std.variant, std.string, std.conv, std.utf, std.traits;

5

import xml.dom.dtdt, xml.dom.domt;

6

import xml.ixml, xml.xmlerror, xml.parse.input, xml.util.read;

7

import std.path, std.stream;

8

import alt.gcstats;

8

9

10

version = ParseDocType;

10

11

12

11

13

alias bool delegate(ref const(dchar)[] buf) MoreInputDg; // For a stream datasource

12

14

private enum BBCount { b1, b2, bend };

13

15

14

16

class DXmlParser(T) {

15

17

version(GC_STATS)

18

{

19

mixin GC_statistics;

20

static this()

21

{

22

setStatsId(typeid(typeof(this)).toString());

23

}

24

}

16

25

private {

17

26

bool validate_;

18

27

bool isStandalone_;

35

44

{

36

45

return this;

37

46

}

38

47

39

48

40

49

41

50

alias immutable(T)[] XmlString;

47

56

alias XmlEvent!T.Attribute Attribute;

48

57

49

58

alias void delegate() ParseStateDg;

50

59

51

60

static struct MarkSlice {

52

61

immutable(T)* ptr_;

53

62

debug immutable(T)* viewptr_;

82

91

int docDeclare;

83

92

dchar front = 0;

84

93

bool empty = true;

85

94

86

95

double docVersion_;

87

96

Buffer!dchar backStack; // oops wrong way buffer

88

97

90

99

uintptr_t mpos; // start char index of front for slicing

91

100

bool slicing; // current context is sliceData_

92

101

immutable(T)[] sliceData; // current input buffer

93

const(dchar)[] streamData; // dynamic stream data

102

const(dchar)[] streamData; // dynamic stream data

94

103

95

MoreInputDg inputDg_; // more stream data

104

MoreInputDg inputDg_; // more stream data

96

105

uintptr_t fpos; // position in buffer

97

106

EntityData entity;

98

107

bool scopePop;

101

110

dchar lastChar_;

102

111

bool isEndOfLine_;

103

112

bool minVersion11_;

104

CharFilter doFilter_;

113

CharFilter doFilter_;

105

114

106

115

107

116

124

133

}

125

134

void explode()

126

135

{

136

fillSource_ = null;

137

events_ = null;

138

errors_ = null;

127

139

}

128

140

129

/// ensure Context state pops when leaving scope.

130

/*

131

struct ContextPopper

132

{

133

DXmlParser _outer;

134

/// Make a new immutable context

135

136

this(immutable(T)[] data, DXmlParser fp)

137

{

138

_outer = fp;

139

_outer.pushContext(data,true,null);

140

}

141

142

/// Ensure existing context is popped on scope exit

143

this(DXmlParser fp)

144

{

145

_outer = fp;

146

}

147

148

/// pop on scope exit

149

~this()

150

{

151

_outer.popContext();

152

}

153

154

155

}*/

156

157

private

141

142

private

158

143

{

159

144

ParseStateDg stateDg_;

160

145

295

280

returnTextContent();

296

281

}

297

282

while(true)

298

{

283

{

299

284

sliceData_ = null;

300

285

if ((contextStack_.length > 0) && !this.scopePop)

301

286

{

310

295

}

311

296

}

312

297

mpos = fpos;

313

front = decode(sliceData_,fpos);

298

front = decode(sliceData_,fpos);

314

299

}

315

300

else {

316

301

if (fpos >= streamData_.length)

317

302

{

318

303

while (true)

319

{

304

{

320

305

if (inputDg_ && inputDg_(streamData_))

321

306

{

322

307

fpos = 0;

337

322

empty = true;

338

323

return;

339

324

}

340

}

325

}

341

326

front = streamData_[fpos++];

342

327

}

343

328

344

329

if (doFilter_ != CharFilter.filterOn)

345

lineChar_++;

330

lineChar_++;

346

331

else

347

332

filterFront();

348

333

return;

376

361

{

377

362

front = 0x0A;

378

363

return;

379

}

364

}

380

365

}

381

366

}

382

367

lastChar_ = 0;

383

}

368

}

384

369

if ((c >= 0x20) && (c < 0x7F))

385

370

{

386

371

lineChar_++;

387

372

return;

388

373

}

389

else

374

else

390

375

{ // detect endOfLine or crazy characters

391

376

switch(c)

392

377

{

463

448

doFilter_ = CharFilter.filterOff;

464

449

}

465

450

466

451

467

452

// The entire parser , uses a popFront after recognition policy.

468

453

// No need to popFront, to check a character match, then unpop it again if not matching

469

454

// This causes some issues to handle for Entity context push and pop.

470

//

455

//

471

456

// eat matching input or restore

472

457

final bool matchInput(dchar val)

473

458

{

560

545

uintptr_t fpos;

561

546

EntityData entity;

562

547

bool scopePop;

563

564

CharFilter doFilter_;

548

549

CharFilter doFilter_;

565

550

566

551

size_t lineNumber_;

567

552

size_t lineChar_;

569

554

bool isEndOfLine_;

570

555

bool minVersion11_;

571

556

//.. End Working context

572

557

573

558

XmlCharTypeFn isNameStartFn;

574

559

XmlCharTypeFn isNameCharFn;

575

560

IXmlErrorHandler!T errors_;

594

579

Buffer!T bufContent_;

595

580

596

581

Buffer!(XmlEvent!T.Attribute) attributes_;

597

XmlEvent!T results_;

598

582

XmlEvent!T results_;

583

599

584

StringType[StringType] charEntity;

600

585

601

586

602

587

603

588

Buffer!(XmlContext) contextStack_;

606

591

DocTypeData dtd_;

607

592

intptr_t itemCount;

608

593

uintptr_t attributeCount;

609

594

610

595

// manage own primary source

611

596

BufferFill!(dchar) fillSource_;

612

597

ulong fillPos_;

613

598

int bomMark_;

614

599

// introduced for the DOCTYPE

615

//ImmuteAlloc!(T,true) stringAlloc_;

600

//ImmuteAlloc!(T,true) stringAlloc_;

616

601

617

602

bool fillData(ref const(dchar)[] data)

618

603

{

628

613

// read the data in a block, translate if necessary to T.

629

614

public void sliceFile(string srcPath)

630

615

{

631

632

616

auto s = alt.bomstring.readTextBom!T(srcPath,bomMark_);

633

617

auto xml = (cast(immutable(T)*) s.ptr)[0..s.length];

634

618

addSystemPath(normalizedDirName(srcPath));

832

816

}

833

817

return 0;

834

818

}

835

819

836

820

/// must be after a <?

837

821

void doProcessingInstruction(uint spaceCt = 0)

838

822

{

877

861

getPIData(bufContent_);

878

862

879

863

//XmlEvent!T results_;

880

with (results_)

864

with (results_)

881

865

{

882

866

eventId = XmlResult.XML_PI;

883

867

data = target;

884

868

attributes = Attribute(target,bufContent_.idup);

885

}

869

}

886

870

if(eventMode_)

887

871

events_.instruction(results_);

888

872

946

930

947

931

if (empty)

948

932

break;

949

933

950

934

if (front == '<')

951

935

{

952

936

markupDepth++;

972

956

{

973

957

doCommentContent();

974

958

break;

975

}

959

}

976

960

else

977

961

throw errors_.makeException("Illegal in prolog");

978

962

//goto default;

979

963

default:

980

964

if (getXmlName(tag_))

981

{

965

{

982

966

if (!hasDeclaration)

983

967

{

984

968

if (validate_)

985

969

{

986

970

errors_.pushError("No xml declaration",XmlErrorLevel.INVALID);

987

971

}

988

}

972

}

989

973

elementDepth++;

990

974

state_ = PState.P_DATA; // into the document proper

991

975

stateDg_ = &doContent;

1240

1224

setXmlStandalone(standaloneValue);

1241

1225

attributes_ ~= Attribute("standalone", standaloneValue);

1242

1226

}

1243

// send event

1227

// send event

1244

1228

with(results_)

1245

1229

{

1246

1230

type = XmlResult.XML_DEC;

1256

1240

itemCount++;

1257

1241

}

1258

1242

1259

/// Chicken and egg. Start off by not knowing encoding, except by BOM

1243

/// Chicken and egg. Start off by not knowing encoding, except by BOM

1260

1244

/// After reading coding declaration, may need to change the source processing.

1261

1245

/// Do by events call back? Throw and catch own exception?

1262

1246

void setXmlEncoding(const(T)[] encoding)

1273

1257

// prepare to fail low level

1274

1258

try

1275

1259

{

1276

// encoding stuff in dchar ?

1260

// encoding stuff in dchar ?

1277

1261

errors_.setEncoding(encoding);

1278

1262

}

1279

1263

catch (XmlError ex)

1343

1327

}

1344

1328

throw errors_.makeException("end tag expected");

1345

1329

}

1346

// called if getXmlName(bufTag_)

1330

// called if getXmlName(bufTag_)

1347

1331

final void doStartTag()

1348

1332

{

1349

1333

int attSpaceCt = 0;

1361

1345

data = tag_;

1362

1346

attributes = (attrCount>0)?attributes_.peek:null;

1363

1347

}

1364

1348

1365

1349

if (eventMode_)

1366

1350

{

1367

1351

if (id == XmlResult.TAG_START)

1385

1369

frontFilterOn();

1386

1370

if (elementDepth + stackElementDepth == 1) // very first tag

1387

1371

{

1388

// delayed popFront,

1372

// delayed popFront,

1389

1373

// processing as startTag

1390

1374

setResults(XmlResult.TAG_START);

1391

1375

popFront();

1392

return;

1376

return;

1393

1377

}

1394

1378

1395

1379

popFront();

1396

1380

1397

1381

if (empty)

1482

1466

}

1483

1467

else if (attrValue_ == "default")

1484

1468

{

1485

1469

1486

1470

}

1487

1471

else

1488

1472

{

1490

1474

}

1491

1475

}

1492

1476

/// TODO: actually implement space instructions?

1493

1477

1494

1478

1495

1479

1496

1480

}

1523

1507

attributes.clear();

1524

1508

}

1525

1509

if (eventMode_)

1526

events_.text(results_);

1510

events_.text(results_);

1527

1511

bufContent_.length = 0;

1528

1512

inCharData = false;

1529

1513

deviantData_ = false;

1597

1581

return;

1598

1582

}

1599

1583

else if (front=='&')

1600

{

1584

{

1601

1585

if(!deviantData_)

1602

1586

{

1603

deviantData_ = true;

1587

deviantData_ = true;

1604

1588

//returnTextContent();

1605

1589

}

1606

1590

// Cannot determine content in advance. Invalidates content so far, ? return text so far?

1645

1629

continue;

1646

1630

else

1647

1631

return;

1648

1632

1649

1633

}

1650

1634

continue;

1651

1635

}

1652

1636

}

1653

1637

errors_.makeException("expected entity");

1654

1638

}

1655

else

1639

else

1656

1640

{

1657

1641

if (!inCharData)

1658

1642

{

1663

1647

deviantData_ = false;

1664

1648

}

1665

1649

}

1666

if (lastChar_)

1650

if (lastChar_)

1667

1651

deviantData_ = true;

1668

1652

bufContent_.put(front);

1669

1653

final switch(bbct)

1679

1663

throw errors_.makeException("illegal CDATA end ]]>");

1680

1664

break;

1681

1665

}

1682

1666

1683

1667

popFront();

1684

1668

1685

1669

} // end switch

1686

1670

} // While not epilog

1687

1671

}

1821

1805

}

1822

1806

/// Return value is the radix of the encoding for the character

1823

1807

/// Throws exception for invalid encoding

1824

final uintptr_t refToChar(ref dchar c)

1808

final uint refToChar(ref dchar c)

1825

1809

{

1826

1810

dchar test;

1827

1811

int digits = 0;

1828

uintptr_t radix = 10;

1812

uint radix = 10;

1829

1813

1830

1814

if (empty)

1831

1815

throw makeEmpty();

1907

1891

}

1908

1892

}// end private {

1909

1893

1910

/// IXMLParser

1894

/// IXMLParser

1911

1895

bool inDTD()

1912

1896

{

1913

1897

return inDTD_;

1914

1898

}

1915

1899

1916

1900

DocTypeData DTD()

1917

1901

{

1918

1902

return dtd_;

1981

1965

{

1982

1966

systemPaths_ ~= dir;

1983

1967

}

1968

version(GC_STATS)

1969

{

1970

~this()

1971

{

1972

gcStatsSum.dec();

1973

}

1974

}

1984

1975

1985

1976

this()

1986

1977

{

1978

version(GC_STATS)

1979

gcStatsSum.inc();

1980

1987

1981

stateDg_ = &doProlog;

1988

1982

setEntityValue("lt","<");

1989

1983

setEntityValue("gt",">");

2002

1996

{

2003

1997

return stackElementDepth + elementDepth;

2004

1998

}

2005

final void eventReturn(XmlEvent!T ret)

1999

final void eventReturn(XmlEvent!T ret)

2006

2000

{

2007

2001

results_ = ret;

2008

2002

}

2074

2068

2075

2069

if (inParamEntity)

2076

2070

{

2077

docDeclare += ctx_.docDeclare;

2071

docDeclare += ctx_.docDeclare;

2078

2072

}

2079

2073

else {

2080

2074

stackElementDepth -= elementDepth;

2096

2090

2097

2091

fpos = ctx_.fpos;

2098

2092

entity = ctx_.entity;

2099

scopePop = ctx_.scopePop;

2093

scopePop = ctx_.scopePop;

2100

2094

lineNumber_ = ctx_.lineNumber_;

2101

2095

lineChar_ = ctx_.lineChar_;

2102

2096

lastChar_ = ctx_.lastChar_;

2103

2097

isEndOfLine_ = ctx_.isEndOfLine_;

2104

2098

minVersion11_ = ctx_.minVersion11_;

2105

2099

2106

doFilter_ = ctx_.doFilter_;

2100

doFilter_ = ctx_.doFilter_;

2107

2101

contextStack_.popBack();

2108

2102

}

2109

2103

if (slen==0)

2121

2115

contextStack_.length = contextStack_.length + 1;

2122

2116

2123

2117

//XmlContext!T ctx_;

2124

auto ctx_ = contextStack_.last();

2118

auto ctx_ = contextStack_.last();

2125

2119

ctx_.markupDepth = markupDepth;

2126

2120

ctx_.elementDepth = elementDepth;

2127

2121

ctx_.squareDepth = squareDepth;

2132

2126

ctx_.front = front;

2133

2127

ctx_.empty = empty;

2134

2128

ctx_.docVersion_ = docVersion_;

2135

ctx_.backStack.takeOver(backStack);

2129

ctx_.backStack.takeOver(backStack);

2136

2130

ctx_.sliceData = sliceData_; // current input buffer

2137

2131

ctx_.streamData = streamData_;

2138

2132

ctx_.slicing = slicing_;

2139

2133

ctx_.fpos = fpos;

2140

2134

2141

2135

ctx_.entity = entity;

2142

ctx_.scopePop = scopePop;

2136

ctx_.scopePop = scopePop;

2143

2137

ctx_.lineNumber_ = lineNumber_;

2144

2138

ctx_.lineChar_ = lineChar_;

2145

2139

ctx_.lastChar_ = lastChar_;

2146

2140

ctx_.isEndOfLine_ = isEndOfLine_;

2147

ctx_.minVersion11_ = minVersion11_;

2141

ctx_.minVersion11_ = minVersion11_;

2148

2142

2149

ctx_.doFilter_ = doFilter_;

2143

ctx_.doFilter_ = doFilter_;

2150

2144

2151

2145

// setup new defaults

2152

2146

lineNumber_ = 0;

2159

2153

streamData_ = null;

2160

2154

sliceData_ = null;

2161

2155

slicing_ = false;

2162

2156

2163

2157

markupDepth = 0;

2164

2158

stackElementDepth += elementDepth;

2165

2159

elementDepth = 0;

2233

2227

if (ge.src_.systemId_.length > 0)

2234

2228

throw errors_.makeException("Entity with System reference in attribute");

2235

2229

StringSet eset;

2236

2230

2237

2231

int reftype = RefTagType.UNKNOWN_REF;

2238

2232

if (! deriveEntityContent(ge, eset, reftype))

2239

2233

return null;

2250

2244

}

2251

2245

/*if(isAttribute && this.isStandalone_ && reftype==RefTagType.SYSTEM_REF)

2252

2246

throw errors_.makeException("External entity in attribute of standalone document");*/

2253

2247

2254

2248

return ge;

2255

2249

}

2256

2250

2399

2393

pe.isInternal_ = false;

2400

2394

pe.baseDir_ = baseDir;

2401

2395

pe.value_ = srcput;

2402

pe.status_ = EntityData.Found;

2396

pe.status_ = EntityData.Found;

2403

2397

*/

2404

2398

2405

2399

}

2406

2400

if (pe.value_.length > 0)

2407

2401

{

2408

2402

Buffer!T buf1;

2409

2403

2410

2404

{

2411

2405

this.pushContext(to!(immutable(T)[])(pe.value_));

2412

2406

scope(exit)

2509

2503

}

2510

2504

else

2511

2505

{

2512

2506

2513

2507

string evalue;

2514

2508

EntityData ed = getEntityData(bufTag_.peek, true);

2515

2509

if (ed !is null)

2573

2567

pushContext(src[epos..$], true, null);

2574

2568

scope(exit)

2575

2569

popContext();

2576

2577

SCAN:

2570

2571

SCAN:

2578

2572

while(!empty)

2579

2573

{

2580

2574

immutable test = front;

2764

2758

markupDepth--;

2765

2759

docDeclare--;

2766

2760

popFront();

2767

2761

2768

2762

}

2769

2763

XmlErrorLevel parseDtdElement()

2770

2764

{

2872

2866

2873

2867

bool getPublicLiteral(ref XmlString opt)

2874

2868

{

2875

2869

2876

2870

if (!empty && (front == '\"' || front == '\''))

2877

2871

{

2878

2872

unquoteValue(opt);

2884

2878

}

2885

2879

else

2886

2880

throw errors_.makeException("Quoted PUBLIC id expected");

2887

2881

2888

2882

int ct = 0;

2889

2883

bufAttr_.length = 0;

2890

2884

auto hasSpace = false;

2956

2950

hasDeclaration = true;

2957

2951

2958

2952

int spacect = munchSpace();

2959

2953

2960

2954

if (! getXmlName(xmlName) )

2961

2955

throw errors_.makeException("DOCTYPE name expected");

2962

2956

if (!spacect)

3125

3119

if (empty)

3126

3120

throw makeEmpty();

3127

3121

if (matchInput("--"))

3128

{

3122

{

3129

3123

parseComment(); // TODO: stick comment somewhere ?, child of DocumentType? event?'

3130

3124

}

3131

3125

else {

3187

3181

throw errors_.makeException(XmlErrorCode.BAD_ENTITY_REFERENCE);

3188

3182

popFront(); // pop ;

3189

3183

}

3190

// got a %, so parse the name and fetch the EntityData.

3184

// got a %, so parse the name and fetch the EntityData.

3191

3185

// Uses recursive buffer, and name set to detect circular references.

3192

3186

// Entity must exist, or exception thrown.

3193

3187

EntityData parseParameterEntity(bool isValue, bool allowed = true)

3218

3212

unpop(sep); // to come after

3219

3213

pushContext(content,false,ed);

3220

3214

if (sep)

3221

unpop(sep); // to come before

3215

unpop(sep); // to come before

3222

3216

}

3223

3217

return true;

3224

3218

}

3307

3301

throw errors_.makeException("NDATA cannot be in Parameter entity");

3308

3302

auto note = dtd_.notationMap.get(ndata_name,null);

3309

3303

if (note is null)//not-wf-sa-083

3310

throw errors_.makeException(format("No notation named %s", ndata_name));

3304

throw errors_.makeException(format("No notation named %s", ndata_name));

3311

3305

edef.ndataref_ = ndata_name;

3312

3306

}

3313

3307

}

3983

3977

return (app.length > 0);

3984

3978

}

3985

3979

3986

3987

3980

bool expandEntityData(ref Buffer!T data, StringSet entityNameSet, out int refType)

3988

3981

{

3989

3982

4002

3995

app.put("&#");

4003

3996

if (radix==16)

4004

3997

app.put('x');

4005

app.put(to!XmlString(cast(uint)cref,radix));

3998

app.putInteger(cast(uint)cref,radix);

4006

3999

app.put(';');

4007

4000

}

4008

4001

4143

4136

void parseExternalDTD(ref ExternalID edtd)

4144

4137

{

4145

4138

string uri = to!string(edtd.systemId_);

4146

4139

4147

4140

if (!findSystemPath(systemPaths_, uri, uri))

4148

4141

{

4149

4142

errors_.pushError(format("Cannot find file %s",uri),XmlErrorLevel.ERROR);

4178

4171

ep.initSource(&getData);

4179

4172

4180

4173

events_.startDoctype(this);

4181

ep.docTypeInnards(DocEndType.noDocEnd);

4174

ep.docTypeInnards(DocEndType.noDocEnd);

4182

4175

events_.endDoctype(this);

4183

4176

}

4184

4177

4185

4178

// Set up a new parser, using same IXMLEvents. Use current Xml version

4186

4179

bool readSystemEntity(EntityData entity)

4187

4180

{

4188

4181

4189

4182

// first locate the file, using this parsers system paths.

4190

4183

string uri = to!string(entity.src_.systemId_);

4191

4184

4195

4188

entity.status_ = EntityData.Failed;

4196

4189

return false;

4197

4190

}

4198

4191

4199

4192

/*if (!isReadable(uri))

4200

4193

throw errors_.makeException(format("%s is empty or unreadable",uri));*/

4201

4194

4222

4215

paths ~= systemPaths_;

4223

4216

ep.systemPaths_ = paths;

4224

4217

ep.events_ = events_;

4225

4218

4226

4219

auto s = new BufferedFile(uri);

4227

4220

auto sf = new XmlStreamFiller(s);

4228

4221

ulong pos;

4232

4225

return sf.fillData(data,pos);

4233

4226

}

4234

4227

ep.initSource(&getData);

4235

4228

4236

4229

if (ep.matchInput("<?xml"))

4237

4230

{

4238

4231

ep.markupDepth++;

4293

4286

ignoreDocType();

4294

4287

}

4295

4288

else

4296

{

4289

{

4297

4290

errors_.pushError(format("Unexpected %s",peek),XmlErrorLevel.FATAL);

4298

4291

throw errors_.makeException(XmlErrorCode.EXPECT_INCLUDE);

4299

4292

}

4386

4379

}

4387

4380

else if (validate_)

4388

4381

errors_.pushError(text("Bad content nesting in entity ",balanceMismatchMsg()),XmlErrorLevel.INVALID);

4389

4382

4390

4383

}

4391

4384

}

4392

4385

}

4847

4840

}

4848

4841

}

4849

4842

4843