~ubuntu-branches/ubuntu/precise/mysql-5.1/precise : revision 1

storage/ndb/tools/restore/Restore.cpp

This program is free software; you can redistribute it and/or modify

it under the terms of the GNU General Public License as published by

the Free Software Foundation; version 2 of the License.

This program is distributed in the hope that it will be useful,

but WITHOUT ANY WARRANTY; without even the implied warranty of

MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

GNU General Public License for more details.

You should have received a copy of the GNU General Public License

along with this program; if not, write to the Free Software

Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */

#include "Restore.hpp"

#include <NdbTCP.h>

#include <NdbMem.h>

#include <OutputStream.hpp>

#include <Bitmask.hpp>

#include <AttributeHeader.hpp>

#include <trigger_definitions.h>

#include <SimpleProperties.hpp>

#include <signaldata/DictTabInfo.hpp>

#include <ndb_limits.h>

#include <NdbAutoPtr.hpp>

#include "../../../../sql/ha_ndbcluster_tables.h"

extern NdbRecordPrintFormat g_ndbrecord_print_format;

Uint16 Twiddle16(Uint16 in); // Byte shift 16-bit data

Uint32 Twiddle32(Uint32 in); // Byte shift 32-bit data

Uint64 Twiddle64(Uint64 in); // Byte shift 64-bit data

bool

BackupFile::Twiddle(const AttributeDesc* attr_desc, AttributeData* attr_data, Uint32 arraySize){

Uint32 i;

if(m_hostByteOrder)

return true;

if(arraySize == 0){

arraySize = attr_desc->arraySize;

}

switch(attr_desc->size){

case 8:

return true;

case 16:

for(i = 0; i<arraySize; i++){

attr_data->u_int16_value[i] = Twiddle16(attr_data->u_int16_value[i]);

}

return true;

case 32:

for(i = 0; i<arraySize; i++){

attr_data->u_int32_value[i] = Twiddle32(attr_data->u_int32_value[i]);

}

return true;

case 64:

for(i = 0; i<arraySize; i++){

// allow unaligned

char* p = (char*)&attr_data->u_int64_value[i];

Uint64 x;

memcpy(&x, p, sizeof(Uint64));

x = Twiddle64(x);

memcpy(p, &x, sizeof(Uint64));

}

return true;

default:

return false;

} // switch

} // Twiddle

FilteredNdbOut err(* new FileOutputStream(stderr), 0, 0);

FilteredNdbOut info(* new FileOutputStream(stdout), 1, 1);

FilteredNdbOut debug(* new FileOutputStream(stdout), 2, 0);

// To decide in what byte order data is

const Uint32 magicByteOrder = 0x12345678;

const Uint32 swappedMagicByteOrder = 0x78563412;

RestoreMetaData::RestoreMetaData(const char* path, Uint32 nodeId, Uint32 bNo) {

debug << "RestoreMetaData constructor" << endl;

setCtlFile(nodeId, bNo, path);

}

RestoreMetaData::~RestoreMetaData(){

for(Uint32 i= 0; i < allTables.size(); i++)

{

TableS *table = allTables[i];

for(Uint32 j= 0; j < table->m_fragmentInfo.size(); j++)

delete table->m_fragmentInfo[j];

delete table;

}

allTables.clear();

100

}

101

102

TableS *

103

RestoreMetaData::getTable(Uint32 tableId) const {

104

for(Uint32 i= 0; i < allTables.size(); i++)

105

if(allTables[i]->getTableId() == tableId)

106

return allTables[i];

107

return NULL;

108

}

109

110

Uint32

111

RestoreMetaData::getStopGCP() const {

112

return m_stopGCP;

113

}

114

115

int

116

RestoreMetaData::loadContent()

117

{

118

Uint32 noOfTables = readMetaTableList();

119

if(noOfTables == 0) {

120

return 1;

121

}

122

for(Uint32 i = 0; i<noOfTables; i++){

123

if(!readMetaTableDesc()){

124

return 0;

125

}

126

}

127

if (! markSysTables())

128

return 0;

129

if(!readGCPEntry())

130

return 0;

131

132

if(!readFragmentInfo())

133

return 0;

134

return 1;

135

}

136

137

Uint32

138

RestoreMetaData::readMetaTableList() {

139

140

Uint32 sectionInfo[2];

141

142

if (buffer_read(&sectionInfo, sizeof(sectionInfo), 1) != 1){

143

err << "readMetaTableList read header error" << endl;

144

return 0;

145

}

146

sectionInfo[0] = ntohl(sectionInfo[0]);

147

sectionInfo[1] = ntohl(sectionInfo[1]);

148

149

const Uint32 tabCount = sectionInfo[1] - 2;

150

151

void *tmp;

152

if (buffer_get_ptr(&tmp, 4, tabCount) != tabCount){

153

err << "readMetaTableList read tabCount error" << endl;

154

return 0;

155

}

156

157

return tabCount;

158

}

159

160

bool

161

RestoreMetaData::readMetaTableDesc() {

162

163

Uint32 sectionInfo[3];

164

165

// Read section header

166

Uint32 sz = sizeof(sectionInfo) >> 2;

167

if (m_fileHeader.NdbVersion < NDBD_ROWID_VERSION)

168

{

169

sz = 2;

170

sectionInfo[2] = htonl(DictTabInfo::UserTable);

171

}

172

if (buffer_read(&sectionInfo, 4*sz, 1) != 1){

173

err << "readMetaTableDesc read header error" << endl;

174

return false;

175

} // if

176

sectionInfo[0] = ntohl(sectionInfo[0]);

177

sectionInfo[1] = ntohl(sectionInfo[1]);

178

sectionInfo[2] = ntohl(sectionInfo[2]);

179

180

assert(sectionInfo[0] == BackupFormat::TABLE_DESCRIPTION);

181

182

// Read dictTabInfo buffer

183

const Uint32 len = (sectionInfo[1] - sz);

184

void *ptr;

185

if (buffer_get_ptr(&ptr, 4, len) != len){

186

err << "readMetaTableDesc read error" << endl;

187

return false;

188

} // if

189

190

int errcode = 0;

191

DictObject obj = { sectionInfo[2], 0 };

192

switch(obj.m_objType){

193

case DictTabInfo::SystemTable:

194

case DictTabInfo::UserTable:

195

case DictTabInfo::UniqueHashIndex:

196

case DictTabInfo::OrderedIndex:

197

return parseTableDescriptor((Uint32*)ptr, len);

198

break;

199

case DictTabInfo::Tablespace:

200

{

201

NdbDictionary::Tablespace * dst = new NdbDictionary::Tablespace;

202

errcode =

203

NdbDictInterface::parseFilegroupInfo(NdbTablespaceImpl::getImpl(* dst),

204

(Uint32*)ptr, len);

205

if (errcode)

206

delete dst;

207

obj.m_objPtr = dst;

208

debug << hex << obj.m_objPtr << " "

209

<< dec << dst->getObjectId() << " " << dst->getName() << endl;

210

break;

211

}

212

case DictTabInfo::LogfileGroup:

213

{

214

NdbDictionary::LogfileGroup * dst = new NdbDictionary::LogfileGroup;

215

errcode =

216

NdbDictInterface::parseFilegroupInfo(NdbLogfileGroupImpl::getImpl(* dst),

217

(Uint32*)ptr, len);

218

if (errcode)

219

delete dst;

220

obj.m_objPtr = dst;

221

debug << hex << obj.m_objPtr << " "

222

<< dec << dst->getObjectId() << " " << dst->getName() << endl;

223

break;

224

}

225

case DictTabInfo::Datafile:

226

{

227

NdbDictionary::Datafile * dst = new NdbDictionary::Datafile;

228

errcode =

229

NdbDictInterface::parseFileInfo(NdbDatafileImpl::getImpl(* dst),

230

(Uint32*)ptr, len);

231

if (errcode)

232

delete dst;

233

obj.m_objPtr = dst;

234

debug << hex << obj.m_objPtr << " "

235

<< dec << dst->getObjectId() << " " << dst->getPath() << endl;

236

break;

237

}

238

case DictTabInfo::Undofile:

239

{

240

NdbDictionary::Undofile * dst = new NdbDictionary::Undofile;

241

errcode =

242

NdbDictInterface::parseFileInfo(NdbUndofileImpl::getImpl(* dst),

243

(Uint32*)ptr, len);

244

if (errcode)

245

delete dst;

246

obj.m_objPtr = dst;

247

debug << hex << obj.m_objPtr << " "

248

<< dec << dst->getObjectId() << " " << dst->getPath() << endl;

249

break;

250

}

251

default:

252

err << "Unsupported table type!! " << sectionInfo[2] << endl;

253

return false;

254

}

255

if (errcode)

256

{

257

err << "Unable to parse dict info..."

258

<< sectionInfo[2] << " " << errcode << endl;

259

return false;

260

}

261

262

/**

263

* DD objects need to be sorted...

264

265

for(Uint32 i = 0; i<m_objects.size(); i++)

266

{

267

switch(sectionInfo[2]){

268

case DictTabInfo::Tablespace:

269

if (DictTabInfo::isFile(m_objects[i].m_objType))

270

{

271

m_objects.push(obj, i);

272

goto end;

273

}

274

break;

275

case DictTabInfo::LogfileGroup:

276

{

277

if (DictTabInfo::isFile(m_objects[i].m_objType) ||

278

m_objects[i].m_objType == DictTabInfo::Tablespace)

279

{

280

m_objects.push(obj, i);

281

goto end;

282

}

283

break;

284

}

285

default:

286

m_objects.push_back(obj);

287

goto end;

288

}

289

}

290

m_objects.push_back(obj);

291

292

end:

293

return true;

294

}

295

296

bool

297

RestoreMetaData::markSysTables()

298

{

299

Uint32 i;

300

for (i = 0; i < getNoOfTables(); i++) {

301

TableS* table = allTables[i];

302

table->m_local_id = i;

303

const char* tableName = table->getTableName();

304

if ( // XXX should use type

305

strcmp(tableName, "SYSTAB_0") == 0 ||

306

strcmp(tableName, "NDB$EVENTS_0") == 0 ||

307

strcmp(tableName, "sys/def/SYSTAB_0") == 0 ||

308

strcmp(tableName, "sys/def/NDB$EVENTS_0") == 0 ||

309

310

The following is for old MySQL versions,

311

before we changed the database name of the tables from

312

"cluster_replication" -> "cluster" -> "mysql"

313

314

strcmp(tableName, "cluster_replication/def/" OLD_NDB_APPLY_TABLE) == 0 ||

315

strcmp(tableName, OLD_NDB_REP_DB "/def/" OLD_NDB_APPLY_TABLE) == 0 ||

316

strcmp(tableName, OLD_NDB_REP_DB "/def/" OLD_NDB_SCHEMA_TABLE) == 0 ||

317

strcmp(tableName, NDB_REP_DB "/def/" NDB_APPLY_TABLE) == 0 ||

318

strcmp(tableName, NDB_REP_DB "/def/" NDB_SCHEMA_TABLE)== 0 )

319

320

table->isSysTable = true;

321

}

322

for (i = 0; i < getNoOfTables(); i++) {

323

TableS* blobTable = allTables[i];

324

const char* blobTableName = blobTable->getTableName();

325

// yet another match blob

326

int cnt, id1, id2;

327

char buf[256];

328

cnt = sscanf(blobTableName, "%[^/]/%[^/]/NDB$BLOB_%d_%d",

329

buf, buf, &id1, &id2);

330

if (cnt == 4) {

331

Uint32 j;

332

for (j = 0; j < getNoOfTables(); j++) {

333

TableS* table = allTables[j];

334

if (table->getTableId() == (Uint32) id1) {

335

if (table->isSysTable)

336

blobTable->isSysTable = true;

337

blobTable->m_main_table = table;

338

break;

339

}

340

}

341

if (j == getNoOfTables()) {

342

err << "Restore: Bad primary table id in " << blobTableName << endl;

343

return false;

344

}

345

}

346

}

347

return true;

348

}

349

350

bool

351

RestoreMetaData::readGCPEntry() {

352

353

Uint32 data[4];

354

355

BackupFormat::CtlFile::GCPEntry * dst =

356

(BackupFormat::CtlFile::GCPEntry *)&data[0];

357

358

if(buffer_read(dst, 4, 4) != 4){

359

err << "readGCPEntry read error" << endl;

360

return false;

361

}

362

363

dst->SectionType = ntohl(dst->SectionType);

364

dst->SectionLength = ntohl(dst->SectionLength);

365

366

if(dst->SectionType != BackupFormat::GCP_ENTRY){

367

err << "readGCPEntry invalid format" << endl;

368

return false;

369

}

370

371

dst->StartGCP = ntohl(dst->StartGCP);

372

dst->StopGCP = ntohl(dst->StopGCP);

373

374

m_startGCP = dst->StartGCP;

375

m_stopGCP = dst->StopGCP;

376

return true;

377

}

378

379

bool

380

RestoreMetaData::readFragmentInfo()

381

{

382

BackupFormat::CtlFile::FragmentInfo fragInfo;

383

TableS * table = 0;

384

Uint32 tableId = RNIL;

385

386

while (buffer_read(&fragInfo, 4, 2) == 2)

387

{

388

fragInfo.SectionType = ntohl(fragInfo.SectionType);

389

fragInfo.SectionLength = ntohl(fragInfo.SectionLength);

390

391

if (fragInfo.SectionType != BackupFormat::FRAGMENT_INFO)

392

{

393

err << "readFragmentInfo invalid section type: " <<

394

fragInfo.SectionType << endl;

395

return false;

396

}

397

398

if (buffer_read(&fragInfo.TableId, (fragInfo.SectionLength-2)*4, 1) != 1)

399

{

400

err << "readFragmentInfo invalid section length: " <<

401

fragInfo.SectionLength << endl;

402

return false;

403

}

404

405

fragInfo.TableId = ntohl(fragInfo.TableId);

406

if (fragInfo.TableId != tableId)

407

{

408

tableId = fragInfo.TableId;

409

table = getTable(tableId);

410

}

411

412

FragmentInfo * tmp = new FragmentInfo;

413

tmp->fragmentNo = ntohl(fragInfo.FragmentNo);

414

tmp->noOfRecords = ntohl(fragInfo.NoOfRecordsLow) +

415

(((Uint64)ntohl(fragInfo.NoOfRecordsHigh)) << 32);

416

tmp->filePosLow = ntohl(fragInfo.FilePosLow);

417

tmp->filePosHigh = ntohl(fragInfo.FilePosHigh);

418

419

table->m_fragmentInfo.push_back(tmp);

420

table->m_noOfRecords += tmp->noOfRecords;

421

}

422

return true;

423

}

424

425

TableS::TableS(Uint32 version, NdbTableImpl* tableImpl)

426

: m_dictTable(tableImpl)

427

{

428

m_dictTable = tableImpl;

429

m_noOfNullable = m_nullBitmaskSize = 0;

430

m_auto_val_id= ~(Uint32)0;

431

m_max_auto_val= 0;

432

m_noOfRecords= 0;

433

backupVersion = version;

434

isSysTable = false;

435

m_main_table = NULL;

436

437

for (int i = 0; i < tableImpl->getNoOfColumns(); i++)

438

createAttr(tableImpl->getColumn(i));

439

}

440

441

TableS::~TableS()

442

{

443

for (Uint32 i= 0; i < allAttributesDesc.size(); i++)

444

delete allAttributesDesc[i];

445

}

446

447

448

// Parse dictTabInfo buffer and pushback to to vector storage

449

bool

450

RestoreMetaData::parseTableDescriptor(const Uint32 * data, Uint32 len)

451

{

452

NdbTableImpl* tableImpl = 0;

453

int ret = NdbDictInterface::parseTableInfo(&tableImpl, data, len, false,

454

m_fileHeader.NdbVersion);

455

456

if (ret != 0) {

457

err << "parseTableInfo " << " failed" << endl;

458

return false;

459

}

460

if(tableImpl == 0)

461

return false;

462

463

debug << "parseTableInfo " << tableImpl->getName() << " done" << endl;

464

TableS * table = new TableS(m_fileHeader.NdbVersion, tableImpl);

465

if(table == NULL) {

466

return false;

467

}

468

469

debug << "Parsed table id " << table->getTableId() << endl;

470

debug << "Parsed table #attr " << table->getNoOfAttributes() << endl;

471

debug << "Parsed table schema version not used " << endl;

472

473

debug << "Pushing table " << table->getTableName() << endl;

474

debug << " with " << table->getNoOfAttributes() << " attributes" << endl;

475

476

allTables.push_back(table);

477

478

return true;

479

}

480

481

// Constructor

482

RestoreDataIterator::RestoreDataIterator(const RestoreMetaData & md, void (* _free_data_callback)())

483

: BackupFile(_free_data_callback), m_metaData(md)

484

{

485

debug << "RestoreDataIterator constructor" << endl;

486

setDataFile(md, 0);

487

}

488

489

TupleS & TupleS::operator=(const TupleS& tuple)

490

{

491

prepareRecord(*tuple.m_currentTable);

492

493

if (allAttrData)

494

memcpy(allAttrData, tuple.allAttrData, getNoOfAttributes()*sizeof(AttributeData));

495

496

return *this;

497

}

498

int TupleS::getNoOfAttributes() const {

499

if (m_currentTable == 0)

500

return 0;

501

return m_currentTable->getNoOfAttributes();

502

}

503

504

TableS * TupleS::getTable() const {

505

return m_currentTable;

506

}

507

508

const AttributeDesc * TupleS::getDesc(int i) const {

509

return m_currentTable->allAttributesDesc[i];

510

}

511

512

AttributeData * TupleS::getData(int i) const{

513

return &(allAttrData[i]);

514

}

515

516

bool

517

TupleS::prepareRecord(TableS & tab){

518

if (allAttrData) {

519

if (getNoOfAttributes() == tab.getNoOfAttributes())

520

{

521

m_currentTable = &tab;

522

return true;

523

}

524

delete [] allAttrData;

525

m_currentTable= 0;

526

}

527

528

allAttrData = new AttributeData[tab.getNoOfAttributes()];

529

if (allAttrData == 0)

530

return false;

531

532

m_currentTable = &tab;

533

534

return true;

535

}

536

537

int

538

RestoreDataIterator::readTupleData(Uint32 *buf_ptr, Uint32 *ptr,

539

Uint32 dataLength)

540

{

541

while (ptr + 2 < buf_ptr + dataLength)

542

{

543

typedef BackupFormat::DataFile::VariableData VarData;

544

VarData * data = (VarData *)ptr;

545

Uint32 sz = ntohl(data->Sz);

546

Uint32 attrId = ntohl(data->Id); // column_no

547

548

AttributeData * attr_data = m_tuple.getData(attrId);

549

const AttributeDesc * attr_desc = m_tuple.getDesc(attrId);

550

551

// just a reminder - remove when backwards compat implemented

552

if (m_currentTable->backupVersion < MAKE_VERSION(5,1,3) &&

553

attr_desc->m_column->getNullable())

554

{

555

const Uint32 ind = attr_desc->m_nullBitIndex;

556

if(BitmaskImpl::get(m_currentTable->m_nullBitmaskSize,

557

buf_ptr,ind))

558

{

559

attr_data->null = true;

560

attr_data->void_value = NULL;

561

continue;

562

}

563

}

564

565

if (m_currentTable->backupVersion < MAKE_VERSION(5,1,3))

566

{

567

sz *= 4;

568

}

569

570

attr_data->null = false;

571

attr_data->void_value = &data->Data[0];

572

attr_data->size = sz;

573

574

//if (m_currentTable->getTableId() >= 2) { ndbout << "var off=" << ptr-buf_ptr << " attrId=" << attrId << endl; }

575

576

/**

577

* Compute array size

578

579

const Uint32 arraySize = sz / (attr_desc->size / 8);

580

assert(arraySize <= attr_desc->arraySize);

581

582

//convert the length of blob(v1) and text(v1)

583

if(!m_hostByteOrder

584

&& (attr_desc->m_column->getType() == NdbDictionary::Column::Blob

585

|| attr_desc->m_column->getType() == NdbDictionary::Column::Text)

586

&& attr_desc->m_column->getArrayType() == NdbDictionary::Column::ArrayTypeFixed)

587

{

588

char* p = (char*)&attr_data->u_int64_value[0];

589

Uint64 x;

590

memcpy(&x, p, sizeof(Uint64));

591

x = Twiddle64(x);

592

memcpy(p, &x, sizeof(Uint64));

593

}

594

595

//convert datetime type

596

if(!m_hostByteOrder

597

&& attr_desc->m_column->getType() == NdbDictionary::Column::Datetime)

598

{

599

char* p = (char*)&attr_data->u_int64_value[0];

600

Uint64 x;

601

memcpy(&x, p, sizeof(Uint64));

602

x = Twiddle64(x);

603

memcpy(p, &x, sizeof(Uint64));

604

}

605

606

if(!Twiddle(attr_desc, attr_data, attr_desc->arraySize))

607

{

608

return -1;

609

}

610

611

ptr += ((sz + 3) >> 2) + 2;

612

}

613

614

assert(ptr == buf_ptr + dataLength);

615

616

return 0;

617

}

618

619

const TupleS *

620

RestoreDataIterator::getNextTuple(int & res)

621

{

622

Uint32 dataLength = 0;

623

// Read record length

624

if (buffer_read(&dataLength, sizeof(dataLength), 1) != 1){

625

err << "getNextTuple:Error reading length of data part" << endl;

626

res = -1;

627

return NULL;

628

} // if

629

630

// Convert length from network byte order

631

dataLength = ntohl(dataLength);

632

const Uint32 dataLenBytes = 4 * dataLength;

633

634

if (dataLength == 0) {

635

// Zero length for last tuple

636

// End of this data fragment

637

debug << "End of fragment" << endl;

638

res = 0;

639

return NULL;

640

} // if

641

642

// Read tuple data

643

void *_buf_ptr;

644

if (buffer_get_ptr(&_buf_ptr, 1, dataLenBytes) != dataLenBytes) {

645

err << "getNextTuple:Read error: " << endl;

646

res = -1;

647

return NULL;

648

}

649

650

//if (m_currentTable->getTableId() >= 2) { for (uint ii=0; ii<dataLenBytes; ii+=4) ndbout << "*" << hex << *(Uint32*)( (char*)_buf_ptr+ii ); ndbout << endl; }

651

652

Uint32 *buf_ptr = (Uint32*)_buf_ptr, *ptr = buf_ptr;

653

ptr += m_currentTable->m_nullBitmaskSize;

654

Uint32 i;

655

for(i= 0; i < m_currentTable->m_fixedKeys.size(); i++){

656

assert(ptr < buf_ptr + dataLength);

657

658

const Uint32 attrId = m_currentTable->m_fixedKeys[i]->attrId;

659

660

AttributeData * attr_data = m_tuple.getData(attrId);

661

const AttributeDesc * attr_desc = m_tuple.getDesc(attrId);

662

663

const Uint32 sz = attr_desc->getSizeInWords();

664

665

attr_data->null = false;

666

attr_data->void_value = ptr;

667

attr_data->size = 4*sz;

668

669

if(!Twiddle(attr_desc, attr_data))

670

{

671

res = -1;

672

return NULL;

673

}

674

ptr += sz;

675

}

676

677

for(i = 0; i < m_currentTable->m_fixedAttribs.size(); i++){

678

assert(ptr < buf_ptr + dataLength);

679

680

const Uint32 attrId = m_currentTable->m_fixedAttribs[i]->attrId;

681

682

AttributeData * attr_data = m_tuple.getData(attrId);

683

const AttributeDesc * attr_desc = m_tuple.getDesc(attrId);

684

685

const Uint32 sz = attr_desc->getSizeInWords();

686

687

attr_data->null = false;

688

attr_data->void_value = ptr;

689

attr_data->size = 4*sz;

690

691

//if (m_currentTable->getTableId() >= 2) { ndbout << "fix i=" << i << " off=" << ptr-buf_ptr << " attrId=" << attrId << endl; }

692

if(!m_hostByteOrder

693

&& attr_desc->m_column->getType() == NdbDictionary::Column::Timestamp)

694

attr_data->u_int32_value[0] = Twiddle32(attr_data->u_int32_value[0]);

695

696

if(!Twiddle(attr_desc, attr_data))

697

{

698

res = -1;

699

return NULL;

700

}

701

702

ptr += sz;

703

}

704

705

// init to NULL

706

for(i = 0; i < m_currentTable->m_variableAttribs.size(); i++){

707

const Uint32 attrId = m_currentTable->m_variableAttribs[i]->attrId;

708

709

AttributeData * attr_data = m_tuple.getData(attrId);

710

711

attr_data->null = true;

712

attr_data->void_value = NULL;

713

}

714

715

if ((res = readTupleData(buf_ptr, ptr, dataLength)))

716

return NULL;

717

718

m_count ++;

719

res = 0;

720

return &m_tuple;

721

} // RestoreDataIterator::getNextTuple

722

723

BackupFile::BackupFile(void (* _free_data_callback)())

724

: free_data_callback(_free_data_callback)

725

{

726

m_file = 0;

727

m_path[0] = 0;

728

m_fileName[0] = 0;

729

730

m_buffer_sz = 64*1024;

731

m_buffer = malloc(m_buffer_sz);

732

m_buffer_ptr = m_buffer;

733

m_buffer_data_left = 0;

734

}

735

736

BackupFile::~BackupFile(){

737

if(m_file != 0)

738

fclose(m_file);

739

if(m_buffer != 0)

740

free(m_buffer);

741

}

742

743

bool

744

BackupFile::openFile(){

745

if(m_file != NULL){

746

fclose(m_file);

747

m_file = 0;

748

}

749

750

m_file = fopen(m_fileName, "r");

751

return m_file != 0;

752

}

753

754

Uint32 BackupFile::buffer_get_ptr_ahead(void **p_buf_ptr, Uint32 size, Uint32 nmemb)

755

{

756

Uint32 sz = size*nmemb;

757

if (sz > m_buffer_data_left) {

758

759

if (free_data_callback)

760

(*free_data_callback)();

761

762

memcpy(m_buffer, m_buffer_ptr, m_buffer_data_left);

763

764

size_t r = fread(((char *)m_buffer) + m_buffer_data_left, 1, m_buffer_sz - m_buffer_data_left, m_file);

765

m_buffer_data_left += r;

766

m_buffer_ptr = m_buffer;

767

768

if (sz > m_buffer_data_left)

769

sz = size * (m_buffer_data_left / size);

770

}

771

772

*p_buf_ptr = m_buffer_ptr;

773

774

return sz/size;

775

}

776

Uint32 BackupFile::buffer_get_ptr(void **p_buf_ptr, Uint32 size, Uint32 nmemb)

777

{

778

Uint32 r = buffer_get_ptr_ahead(p_buf_ptr, size, nmemb);

779

780

m_buffer_ptr = ((char*)m_buffer_ptr)+(r*size);

781

m_buffer_data_left -= (r*size);

782

783

return r;

784

}

785

786

Uint32 BackupFile::buffer_read_ahead(void *ptr, Uint32 size, Uint32 nmemb)

787

{

788

void *buf_ptr;

789

Uint32 r = buffer_get_ptr_ahead(&buf_ptr, size, nmemb);

790

memcpy(ptr, buf_ptr, r*size);

791

792

return r;

793

}

794

795

Uint32 BackupFile::buffer_read(void *ptr, Uint32 size, Uint32 nmemb)

796

{

797

void *buf_ptr;

798

Uint32 r = buffer_get_ptr(&buf_ptr, size, nmemb);

799

memcpy(ptr, buf_ptr, r*size);

800

801

return r;

802

}

803

804

void

805

BackupFile::setCtlFile(Uint32 nodeId, Uint32 backupId, const char * path){

806

m_nodeId = nodeId;

807

m_expectedFileHeader.BackupId = backupId;

808

m_expectedFileHeader.FileType = BackupFormat::CTL_FILE;

809

810

char name[PATH_MAX]; const Uint32 sz = sizeof(name);

811

BaseString::snprintf(name, sz, "BACKUP-%d.%d.ctl", backupId, nodeId);

812

setName(path, name);

813

}

814

815

void

816

BackupFile::setDataFile(const BackupFile & bf, Uint32 no){

817

m_nodeId = bf.m_nodeId;

818

m_expectedFileHeader = bf.m_fileHeader;

819

m_expectedFileHeader.FileType = BackupFormat::DATA_FILE;

820

821

char name[PATH_MAX]; const Uint32 sz = sizeof(name);

822

BaseString::snprintf(name, sz, "BACKUP-%d-%d.%d.Data",

823

m_expectedFileHeader.BackupId, no, m_nodeId);

824

setName(bf.m_path, name);

825

}

826

827

void

828

BackupFile::setLogFile(const BackupFile & bf, Uint32 no){

829

m_nodeId = bf.m_nodeId;

830

m_expectedFileHeader = bf.m_fileHeader;

831

m_expectedFileHeader.FileType = BackupFormat::LOG_FILE;

832

833

char name[PATH_MAX]; const Uint32 sz = sizeof(name);

834

BaseString::snprintf(name, sz, "BACKUP-%d.%d.log",

835

m_expectedFileHeader.BackupId, m_nodeId);

836

setName(bf.m_path, name);

837

}

838

839

void

840

BackupFile::setName(const char * p, const char * n){

841

const Uint32 sz = sizeof(m_path);

842

if(p != 0 && strlen(p) > 0){

843

if(p[strlen(p)-1] == '/'){

844

BaseString::snprintf(m_path, sz, "%s", p);

845

} else {

846

BaseString::snprintf(m_path, sz, "%s%s", p, "/");

847

}

848

} else {

849

m_path[0] = 0;

850

}

851

852

BaseString::snprintf(m_fileName, sizeof(m_fileName), "%s%s", m_path, n);

853

debug << "Filename = " << m_fileName << endl;

854

}

855

856

bool

857

BackupFile::readHeader(){

858

if(!openFile()){

859

return false;

860

}

861

862

if(buffer_read(&m_fileHeader, sizeof(m_fileHeader), 1) != 1){

863

err << "readDataFileHeader: Error reading header" << endl;

864

return false;

865

}

866

867

// Convert from network to host byte order for platform compatibility

868

m_fileHeader.NdbVersion = ntohl(m_fileHeader.NdbVersion);

869

m_fileHeader.SectionType = ntohl(m_fileHeader.SectionType);

870

m_fileHeader.SectionLength = ntohl(m_fileHeader.SectionLength);

871

m_fileHeader.FileType = ntohl(m_fileHeader.FileType);

872

m_fileHeader.BackupId = ntohl(m_fileHeader.BackupId);

873

m_fileHeader.BackupKey_0 = ntohl(m_fileHeader.BackupKey_0);

874

m_fileHeader.BackupKey_1 = ntohl(m_fileHeader.BackupKey_1);

875

876

debug << "FileHeader: " << m_fileHeader.Magic << " " <<

877

m_fileHeader.NdbVersion << " " <<

878

m_fileHeader.SectionType << " " <<

879

m_fileHeader.SectionLength << " " <<

880

m_fileHeader.FileType << " " <<

881

m_fileHeader.BackupId << " " <<

882

m_fileHeader.BackupKey_0 << " " <<

883

m_fileHeader.BackupKey_1 << " " <<

884

m_fileHeader.ByteOrder << endl;

885

886

debug << "ByteOrder is " << m_fileHeader.ByteOrder << endl;

887

debug << "magicByteOrder is " << magicByteOrder << endl;

888

889

if (m_fileHeader.FileType != m_expectedFileHeader.FileType){

890

abort();

891

}

892

893

// Check for BackupFormat::FileHeader::ByteOrder if swapping is needed

894

if (m_fileHeader.ByteOrder == magicByteOrder) {

895

m_hostByteOrder = true;

896

} else if (m_fileHeader.ByteOrder == swappedMagicByteOrder){

897

m_hostByteOrder = false;

898

} else {

899

abort();

900

}

901

902

return true;

903

} // BackupFile::readHeader

904

905

bool

906

BackupFile::validateFooter(){

907

return true;

908

}

909

910

bool RestoreDataIterator::readFragmentHeader(int & ret, Uint32 *fragmentId)

911

{

912

BackupFormat::DataFile::FragmentHeader Header;

913

914

debug << "RestoreDataIterator::getNextFragment" << endl;

915

916

while (1)

917

{

918

/* read first part of header */

919

if (buffer_read(&Header, 8, 1) != 1)

920

{

921

ret = 0;

922

return false;

923

} // if

924

925

/* skip if EMPTY_ENTRY */

926

Header.SectionType = ntohl(Header.SectionType);

927

Header.SectionLength = ntohl(Header.SectionLength);

928

if (Header.SectionType == BackupFormat::EMPTY_ENTRY)

929

{

930

void *tmp;

931

buffer_get_ptr(&tmp, Header.SectionLength*4-8, 1);

932

continue;

933

}

934

break;

935

}

936

/* read rest of header */

937

if (buffer_read(((char*)&Header)+8, sizeof(Header)-8, 1) != 1)

938

{

939

ret = 0;

940

return false;

941

}

942

Header.TableId = ntohl(Header.TableId);

943

Header.FragmentNo = ntohl(Header.FragmentNo);

944

Header.ChecksumType = ntohl(Header.ChecksumType);

945

946

debug << "FragmentHeader: " << Header.SectionType

947

<< " " << Header.SectionLength

948

<< " " << Header.TableId

949

<< " " << Header.FragmentNo

950

<< " " << Header.ChecksumType << endl;

951

952

m_currentTable = m_metaData.getTable(Header.TableId);

953

if(m_currentTable == 0){

954

ret = -1;

955

return false;

956

}

957

958

if(!m_tuple.prepareRecord(*m_currentTable))

959

{

960

ret =-1;

961

return false;

962

}

963

964

info.setLevel(254);

965

info << "_____________________________________________________" << endl

966

<< "Processing data in table: " << m_currentTable->getTableName()

967

<< "(" << Header.TableId << ") fragment "

968

<< Header.FragmentNo << endl;

969

970

m_count = 0;

971

ret = 0;

972

*fragmentId = Header.FragmentNo;

973

return true;

974

} // RestoreDataIterator::getNextFragment

975

976

977

bool

978

RestoreDataIterator::validateFragmentFooter() {

979

BackupFormat::DataFile::FragmentFooter footer;

980

981

if (buffer_read(&footer, sizeof(footer), 1) != 1){

982

err << "getFragmentFooter:Error reading fragment footer" << endl;

983

return false;

984

}

985

986

// TODO: Handle footer, nothing yet

987

footer.SectionType = ntohl(footer.SectionType);

988

footer.SectionLength = ntohl(footer.SectionLength);

989

footer.TableId = ntohl(footer.TableId);

990

footer.FragmentNo = ntohl(footer.FragmentNo);

991

footer.NoOfRecords = ntohl(footer.NoOfRecords);

992

footer.Checksum = ntohl(footer.Checksum);

993

994

assert(m_count == footer.NoOfRecords);

995

996

return true;

997

} // RestoreDataIterator::getFragmentFooter

998

999

AttributeDesc::AttributeDesc(NdbDictionary::Column *c)

1000

: m_column(c)

1001

{

1002

size = 8*NdbColumnImpl::getImpl(* c).m_attrSize;

1003

arraySize = NdbColumnImpl::getImpl(* c).m_arraySize;

1004

}

1005

1006

void TableS::createAttr(NdbDictionary::Column *column)

1007

{

1008

AttributeDesc * d = new AttributeDesc(column);

1009

if(d == NULL) {

1010

ndbout_c("Restore: Failed to allocate memory");

1011

abort();

1012

}

1013

d->attrId = allAttributesDesc.size();

1014

allAttributesDesc.push_back(d);

1015

1016

if (d->m_column->getAutoIncrement())

1017

m_auto_val_id= d->attrId;

1018

1019

if(d->m_column->getPrimaryKey() && backupVersion <= MAKE_VERSION(4,1,7))

1020

{

1021

m_fixedKeys.push_back(d);

1022

return;

1023

}

1024

1025

if (d->m_column->getArrayType() == NDB_ARRAYTYPE_FIXED &&

1026

! d->m_column->getNullable())

1027

{

1028

m_fixedAttribs.push_back(d);

1029

return;

1030

}

1031

1032

// just a reminder - does not solve backwards compat

1033

if (backupVersion < MAKE_VERSION(5,1,3))

1034

{

1035

d->m_nullBitIndex = m_noOfNullable;

1036

m_noOfNullable++;

1037

m_nullBitmaskSize = (m_noOfNullable + 31) / 32;

1038

}

1039

m_variableAttribs.push_back(d);

1040

} // TableS::createAttr

1041

1042

Uint16 Twiddle16(Uint16 in)

1043

{

1044

Uint16 retVal = 0;

1045

1046

retVal = ((in & 0xFF00) >> 8) |

1047

((in & 0x00FF) << 8);

1048

1049

return(retVal);

1050

} // Twiddle16

1051

1052

Uint32 Twiddle32(Uint32 in)

1053

{

1054

Uint32 retVal = 0;

1055

1056

retVal = ((in & 0x000000FF) << 24) |

1057

((in & 0x0000FF00) << 8) |

1058

((in & 0x00FF0000) >> 8) |

1059

((in & 0xFF000000) >> 24);

1060

1061

return(retVal);

1062

} // Twiddle32

1063

1064

Uint64 Twiddle64(Uint64 in)

1065

{

1066

Uint64 retVal = 0;

1067

1068

retVal =

1069

((in & (Uint64)0x00000000000000FFLL) << 56) |

1070

((in & (Uint64)0x000000000000FF00LL) << 40) |

1071

((in & (Uint64)0x0000000000FF0000LL) << 24) |

1072

((in & (Uint64)0x00000000FF000000LL) << 8) |

1073

((in & (Uint64)0x000000FF00000000LL) >> 8) |

1074

((in & (Uint64)0x0000FF0000000000LL) >> 24) |

1075

((in & (Uint64)0x00FF000000000000LL) >> 40) |

1076

((in & (Uint64)0xFF00000000000000LL) >> 56);

1077

1078

return(retVal);

1079

} // Twiddle64

1080

1081

1082

RestoreLogIterator::RestoreLogIterator(const RestoreMetaData & md)

1083

: m_metaData(md)

1084

{

1085

debug << "RestoreLog constructor" << endl;

1086

setLogFile(md, 0);

1087

1088

m_count = 0;

1089

m_last_gci = 0;

1090

}

1091

1092

const LogEntry *

1093

RestoreLogIterator::getNextLogEntry(int & res) {

1094

// Read record length

1095

const Uint32 stopGCP = m_metaData.getStopGCP();

1096

Uint32 tableId;

1097

Uint32 triggerEvent;

1098

Uint32 frag_id;

1099

Uint32 *attr_data;

1100

Uint32 attr_data_len;

1101

do {

1102

Uint32 len;

1103

Uint32 *logEntryPtr;

1104

if (buffer_read_ahead(&len, sizeof(Uint32), 1) != 1){

1105

res= -1;

1106

return 0;

1107

}

1108

len= ntohl(len);

1109

1110

Uint32 data_len = sizeof(Uint32) + len*4;

1111

if (buffer_get_ptr((void **)(&logEntryPtr), 1, data_len) != data_len) {

1112

res= -2;

1113

return 0;

1114

}

1115

1116

if(len == 0){

1117

res= 0;

1118

return 0;

1119

}

1120

1121

if (unlikely(m_metaData.getFileHeader().NdbVersion < NDBD_FRAGID_VERSION))

1122

{

1123

1124

FragId was introduced in LogEntry in version

1125

5.1.6

1126

We set FragId to 0 in older versions (these versions

1127

do not support restore of user defined partitioned

1128

tables.

1129

1130

typedef BackupFormat::LogFile::LogEntry_no_fragid LogE_no_fragid;

1131

LogE_no_fragid * logE_no_fragid= (LogE_no_fragid *)logEntryPtr;

1132

tableId= ntohl(logE_no_fragid->TableId);

1133

triggerEvent= ntohl(logE_no_fragid->TriggerEvent);

1134

frag_id= 0;

1135

attr_data= &logE_no_fragid->Data[0];

1136

attr_data_len= len - ((offsetof(LogE_no_fragid, Data) >> 2) - 1);

1137

}

1138

else /* normal case */

1139

{

1140

typedef BackupFormat::LogFile::LogEntry LogE;

1141

LogE * logE= (LogE *)logEntryPtr;

1142

tableId= ntohl(logE->TableId);

1143

triggerEvent= ntohl(logE->TriggerEvent);

1144

frag_id= ntohl(logE->FragId);

1145

attr_data= &logE->Data[0];

1146

attr_data_len= len - ((offsetof(LogE, Data) >> 2) - 1);

1147

}

1148

1149

const bool hasGcp= (triggerEvent & 0x10000) != 0;

1150

triggerEvent &= 0xFFFF;

1151

1152

if(hasGcp){

1153

// last attr_data is gci info

1154

attr_data_len--;

1155

m_last_gci = ntohl(*(attr_data + attr_data_len));

1156

}

1157

} while(m_last_gci > stopGCP + 1);

1158

1159

m_logEntry.m_table = m_metaData.getTable(tableId);

1160

switch(triggerEvent){

1161

case TriggerEvent::TE_INSERT:

1162

m_logEntry.m_type = LogEntry::LE_INSERT;

1163

break;

1164

case TriggerEvent::TE_UPDATE:

1165

m_logEntry.m_type = LogEntry::LE_UPDATE;

1166

break;

1167

case TriggerEvent::TE_DELETE:

1168

m_logEntry.m_type = LogEntry::LE_DELETE;

1169

break;

1170

default:

1171

res = -1;

1172

return NULL;

1173

}

1174

1175

const TableS * tab = m_logEntry.m_table;

1176

m_logEntry.clear();

1177

1178

AttributeHeader * ah = (AttributeHeader *)attr_data;

1179

AttributeHeader *end = (AttributeHeader *)(attr_data + attr_data_len);

1180

AttributeS * attr;

1181

m_logEntry.m_frag_id = frag_id;

1182

while(ah < end){

1183

attr= m_logEntry.add_attr();

1184

if(attr == NULL) {

1185

ndbout_c("Restore: Failed to allocate memory");

1186

res = -1;

1187

return 0;

1188

}

1189

1190

if(unlikely(!m_hostByteOrder))

1191

*(Uint32*)ah = Twiddle32(*(Uint32*)ah);

1192

1193

attr->Desc = (* tab)[ah->getAttributeId()];

1194

assert(attr->Desc != 0);

1195

1196

const Uint32 sz = ah->getDataSize();

1197

if(sz == 0){

1198

attr->Data.null = true;

1199

attr->Data.void_value = NULL;

1200

} else {

1201

attr->Data.null = false;

1202

attr->Data.void_value = ah->getDataPtr();

1203

}

1204

1205

Twiddle(attr->Desc, &(attr->Data));

1206

1207

ah = ah->getNext();

1208

}

1209

1210

m_count ++;

1211

res = 0;

1212

return &m_logEntry;

1213

}

1214

1215

NdbOut &

1216

operator<<(NdbOut& ndbout, const AttributeS& attr){

1217

const AttributeData & data = attr.Data;

1218

const AttributeDesc & desc = *(attr.Desc);

1219

1220

if (data.null)

1221

{

1222

ndbout << g_ndbrecord_print_format.null_string;

1223

return ndbout;

1224

}

1225

1226

NdbRecAttr tmprec(0);

1227

tmprec.setup(desc.m_column, 0);

1228

tmprec.receive_data((Uint32*)data.void_value, data.size);

1229

ndbrecattr_print_formatted(ndbout, tmprec, g_ndbrecord_print_format);

1230

1231

return ndbout;

1232

}

1233

1234

// Print tuple data

1235

NdbOut&

1236

operator<<(NdbOut& ndbout, const TupleS& tuple)

1237

{

1238

for (int i = 0; i < tuple.getNoOfAttributes(); i++)

1239

{

1240

if (i > 0)

1241

ndbout << g_ndbrecord_print_format.fields_terminated_by;

1242

AttributeData * attr_data = tuple.getData(i);

1243

const AttributeDesc * attr_desc = tuple.getDesc(i);

1244

const AttributeS attr = {attr_desc, *attr_data};

1245

debug << i << " " << attr_desc->m_column->getName();

1246

ndbout << attr;

1247

} // for

1248

return ndbout;

1249

}

1250

1251

// Print tuple data

1252

NdbOut&

1253

operator<<(NdbOut& ndbout, const LogEntry& logE)

1254

{

1255

switch(logE.m_type)

1256

{

1257

case LogEntry::LE_INSERT:

1258

ndbout << "INSERT " << logE.m_table->getTableName() << " ";

1259

break;

1260

case LogEntry::LE_DELETE:

1261

ndbout << "DELETE " << logE.m_table->getTableName() << " ";

1262

break;

1263

case LogEntry::LE_UPDATE:

1264

ndbout << "UPDATE " << logE.m_table->getTableName() << " ";

1265

break;

1266

default:

1267

ndbout << "Unknown log entry type (not insert, delete or update)" ;

1268

}

1269

1270

for (Uint32 i= 0; i < logE.size();i++)

1271

{

1272

const AttributeS * attr = logE[i];

1273

ndbout << attr->Desc->m_column->getName() << "=";

1274

ndbout << (* attr);

1275

if (i < (logE.size() - 1))

1276

ndbout << ", ";

1277

}

1278

return ndbout;

1279

}

1280

1281

#include <NDBT.hpp>

1282

1283

NdbOut &

1284

operator<<(NdbOut& ndbout, const TableS & table){

1285

1286

ndbout << (* (NDBT_Table*)table.m_dictTable) << endl;

1287

return ndbout;

1288

}

1289

1290

template class Vector<TableS*>;

1291

template class Vector<AttributeS*>;

1292

template class Vector<AttributeDesc*>;

1293

template class Vector<FragmentInfo*>;

1294

template class Vector<DictObject>;

Older »