~ubuntu-branches/ubuntu/vivid/python-ghost/vivid-proposed : revision 2

1

# -*- coding: utf-8 -*-

2

import sys

3

import os

4

import time

5

import codecs

6

import logging

7

import subprocess

8

import tempfile

9

from functools import wraps

10

from cookielib import Cookie, LWPCookieJar

11

12

__version__ = "0.1b3"

13

14

bindings = ["PySide", "PyQt4"]

15

16

binding = None

17

for name in bindings:

18

try:

19

binding = __import__(name)

20

break

21

except ImportError:

22

continue

23

24

25

if not binding:

26

raise Exception("Ghost.py requires PySide or PyQt4")

27

28

29

PYSIDE = binding.__name__ == 'PySide'

30

31

if not PYSIDE:

32

import sip

33

sip.setapi('QVariant', 2)

34

35

36

def _import(name):

37

name = "%s.%s" % (binding.__name__, name)

38

module = __import__(name)

39

for n in name.split(".")[1:]:

40

module = getattr(module, n)

41

return module

42

43

44

QtCore = _import("QtCore")

45

QSize = QtCore.QSize

46

QByteArray = QtCore.QByteArray

47

QUrl = QtCore.QUrl

48

QDateTime = QtCore.QDateTime

49

QtCriticalMsg = QtCore.QtCriticalMsg

50

QtDebugMsg = QtCore.QtDebugMsg

51

QtFatalMsg = QtCore.QtFatalMsg

52

QtWarningMsg = QtCore.QtWarningMsg

53

qInstallMsgHandler = QtCore.qInstallMsgHandler

54

55

QtGui = _import("QtGui")

56

QApplication = QtGui.QApplication

57

QImage = QtGui.QImage

58

QPainter = QtGui.QPainter

59

QPrinter = QtGui.QPrinter

60

61

QtNetwork = _import("QtNetwork")

62

QNetworkRequest = QtNetwork.QNetworkRequest

63

QNetworkAccessManager = QtNetwork.QNetworkAccessManager

64

QNetworkCookieJar = QtNetwork.QNetworkCookieJar

65

QNetworkDiskCache = QtNetwork.QNetworkDiskCache

66

QNetworkProxy = QtNetwork.QNetworkProxy

67

QNetworkCookie = QtNetwork.QNetworkCookie

68

69

QtWebKit = _import('QtWebKit')

70

71

72

default_user_agent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.2 " +\

73

"(KHTML, like Gecko) Chrome/15.0.874.121 Safari/535.2"

74

75

76

logger = logging.getLogger('ghost')

77

78

79

class Error(Exception):

80

"""Base class for Ghost exceptions."""

81

pass

82

83

84

class TimeoutError(Error):

85

"""Raised when a request times out"""

86

pass

87

88

89

class Logger(logging.Logger):

90

@staticmethod

91

def log(message, sender="Ghost", level="info"):

92

if not hasattr(logger, level):

93

raise Error('invalid log level')

94

getattr(logger, level)("%s: %s", sender, message)

95

96

97

class QTMessageProxy(object):

98

def __init__(self, debug=False):

99

self.debug = debug

100

101

def __call__(self, msgType, msg):

102

if msgType == QtDebugMsg and self.debug:

103

Logger.log(msg, sender='QT', level='debug')

104

elif msgType == QtWarningMsg and self.debug:

105

Logger.log(msg, sender='QT', level='warning')

106

elif msgType == QtCriticalMsg:

107

Logger.log(msg, sender='QT', level='critical')

108

elif msgType == QtFatalMsg:

109

Logger.log(msg, sender='QT', level='fatal')

110

elif self.debug:

111

Logger.log(msg, sender='QT', level='info')

112

113

114

class GhostWebPage(QtWebKit.QWebPage):

115

"""Overrides QtWebKit.QWebPage in order to intercept some graphical

116

behaviours like alert(), confirm().

117

Also intercepts client side console.log().

118

"""

119

def __init__(self, app, ghost):

120

self.ghost = ghost

121

super(GhostWebPage, self).__init__(app)

122

123

def chooseFile(self, frame, suggested_file=None):

124

return Ghost._upload_file

125

126

def javaScriptConsoleMessage(self, message, line, source):

127

"""Prints client console message in current output stream."""

128

super(GhostWebPage, self).javaScriptConsoleMessage(message, line,

129

source)

130

log_type = "error" if "Error" in message else "info"

131

Logger.log("%s(%d): %s" % (source or '<unknown>', line, message),

132

sender="Frame", level=log_type)

133

134

def javaScriptAlert(self, frame, message):

135

"""Notifies ghost for alert, then pass."""

136

Ghost._alert = message

137

self.ghost.append_popup_message(message)

138

Logger.log("alert('%s')" % message, sender="Frame")

139

140

def javaScriptConfirm(self, frame, message):

141

"""Checks if ghost is waiting for confirm, then returns the right

142

value.

143

"""

144

if Ghost._confirm_expected is None:

145

raise Error('You must specified a value to confirm "%s"' %

146

message)

147

self.ghost.append_popup_message(message)

148

confirmation, callback = Ghost._confirm_expected

149

Logger.log("confirm('%s')" % message, sender="Frame")

150

if callback is not None:

151

return callback()

152

return confirmation

153

154

def javaScriptPrompt(self, frame, message, defaultValue, result=None):

155

"""Checks if ghost is waiting for prompt, then enters the right

156

value.

157

"""

158

if Ghost._prompt_expected is None:

159

raise Error('You must specified a value for prompt "%s"' %

160

message)

161

self.ghost.append_popup_message(message)

162

result_value, callback = Ghost._prompt_expected

163

Logger.log("prompt('%s')" % message, sender="Frame")

164

if callback is not None:

165

result_value = callback()

166

if result_value == '':

167

Logger.log("'%s' prompt filled with empty string" % message,

168

level='warning')

169

170

if result is None:

171

# PySide

172

return True, result_value

173

result.append(unicode(result_value))

174

return True

175

176

def setUserAgent(self, user_agent):

177

self.user_agent = user_agent

178

179

def userAgentForUrl(self, url):

180

return self.user_agent

181

182

183

def can_load_page(func):

184

"""Decorator that specifies if user can expect page loading from

185

this action. If expect_loading is set to True, ghost will wait

186

for page_loaded event.

187

"""

188

@wraps(func)

189

def wrapper(self, *args, **kwargs):

190

expect_loading = False

191

if 'expect_loading' in kwargs:

192

expect_loading = kwargs['expect_loading']

193

del kwargs['expect_loading']

194

if expect_loading:

195

self.loaded = False

196

func(self, *args, **kwargs)

197

return self.wait_for_page_loaded()

198

return func(self, *args, **kwargs)

199

return wrapper

200

201

202

class HttpResource(object):

203

"""Represents an HTTP resource.

204

"""

205

def __init__(self, reply, cache, content=None):

206

self.url = reply.url().toString()

207

self.content = content

208

if cache and self.content is None:

209

# Tries to get back content from cache

210

buffer = None

211

if PYSIDE:

212

buffer = cache.data(reply.url().toString())

213

else:

214

buffer = cache.data(reply.url())

215

if buffer is not None:

216

content = buffer.readAll()

217

try:

218

self.content = unicode(content)

219

except UnicodeDecodeError:

220

self.content = content

221

self.http_status = reply.attribute(

222

QNetworkRequest.HttpStatusCodeAttribute)

223

Logger.log("Resource loaded: %s %s" % (self.url, self.http_status))

224

self.headers = {}

225

for header in reply.rawHeaderList():

226

try:

227

self.headers[unicode(header)] = unicode(

228

reply.rawHeader(header))

229

except UnicodeDecodeError:

230

# it will lose the header value,

231

# but at least not crash the whole process

232

logger.error(

233

"Invalid characters in header {0}={1}".format(header, reply.rawHeader(header))

234

)

235

self._reply = reply

236

237

238

class Ghost(object):

239

"""Ghost manages a QWebPage.

240

241

:param user_agent: The default User-Agent header.

242

:param wait_timeout: Maximum step duration in second.

243

:param wait_callback: An optional callable that is periodically

244

executed until Ghost stops waiting.

245

:param log_level: The optional logging level.

246

:param display: A boolean that tells ghost to displays UI.

247

:param viewport_size: A tuple that sets initial viewport size.

248

:param ignore_ssl_errors: A boolean that forces ignore ssl errors.

249

:param cache_dir: A directory path where to store cache datas.

250

:param plugins_enabled: Enable plugins (like Flash).

251

:param java_enabled: Enable Java JRE.

252

:param plugin_path: Array with paths to plugin directories

253

(default ['/usr/lib/mozilla/plugins'])

254

:param download_images: Indicate if the browser should download images

255

"""

256

_alert = None

257

_confirm_expected = None

258

_prompt_expected = None

259

_upload_file = None

260

_app = None

261

262

def __init__(self,

263

user_agent=default_user_agent,

264

wait_timeout=8,

265

wait_callback=None,

266

log_level=logging.WARNING,

267

display=False,

268

viewport_size=(800, 600),

269

ignore_ssl_errors=True,

270

cache_dir=os.path.join(tempfile.gettempdir(), "ghost.py"),

271

plugins_enabled=False,

272

java_enabled=False,

273

plugin_path=['/usr/lib/mozilla/plugins', ],

274

download_images=True,

275

qt_debug=False,

276

show_scrollbars=True,

277

network_access_manager_class=None):

278

279

self.http_resources = []

280

281

self.user_agent = user_agent

282

self.wait_timeout = wait_timeout

283

self.wait_callback = wait_callback

284

self.ignore_ssl_errors = ignore_ssl_errors

285

self.loaded = True

286

287

if sys.platform.startswith('linux') and not 'DISPLAY' in os.environ\

288

and not hasattr(Ghost, 'xvfb'):

289

try:

290

os.environ['DISPLAY'] = ':99'

291

Ghost.xvfb = subprocess.Popen(['Xvfb', ':99'])

292

except OSError:

293

raise Error('Xvfb is required to a ghost run outside ' +

294

'an X instance')

295

296

self.display = display

297

298

if not Ghost._app:

299

Ghost._app = QApplication.instance() or QApplication(['ghost'])

300

qInstallMsgHandler(QTMessageProxy(qt_debug))

301

if plugin_path:

302

for p in plugin_path:

303

Ghost._app.addLibraryPath(p)

304

305

self.popup_messages = []

306

self.page = GhostWebPage(Ghost._app, self)

307

308

if network_access_manager_class is not None:

309

self.page.setNetworkAccessManager(network_access_manager_class())

310

311

QtWebKit.QWebSettings.setMaximumPagesInCache(0)

312

QtWebKit.QWebSettings.setObjectCacheCapacities(0, 0, 0)

313

QtWebKit.QWebSettings.globalSettings().setAttribute(

314

QtWebKit.QWebSettings.LocalStorageEnabled, True)

315

316

self.page.setForwardUnsupportedContent(True)

317

self.page.settings().setAttribute(

318

QtWebKit.QWebSettings.AutoLoadImages, download_images)

319

self.page.settings().setAttribute(

320

QtWebKit.QWebSettings.PluginsEnabled, plugins_enabled)

321

self.page.settings().setAttribute(QtWebKit.QWebSettings.JavaEnabled,

322

java_enabled)

323

324

if not show_scrollbars:

325

self.page.mainFrame().setScrollBarPolicy(QtCore.Qt.Vertical,

326

QtCore.Qt.ScrollBarAlwaysOff)

327

self.page.mainFrame().setScrollBarPolicy(QtCore.Qt.Horizontal,

328

QtCore.Qt.ScrollBarAlwaysOff)

329

330

self.set_viewport_size(*viewport_size)

331

332

# Page signals

333

self.page.loadFinished.connect(self._page_loaded)

334

self.page.loadStarted.connect(self._page_load_started)

335

self.page.unsupportedContent.connect(self._unsupported_content)

336

337

self.manager = self.page.networkAccessManager()

338

self.manager.finished.connect(self._request_ended)

339

self.manager.sslErrors.connect(self._on_manager_ssl_errors)

340

# Cache

341

if cache_dir:

342

self.cache = QNetworkDiskCache()

343

self.cache.setCacheDirectory(cache_dir)

344

self.manager.setCache(self.cache)

345

else:

346

self.cache = None

347

# Cookie jar

348

self.cookie_jar = QNetworkCookieJar()

349

self.manager.setCookieJar(self.cookie_jar)

350

# User Agent

351

self.page.setUserAgent(self.user_agent)

352

353

self.page.networkAccessManager().authenticationRequired\

354

.connect(self._authenticate)

355

self.page.networkAccessManager().proxyAuthenticationRequired\

356

.connect(self._authenticate)

357

358

self.main_frame = self.page.mainFrame()

359

360

logger.setLevel(log_level)

361

362

class GhostQWebView(QtWebKit.QWebView):

363

def sizeHint(self):

364

return QSize(*viewport_size)

365

366

self.webview = GhostQWebView()

367

368

if plugins_enabled:

369

self.webview.settings().setAttribute(

370

QtWebKit.QWebSettings.PluginsEnabled, True)

371

if java_enabled:

372

self.webview.settings().setAttribute(

373

QtWebKit.QWebSettings.JavaEnabled, True)

374

375

self.webview.setPage(self.page)

376

377

if self.display:

378

self.webview.show()

379

380

def __del__(self):

381

self.exit()

382

383

def ascend_to_root_frame(self):

384

""" Set main frame as current main frame's parent.

385

"""

386

# we can't ascend directly to parent frame because it might have been

387

# deleted

388

self.main_frame = self.page.mainFrame()

389

390

def descend_frame(self, child_name):

391

""" Set main frame as one of current main frame's children.

392

393

:param child_name: The name of the child to descend to.

394

"""

395

for frame in self.main_frame.childFrames():

396

if frame.frameName() == child_name:

397

self.main_frame = frame

398

return

399

# frame not found so we throw an exception

400

raise LookupError("Child frame '%s' not found." % child_name)

401

402

def capture(self, region=None, selector=None,

403

format=QImage.Format_ARGB32_Premultiplied):

404

"""Returns snapshot as QImage.

405

406

:param region: An optional tuple containing region as pixel

407

coodinates.

408

:param selector: A selector targeted the element to crop on.

409

:param format: The output image format.

410

"""

411

412

self.main_frame.setScrollBarPolicy(QtCore.Qt.Vertical,

413

QtCore.Qt.ScrollBarAlwaysOff)

414

self.main_frame.setScrollBarPolicy(QtCore.Qt.Horizontal,

415

QtCore.Qt.ScrollBarAlwaysOff)

416

self.page.setViewportSize(self.main_frame.contentsSize())

417

image = QImage(self.page.viewportSize(), format)

418

painter = QPainter(image)

419

self.main_frame.render(painter)

420

painter.end()

421

422

if region is None and selector is not None:

423

region = self.region_for_selector(selector)

424

425

if region:

426

x1, y1, x2, y2 = region

427

w, h = (x2 - x1), (y2 - y1)

428

image = image.copy(x1, y1, w, h)

429

430

return image

431

432

def capture_to(self, path, region=None, selector=None,

433

format=QImage.Format_ARGB32_Premultiplied):

434

"""Saves snapshot as image.

435

436

:param path: The destination path.

437

:param region: An optional tuple containing region as pixel

438

coodinates.

439

:param selector: A selector targeted the element to crop on.

440

:param format: The output image format.

441

"""

442

self.capture(region=region, format=format,

443

selector=selector).save(path)

444

445

def print_to_pdf(self, path, paper_size=(8.5, 11.0),

446

paper_margins=(0, 0, 0, 0), paper_units=QPrinter.Inch,

447

zoom_factor=1.0):

448

"""Saves page as a pdf file.

449

450

See qt4 QPrinter documentation for more detailed explanations

451

of options.

452

453

:param path: The destination path.

454

:param paper_size: A 2-tuple indicating size of page to print to.

455

:param paper_margins: A 4-tuple indicating size of each margin.

456

:param paper_units: Units for pager_size, pager_margins.

457

:param zoom_factor: Scale the output content.

458

"""

459

assert len(paper_size) == 2

460

assert len(paper_margins) == 4

461

printer = QPrinter(mode=QPrinter.ScreenResolution)

462

printer.setOutputFormat(QPrinter.PdfFormat)

463

printer.setPaperSize(QtCore.QSizeF(*paper_size), paper_units)

464

printer.setPageMargins(*(paper_margins + (paper_units,)))

465

printer.setFullPage(True)

466

printer.setOutputFileName(path)

467

if self.webview is None:

468

self.webview = QtWebKit.QWebView()

469

self.webview.setPage(self.page)

470

self.webview.setZoomFactor(zoom_factor)

471

self.webview.print_(printer)

472

473

@can_load_page

474

def click(self, selector):

475

"""Click the targeted element.

476

477

:param selector: A CSS3 selector to targeted element.

478

"""

479

if not self.exists(selector):

480

raise Error("Can't find element to click")

481

return self.evaluate("""

482

(function () {

483

var element = document.querySelector(%s);

484

var evt = document.createEvent("MouseEvents");

485

evt.initMouseEvent("click", true, true, window, 1, 1, 1, 1, 1,

486

false, false, false, false, 0, element);

487

return element.dispatchEvent(evt);

488

})();

489

""" % repr(selector))

490

491

class confirm:

492

"""Statement that tells Ghost how to deal with javascript confirm().

493

494

:param confirm: A boolean to set confirmation.

495

:param callable: A callable that returns a boolean for confirmation.

496

"""

497

def __init__(self, confirm=True, callback=None):

498

self.confirm = confirm

499

self.callback = callback

500

501

def __enter__(self):

502

Ghost._confirm_expected = (self.confirm, self.callback)

503

504

def __exit__(self, type, value, traceback):

505

Ghost._confirm_expected = None

506

507

@property

508

def content(self, to_unicode=True):

509

"""Returns current frame HTML as a string.

510

511

:param to_unicode: Whether to convert html to unicode or not

512

"""

513

if to_unicode:

514

return unicode(self.main_frame.toHtml())

515

else:

516

return self.main_frame.toHtml()

517

518

@property

519

def cookies(self):

520

"""Returns all cookies."""

521

return self.cookie_jar.allCookies()

522

523

def delete_cookies(self):

524

"""Deletes all cookies."""

525

self.cookie_jar.setAllCookies([])

526

527

def clear_alert_message(self):

528

"""Clears the alert message"""

529

self._alert = None

530

531

@can_load_page

532

def evaluate(self, script):

533

"""Evaluates script in page frame.

534

535

:param script: The script to evaluate.

536

"""

537

return (self.main_frame.evaluateJavaScript("%s" % script),

538

self._release_last_resources())

539

540

def evaluate_js_file(self, path, encoding='utf-8', **kwargs):

541

"""Evaluates javascript file at given path in current frame.

542

Raises native IOException in case of invalid file.

543

544

:param path: The path of the file.

545

:param encoding: The file's encoding.

546

"""

547

with codecs.open(path, encoding=encoding) as f:

548

return self.evaluate(f.read(), **kwargs)

549

550

def exists(self, selector):

551

"""Checks if element exists for given selector.

552

553

:param string: The element selector.

554

"""

555

return not self.main_frame.findFirstElement(selector).isNull()

556

557

def exit(self):

558

"""Exits application and related."""

559

if self.display:

560

self.webview.close()

561

Ghost._app.quit()

562

del self.manager

563

del self.page

564

del self.main_frame

565

if hasattr(self, 'xvfb'):

566

self.xvfb.terminate()

567

568

@can_load_page

569

def fill(self, selector, values):

570

"""Fills a form with provided values.

571

572

:param selector: A CSS selector to the target form to fill.

573

:param values: A dict containing the values.

574

"""

575

if not self.exists(selector):

576

raise Error("Can't find form")

577

resources = []

578

for field in values:

579

r, res = self.set_field_value(

580

"%s [name=%s]" % (selector, repr(field)), values[field])

581

resources.extend(res)

582

return True, resources

583

584

@can_load_page

585

def fire_on(self, selector, method):

586

"""Call method on element matching given selector.

587

588

:param selector: A CSS selector to the target element.

589

:param method: The name of the method to fire.

590

:param expect_loading: Specifies if a page loading is expected.

591

"""

592

return self.evaluate('document.querySelector(%s)[%s]();' % \

593

(repr(selector), repr(method)))

594

595

def global_exists(self, global_name):

596

"""Checks if javascript global exists.

597

598

:param global_name: The name of the global.

599

"""

600

return self.evaluate('!(typeof this[%s] === "undefined");' %

601

repr(global_name))[0]

602

603

def hide(self):

604

"""Close the webview."""

605

try:

606

self.webview.close()

607

except:

608

raise Error("no webview to close")

609

610

def load_cookies(self, cookie_storage, keep_old=False):

611

"""load from cookielib's CookieJar or Set-Cookie3 format text file.

612

613

:param cookie_storage: file location string on disk or CookieJar

614

instance.

615

:param keep_old: Don't reset, keep cookies not overridden.

616

"""

617

def toQtCookieJar(PyCookieJar, QtCookieJar):

618

allCookies = QtCookieJar.allCookies() if keep_old else []

619

for pc in PyCookieJar:

620

qc = toQtCookie(pc)

621

allCookies.append(qc)

622

QtCookieJar.setAllCookies(allCookies)

623

624

def toQtCookie(PyCookie):

625

qc = QNetworkCookie(PyCookie.name, PyCookie.value)

626

qc.setSecure(PyCookie.secure)

627

if PyCookie.path_specified:

628

qc.setPath(PyCookie.path)

629

if PyCookie.domain != "":

630

qc.setDomain(PyCookie.domain)

631

if PyCookie.expires and PyCookie.expires != 0:

632

t = QDateTime()

633

t.setTime_t(PyCookie.expires)

634

qc.setExpirationDate(t)

635

# not yet handled(maybe less useful):

636

# py cookie.rest / QNetworkCookie.setHttpOnly()

637

return qc

638

639

if cookie_storage.__class__.__name__ == 'str':

640

cj = LWPCookieJar(cookie_storage)

641

cj.load()

642

toQtCookieJar(cj, self.cookie_jar)

643

elif cookie_storage.__class__.__name__.endswith('CookieJar'):

644

toQtCookieJar(cookie_storage, self.cookie_jar)

645

else:

646

raise ValueError('unsupported cookie_storage type.')

647

648

def open(self, address, method='get', headers={}, auth=None, body=None,

649

default_popup_response=None, wait=True):

650

"""Opens a web page.

651

652

:param address: The resource URL.

653

:param method: The Http method.

654

:param headers: An optional dict of extra request hearders.

655

:param auth: An optional tuple of HTTP auth (username, password).

656

:param body: An optional string containing a payload.

657

:param default_popup_response: the default response for any confirm/

658

alert/prompt popup from the Javascript (replaces the need for the with

659

blocks)

660

:param wait: If set to True (which is the default), this

661

method call waits for the page load to complete before

662

returning. Otherwise, it just starts the page load task and

663

it is the caller's responsibilty to wait for the load to

664

finish by other means (e.g. by calling wait_for_page_loaded()).

665

:return: Page resource, and all loaded resources, unless wait

666

is False, in which case it returns None.

667

"""

668

body = body or QByteArray()

669

try:

670

method = getattr(QNetworkAccessManager,

671

"%sOperation" % method.capitalize())

672

except AttributeError:

673

raise Error("Invalid http method %s" % method)

674

request = QNetworkRequest(QUrl(address))

675

request.CacheLoadControl(0)

676

for header in headers:

677

request.setRawHeader(header, headers[header])

678

self._auth = auth

679

self._auth_attempt = 0 # Avoids reccursion

680

681

self.main_frame.load(request, method, body)

682

self.loaded = False

683

684

if default_popup_response is not None:

685

Ghost._prompt_expected = (default_popup_response, None)

686

Ghost._confirm_expected = (default_popup_response, None)

687

688

if wait:

689

return self.wait_for_page_loaded()

690

691

def scroll_to_anchor(self, anchor):

692

self.main_frame.scrollToAnchor(anchor)

693

694

class prompt:

695

"""Statement that tells Ghost how to deal with javascript prompt().

696

697

:param value: A string value to fill in prompt.

698

:param callback: A callable that returns the value to fill in.

699

"""

700

def __init__(self, value='', callback=None):

701

self.value = value

702

self.callback = callback

703

704

def __enter__(self):

705

Ghost._prompt_expected = (self.value, self.callback)

706

707

def __exit__(self, type, value, traceback):

708

Ghost._prompt_expected = None

709

710

def region_for_selector(self, selector):

711

"""Returns frame region for given selector as tuple.

712

713

:param selector: The targeted element.

714

"""

715

geo = self.main_frame.findFirstElement(selector).geometry()

716

try:

717

region = (geo.left(), geo.top(), geo.right(), geo.bottom())

718

except:

719

raise Error("can't get region for selector '%s'" % selector)

720

return region

721

722

def save_cookies(self, cookie_storage):

723

"""Save to cookielib's CookieJar or Set-Cookie3 format text file.

724

725

:param cookie_storage: file location string or CookieJar instance.

726

"""

727

def toPyCookieJar(QtCookieJar, PyCookieJar):

728

for c in QtCookieJar.allCookies():

729

PyCookieJar.set_cookie(toPyCookie(c))

730

731

def toPyCookie(QtCookie):

732

port = None

733

port_specified = False

734

secure = QtCookie.isSecure()

735

name = str(QtCookie.name())

736

value = str(QtCookie.value())

737

v = str(QtCookie.path())

738

path_specified = bool(v != "")

739

path = v if path_specified else None

740

v = str(QtCookie.domain())

741

domain_specified = bool(v != "")

742

domain = v

743

if domain_specified:

744

domain_initial_dot = v.startswith('.')

745

else:

746

domain_initial_dot = None

747

v = long(QtCookie.expirationDate().toTime_t())

748

# Long type boundary on 32bit platfroms; avoid ValueError

749

expires = 2147483647 if v > 2147483647 else v

750

rest = {}

751

discard = False

752

return Cookie(0, name, value, port, port_specified, domain,

753

domain_specified, domain_initial_dot, path, path_specified,

754

secure, expires, discard, None, None, rest)

755

756

if cookie_storage.__class__.__name__ == 'str':

757

cj = LWPCookieJar(cookie_storage)

758

toPyCookieJar(self.cookie_jar, cj)

759

cj.save()

760

elif cookie_storage.__class__.__name__.endswith('CookieJar'):

761

toPyCookieJar(self.cookie_jar, cookie_storage)

762

else:

763

raise ValueError('unsupported cookie_storage type.')

764

765

@can_load_page

766

def set_field_value(self, selector, value, blur=True):

767

"""Sets the value of the field matched by given selector.

768

769

:param selector: A CSS selector that target the field.

770

:param value: The value to fill in.

771

:param blur: An optional boolean that force blur when filled in.

772

"""

773

def _set_checkbox_value(el, value):

774

el.setFocus()

775

if value is True:

776

el.setAttribute('checked', 'checked')

777

else:

778

el.removeAttribute('checked')

779

780

def _set_checkboxes_value(els, value):

781

for el in els:

782

if el.attribute('value') == value:

783

_set_checkbox_value(el, True)

784

else:

785

_set_checkbox_value(el, False)

786

787

def _set_radio_value(els, value):

788

for el in els:

789

if el.attribute('value') == value:

790

el.setFocus()

791

el.setAttribute('checked', 'checked')

792

793

def _set_text_value(el, value):

794

el.setFocus()

795

el.setAttribute('value', value)

796

797

def _set_select_value(el, value):

798

el.setFocus()

799

self.evaluate('document.querySelector(%s).value = %s;' %

800

(repr(selector), repr(value)))

801

802

def _set_textarea_value(el, value):

803

el.setFocus()

804

el.setPlainText(value)

805

806

res, ressources = None, []

807

element = self.main_frame.findFirstElement(selector)

808

if element.isNull():

809

raise Error('can\'t find element for %s"' % selector)

810

811

tag_name = str(element.tagName()).lower()

812

813

if tag_name == "select":

814

_set_select_value(element, value)

815

elif tag_name == "textarea":

816

_set_textarea_value(element, value)

817

elif tag_name == "input":

818

type_ = str(element.attribute('type')).lower()

819

if type_ in [

820

"color", "date", "datetime",

821

"datetime-local", "email", "hidden", "month", "number",

822

"password", "range", "search", "tel", "text", "time",

823

"url", "week", ""]:

824

_set_text_value(element, value)

825

elif type_ == "checkbox":

826

els = self.main_frame.findAllElements(selector)

827

if els.count() > 1:

828

_set_checkboxes_value(els, value)

829

else:

830

_set_checkbox_value(element, value)

831

elif type_ == "radio":

832

_set_radio_value(self.main_frame.findAllElements(selector),

833

value)

834

elif type_ == "file":

835

Ghost._upload_file = value

836

res, resources = self.click(selector)

837

Ghost._upload_file = None

838

else:

839

raise Error('unsuported field tag')

840

if blur:

841

self.fire_on(selector, 'blur')

842

return res, ressources

843

844

def set_proxy(self, type_, host='localhost', port=8888, user='',

845

password=''):

846

"""Set up proxy for FURTHER connections.

847

848

:param type_: proxy type to use: \

849

none/default/socks5/https/http.

850

:param host: proxy server ip or host name.

851

:param port: proxy port.

852

"""

853

_types = {

854

'default': QNetworkProxy.DefaultProxy,

855

'none': QNetworkProxy.NoProxy,

856

'socks5': QNetworkProxy.Socks5Proxy,

857

'https': QNetworkProxy.HttpProxy,

858

'http': QNetworkProxy.HttpCachingProxy

859

}

860

861

if type_ is None:

862

type_ = 'none'

863

type_ = type_.lower()

864

if type_ in ['none', 'default']:

865

self.manager.setProxy(QNetworkProxy(_types[type_]))

866

return

867

elif type_ in _types:

868

proxy = QNetworkProxy(_types[type_], hostName=host, port=port,

869

user=user, password=password)

870

self.manager.setProxy(proxy)

871

else:

872

raise ValueError('Unsupported proxy type:' + type_ \

873

+ '\nsupported types are: none/socks5/http/https/default')

874

875

def set_viewport_size(self, width, height):

876

"""Sets the page viewport size.

877

878

:param width: An integer that sets width pixel count.

879

:param height: An integer that sets height pixel count.

880

"""

881

self.page.setViewportSize(QSize(width, height))

882

883

def append_popup_message(self, message):

884

self.popup_messages.append(unicode(message))

885

886

def show(self):

887

"""Show current page inside a QWebView.

888

"""

889

self.webview.show()

890

891

def sleep(self, value):

892

started_at = time.time()

893

894

time.sleep(0)

895

Ghost._app.processEvents()

896

while time.time() <= (started_at + value):

897

time.sleep(0.01)

898

Ghost._app.processEvents()

899

900

def wait_for(self, condition, timeout_message):

901

"""Waits until condition is True.

902

903

:param condition: A callable that returns the condition.

904

:param timeout_message: The exception message on timeout.

905

"""

906

started_at = time.time()

907

while not condition():

908

if time.time() > (started_at + self.wait_timeout):

909

raise TimeoutError(timeout_message)

910

time.sleep(0.01)

911

Ghost._app.processEvents()

912

if self.wait_callback is not None:

913

self.wait_callback()

914

915

def wait_for_alert(self):

916

"""Waits for main frame alert().

917

"""

918

self.wait_for(lambda: Ghost._alert is not None,

919

'User has not been alerted.')

920

msg = Ghost._alert

921

Ghost._alert = None

922

return msg, self._release_last_resources()

923

924

def wait_for_page_loaded(self):

925

"""Waits until page is loaded, assumed that a page as been requested.

926

"""

927

self.wait_for(lambda: self.loaded,

928

'Unable to load requested page')

929

resources = self._release_last_resources()

930

page = None

931

932

url = self.main_frame.url().toString()

933

url_without_hash = url.split("#")[0]

934

935

for resource in resources:

936

if url == resource.url or url_without_hash == resource.url:

937

page = resource

938

return page, resources

939

940

def wait_for_selector(self, selector):

941

"""Waits until selector match an element on the frame.

942

943

:param selector: The selector to wait for.

944

"""

945

self.wait_for(lambda: self.exists(selector),

946

'Can\'t find element matching "%s"' % selector)

947

return True, self._release_last_resources()

948

949

def wait_while_selector(self, selector):

950

"""Waits until the selector no longer matches an element on the frame.

951

952

:param selector: The selector to wait for.

953

"""

954

self.wait_for(lambda: not self.exists(selector),

955

'Element matching "%s" is still available' % selector)

956

return True, self._release_last_resources()

957

958

def wait_for_text(self, text):

959

"""Waits until given text appear on main frame.

960

961

:param text: The text to wait for.

962

"""

963

self.wait_for(lambda: text in self.content,

964

'Can\'t find "%s" in current frame' % text)

965

return True, self._release_last_resources()

966

967

def _authenticate(self, mix, authenticator):

968

"""Called back on basic / proxy http auth.

969

970

:param mix: The QNetworkReply or QNetworkProxy object.

971

:param authenticator: The QAuthenticator object.

972

"""

973

if self._auth is not None and self._auth_attempt == 0:

974

username, password = self._auth

975

authenticator.setUser(username)

976

authenticator.setPassword(password)

977

self._auth_attempt += 1

978

979

def _page_loaded(self):

980

"""Called back when page is loaded.

981

"""

982

self.loaded = True

983

if self.cache:

984

self.cache.clear()

985

986

def _page_load_started(self):

987

"""Called back when page load started.

988

"""

989

self.loaded = False

990

991

def _release_last_resources(self):

992

"""Releases last loaded resources.

993

994

:return: The released resources.

995

"""

996

last_resources = self.http_resources

997

self.http_resources = []

998

return last_resources

999

1000

def _request_ended(self, reply):

1001

"""Adds an HttpResource object to http_resources.

1002

1003

:param reply: The QNetworkReply object.

1004

"""

1005

1006

if reply.attribute(QNetworkRequest.HttpStatusCodeAttribute):

1007

Logger.log("[%s] bytesAvailable()= %s" % (str(reply.url()),

1008

reply.bytesAvailable()), level="debug")

1009

1010

# Some web pages return cache headers that mandates not to cache

1011

# the reply, which means we won't find this QNetworkReply in

1012

# the cache object. In this case bytesAvailable will return > 0.

1013

# Such pages are www.etsy.com

1014

# This is a bit of a hack and due to the async nature of QT, might

1015

# not work at times. We should move to using some proxied

1016

# implementation of QNetworkManager and QNetworkReply in order to

1017

# get the contents of the requests properly rather than relying

1018

# on the cache.

1019

if reply.bytesAvailable() > 0:

1020

content = reply.peek(reply.bytesAvailable())

1021

else:

1022

content = None

1023

self.http_resources.append(HttpResource(reply, self.cache,

1024

content=content))

1025

1026

def _unsupported_content(self, reply):

1027

reply.readyRead.connect(

1028

lambda reply=reply: self._reply_download_content(reply))

1029

1030

def _reply_download_content(self, reply):

1031

"""Adds an HttpResource object to http_resources with unsupported

1032

content.

1033

1034

:param reply: The QNetworkReply object.

1035

"""

1036

if reply.attribute(QNetworkRequest.HttpStatusCodeAttribute):

1037

self.http_resources.append(HttpResource(reply, self.cache,

1038

reply.readAll()))

1039

1040

def _on_manager_ssl_errors(self, reply, errors):

1041

url = unicode(reply.url().toString())

1042

if self.ignore_ssl_errors:

1043

reply.ignoreSslErrors()

1044

else:

1045

Logger.log('SSL certificate error: %s' % url, level='warning')

1046

1047

def __enter__(self):

1048

return self

1049

1050

def __exit__(self, exc_type, exc_val, exc_tb):

1051

self.exit()

1052