~canonical-ci-engineering/charms/trusty/gunicorn/gunicorn-py3-support : revision 27

1

#!/usr/bin/env python

2

# vim: et ai ts=4 sw=4:

3

4

import json

5

import os

6

import re

7

import subprocess

8

import sys

9

import time

10

from pwd import getpwnam

11

from grp import getgrnam

12

13

CHARM_PACKAGES = ["gunicorn"]

14

15

INJECTED_WARNING = """

16

#------------------------------------------------------------------------------

17

# The following is the import code for the settings directory injected by Juju

18

#------------------------------------------------------------------------------

19

"""

20

21

22

###############################################################################

23

# Supporting functions

24

###############################################################################

25

MSG_CRITICAL = "CRITICAL"

26

MSG_DEBUG = "DEBUG"

27

MSG_INFO = "INFO"

28

MSG_ERROR = "ERROR"

29

MSG_WARNING = "WARNING"

30

31

32

def juju_log(level, msg):

33

subprocess.call(['juju-log', '-l', level, msg])

34

35

#------------------------------------------------------------------------------

36

# run: Run a command, return the output

37

#------------------------------------------------------------------------------

38

def run(command, exit_on_error=True, cwd=None):

39

try:

40

juju_log(MSG_DEBUG, command)

41

return subprocess.check_output(

42

command, stderr=subprocess.STDOUT, shell=True, cwd=cwd)

43

except subprocess.CalledProcessError, e:

44

juju_log(MSG_ERROR, "status=%d, output=%s" % (e.returncode, e.output))

45

if exit_on_error:

46

sys.exit(e.returncode)

47

else:

48

raise

49

50

51

#------------------------------------------------------------------------------

52

# install_file: install a file resource. overwites existing files.

53

#------------------------------------------------------------------------------

54

def install_file(contents, dest, owner="root", group="root", mode=0600):

55

uid = getpwnam(owner)[2]

56

gid = getgrnam(group)[2]

57

dest_fd = os.open(dest, os.O_WRONLY | os.O_TRUNC | os.O_CREAT, mode)

58

os.fchown(dest_fd, uid, gid)

59

with os.fdopen(dest_fd, 'w') as destfile:

60

destfile.write(str(contents))

61

62

63

#------------------------------------------------------------------------------

64

# install_dir: create a directory

65

#------------------------------------------------------------------------------

66

def install_dir(dirname, owner="root", group="root", mode=0700):

67

command = \

68

'/usr/bin/install -o {} -g {} -m {} -d {}'.format(owner, group, oct(mode),

69

dirname)

70

return run(command)

71

72

#------------------------------------------------------------------------------

73

# config_get: Returns a dictionary containing all of the config information

74

# Optional parameter: scope

75

# scope: limits the scope of the returned configuration to the

76

# desired config item.

77

#------------------------------------------------------------------------------

78

def config_get(scope=None):

79

try:

80

config_cmd_line = ['config-get']

81

if scope is not None:

82

config_cmd_line.append(scope)

83

config_cmd_line.append('--format=json')

84

config_data = json.loads(subprocess.check_output(config_cmd_line))

85

except:

86

config_data = None

87

finally:

88

return(config_data)

89

90

91

#------------------------------------------------------------------------------

92

# relation_json: Returns json-formatted relation data

93

# Optional parameters: scope, relation_id

94

# scope: limits the scope of the returned data to the

95

# desired item.

96

# unit_name: limits the data ( and optionally the scope )

97

# to the specified unit

98

# relation_id: specify relation id for out of context usage.

99

#------------------------------------------------------------------------------

100

def relation_json(scope=None, unit_name=None, relation_id=None):

101

command = ['relation-get', '--format=json']

102

if relation_id is not None:

103

command.extend(('-r', relation_id))

104

if scope is not None:

105

command.append(scope)

106

else:

107

command.append('-')

108

if unit_name is not None:

109

command.append(unit_name)

110

output = subprocess.check_output(command, stderr=subprocess.STDOUT)

111

return output or None

112

113

114

#------------------------------------------------------------------------------

115

# relation_get: Returns a dictionary containing the relation information

116

# Optional parameters: scope, relation_id

117

# scope: limits the scope of the returned data to the

118

# desired item.

119

# unit_name: limits the data ( and optionally the scope )

120

# to the specified unit

121

#------------------------------------------------------------------------------

122

def relation_get(scope=None, unit_name=None, relation_id=None):

123

j = relation_json(scope, unit_name, relation_id)

124

if j:

125

return json.loads(j)

126

else:

127

return None

128

129

130

def relation_set(keyvalues, relation_id=None):

131

args = []

132

if relation_id:

133

args.extend(['-r', relation_id])

134

args.extend(["{}='{}'".format(k, v or '') for k, v in keyvalues.items()])

135

run("relation-set {}".format(' '.join(args)))

136

137

## Posting json to relation-set doesn't seem to work as documented?

138

## Bug #1116179

139

##

140

## cmd = ['relation-set']

141

## if relation_id:

142

## cmd.extend(['-r', relation_id])

143

## p = Popen(

144

## cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE,

145

## stderr=subprocess.PIPE)

146

## (out, err) = p.communicate(json.dumps(keyvalues))

147

## if p.returncode:

148

## juju_log(MSG_ERROR, err)

149

## sys.exit(1)

150

## juju_log(MSG_DEBUG, "relation-set {}".format(repr(keyvalues)))

151

152

153

def relation_list(relation_id=None):

154

"""Return the list of units participating in the relation."""

155

if relation_id is None:

156

relation_id = os.environ['JUJU_RELATION_ID']

157

cmd = ['relation-list', '--format=json', '-r', relation_id]

158

json_units = subprocess.check_output(cmd).strip()

159

if json_units:

160

return json.loads(subprocess.check_output(cmd))

161

return []

162

163

164

#------------------------------------------------------------------------------

165

# relation_ids: Returns a list of relation ids

166

# optional parameters: relation_type

167

# relation_type: return relations only of this type

168

#------------------------------------------------------------------------------

169

def relation_ids(relation_types=('db',)):

170

# accept strings or iterators

171

if isinstance(relation_types, basestring):

172

reltypes = [relation_types, ]

173

else:

174

reltypes = relation_types

175

relids = []

176

for reltype in reltypes:

177

relid_cmd_line = ['relation-ids', '--format=json', reltype]

178

json_relids = subprocess.check_output(relid_cmd_line).strip()

179

if json_relids:

180

relids.extend(json.loads(json_relids))

181

return relids

182

183

184

#------------------------------------------------------------------------------

185

# relation_get_all: Returns a dictionary containing the relation information

186

# optional parameters: relation_type

187

# relation_type: limits the scope of the returned data to the

188

# desired item.

189

#------------------------------------------------------------------------------

190

def relation_get_all(*args, **kwargs):

191

relation_data = []

192

relids = relation_ids(*args, **kwargs)

193

for relid in relids:

194

units_cmd_line = ['relation-list', '--format=json', '-r', relid]

195

json_units = subprocess.check_output(units_cmd_line).strip()

196

if json_units:

197

for unit in json.loads(json_units):

198

unit_data = \

199

json.loads(relation_json(relation_id=relid,

200

unit_name=unit))

201

for key in unit_data:

202

if key.endswith('-list'):

203

unit_data[key] = unit_data[key].split()

204

unit_data['relation-id'] = relid

205

unit_data['unit'] = unit

206

relation_data.append(unit_data)

207

return relation_data

208

209

def apt_get_update():

210

cmd_line = ['apt-get', 'update']

211

return(subprocess.call(cmd_line))

212

213

214

#------------------------------------------------------------------------------

215

# apt_get_install( packages ): Installs package(s)

216

#------------------------------------------------------------------------------

217

def apt_get_install(packages=None):

218

if packages is None:

219

return(False)

220

cmd_line = ['apt-get', '-y', 'install', '-qq']

221

if isinstance(packages, list):

222

cmd_line.extend(packages)

223

else:

224

cmd_line.append(packages)

225

return(subprocess.call(cmd_line))

226

227

228

#------------------------------------------------------------------------------

229

# pip_install( package ): Installs a python package

230

#------------------------------------------------------------------------------

231

def pip_install(packages=None, upgrade=False):

232

cmd_line = ['pip', 'install']

233

if packages is None:

234

return(False)

235

if upgrade:

236

cmd_line.append('-u')

237

if packages.startswith('svn+') or packages.startswith('git+') or \

238

packages.startswith('hg+') or packages.startswith('bzr+'):

239

cmd_line.append('-e')

240

cmd_line.append(packages)

241

return run(cmd_line)

242

243

#------------------------------------------------------------------------------

244

# pip_install_req( path ): Installs a requirements file

245

#------------------------------------------------------------------------------

246

def pip_install_req(path=None, upgrade=False):

247

cmd_line = ['pip', 'install']

248

if path is None:

249

return(False)

250

if upgrade:

251

cmd_line.append('-u')

252

cmd_line.append('-r')

253

cmd_line.append(path)

254

cwd = os.path.dirname(path)

255

return run(cmd_line)

256

257

#------------------------------------------------------------------------------

258

# open_port: Convenience function to open a port in juju to

259

# expose a service

260

#------------------------------------------------------------------------------

261

def open_port(port=None, protocol="TCP"):

262

if port is None:

263

return(None)

264

return(subprocess.call(['open-port', "%d/%s" %

265

(int(port), protocol)]))

266

267

268

#------------------------------------------------------------------------------

269

# close_port: Convenience function to close a port in juju to

270

# unexpose a service

271

#------------------------------------------------------------------------------

272

def close_port(port=None, protocol="TCP"):

273

if port is None:

274

return(None)

275

return(subprocess.call(['close-port', "%d/%s" %

276

(int(port), protocol)]))

277

278

279

#------------------------------------------------------------------------------

280

# update_service_ports: Convenience function that evaluate the old and new

281

# service ports to decide which ports need to be

282

# opened and which to close

283

#------------------------------------------------------------------------------

284

def update_service_port(old_service_port=None, new_service_port=None):

285

if old_service_port is None or new_service_port is None:

286

return(None)

287

if new_service_port != old_service_port:

288

close_port(old_service_port)

289

open_port(new_service_port)

290

291

#

292

# Utils

293

#

294

295

def install_or_append(contents, dest, owner="root", group="root", mode=0600):

296

if os.path.exists(dest):

297

uid = getpwnam(owner)[2]

298

gid = getgrnam(group)[2]

299

dest_fd = os.open(dest, os.O_APPEND, mode)

300

os.fchown(dest_fd, uid, gid)

301

with os.fdopen(dest_fd, 'a') as destfile:

302

destfile.write(str(contents))

303

else:

304

install_file(contents, dest, owner, group, mode)

305

306

def token_sql_safe(value):

307

# Only allow alphanumeric + underscore in database identifiers

308

if re.search('[^A-Za-z0-9_]', value):

309

return False

310

return True

311

312

def sanitize(s):

313

s = s.replace(':', '_')

314

s = s.replace('-', '_')

315

s = s.replace('/', '_')

316

s = s.replace('"', '_')

317

s = s.replace("'", '_')

318

return s

319

320

def user_name(relid, remote_unit, admin=False, schema=False):

321

components = [sanitize(relid), sanitize(remote_unit)]

322

if admin:

323

components.append("admin")

324

elif schema:

325

components.append("schema")

326

return "_".join(components)

327

328

def get_relation_host():

329

remote_host = run("relation-get ip")

330

if not remote_host:

331

# remote unit $JUJU_REMOTE_UNIT uses deprecated 'ip=' component of

332

# interface.

333

remote_host = run("relation-get private-address")

334

return remote_host

335

336

337

def get_unit_host():

338

this_host = run("unit-get private-address")

339

return this_host.strip()

340

341

def process_template(template_name, template_vars, destination):

342

# --- exported service configuration file

343

from jinja2 import Environment, FileSystemLoader

344

template_env = Environment(

345

loader=FileSystemLoader(os.path.join(os.environ['CHARM_DIR'],

346

'templates')))

347

348

template = \

349

template_env.get_template(template_name).render(template_vars)

350

351

with open(destination, 'w') as inject_file:

352

inject_file.write(str(template))

353

354

def append_template(template_name, template_vars, path, try_append=False):

355

356

# --- exported service configuration file

357

from jinja2 import Environment, FileSystemLoader

358

template_env = Environment(

359

loader=FileSystemLoader(os.path.join(os.environ['CHARM_DIR'],

360

'templates')))

361

362

template = \

363

template_env.get_template(template_name).render(template_vars)

364

365

append = False

366

if os.path.exists(path):

367

with open(path, 'r') as inject_file:

368

if not str(template) in inject_file:

369

append = True

370

else:

371

append = True

372

373

if append == True:

374

with open(path, 'a') as inject_file:

375

inject_file.write(INJECTED_WARNING)

376

inject_file.write(str(template))

377

378

379

380

###############################################################################

381

# Hook functions

382

###############################################################################

383

def install():

384

385

for retry in xrange(0,24):

386

if apt_get_install(CHARM_PACKAGES):

387

time.sleep(10)

388

else:

389

break

390

391

def upgrade():

392

393

apt_get_update()

394

for retry in xrange(0,24):

395

if apt_get_install(CHARM_PACKAGES):

396

time.sleep(10)

397

else:

398

break

399

400

def wsgi_file_relation_joined_changed():

401

wsgi_config = config_data

402

relation_id = os.environ['JUJU_RELATION_ID']

403

juju_log(MSG_INFO, "JUJU_RELATION_ID: %s".format(relation_id))

404

405

remote_unit_name = sanitize(os.environ['JUJU_REMOTE_UNIT'].split('/')[0])

406

juju_log(MSG_INFO, "JUJU_REMOTE_UNIT: %s".format(remote_unit_name))

407

wsgi_config['unit_name'] = remote_unit_name

408

409

project_conf = '/etc/init/%s.conf' % remote_unit_name

410

411

working_dir = relation_get('working_dir')

412

if not working_dir:

413

return

414

415

wsgi_config['working_dir'] = working_dir

416

wsgi_config['project_name'] = remote_unit_name

417

418

for v in wsgi_config.keys():

419

if v.startswith('wsgi_') or v in ['python_path', 'listen_ip', 'port']:

420

upstream_value = relation_get(v)

421

if upstream_value:

422

wsgi_config[v] = upstream_value

423

424

if wsgi_config['wsgi_worker_class'] == 'eventlet':

425

apt_get_install('python-eventlet')

426

elif wsgi_config['wsgi_worker_class'] == 'gevent':

427

apt_get_install('python-gevent')

428

elif wsgi_config['wsgi_worker_class'] == 'tornado':

429

apt_get_install('python-tornado')

430

431

if wsgi_config['wsgi_workers'] == 0:

432

res = run('python -c "import multiprocessing ; print(multiprocessing.cpu_count())"')

433

wsgi_config['wsgi_workers'] = int(res) + 1

434

435

if wsgi_config['wsgi_access_logfile']:

436

wsgi_config['wsgi_extra'] = " ".join([

437

wsgi_config['wsgi_extra'],

438

'--access-logformat=%s' % wsgi_config['wsgi_access_logfile'],

439

'--access-logformat="%s"' % wsgi_config['wsgi_access_logformat']

440

])

441

442

wsgi_config['wsgi_wsgi_file'] = wsgi_config['wsgi_wsgi_file']

443

444

process_template('upstart.tmpl', wsgi_config, project_conf)

445

446

447

# We need this because when the contained charm configuration or code changed

448

# Gunicorn needs to restart to run the new code.

449

run("service %s restart || service %s start" % (remote_unit_name, remote_unit_name))

450

451

452

def wsgi_file_relation_broken():

453

remote_unit_name = sanitize(os.environ['JUJU_REMOTE_UNIT'].split('/')[0])

454

455

run('service %s stop' % remote_unit_name)

456

run('rm /etc/init/%s.conf' % remote_unit_name)

457

458

459

###############################################################################

460

# Global variables

461

###############################################################################

462

config_data = config_get()

463

juju_log(MSG_DEBUG, "got config: %s" % str(config_data))

464

465

unit_name = os.environ['JUJU_UNIT_NAME'].split('/')[0]

466

467

hook_name = os.path.basename(sys.argv[0])

468

469

###############################################################################

470

# Main section

471

###############################################################################

472

def main():

473

juju_log(MSG_INFO, "Running {} hook".format(hook_name))

474

if hook_name == "install":

475

install()

476

477

elif hook_name == "upgrade-charm":

478

upgrade()

479

480

elif hook_name in ["wsgi-file-relation-joined", "wsgi-file-relation-changed"]:

481

wsgi_file_relation_joined_changed()

482

483

elif hook_name == "wsgi-file-relation-broken":

484

wsgi_file_relation_broken()

485

486

else:

487

print "Unknown hook {}".format(hook_name)

488

raise SystemExit(1)

489

490

if __name__ == '__main__':

491

raise SystemExit(main())