~ubuntu-branches/ubuntu/wily/openstack-pkg-tools/wily : revision 22

1

#!/bin/sh

2

3

set -e

4

set -x

5

6

#################################

7

# AUTOMATICALLY DETECT ENV HERE #

8

#################################

9

10

if ! [ -r /etc/pkgos/pkgos.conf ] ; then

11

echo "Could not read /etc/pkgos/pkgos.conf"

12

exit 1

13

else

14

. /etc/pkgos/pkgos.conf

15

fi

16

17

detect_env () {

18

DEB_RELEASE=`lsb_release -a | grep Codename: | awk '{print $2}'`

19

DEB_RELEASE_NUM=`lsb_release -a | grep Release: | awk '{print $2}'`

20

APT="apt-get install -y"

21

echo 'APT::Install-Recommends "0";' >/etc/apt/apt.conf.d/80norecommends

22

}

23

24

configure_hostname () {

25

DEFROUTE_IF=`LC_ALL=C /sbin/route | grep default |awk -- '{ print $8 }' | cut -d" " -f1`

26

if [ -n "${DEFROUTE_IF}" ] ; then

27

DEFROUTE_IP=`LC_ALL=C ip addr show "${DEFROUTE_IF}" | grep inet | head -n 1 | awk '{print $2}' | cut -d/ -f1 | grep -E '^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$'`

28

if [ -n "${DEFROUTE_IP}" ] ; then

29

echo "Detected IP: ${DEFROUTE_IP}"

30

echo "127.0.0.1 localhost.localdomain localhost

31

${DEFROUTE_IP} ${HOST_FQDN} ${TARGET_OPENSTACK_REL}-${TARGET_DISTRO}

32

33

# The following lines are desirable for IPv6 capable hosts

34

::1 ip6-localhost ip6-loopback

35

fe00::0 ip6-localnet

36

ff00::0 ip6-mcastprefix

37

ff02::1 ip6-allnodes

38

ff02::2 ip6-allrouters

39

ff02::3 ip6-allhosts

40

" >/etc/hosts

41

fi

42

fi

43

echo $HOST_FQDN >/etc/hostname

44

hostname `cat /etc/hostname`

45

}

46

47

install_all_software () {

48

${APT} linux-image-amd64 sbuild apache2 screen joe apache2 pure-ftpd ftp most \

49

man-db git-buildpackage debhelper eatmydata build-essential python-setuptools \

50

fakeroot python3-all python-all python3-setuptools pristine-tar dh-autoreconf ssl-cert \

51

dh-python dh-systemd python-sphinx sudo debootstrap openstack-pkg-tools \

52

lintian lsb-release postfix

53

if [ "${DEV_OR_JENKINS}" = "jenkins" ] ; then

54

${APT} jenkins jenkins-job-builder jenkins-cli

55

fi

56

}

57

58

configure_apache () {

59

a2enmod proxy

60

a2enmod proxy_http

61

a2enmod ssl

62

a2enmod headers

63

if [ "${DEB_RELEASE}" = "precise" ] ; then

64

APACHE_SSL_VHOST_CONF=default-ssl

65

FORWARD_TO_ADDR=ip6-localhost

66

DEFAULT_SITE=default

67

DEFAULT_SSL_SITE=default-ssl

68

elif [ "${DEB_RELEASE}" = "wheezy" ] ; then

69

APACHE_SSL_VHOST_CONF=default-ssl

70

FORWARD_TO_ADDR=localhost

71

DEFAULT_SITE=default

72

DEFAULT_SSL_SITE=default-ssl

73

else

74

APACHE_SSL_VHOST_CONF=default-ssl.conf

75

FORWARD_TO_ADDR=localhost

76

DEFAULT_SITE=000-default.conf

77

DEFAULT_SSL_SITE=default-ssl.conf

78

fi

79

a2ensite ${DEFAULT_SSL_SITE}

80

APACHE_SSL_VHOST_CONF_FULL_PATH=/etc/apache2/sites-available/${APACHE_SSL_VHOST_CONF}

81

echo "<IfModule mod_ssl.c>

82

83

ServerAdmin webmaster@localhost

84

ErrorLog \${APACHE_LOG_DIR}/error.log

85

CustomLog \${APACHE_LOG_DIR}/access.log combined

86

87

SSLEngine on

88

SSLCertificateFile /etc/ssl/certs/ssl-cert-snakeoil.pem

89

SSLCertificateKeyFile /etc/ssl/private/ssl-cert-snakeoil.key

90

91

92

SSLOptions +StdEnvVars

93

</FilesMatch>

94

95

SSLOptions +StdEnvVars

96

</Directory>

97

98

# Jenkins proxy (and reverse)

99

ProxyPass / http://${FORWARD_TO_ADDR}:8080/ nocanon

100

ProxyPassReverse / http://${FORWARD_TO_ADDR}:8080/

101

ProxyRequests Off

102

AllowEncodedSlashes NoDecode

103

Header edit Location ^http://${HOST_FQDN}/ https://${HOST_FQDN}/

104

RequestHeader set X-Forwarded-Proto \"https\"

105

RequestHeader set X-Forwarded-Port \"443\"

106

ProxyPreserveHost on

107

SetOutputFilter INFLATE;proxy-html;DEFLATE

108

SetEnv proxy-nokeepalive 1

109

110

ServerAdmin webmaster@localhost

111

</VirtualHost>

112

</IfModule>

113

" >${APACHE_SSL_VHOST_CONF_FULL_PATH}

114

a2ensite ${APACHE_SSL_VHOST_CONF}

115

service apache2 restart

116

if [ ! -e /var/www/html/debian ] && [ ! -h /var/www/html/debian ] ; then

117

ln -s /home/ftp/debian /var/www/html/debian

118

fi

119

}

120

121

configure_pure () {

122

if getent passwd ftp >/dev/null ; then

123

echo "FTP user already existing"

124

else

125

/usr/sbin/useradd -m ftp

126

fi

127

chown ${THE_DEV_USER}:${THE_DEV_USER} /home/ftp

128

rm -f /etc/pure-ftpd/conf/NoAnonymous

129

service pure-ftpd stop

130

sleep 2

131

service pure-ftpd start

132

}

133

134

install_jenkins_plugins () {

135

jenkins-cli -s https://${HOST_FQDN}/ -noCertificateCheck install-plugin instant-messaging

136

jenkins-cli -s https://${HOST_FQDN}/ -noCertificateCheck install-plugin ircbot

137

}

138

139

configure_jenkins_dotgitconfig () {

140

echo "[user]

141

email = zigo@debian.org

142

name = Thomas Goirand

143

[gitreview]

144

username = thomas-goirand

145

[alias]

146

wdiff = diff --color-words

147

wshow = show --color-words

148

[color]

149

ui = true" >/var/lib/jenkins/.gitconfig

150

}

151

152

configure_jenkins_sudoers () {

153

echo "jenkins ALL = NOPASSWD: /usr/bin/sbuild-update -udcar ${TARGET_DISTRO}-amd64" >/etc/sudoers.d/jenkins

154

chmod 440 /etc/sudoers.d/jenkins

155

}

156

157

# @param: $1 homedir of the GPG user

158

gen_the_dev_user_gpg_key () {

159

GPG_USER_HOMEDIR=${1}

160

mkdir -p ${GPG_USER_HOMEDIR}/.gnupg

161

chmod 700 ${GPG_USER_HOMEDIR}/.gnupg

162

chmod 700 ${GPG_USER_HOMEDIR}/.gnupg

163

if ! [ -r ${GPG_USER_HOMEDIR}/.gnupg/gpg.conf ] ; then

164

echo "keyserver hkp://pool.sks-keyservers.net

165

personal-digest-preferences SHA256

166

default-preference-list SHA512 SHA384 SHA256 SHA224 AES256 AES192 AES CAST5 ZLIB BZIP2 ZIP Uncompressed

167

cert-digest-algo SHA256

168

" >${GPG_USER_HOMEDIR}/.gnupg/gpg.conf

169

fi

170

if ! [ -r ${GPG_USER_HOMEDIR}/.gnupg/pkgos-gen-key-batchfile ] ; then

171

echo "

172

%echo Generating a basic OpenPGP key

173

Key-Type: RSA

174

Key-Length: 4096

175

Name-Real: Autogenerated key

176

Name-Email: ${THE_DEV_USER}@"`hostname --fqdn`"

177

Expire-Date: 0

178

" >${GPG_USER_HOMEDIR}/.gnupg/pkgos-gen-key-batchfile

179

fi

180

chown -R ${THE_DEV_USER}:${THE_DEV_USER} ${GPG_USER_HOMEDIR}/.gnupg

181

su ${THE_DEV_USER} -c 'gpg --gen-key --batch '${GPG_USER_HOMEDIR}'/.gnupg/pkgos-gen-key-batchfile'

182

}

183

184

configure_jenkins_gpg_key () {

185

# Generate a self-signed gpg key so that we can sign packages

186

if ! [ -e /var/lib/jenkins/.gnupg/secring.gpg ] ; then

187

gen_the_dev_user_gpg_key /var/lib/jenkins

188

fi

189

}

190

191

192

check_user_gpg_key () {

193

if ! [ -d /home/${THE_DEV_USER}/.gnupg ] ; then

194

echo "There's no /home/${THE_DEV_USER}/.gnupg folder,"

195

echo "do you want this script to generate one? If you"

196

echo "don't, the script will exit."

197

echo -n "Generate a GPG key (Y/n)?"

198

read GEN_KEY

199

if [ -z "${GEN_KEY}" ] || [ "${GEN_KEY}" = "y" ] || [ "${GEN_KEY}" = "Y" ] ; then

200

echo "Genrating a gpg key for you..."

201

gen_the_dev_user_gpg_key /home/${THE_DEV_USER}

202

else

203

echo "No gpg key: exiting..."

204

exit 1

205

fi

206

fi

207

GPG_KEY_ID=$(su ${THE_DEV_USER} -c "gpg --list-keys ${THE_DEV_USER} | grep ^pub | awk '{print \$2}' | cut -d/ -f2 | head -n 1")

208

if [ -z "${GPG_KEY_ID}" ] ; then

209

echo "Cloud not find key ID, but there's a /home/${THE_DEV_USER}/.gnupg"

210

echo -n "Generate a GPG key (Y/n)?"

211

read GEN_KEY

212

if [ -z "${GEN_KEY}" ] || [ "${GEN_KEY}" = "y" ] || [ "${GEN_KEY}" = "Y" ] ; then

213

echo "Genrating a gpg key for you..."

214

gen_the_dev_user_gpg_key /home/${THE_DEV_USER}

215

else

216

echo "No gpg key: exiting..."

217

exit 1

218

fi

219

fi

220

GPG_KEY_ID=$(su ${THE_DEV_USER} -c "gpg --list-keys ${THE_DEV_USER} | grep ^pub | awk '{print \$2}' | cut -d/ -f2 | head -n 1")

221

echo "===> Key ID: ${GPG_KEY_ID}"

222

}

223

224

build_ostack_archive_keyring_package () {

225

# Export the jenkins GPG key as pubkey.gpg in debian/dists/pubkey.gpg

226

mkdir -p ${REPO_ROOT}/debian/dists

227

chown -R ${THE_DEV_USER}:${THE_DEV_USER} ${REPO_ROOT}/debian/dists

228

chown ${THE_DEV_USER}:${THE_DEV_USER} ${REPO_ROOT}/debian

229

su ${THE_DEV_USER} -c "gpg --export -a ${THE_DEV_USER}" >${REPO_ROOT}/debian/dists/pubkey.gpg

230

# Create a Debian package out of it, called openstack-debian-archive-keyring

231

# and put it in the newly created Debian repository.

232

# Yes, a working, Debian-policy-compliant package is really only a few lines of shell... :)

233

TMPDIR=`mktemp -d`

234

MYCWD=`pwd`

235

cd ${TMPDIR}

236

chown ${THE_DEV_USER}:${THE_DEV_USER} .

237

VER=0.1

238

NAME=${TARGET_OPENSTACK_REL}-${TARGET_DISTRO}-archive-keyring

239

rm -rf ${NAME}-${VER}

240

mkdir -p ${NAME}-${VER}/debian/source

241

cd ${NAME}-${VER}

242

export DEBFULLNAME="Debian OpenStack Jenkins"

243

export DEBEMAIL="${THE_DEV_USER}@${HOST_FQDN}"

244

dch --create --package ${NAME} -D unstable --noquery --newversion 0.1 -m "Automatic archive package build."

245

sed -i 's/MAINTAINER <EMAIL>/Debian OpenStack <'${THE_DEV_USER}'@'${HOST_FQDN}'>/' debian/changelog

246

echo "3.0 (native)" >debian/source/format

247

echo 9 >debian/compat

248

echo "#!/usr/bin/make -f

249

%:

250

dh \$@

251

" >debian/rules

252

echo "Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/

253

Upstream-Name: ${NAME}

254

Source: See the openstack-pkg-tools package

255

256

Files: *

257

258

License: Apache-2

259

Licensed under the Apache License, Version 2.0 (the \"License\");

260

you may not use this file except in compliance with the License.

261

You may obtain a copy of the License at

262

.

263

http://www.apache.org/licenses/LICENSE-2.0

264

.

265

Unless required by applicable law or agreed to in writing, software

266

distributed under the License is distributed on an \"AS IS\" BASIS,

267

WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

268

See the License for the specific language governing permissions and

269

limitations under the License.

270

.

271

On Debian-based systems the full text of the Apache version 2.0 license

272

can be found in /usr/share/common-licenses/Apache-2.0.

273

" >debian/copyright

274

echo "Source: ${NAME}

275

Section: net

276

Priority: extra

277

Maintainer: PKG OpenStack <openstack-devel@lists.alioth.debian.org>

278

Uploaders: Thomas Goirand <zigo@debian.org>

279

Build-Depends: debhelper (>= 9)

280

Standards-Version: 3.9.6

281

282

Package: ${NAME}

283

Architecture: all

284

Depends: \${misc:Depends}

285

Description: OpenStack ${TARGET_OPENSTACK_REL} ${TARGET_DISTRO} archive keyring

286

OpenStack ${TARGET_OPENSTACK_REL} ${TARGET_DISTRO} archive keyring

287

">debian/control

288

cp ${REPO_ROOT}/debian/dists/pubkey.gpg .

289

echo "pubkey.gpg /usr/share/${NAME}" >debian/${NAME}.install

290

echo "#!/bin/sh

291

set -e

292

293

if [ \"\$1\" = \"configure\" ] || [ \"\$1\" = \"reconfigure\" ] ; then

294

apt-key add /usr/share/${NAME}/pubkey.gpg

295

fi

296

297

#DEBHELPER#

298

299

exit 0

300

" >debian/${NAME}.postinst

301

chown -R ${THE_DEV_USER}:${THE_DEV_USER} .

302

su ${THE_DEV_USER} -c dpkg-buildpackage || true

303

FIRST_LETTER=`echo ${TARGET_OPENSTACK_REL} | awk '{print substr($0,0,2)}'`

304

cd ..

305

MYDEST=${REPO_ROOT}/debian/pool/${REPO_DEST}/main/${FIRST_LETTER}/${NAME}

306

MYDEST2=${REPO_ROOT}/debian/pool/${REPO_NOCHANGE_BACKPORT_DEST}/main/${FIRST_LETTER}/${NAME}

307

for i in ${MYDEST} ${MYDEST2} ; do

308

su ${THE_DEV_USER} -c "mkdir -p ${i}"

309

cp *.changes *.tar.xz *.deb *.dsc ${i}

310

chown -R ${THE_DEV_USER}:${THE_DEV_USER} ${i}

311

done

312

cd ${MYCWD}

313

rm -r ${TMPDIR}

314

}

315

316

configure_sbuildrc () {

317

GPG_KEY_ID=$(su ${THE_DEV_USER} -c "gpg --list-keys ${THE_DEV_USER} | grep ^pub | awk '{print \$2}' | cut -d/ -f2")

318

DOT_SBUILDRC_PATH=${THE_USER_HOMEDIR}/.sbuildrc

319

if ! [ -r "${DOT_SBUILDRC_PATH}" ] ; then

320

echo "# don't remove this, Perl needs it:

321

322

\$build_arch_all = 1;

323

\$build_source = 1;

324

\$distribution = '"${TARGET_DISTRO}"';

325

\$run_lintian = 0;

326

327

# Don't sign packages:

328

#\$pgp_options = '-us -uc';

329

330

# FIX THIS !!!

331

\$key_id = '${GPG_KEY_ID}';

332

333

1;

334

" >${DOT_SBUILDRC_PATH}

335

fi

336

}

337

338

configure_sbuild () {

339

# Setup the gpg key for sbuild

340

mkdir -p /root/.gnupg

341

chmod 600 /root/.gnupg

342

gpg --list-keys

343

sbuild-update --keygen

344

345

# Add jenkins as a sbuild user

346

sbuild-adduser ${THE_DEV_USER}

347

348

# Create the actual schroot env

349

if ! [ -e /var/lib/sbuild/${TARGET_DISTRO}-amd64.tar.gz ] ; then

350

sbuild-createchroot --make-sbuild-tarball=/var/lib/sbuild/${TARGET_DISTRO}-amd64.tar.gz ${TARGET_DISTRO} `mktemp -d` ${CLOSEST_DEBIAN_MIRROR}

351

fi

352

353

# Make sure git-buildpackage is using sbuild

354

sed -i 's/^[ #\t]*builder[ #\t]*=.*/builder = sbuild -v --no-apt-update/' /etc/git-buildpackage/gbp.conf

355

sed -i 's/^[ #\t]*cleaner[ #\t]*=.*/cleaner = \/bin\/true/' /etc/git-buildpackage/gbp.conf

356

357

# Make sure that /dev/shm is mounted in the chroot, otherwise anything which

358

# uses /dev/shm (like python-taskflow, etc.) will fail to build

359

sed -i 's|#/dev/shm|/dev/shm|' /etc/schroot/default/fstab

360

361

# Install the juno-jessie-archive-keyring package in the repository

362

FIRST_LETTER=`echo ${TARGET_OPENSTACK_REL} | awk '{print substr($0,0,2)}'`

363

NAME=${TARGET_OPENSTACK_REL}-${TARGET_DISTRO}-archive-keyring

364

VERS=0.1

365

MYPKG_FILE_NAME=${NAME}_0.1_all.deb

366

LOCATION=debian/pool/${TARGET_DISTRO}-${TARGET_OPENSTACK_REL}-backports/main/${FIRST_LETTER}/${NAME}/${MYPKG_FILE_NAME}

367

schroot -c source:${TARGET_DISTRO}-amd64-sbuild -u root -- apt-get install -y wget

368

schroot -c source:${TARGET_DISTRO}-amd64-sbuild -u root -- wget http://localhost/${LOCATION}

369

schroot -c source:${TARGET_DISTRO}-amd64-sbuild -u root -- dpkg -i ${MYPKG_FILE_NAME}

370

371

# Since we have already a first package (the archive-keyring one),

372

# let's scan the tree in /home/ftp/debian/pool, and build a valid Debian repo

373

374

MYCWD=`pwd`

375

cd ${THE_USER_HOMEDIR}

376

su ${THE_DEV_USER} -c pkgos-scan-repo

377

su ${THE_DEV_USER} -c pkgos-scan-repo ${REPO_NOCHANGE_BACKPORT_DEST}

378

cd ${MYCWD}

379

380

# Add a hook to have both our repositories used inside the sbuild chroot

381

echo "#!/bin/sh

382

383

set -e

384

385

. \"\$SETUP_DATA_DIR/common-data\"

386

. \"\$SETUP_DATA_DIR/common-functions\"

387

. \"\$SETUP_DATA_DIR/common-config\"

388

389

if [ \$STAGE = \"setup-start\" ] || [ \$STAGE = \"setup-recover\" ]; then

390

echo \"deb http://localhost/debian ${TARGET_DISTRO}-${TARGET_OPENSTACK_REL}-backports main\" >\${CHROOT_PATH}/etc/apt/sources.list.d/openstack.list

391

echo \"deb http://localhost/debian ${REPO_NOCHANGE_BACKPORT_DEST} main\" >\${CHROOT_PATH}/etc/apt/sources.list.d/openstack-backports.list

392

fi" >>/etc/schroot/setup.d/80sources

393

chmod +x /etc/schroot/setup.d/80sources

394

}

395

396

configure_jenkins_job_builder () {

397

GENERATED_PASSWORD=$(dd if=/dev/random bs=64 count=1 2>|/dev/null | md5sum | awk '{print $1}')

398

echo "[job_builder]

399

ignore_cache=True

400

keep_descriptions=False

401

include_path=/usr/local/bin

402

recursive=False

403

allow_duplicates=False

404

405

[jenkins]

406

user=zigo

407

password=${JENKINS_JOBS_BUILDER_PASS}

408

url=http://localhost:8080/

409

" >/etc/jenkins_jobs/jenkins_jobs.ini

410

echo "" >job.yaml

411

for i in `cat /etc/pkgos/build-list` ; do

412

echo "- job:

413

name: $i

414

builders:

415

- shell: 'pkgos-bop-jenkins "${i}"'

416

auth-token: g5rjtpms5emw

417

logrotate:

418

numToKeep: 4

419

publishers:

420

- ircbot:

421

strategy: all

422

notify-start: true

423

message-type: summary

424

wrappers:

425

- ansicolor:

426

colormap: xterm

427

" >>job.yaml

428

done

429

jenkins-jobs update job.yaml

430

}

431

432

restart_jenkins () {

433

service jenkins restart

434

}

435

436

##############################

437

# ACTUAL START OF THE SCRIPT #

438

##############################

439

440

usage () {

441

echo "Wrong usage: $0 [dev|jenkins] <username>"

442

echo "the username 2nd param is only if you use the dev mode"

443

exit 1

444

}

445

446

if [ "${1}" = "dev" ] ; then

447

echo "Setting-up a developer machine."

448

DEV_OR_JENKINS=dev

449

shift

450

if [ -z "${1}" ] ; then

451

usage

452

fi

453

THE_DEV_USER=${1}

454

shift

455

if [ -n "${1}" ] ; then

456

usage

457

fi

458

THE_USER_HOMEDIR=/home/${THE_DEV_USER}

459

elif [ "${1}" = "jenkins" ] ; then

460

echo "Setting-up a Jenkins build server."

461

DEV_OR_JENKINS=jenkins

462

THE_DEV_USER=jenkins

463

shift

464

if [ -n "${1}" ] ; then

465

usage

466

fi

467

THE_USER_HOMEDIR=/var/lib/jenkins

468

else

469

usage

470

fi

471

472

detect_env

473

[ "${DEV_OR_JENKINS}" = "jenkins" ] && configure_hostname

474

install_all_software

475

configure_apache

476

configure_pure

477

# This often fails because the list of plugins isn't fetched by Jenkins

478

#install_jenkins_plugins

479

[ "${DEV_OR_JENKINS}" = "jenkins" ] && configure_jenkins_dotgitconfig

480

[ "${DEV_OR_JENKINS}" = "jenkins" ] && configure_jenkins_sudoers

481

[ "${DEV_OR_JENKINS}" = "jenkins" ] && configure_jenkins_gpg_key

482

[ "${DEV_OR_JENKINS}" = "dev" ] && check_user_gpg_key

483

configure_sbuildrc

484

build_ostack_archive_keyring_package

485

configure_sbuild

486

[ "${DEV_OR_JENKINS}" = "jenkins" ] && configure_jenkins_job_builder

487

# This is needed so that jenkins can login into sbuild

488

[ "${DEV_OR_JENKINS}" = "jenkins" ] && restart_jenkins