~ubuntu-core-dev/update-manager/main : revision 2884

1

#!/usr/bin/python3

2

3

import apt

4

import argparse

5

import distro_info

6

import os

7

import sys

8

import gettext

9

import subprocess

10

11

from UpdateManager.Core.utils import get_dist

12

13

from datetime import datetime

14

from textwrap import wrap

15

from urllib.error import URLError, HTTPError

16

from urllib.request import urlopen

17

18

# TODO make DEBUG an environmental variable

19

DEBUG = False

20

21

22

class PatchStats:

23

"""Tracks overall patch status

24

25

The relationship between archives enabled and whether a patch is eligible

26

for receiving updates is non-trivial. We track here all the important

27

buckets a package can be in:

28

29

- Whether it is set to expire with no ESM coverage

30

- Whether it is in an archive covered by ESM

31

- Whether it received LTS patches

32

- whether it received ESM patches

33

34

We also track the total packages covered and uncovered, and for the

35

uncovered packages, we track where they originate from.

36

37

The Ubuntu main archive receives patches for 5 years.

38

Canonical-owned archives (excluding partner) receive patches for 10 years.

39

patches for 10 years.

40

"""

41

def __init__(self):

42

# TODO no-update FIPS is never patched

43

self.pkgs_uncovered_fips = set()

44

45

# list of package names available in ESM

46

self.pkgs_updated_in_esmi = set()

47

self.pkgs_updated_in_esma = set()

48

49

self.pkgs_mr = set()

50

self.pkgs_um = set()

51

self.pkgs_unavailable = set()

52

self.pkgs_thirdparty = set()

53

# the bin of unknowns

54

self.pkgs_uncategorized = set()

55

56

57

def print_debug(s):

58

if DEBUG:

59

print(s)

60

61

62

def whats_in_esm(url):

63

pkgs = set()

64

# return a set of package names in an esm archive

65

try:

66

response = urlopen(url)

67

except (URLError, HTTPError):

68

print_debug('failed to load: %s' % url)

69

return pkgs

70

try:

71

content = response.read().decode('utf-8')

72

except IOError:

73

print('failed to read data at: %s' % url)

74

sys.exit(1)

75

for line in content.split('\n'):

76

if not line.startswith('Package:'):

77

continue

78

else:

79

pkg = line.split(': ')[1]

80

pkgs.add(pkg)

81

return pkgs

82

83

84

def livepatch_is_enabled():

85

""" Check to see if livepatch is enabled on the system"""

86

try:

87

c_livepatch = subprocess.run(["/snap/bin/canonical-livepatch",

88

"status"],

89

stdout=subprocess.PIPE,

90

stderr=subprocess.PIPE)

91

# it can't be enabled if it isn't installed

92

except FileNotFoundError:

93

return False

94

if c_livepatch.returncode == 0:

95

return True

96

elif c_livepatch.returncode == 1:

97

return False

98

99

100

def esm_is_enabled():

101

""" Check to see if esm is an available source"""

102

acp = subprocess.Popen(["apt-cache", "policy"],

103

stdout=subprocess.PIPE, stderr=subprocess.PIPE)

104

grep = subprocess.run(["grep", "-F", "-q", "https://%s" % esm_site],

105

stdin=acp.stdout, stdout=subprocess.PIPE)

106

if grep.returncode == 0:

107

return True

108

elif grep.returncode == -1:

109

return False

110

111

112

def trim_archive(archive):

113

return archive.split("-")[-1]

114

115

116

def trim_site(host):

117

# *.ec2.archive.ubuntu.com -> archive.ubuntu.com

118

if host.endswith("archive.ubuntu.com"):

119

return "archive.ubuntu.com"

120

return host

121

122

123

def mirror_list():

124

m_file = '/usr/share/ubuntu-release-upgrader/mirrors.cfg'

125

if not os.path.exists(m_file):

126

print("Official mirror list not found.")

127

with open(m_file) as f:

128

items = [x.strip() for x in f]

129

mirrors = [s.split('//')[1].split('/')[0] for s in items

130

if not s.startswith("#") and not s == ""]

131

# ddebs.ubuntu.com isn't in mirrors.cfg for every release

132

mirrors.append('ddebs.ubuntu.com')

133

return mirrors

134

135

136

def origins_for(ver: apt.package.Version) -> str:

137

s = []

138

for origin in ver.origins:

139

if not origin.site:

140

# When the package is installed, site is empty, archive/component

141

# are "now/now"

142

continue

143

site = trim_site(origin.site)

144

s.append("%s %s/%s" % (site, origin.archive, origin.component))

145

return ",".join(s)

146

147

148

def print_wrapped(str):

149

print("\n".join(wrap(str, break_on_hyphens=False)))

150

151

152

def print_thirdparty_count():

153

print(gettext.dngettext("update-manager",

154

"%s package is from a third party",

155

"%s packages are from third parties",

156

len(pkgstats.pkgs_thirdparty)) %

157

"{:>{width}}".format(len(pkgstats.pkgs_thirdparty), width=width))

158

159

160

def print_unavailable_count():

161

print(gettext.dngettext("update-manager",

162

"%s package is no longer available for "

163

"download",

164

"%s packages are no longer available for "

165

"download",

166

len(pkgstats.pkgs_unavailable)) %

167

"{:>{width}}".format(len(pkgstats.pkgs_unavailable), width=width))

168

169

170

def parse_options():

171

'''Parse command line arguments.

172

173

Return parser

174

'''

175

parser = argparse.ArgumentParser(

176

description='Return information about security support for packages')

177

parser.add_argument('--thirdparty', action='store_true')

178

parser.add_argument('--unavailable', action='store_true')

179

return parser

180

181

182

if __name__ == "__main__":

183

# gettext

184

APP = "update-manager"

185

DIR = "/usr/share/locale"

186

gettext.bindtextdomain(APP, DIR)

187

gettext.textdomain(APP)

188

189

parser = parse_options()

190

args = parser.parse_args()

191

192

esm_site = "esm.ubuntu.com"

193

194

try:

195

dpkg = subprocess.check_output(['dpkg', '--print-architecture'])

196

arch = dpkg.decode().strip()

197

except subprocess.CalledProcessError:

198

print("failed getting dpkg architecture")

199

sys.exit(1)

200

201

cache = apt.Cache()

202

pkgstats = PatchStats()

203

codename = get_dist()

204

di = distro_info.UbuntuDistroInfo()

205

lts = di.is_lts(codename)

206

release_expired = True

207

if codename in di.supported():

208

release_expired = False

209

# distro-info-data in Ubuntu 16.04 LTS does not have eol-esm data

210

if codename != 'xenial':

211

eol_data = [(r.eol, r.eol_esm)

212

for r in di._releases if r.series == codename][0]

213

elif codename == 'xenial':

214

eol_data = (datetime.strptime('2021-04-21', '%Y-%m-%d'),

215

datetime.strptime('2024-04-21', '%Y-%m-%d'))

216

eol = eol_data[0]

217

eol_esm = eol_data[1]

218

219

all_origins = set()

220

origins_by_package = {}

221

official_mirrors = mirror_list()

222

223

# N.B. only the security pocket is checked because this tool displays

224

# information about security updates

225

esm_url = \

226

'https://%s/%s/ubuntu/dists/%s-%s-%s/main/binary-%s/Packages'

227

pkgs_in_esma = whats_in_esm(esm_url %

228

(esm_site, 'apps', codename, 'apps',

229

'security', arch))

230

pkgs_in_esmi = whats_in_esm(esm_url %

231

(esm_site, 'infra', codename, 'infra',

232

'security', arch))

233

234

for pkg in cache:

235

pkgname = pkg.name

236

237

downloadable = True

238

if not pkg.is_installed:

239

continue

240

if not pkg.candidate or not pkg.candidate.downloadable:

241

downloadable = False

242

pkg_sites = []

243

origins_by_package[pkgname] = set()

244

245

for ver in pkg.versions:

246

# Loop through origins and store all of them. The idea here is that

247

# we don't care where the installed package comes from, provided

248

# there is at least one repository we identify as being

249

# security-assured under either LTS or ESM.

250

for origin in ver.origins:

251

# TODO: in order to handle FIPS and other archives which have

252

# root-level path names, we'll need to loop over ver.uris

253

# instead

254

if not origin.site:

255

continue

256

site = trim_site(origin.site)

257

archive = origin.archive

258

component = origin.component

259

origin = origin.origin

260

official_mirror = False

261

thirdparty = True

262

# thirdparty providers like dl.google.com don't set "Origin"

263

if origin != "Ubuntu":

264

thirdparty = False

265

if site in official_mirrors:

266

site = "official_mirror"

267

if "MY_MIRROR" in os.environ:

268

if site in os.environ["MY_MIRROR"]:

269

site = "official_mirror"

270

t = (site, archive, component, thirdparty)

271

if not site:

272

continue

273

all_origins.add(t)

274

origins_by_package[pkgname].add(t)

275

276

if DEBUG:

277

pkg_sites.append("%s %s/%s" %

278

(site, archive, component))

279

280

print_debug("available versions for %s" % pkgname)

281

print_debug(",".join(pkg_sites))

282

283

# This tracks suites we care about. Sadly, it appears that the way apt

284

# stores origins truncates away the path that comes after the

285

# domainname in the site portion, or maybe I am just clueless, but

286

# there's no way to tell FIPS apart from ESM, for instance.

287

# See 00REPOS.txt for examples

288

289

# 2020-03-18 ver.filename has the path so why is that no good?

290

291

# TODO Need to handle:

292

# MAAS, lxd, juju PPAs

293

# other PPAs

294

# other repos

295

296

# TODO handle partner.c.c

297

298

# main and restricted from release, -updates, -proposed, or -security

299

# pockets

300

suite_main = ("official_mirror", codename, "main", True)

301

suite_main_updates = ("official_mirror", codename + "-updates",

302

"main", True)

303

suite_main_security = ("official_mirror", codename + "-security",

304

"main", True)

305

suite_main_proposed = ("official_mirror", codename + "-proposed",

306

"main", True)

307

308

suite_restricted = ("official_mirror", codename, "restricted",

309

True)

310

suite_restricted_updates = ("official_mirror",

311

codename + "-updates",

312

"restricted", True)

313

suite_restricted_security = ("official_mirror",

314

codename + "-security",

315

"restricted", True)

316

suite_restricted_proposed = ("official_mirror",

317

codename + "-proposed",

318

"restricted", True)

319

320

# universe and multiverse from release, -updates, -proposed, or -security

321

# pockets

322

suite_universe = ("official_mirror", codename, "universe", True)

323

suite_universe_updates = ("official_mirror", codename + "-updates",

324

"universe", True)

325

suite_universe_security = ("official_mirror",

326

codename + "-security",

327

"universe", True)

328

suite_universe_proposed = ("official_mirror",

329

codename + "-proposed",

330

"universe", True)

331

332

suite_multiverse = ("official_mirror", codename, "multiverse",

333

True)

334

suite_multiverse_updates = ("official_mirror",

335

codename + "-updates",

336

"multiverse", True)

337

suite_multiverse_security = ("official_mirror",

338

codename + "-security",

339

"multiverse", True)

340

suite_multiverse_proposed = ("official_mirror",

341

codename + "-proposed",

342

"multiverse", True)

343

344

# packages from the esm respositories

345

# N.B. Origin: Ubuntu is not set for esm

346

suite_esm_main = (esm_site, "%s-infra-updates" % codename,

347

"main")

348

suite_esm_main_security = (esm_site,

349

"%s-infra-security" % codename, "main")

350

suite_esm_universe = (esm_site,

351

"%s-apps-updates" % codename, "main")

352

suite_esm_universe_security = (esm_site,

353

"%s-apps-security" % codename,

354

"main")

355

356

livepatch_enabled = livepatch_is_enabled()

357

esm_enabled = esm_is_enabled()

358

is_esm_infra_used = (suite_esm_main in all_origins) or \

359

(suite_esm_main_security in all_origins)

360

is_esm_apps_used = (suite_esm_universe in all_origins) or \

361

(suite_esm_universe_security in all_origins)

362

363

# Now do the final loop through

364

for pkg in cache:

365

if not pkg.is_installed:

366

continue

367

if not pkg.candidate or not pkg.candidate.downloadable:

368

pkgstats.pkgs_unavailable.add(pkg.name)

369

continue

370

pkgname = pkg.name

371

pkg_origins = origins_by_package[pkgname]

372

373

# This set of is_* booleans tracks specific situations we care about in

374

# the logic below; for instance, if the package has a main origin, or

375

# if the esm repos are enabled.

376

377

# Some packages get added in -updates and don't exist in the release

378

# pocket e.g. ubuntu-advantage-tools and libdrm-updates. To be safe all

379

# pockets are allowed.

380

is_mr_pkg_origin = (suite_main in pkg_origins) or \

381

(suite_main_updates in pkg_origins) or \

382

(suite_main_security in pkg_origins) or \

383

(suite_main_proposed in pkg_origins) or \

384

(suite_restricted in pkg_origins) or \

385

(suite_restricted_updates in pkg_origins) or \

386

(suite_restricted_security in pkg_origins) or \

387

(suite_restricted_proposed in pkg_origins)

388

is_um_pkg_origin = (suite_universe in pkg_origins) or \

389

(suite_universe_updates in pkg_origins) or \

390

(suite_universe_security in pkg_origins) or \

391

(suite_universe_proposed in pkg_origins) or \

392

(suite_multiverse in pkg_origins) or \

393

(suite_multiverse_updates in pkg_origins) or \

394

(suite_multiverse_security in pkg_origins) or \

395

(suite_multiverse_proposed in pkg_origins)

396

397

is_esm_infra_pkg_origin = (suite_esm_main in pkg_origins) or \

398

(suite_esm_main_security in pkg_origins)

399

is_esm_apps_pkg_origin = (suite_esm_universe in pkg_origins) or \

400

(suite_esm_universe_security in pkg_origins)

401

402

# A third party one won't appear in any of the above origins

403

if not is_mr_pkg_origin and not is_um_pkg_origin \

404

and not is_esm_infra_pkg_origin and not is_esm_apps_pkg_origin:

405

pkgstats.pkgs_thirdparty.add(pkgname)

406

407

if False: # TODO package has ESM fips origin

408

# TODO package has ESM fips-updates origin: OK

409

# If user has enabled FIPS, but not updates, BAD, but need some

410

# thought on how to display it, as it can't be patched at all

411

pass

412

elif is_mr_pkg_origin:

413

pkgstats.pkgs_mr.add(pkgname)

414

elif is_um_pkg_origin:

415

pkgstats.pkgs_um.add(pkgname)

416

else:

417

# TODO print information about packages in this category if in

418

# debugging mode

419

pkgstats.pkgs_uncategorized.add(pkgname)

420

421

# Check to see if the package is available in esm-infra or esm-apps

422

# and add it to the right pkgstats category

423

# NB: apps is ordered first for testing the hello package which is both

424

# in esmi and esma

425

if pkgname in pkgs_in_esma:

426

pkgstats.pkgs_updated_in_esma.add(pkgname)

427

elif pkgname in pkgs_in_esmi:

428

pkgstats.pkgs_updated_in_esmi.add(pkgname)

429

430

total_packages = (len(pkgstats.pkgs_mr) +

431

len(pkgstats.pkgs_um) +

432

len(pkgstats.pkgs_thirdparty) +

433

len(pkgstats.pkgs_unavailable))

434

width = len(str(total_packages))

435

print("%s packages installed, of which:" %

436

"{:>{width}}".format(total_packages, width=width))

437

438

# filters first as they provide less information

439

if args.thirdparty:

440

if pkgstats.pkgs_thirdparty:

441

pkgs_thirdparty = sorted(p for p in pkgstats.pkgs_thirdparty)

442

print_thirdparty_count()

443

print_wrapped(' '.join(pkgs_thirdparty))

444

msg = ("Packages from third parties are not provided by the "

445

"official Ubuntu archive, for example packages from "

446

"Personal Package Archives in Launchpad.")

447

print("")

448

print_wrapped(msg)

449

print("")

450

print_wrapped("Run 'apt-cache policy %s' to learn more about "

451

"that package." % pkgs_thirdparty[0])

452

sys.exit(0)

453

else:

454

print_wrapped("You have no packages installed from a third party.")

455

sys.exit(0)

456

if args.unavailable:

457

if pkgstats.pkgs_unavailable:

458

pkgs_unavailable = sorted(p for p in pkgstats.pkgs_unavailable)

459

print_unavailable_count()

460

print_wrapped(' '.join(pkgs_unavailable))

461

msg = ("Packages that are not available for download "

462

"may be left over from a previous release of "

463

"Ubuntu, may have been installed directly from "

464

"a .deb file, or are from a source which has "

465

"been disabled.")

466

print("")

467

print_wrapped(msg)

468

print("")

469

print_wrapped("Run 'apt-cache show %s' to learn more about "

470

"that package." % pkgs_unavailable[0])

471

sys.exit(0)

472

else:

473

print_wrapped("You have no packages installed that are no longer "

474

"available.")

475

sys.exit(0)

476

# Only show LTS patches and expiration notices if the release is not

477

# yet expired; showing LTS patches would give a false sense of

478

# security.

479

if not release_expired:

480

print("%s receive package updates%s until %d/%d" %

481

("{:>{width}}".format(len(pkgstats.pkgs_mr),

482

width=width),

483

" with LTS" if lts else "",

484

eol.month, eol.year))

485

elif release_expired and lts:

486

print("%s %s security updates with ESM Infra "

487

"until %d/%d" %

488

("{:>{width}}".format(len(pkgstats.pkgs_mr),

489

width=width),

490

"are receiving" if esm_enabled else "could receive",

491

eol_esm.month, eol_esm.year))

492

if lts and pkgstats.pkgs_um:

493

print("%s %s security updates with ESM Apps "

494

"until %d/%d" %

495

("{:>{width}}".format(len(pkgstats.pkgs_um),

496

width=width),

497

"are receiving" if esm_enabled else "could receive",

498

eol_esm.month, eol_esm.year))

499

if pkgstats.pkgs_thirdparty:

500

print_thirdparty_count()

501

if pkgstats.pkgs_unavailable:

502

print_unavailable_count()

503

# print the detail messages after the count of packages

504

if pkgstats.pkgs_thirdparty:

505

msg = ("Packages from third parties are not provided by the "

506

"official Ubuntu archive, for example packages from "

507

"Personal Package Archives in Launchpad.")

508

print("")

509

print_wrapped(msg)

510

action = ("For more information on the packages, run "

511

"'ubuntu-security-status --thirdparty'.")

512

print_wrapped(action)

513

if pkgstats.pkgs_unavailable:

514

msg = ("Packages that are not available for download "

515

"may be left over from a previous release of "

516

"Ubuntu, may have been installed directly from "

517

"a .deb file, or are from a source which has "

518

"been disabled.")

519

print("")

520

print_wrapped(msg)

521

action = ("For more information on the packages, run "

522

"'ubuntu-security-status --unavailable'.")

523

print_wrapped(action)

524

# print the ESM calls to action last

525

if lts and not esm_enabled:

526

if release_expired and pkgstats.pkgs_mr:

527

pkgs_updated_in_esmi = pkgstats.pkgs_updated_in_esmi

528

print("")

529

print_wrapped(gettext.dngettext("update-manager",

530

"Enable Extended Security "

531

"Maintenance (ESM Infra) to "

532

"get %i security update (so far) "

533

"and enable coverage of %i "

534

"packages.",

535

"Enable Extended Security "

536

"Maintenance (ESM Infra) to "

537

"get %i security updates (so far) "

538

"and enable coverage of %i "

539

"packages.",

540

len(pkgs_updated_in_esmi)) %

541

(len(pkgs_updated_in_esmi),

542

len(pkgstats.pkgs_mr)))

543

if livepatch_enabled:

544

print("\nEnable ESM Infra with: ua enable esm-infra")

545

if pkgstats.pkgs_um:

546

pkgs_updated_in_esma = pkgstats.pkgs_updated_in_esma

547

print("")

548

print_wrapped(gettext.dngettext("update-manager",

549

"Enable Extended Security "

550

"Maintenance (ESM Apps) to "

551

"get %i security update (so far) "

552

"and enable coverage of %i "

553

"packages.",

554

"Enable Extended Security "

555

"Maintenance (ESM Apps) to "

556

"get %i security updates (so far) "

557

"and enable coverage of %i "

558

"packages.",

559

len(pkgs_updated_in_esma)) %

560

(len(pkgs_updated_in_esma),

561

len(pkgstats.pkgs_um)))

562

if livepatch_enabled:

563

print("\nEnable ESM Apps with: ua enable esm-apps")

564

if lts and not livepatch_enabled:

565

print("\nThis machine is not attached to an Ubuntu Advantage "

566

"subscription.\nSee https://ubuntu.com/advantage")