« back to all changes in this revision
Viewing changes to udd/iconfig.py
- browse files at revision 560
- compare with another revision
- download diff
- download tarball
- view history from revision 560
- Committer: James Westby
- Date: 2011-12-13 21:09:23 UTC
- mfrom: (557.1.1 drop_email_failures)
- Revision ID: james.westby@canonical.com-20111213210923-tfrirlx3xbwmi70u
Merged drop_email_failures into management-commands.
- files added:
- udd/tests/__init__.py
- files removed:
- udd/paths.py
- files renamed:
- udd/tests/__init__.py => udd/tests/test_icommon.py
- files modified:
- README
added
removed
udd/iconfig.py
8
8
transport,
9
9
)
10
10
11
from udd.paths import paths
12
13
11
14
12
_root_dir = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
15
13
"""Where the udd branch is rooted at."""
34
32
return config.NameMatcher(self, None).get_sections()
35
33
36
34
35
class ImporterStack(config.Stack):
36
"""Configuration stack for the importer.
37
38
The ``pkgimport.conf`` file defines the configuration options for the
39
importer itself, especially all the directories that are used at
40
runtime. These options can be overridden in ``locations.conf`` by using a
41
section named after the directory used for deployment.
42
"""
43
44
def __init__(self, possible_transports=None):
45
t = transport.get_transport(_root_dir,
46
possible_transports=possible_transports)
47
location_store = config.LocationStore()
48
loc_matcher = config.LocationMatcher(location_store, _root_dir)
49
pkgimport_store = PkgimportStore(t)
50
super(ImporterStack, self).__init__(
51
[
52
# locations.conf can override any option, based on the tree
53
# location
54
loc_matcher.get_sections,
55
# Otherwise we use pkgimport.conf
56
pkgimport_store.get_common_sections,
57
])
58
59
def get(self, name, expand=True):
60
"""Override base class to expand by default."""
61
return super(ImporterStack, self).get(name, expand)
62
63
37
64
class PackageStack(config.Stack):
38
"""Configuration stack for a package to be imported."""
65
"""Configuration stack for a package to be imported.
66
67
Options specific to a package are defined in a section named after the
68
package. Failing that, the no-name section defines the options common to
69
all packages.
70
"""
39
71
40
72
def __init__(self, package_name, possible_transports=None):
41
73
t = transport.get_transport(_root_dir,
64
96
config.option_registry.register(config.Option(*args, **kwargs))
65
97
66
98
67
register_option('pkgimport.max_duration',
99
# 'pi' stands for Package Import and is used as a prefix for all related
100
# options
101
102
register_option('pi.install_dir', default=_root_dir,
103
help='Base directory where udd scripts are run from.')
104
105
# Common to all scripts:
106
register_option('pi.base_dir',
107
default='/srv/package-import.canonical.com/new',
108
help='Base directory for the udd scripts.')
109
register_option('pi.log_dir', default='{pi.base_dir}/logs',
110
help='Base directory for the log files.')
111
register_option('pi.locks_dir', default='{pi.base_dir}/locks',
112
help='Base directory for the lock files.')
113
register_option('pi.distributions', default='debian,ubuntu',
114
from_unicode=config.list_from_store,
115
help='''Which distributions to operate on.
116
117
Affect the behaviour of add-import-jobs and list-packages.
118
''')
119
register_option('pi.retry_all_failed_jobs', default='False',
120
from_unicode=config.bool_from_store,
121
help='''Whether or not to retry all failed jobs.
122
123
All failed imports are retried when set. Otherwise only the import jobs that
124
were currently running the supervisor died are retried.
125
''')
126
127
128
# Databases (common across multiple components):
129
register_option('pi.sqlite_file', default = '{pi.base_dir}/meta.db',
130
help='Main database for the package importer.')
131
register_option('pi.sqlite_package_file', default = '{pi.base_dir}/packages.db',
132
help='Packages database for the package importer.')
133
register_option('pi.sqlite_history_file', default = '{pi.base_dir}/history.db',
134
help='History database for the package importer.')
135
136
137
# Paths common across multiple components:
138
register_option('pi.package_locks_dir', default = '{pi.locks_dir}/packages',
139
help='Directory for package exclusive lock files.')
140
register_option('pi.script_locks_dir', default = '{pi.locks_dir}/scripts',
141
help='Directory for script exclusive lock files.')
142
register_option('pi.www_dir', default = '{pi.base_dir}/www',
143
help='Directory for publishing web pages.')
144
# The explanations file contains tracebacks - these are highly sensitive to
145
# changes in the code, so are kept alongside the code.
146
register_option('pi.explanations_file', default='{pi.install_dir}/explanations',
147
help='Explanations for import failures.')
148
149
150
# Used by mass_import:
151
register_option('pi.import_command',
152
default='{pi.install_dir}/bin/import-package {package}',
153
help='''Command used to import a package.
154
155
{package} is provided when the command is expanded and contains the package
156
name.
157
''')
158
register_option('pi.driver.log_dir', default='{pi.log_dir}/driver',
159
help='Directory for the import driver log files.')
160
register_option('pi.driver.log.debug', default='{pi.driver.log_dir}/debug_log',
161
help='Import driver debug log file.')
162
register_option('pi.driver.log.progress',
163
default='{pi.driver.log_dir}/progress_log',
164
help='Import package driver progress log file.')
165
register_option('pi.stop_file', default='{pi.base_dir}/STOP_PLEASE',
166
help="A touch'ed file to stop the importer gracefully.")
167
register_option('pi.max_threads_file', default='{pi.base_dir}/max_threads',
168
help='A file containing the maximum number'
169
' of parallel imports.')
170
171
172
# Used by import_package.py:
173
register_option('pi.packages.log_dir', default='{pi.log_dir}/packages',
174
help='Base directory for the log files.')
175
register_option('pi.lists_dir', default='{pi.base_dir}/lists',
176
help='Directory for distribution description mirrors.')
177
register_option('pi.download_cache_dir', default='{pi.base_dir}/download-cache',
178
help='Directory for caching downloaded source packages.')
179
register_option('pi.localbranches_dir', default='{pi.base_dir}/localbranches',
180
help='Directory for caching local branches when debugging.')
181
register_option('pi.updates_dir', default='{pi.base_dir}/updates',
182
help='Working directory to prepare packaging branches.')
183
register_option('pi.debian_diffs_dir', default='{pi.www_dir}/diffs',
184
help='Diffs versus debian versions.')
185
register_option('pi.ubuntu_merges_dir', default='{pi.www_dir}/merges',
186
help='Merges against Ubuntu versions.')
187
188
189
# Used by categorize_failures.py:
190
register_option('pi.web_status_dir', default = '{pi.www_dir}/status',
191
help='Directory for publishing importer status.')
192
193
# Used by lpapi.py:
194
# FIXME: Keeping a local credentials file doesn't feel right, when we have a
195
# newer launchpadlib, we should try to offload this to relying on its APIs.
196
# --maxb 20110615
197
register_option('pi.lp_creds_file', default='{pi.base_dir}/lp_creds.txt',
198
help='Creadentials file to access launchpad.')
199
200
201
# Package specific options
202
register_option('pi.max_duration',
68
203
from_unicode=config.int_from_store,
69
204
default='86400',
70
205
help='Number of seconds before an import is killed.')
71
register_option('pkgimport.kill.grace_period',
206
register_option('pi.kill.grace_period',
72
207
from_unicode=config.int_from_store,
73
208
default='10',
74
209
help='Number of seconds before killing an import'
75
210
' switch from SIGINT to SIGKILL.')
76
77
78
def pkgimportconfig_file_name():
79
new_path = paths.static_file_path('config/pkgimport.conf')
80
old_path = paths.static_file_path('pkgimport.conf')
81
if os.path.exists(new_path):
82
return new_path
83
else:
84
return old_path
85
86
87
# FIXME: We don't inherit from LockableConfig because we don't want to create a
88
# 'lock' dir alonside the scripts (we should not require write access there and
89
# we don't need to modify options in the configuration file) -- vila 2011-03-22
90
class PkgimportConfig(config.IniBasedConfig):
91
92
def __init__(self, file_name=None):
93
if file_name is None:
94
file_name = pkgimportconfig_file_name()
95
super(PkgimportConfig, self).__init__(file_name=file_name)
96
97
def config_id(self):
98
return 'pkgimport'
99
100
def get(self, name):
101
return self.get_user_option(name)
102
103
104
def compatible_get(self, name, default_value=None):
105
value = self.get_user_option(name)
106
if value is None:
107
value = default_value
108
return value
109
110
111
class ConfigStack(object):
112
"""A stack of configurations where an option can be defined"""
113
114
def __init__(self, config_list):
115
self.list = []
116
for c in config_list:
117
if hasattr(c, 'get_user_option') and not hasattr(c, 'get'):
118
c.get = types.MethodType(compatible_get, c, c.__class__)
119
if not hasattr(c, 'get'):
120
raise AssertionError("%r does not provide a 'get' method"
121
% (c,))
122
self.list.append(c)
123
124
def get(self, name):
125
for c in self.list:
126
value = c.get(name)
127
if value is not None:
128
break
129
return value
130
131
132
class Iconfig(ConfigStack):
133
"""An import packager configuration taking ``locations.conf`` into account.
134
135
This allows one to override the values in ``pkgimport.conf`` in
136
``locations.conf`` for specified sections.
137
"""
138
139
def __init__(self, location=None):
140
if location is None:
141
location = paths.root_dir
142
super(Iconfig, self).__init__(
143
[config.LocationConfig(location), PkgimportConfig()])
144
211
register_option('udd.branch_hostname', default='bazaar.launchpad.net',
212
help='Branch hostname.')
213
register_option('udd.branch_owner', default='ubuntu-branches',
214
help='Branch owner as a Launchpad or unix username.')
Loggerhead is a web-based interface for Breezy
Version: 2.0.1