40
from udd import icommon
41
from udd import iconfig
43
from udd.paths import paths
47
# Not ideal to declare this here but this is used by several local functions
48
# below. Refactoring the whole script may help -- vila 2011-12-08
49
conf = None # Will be set by main()
45
52
push_lock = threading.Lock()
80
87
parser.add_option("--verbose", dest="verbose", action="store_true")
81
88
parser.add_option("--keep-temp", dest="keep_temp", action="store_true",
82
89
help="Do not delete the temporary directory at "
83
"BASE_DIR/updates/PACKAGE_NAME when done")
90
"{pi.updates_dir}/PACKAGE when done")
84
91
parser.add_option("-D", dest="debug_flags", action="append", default=[],
85
92
help="Set debug flags, see 'bzr help debug-flags'")
86
93
parser.add_option("--lp-cache", dest="lp_cache", default=None,
87
94
help="Set the location of a Launchpadlib cache")
88
95
parser.add_option("--persistent-download-cache",
89
96
dest="persistent_download_cache", action="store_true",
90
help="Cache downloaded Debian source packages persistently "
91
"between runs of this script. Useful when testing and "
92
"developing to avoid repeatedly downloading the same files.")
93
parser.add_option("--local-branches", dest="local_branches", action="store_true",
94
help="Debugging mode in which branches in "
95
"BASE_DIR/localbranches/PACKAGE are used instead of the "
96
"real Launchpad branches. A local branch will be branched "
97
"from the Launchpad branch on first access if it does not "
97
help="Cache downloaded Debian source packages"
98
" in {pi.download_cache_dir} persistently between runs"
99
" of this script. Useful when testing and developing to"
100
" avoid repeatedly downloading the same files.")
101
parser.add_option("--local-branches", dest="local_branches",
103
help="Debugging mode in which branches in"
104
" {pi.localbranches_dir}/PACKAGE are used instead of the"
105
" real Launchpad branches. A local branch will be"
106
" branched from the Launchpad branch on first access"
107
" if it does not exist locally")
99
108
parser.add_option("--only-before", help="Only import package versions less "
128
137
# for component in ("main", "contrib", "non-free"):
129
138
# source_url = "http://ftp.uk.debian.org/debian/dists/%s/%s/source/Sources.gz" % (release, component)
130
139
# update_list(release, component, source_url)
131
if icommon.lock_update_lists() is None:
140
if icommon.lock_path(conf.get('pi.script_locks_dir'),
141
'update_lists') is None:
133
143
for release in ("woody", "sarge", "etch",):
134
144
for component in ("main", "contrib", "non-free"):
135
source_url = "http://archive.debian.org/debian/dists/%s/%s/source/Sources.gz" % (release, component)
145
source_url = ''.join(['http://archive.debian.org/debian/dists/',
146
release, '/', component, '/source/Sources.gz'])
136
147
icommon.update_list(release, component, source_url)
177
proc = subprocess.Popen(["/usr/bin/madison-lite", "--mirror", paths.lists_dir,
178
package], stdout=subprocess.PIPE, preexec_fn=subprocess_setup)
188
lists_dir = conf.get('pi.lists_dir')
189
proc = subprocess.Popen(
190
["/usr/bin/madison-lite", "--mirror", lists_dir, package],
191
stdout=subprocess.PIPE, preexec_fn=subprocess_setup)
179
192
except OSError, e:
180
193
if e.errno in (errno.ENOENT,):
181
194
sys.stderr.write('You must have "madison-lite" installed.\n'
255
268
new_branch.set_append_revisions_only(False)
256
269
return new_branch
258
def create_updates_branch(updates_branch_path, importp, bstore, possible_transports=None):
259
br_from = bstore.get_branch(importp, possible_transports=possible_transports)
271
def create_updates_branch(updates_branch_path, importp, bstore,
272
possible_transports=None):
273
br_from = bstore.get_branch(importp,
274
possible_transports=possible_transports)
260
275
if br_from is None:
261
276
updates_branch = create_branch(updates_branch_path)
262
277
return updates_branch
755
770
updates_branch_path = os.path.join(temp_dir, suite)
757
branch.Branch.open(updates_branch_path, possible_transports=possible_transports)
772
branch.Branch.open(updates_branch_path,
773
possible_transports=possible_transports)
758
774
except errors.NotBranchError:
759
775
create_updates_branch(updates_branch_path, importp,
760
776
bstore, possible_transports=possible_transports)
825
def check_same(importp, db, revid, name, updates_branch, temp_dir, download_dir):
841
def check_same(importp, db, revid, name, updates_branch,
842
temp_dir, download_dir):
826
843
trace.mutter('Checking if content of revid: %s is the same as %s %s'
827
844
% (revid, name, importp.version))
828
845
operation = cleanup.OperationWithCleanups(_check_same)
956
973
return filtered_versions
959
def main(lp, package, push=True, check=False, extra_debian=None, no_existing=False,
960
keep_temp=False, local_branches=False, persistent_download_cache=False,
962
conf = iconfig.Iconfig()
963
log_dir = conf.get('pkgimport.packages.log_dir')
976
def _import_package(lp, package, push=True, check=False, extra_debian=None,
977
no_existing=False, keep_temp=False, local_branches=False,
978
persistent_download_cache=False, only_before=None):
979
log_dir = conf.get('pi.packages.log_dir')
964
980
icommon.ensure_directory(log_dir)
965
981
log_name = os.path.join(log_dir, package)
966
982
f = open(log_name, "ab", 1)
969
985
trace.mutter("Time (UTC): %s" % str(datetime.datetime.utcnow()))
971
icommon.ensure_directory(paths.updates_dir)
987
updates_dir = conf.get('pi.updates_dir')
988
icommon.ensure_directory(updates_dir)
972
989
# The temp_dir is unique per single package import run.
973
temp_dir = os.path.join(paths.updates_dir, package)
990
temp_dir = os.path.join(updates_dir, package)
974
991
# The download_dir is used to dget Debian source packages. It can be
975
992
# equal to the temp_dir to have these discarded when no longer
976
993
# required, or set to a different directory to persistently store
977
994
# downloaded source between runs.
978
995
if persistent_download_cache:
979
download_dir = os.path.join(paths.download_cache_dir, package)
996
download_dir = os.path.join(
997
conf.get('pi.download_cache_dir'), package)
980
998
icommon.ensure_directory(download_dir)
982
1000
download_dir = temp_dir
983
1001
if not no_existing:
984
revid_db = icommon.RevidDatabase(paths.sqlite_file, package)
1002
sqlite_file = conf.get('pi.sqlite_file')
1003
revid_db = icommon.RevidDatabase(sqlite_file, package)
985
1004
suffix = revid_db.get_suffix()
986
1005
if local_branches:
987
1006
bstore = icommon.LocalShadowBranchStore(package, lp, suffix)
989
1008
bstore = icommon.BranchStore(package, lp, suffix)
990
commit_db = icommon.CommitDatabase(paths.sqlite_file, package)
1009
commit_db = icommon.CommitDatabase(sqlite_file, package)
992
1011
revid_db = icommon.MemoryRevidDatabase(package)
993
1012
bstore = icommon.MemoryBranchStore(package, lp, "")
1105
1124
trace.push_log_file(sys.stderr)
1106
1125
if sys.stderr.isatty():
1107
1126
# Don't create a progress bar if we aren't actually in a terminal
1108
ui.ui_factory = ui.make_ui_for_terminal(sys.stdin, sys.stdout, sys.stderr)
1127
ui.ui_factory = ui.make_ui_for_terminal(
1128
sys.stdin, sys.stdout, sys.stderr)
1109
1129
debug.debug_flags.update(options.debug_flags)
1132
conf = iconfig.ImporterStack()
1112
1134
if len(args) < 1:
1113
1135
print "Must pass the name of a package"
1116
lock = icommon.lock_package(args[0])
1138
lock = icommon.lock_path(conf.get('pi.package_locks_dir'), args[0])
1118
1140
if lock is None:
1119
1141
print "Unable to lock"
1140
1162
print "Must pass the name of one package to import"
1143
sys.exit(main(lp, args[0], push=not options.no_push,
1144
extra_debian=options.extra_debian, check=options.check,
1145
no_existing=options.no_existing,
1146
keep_temp=options.keep_temp,
1147
local_branches=options.local_branches,
1148
persistent_download_cache=options.persistent_download_cache,
1149
only_before=options.only_before))
1165
sys.exit(_import_package(
1166
lp, args[0], push=not options.no_push,
1167
extra_debian=options.extra_debian, check=options.check,
1168
no_existing=options.no_existing,
1169
keep_temp=options.keep_temp,
1170
local_branches=options.local_branches,
1171
persistent_download_cache=options.persistent_download_cache,
1172
only_before=options.only_before))
1150
1173
except HTTPError, e:
1151
1174
print e.content.decode("utf-8", "replace")