~james-w/udd/management-commands

« back to all changes in this revision

Viewing changes to udd/scripts/import_package.py

  • Committer: James Westby
  • Date: 2011-12-13 21:09:23 UTC
  • mfrom: (557.1.1 drop_email_failures)
  • Revision ID: james.westby@canonical.com-20111213210923-tfrirlx3xbwmi70u
Merged drop_email_failures into management-commands.

Show diffs side-by-side

added added

removed removed

Lines of Context:
37
37
    urlutils,
38
38
    )
39
39
 
40
 
from udd import icommon
41
 
from udd import iconfig
42
 
from udd import lpapi
43
 
from udd.paths import paths
 
40
from udd import (
 
41
    icommon,
 
42
    iconfig,
 
43
    lpapi,
 
44
    )
 
45
 
 
46
 
 
47
# Not ideal to declare this here but this is used by several local functions
 
48
# below. Refactoring the whole script may help -- vila 2011-12-08
 
49
conf = None # Will be set by main()
 
50
 
44
51
 
45
52
push_lock = threading.Lock()
46
53
 
80
87
    parser.add_option("--verbose", dest="verbose", action="store_true")
81
88
    parser.add_option("--keep-temp", dest="keep_temp", action="store_true",
82
89
                      help="Do not delete the temporary directory at "
83
 
                      "BASE_DIR/updates/PACKAGE_NAME when done")
 
90
                      "{pi.updates_dir}/PACKAGE when done")
84
91
    parser.add_option("-D", dest="debug_flags", action="append", default=[],
85
92
                      help="Set debug flags, see 'bzr help debug-flags'")
86
93
    parser.add_option("--lp-cache", dest="lp_cache", default=None,
87
94
                      help="Set the location of a Launchpadlib cache")
88
95
    parser.add_option("--persistent-download-cache",
89
96
                      dest="persistent_download_cache", action="store_true",
90
 
                      help="Cache downloaded Debian source packages persistently "
91
 
                      "between runs of this script. Useful when testing and "
92
 
                      "developing to avoid repeatedly downloading the same files.")
93
 
    parser.add_option("--local-branches", dest="local_branches", action="store_true",
94
 
                      help="Debugging mode in which branches in "
95
 
                      "BASE_DIR/localbranches/PACKAGE are used instead of the "
96
 
                      "real Launchpad branches. A local branch will be branched "
97
 
                      "from the Launchpad branch on first access if it does not "
98
 
                      "exist locally")
 
97
                      help="Cache downloaded Debian source packages"
 
98
                      " in {pi.download_cache_dir} persistently between runs"
 
99
                      " of this script. Useful when testing and developing to"
 
100
                      " avoid repeatedly downloading the same files.")
 
101
    parser.add_option("--local-branches", dest="local_branches",
 
102
                      action="store_true",
 
103
                      help="Debugging mode in which branches in"
 
104
                      " {pi.localbranches_dir}/PACKAGE are used instead of the"
 
105
                      " real Launchpad branches. A local branch will be"
 
106
                      " branched from the Launchpad branch on first access"
 
107
                      " if it does not exist locally")
99
108
    parser.add_option("--only-before", help="Only import package versions less "
100
109
                      "than this")
101
110
    return parser
128
137
    #    for component in ("main", "contrib", "non-free"):
129
138
    #        source_url = "http://ftp.uk.debian.org/debian/dists/%s/%s/source/Sources.gz" % (release, component)
130
139
    #        update_list(release, component, source_url)
131
 
    if icommon.lock_update_lists() is None:
 
140
    if icommon.lock_path(conf.get('pi.script_locks_dir'),
 
141
                         'update_lists') is None:
132
142
        return
133
143
    for release in ("woody", "sarge", "etch",):
134
144
        for component in ("main", "contrib", "non-free"):
135
 
            source_url = "http://archive.debian.org/debian/dists/%s/%s/source/Sources.gz" % (release, component)
 
145
            source_url = ''.join(['http://archive.debian.org/debian/dists/',
 
146
                                 release, '/', component, '/source/Sources.gz'])
136
147
            icommon.update_list(release, component, source_url)
137
148
 
138
149
 
174
185
        pb.finished()
175
186
    update_lists()
176
187
    try:
177
 
        proc = subprocess.Popen(["/usr/bin/madison-lite", "--mirror", paths.lists_dir,
178
 
                package], stdout=subprocess.PIPE, preexec_fn=subprocess_setup)
 
188
        lists_dir = conf.get('pi.lists_dir')
 
189
        proc = subprocess.Popen(
 
190
            ["/usr/bin/madison-lite", "--mirror", lists_dir, package],
 
191
            stdout=subprocess.PIPE, preexec_fn=subprocess_setup)
179
192
    except OSError, e:
180
193
        if e.errno in (errno.ENOENT,):
181
194
            sys.stderr.write('You must have "madison-lite" installed.\n'
255
268
    new_branch.set_append_revisions_only(False)
256
269
    return new_branch
257
270
 
258
 
def create_updates_branch(updates_branch_path, importp, bstore, possible_transports=None):
259
 
    br_from = bstore.get_branch(importp, possible_transports=possible_transports)
 
271
def create_updates_branch(updates_branch_path, importp, bstore,
 
272
                          possible_transports=None):
 
273
    br_from = bstore.get_branch(importp,
 
274
                                possible_transports=possible_transports)
260
275
    if br_from is None:
261
276
        updates_branch = create_branch(updates_branch_path)
262
277
        return updates_branch
754
769
            continue
755
770
        updates_branch_path = os.path.join(temp_dir, suite)
756
771
        try:
757
 
            branch.Branch.open(updates_branch_path, possible_transports=possible_transports)
 
772
            branch.Branch.open(updates_branch_path,
 
773
                               possible_transports=possible_transports)
758
774
        except errors.NotBranchError:
759
775
            create_updates_branch(updates_branch_path, importp,
760
776
                    bstore, possible_transports=possible_transports)
822
838
    return True
823
839
 
824
840
 
825
 
def check_same(importp, db, revid, name, updates_branch, temp_dir, download_dir):
 
841
def check_same(importp, db, revid, name, updates_branch,
 
842
               temp_dir, download_dir):
826
843
    trace.mutter('Checking if content of revid: %s is the same as %s %s'
827
844
                 % (revid, name, importp.version))
828
845
    operation = cleanup.OperationWithCleanups(_check_same)
956
973
    return filtered_versions
957
974
 
958
975
 
959
 
def main(lp, package, push=True, check=False, extra_debian=None, no_existing=False,
960
 
        keep_temp=False, local_branches=False, persistent_download_cache=False,
961
 
        only_before=None):
962
 
    conf = iconfig.Iconfig()
963
 
    log_dir = conf.get('pkgimport.packages.log_dir')
 
976
def _import_package(lp, package, push=True, check=False, extra_debian=None,
 
977
                    no_existing=False, keep_temp=False, local_branches=False,
 
978
                    persistent_download_cache=False, only_before=None):
 
979
    log_dir = conf.get('pi.packages.log_dir')
964
980
    icommon.ensure_directory(log_dir)
965
981
    log_name = os.path.join(log_dir, package)
966
982
    f = open(log_name, "ab", 1)
968
984
    try:
969
985
        trace.mutter("Time (UTC): %s" % str(datetime.datetime.utcnow()))
970
986
        f.flush()
971
 
        icommon.ensure_directory(paths.updates_dir)
 
987
        updates_dir = conf.get('pi.updates_dir')
 
988
        icommon.ensure_directory(updates_dir)
972
989
        # The temp_dir is unique per single package import run.
973
 
        temp_dir = os.path.join(paths.updates_dir, package)
 
990
        temp_dir = os.path.join(updates_dir, package)
974
991
        # The download_dir is used to dget Debian source packages. It can be
975
992
        # equal to the temp_dir to have these discarded when no longer
976
993
        # required, or set to a different directory to persistently store
977
994
        # downloaded source between runs.
978
995
        if persistent_download_cache:
979
 
            download_dir = os.path.join(paths.download_cache_dir, package)
 
996
            download_dir = os.path.join(
 
997
                conf.get('pi.download_cache_dir'), package)
980
998
            icommon.ensure_directory(download_dir)
981
999
        else:
982
1000
            download_dir = temp_dir
983
1001
        if not no_existing:
984
 
            revid_db = icommon.RevidDatabase(paths.sqlite_file, package)
 
1002
            sqlite_file = conf.get('pi.sqlite_file')
 
1003
            revid_db = icommon.RevidDatabase(sqlite_file, package)
985
1004
            suffix = revid_db.get_suffix()
986
1005
            if local_branches:
987
1006
                bstore = icommon.LocalShadowBranchStore(package, lp, suffix)
988
1007
            else:
989
1008
                bstore = icommon.BranchStore(package, lp, suffix)
990
 
            commit_db = icommon.CommitDatabase(paths.sqlite_file, package)
 
1009
            commit_db = icommon.CommitDatabase(sqlite_file, package)
991
1010
        else:
992
1011
            revid_db = icommon.MemoryRevidDatabase(package)
993
1012
            bstore = icommon.MemoryBranchStore(package, lp, "")
1094
1113
    import_dsc = _import_dsc
1095
1114
 
1096
1115
 
1097
 
def run_script():
 
1116
def main():
1098
1117
    load_import_dsc()
1099
1118
    parser = get_options()
1100
1119
    options, args = parser.parse_args()
1105
1124
        trace.push_log_file(sys.stderr)
1106
1125
    if sys.stderr.isatty():
1107
1126
        # Don't create a progress bar if we aren't actually in a terminal
1108
 
        ui.ui_factory = ui.make_ui_for_terminal(sys.stdin, sys.stdout, sys.stderr)
 
1127
        ui.ui_factory = ui.make_ui_for_terminal(
 
1128
            sys.stdin, sys.stdout, sys.stderr)
1109
1129
    debug.debug_flags.update(options.debug_flags)
1110
1130
 
 
1131
    global conf
 
1132
    conf = iconfig.ImporterStack()
1111
1133
 
1112
1134
    if len(args) < 1:
1113
1135
        print "Must pass the name of a package"
1114
1136
        sys.exit(1)
1115
1137
 
1116
 
    lock = icommon.lock_package(args[0])
 
1138
    lock = icommon.lock_path(conf.get('pi.package_locks_dir'), args[0])
1117
1139
 
1118
1140
    if lock is None:
1119
1141
        print "Unable to lock"
1140
1162
            print "Must pass the name of one package to import"
1141
1163
            sys.exit(1)
1142
1164
        try:
1143
 
            sys.exit(main(lp, args[0], push=not options.no_push,
1144
 
                        extra_debian=options.extra_debian, check=options.check,
1145
 
                        no_existing=options.no_existing,
1146
 
                        keep_temp=options.keep_temp,
1147
 
                        local_branches=options.local_branches,
1148
 
                        persistent_download_cache=options.persistent_download_cache,
1149
 
                        only_before=options.only_before))
 
1165
            sys.exit(_import_package(
 
1166
                    lp, args[0], push=not options.no_push,
 
1167
                    extra_debian=options.extra_debian, check=options.check,
 
1168
                    no_existing=options.no_existing,
 
1169
                    keep_temp=options.keep_temp,
 
1170
                    local_branches=options.local_branches,
 
1171
                    persistent_download_cache=options.persistent_download_cache,
 
1172
                    only_before=options.only_before))
1150
1173
        except HTTPError, e:
1151
1174
            print e.content.decode("utf-8", "replace")
1152
1175
            raise