1
(ns cemerick.pomegranate.aether
2
(:refer-clojure :exclude [type proxy])
3
(:require [clojure.java.io :as io]
5
[clojure.string :as str])
6
(:import (org.apache.maven.repository.internal DefaultServiceLocator MavenRepositorySystemSession)
7
(org.sonatype.aether RepositorySystem)
8
(org.sonatype.aether.transfer TransferListener)
9
(org.sonatype.aether.artifact Artifact)
10
(org.sonatype.aether.connector.file FileRepositoryConnectorFactory)
11
(org.sonatype.aether.connector.wagon WagonProvider WagonRepositoryConnectorFactory)
12
(org.sonatype.aether.spi.connector RepositoryConnectorFactory)
13
(org.sonatype.aether.repository Proxy ArtifactRepository Authentication
14
RepositoryPolicy LocalRepository RemoteRepository
16
(org.sonatype.aether.util.repository DefaultProxySelector DefaultMirrorSelector)
17
(org.sonatype.aether.graph Dependency Exclusion DependencyNode)
18
(org.sonatype.aether.collection CollectRequest)
19
(org.sonatype.aether.resolution DependencyRequest ArtifactRequest)
20
(org.sonatype.aether.util.graph PreorderNodeListGenerator)
21
(org.sonatype.aether.util.artifact DefaultArtifact SubArtifact
23
(org.sonatype.aether.deployment DeployRequest)
24
(org.sonatype.aether.installation InstallRequest)
25
(org.sonatype.aether.util.version GenericVersionScheme)))
27
(def ^{:private true} default-local-repo
28
(io/file (System/getProperty "user.home") ".m2" "repository"))
30
(def maven-central {"central" "http://repo1.maven.org/maven2/"})
32
; Using HttpWagon (which uses apache httpclient) because the "LightweightHttpWagon"
33
; (which just uses JDK HTTP) reliably flakes if you attempt to resolve SNAPSHOT
34
; artifacts from an HTTPS password-protected repository (like a nexus instance)
35
; when other un-authenticated repositories are included in the resolution.
36
; My theory is that the JDK HTTP impl is screwing up connection pooling or something,
37
; and reusing the same connection handle for the HTTPS repo as it used for e.g.
38
; central, without updating the authentication info.
39
; In any case, HttpWagon is what Maven 3 uses, and it works.
40
(def ^{:private true} wagon-factories (atom {"http" #(org.apache.maven.wagon.providers.http.HttpWagon.)
41
"https" #(org.apache.maven.wagon.providers.http.HttpWagon.)}))
43
(defn register-wagon-factory!
44
"Registers a new no-arg factory function for the given scheme. The function must return
45
an implementation of org.apache.maven.wagon.Wagon."
47
(swap! wagon-factories (fn [m]
48
(when-let [fn (m scheme)]
49
(println (format "Warning: replacing existing support for %s repositories (%s) with %s" scheme fn factory-fn)))
50
(assoc m scheme factory-fn))))
52
(deftype PomegranateWagonProvider []
56
(when-let [f (get @wagon-factories role-hint)]
59
(deftype TransferListenerProxy [listener-fn]
61
(transferCorrupted [_ e] (listener-fn e))
62
(transferFailed [_ e] (listener-fn e))
63
(transferInitiated [_ e] (listener-fn e))
64
(transferProgressed [_ e] (listener-fn e))
65
(transferStarted [_ e] (listener-fn e))
66
(transferSucceeded [_ e] (listener-fn e)))
69
[^org.sonatype.aether.transfer.TransferEvent e]
70
; INITIATED, STARTED, PROGRESSED, CORRUPTED, SUCCEEDED, FAILED
71
{:type (-> e .getType .name str/lower-case keyword)
73
:method (-> e .getRequestType str/lower-case keyword)
74
:transferred (.getTransferredBytes e)
75
:error (.getException e)
76
:data-buffer (.getDataBuffer e)
77
:data-length (.getDataLength e)
78
:resource (let [r (.getResource e)]
79
{:repository (.getRepositoryUrl r)
80
:name (.getResourceName r)
82
:size (.getContentLength r)
83
:transfer-start-time (.getTransferStartTime r)
84
:trace (.getTrace r)})})
86
(defn- default-listener-fn
87
[{:keys [type method transferred resource error] :as evt}]
88
(let [{:keys [name size repository transfer-start-time]} resource]
91
(print (case method :get "Retrieving" :put "Sending")
95
(format "(%sk)" (Math/round (double (max 1 (/ size 1024)))))))
96
(when (< 70 (+ 10 (count name) (count repository)))
97
(println) (print " "))
98
(println (case method :get "from" :put "to") repository))
99
(:corrupted :failed) (when error (println (.getMessage error)))
102
(defn- repository-system
104
(.getService (doto (DefaultServiceLocator.)
105
(.addService RepositoryConnectorFactory FileRepositoryConnectorFactory)
106
(.addService RepositoryConnectorFactory WagonRepositoryConnectorFactory)
107
(.addService WagonProvider PomegranateWagonProvider))
108
org.sonatype.aether.RepositorySystem))
110
(defn- construct-transfer-listener
113
(instance? TransferListener transfer-listener) transfer-listener
115
(= transfer-listener :stdout)
116
(TransferListenerProxy. (comp default-listener-fn transfer-event))
118
(fn? transfer-listener)
119
(TransferListenerProxy. (comp transfer-listener transfer-event))
121
:else (TransferListenerProxy. (fn [_]))))
123
(defn repository-session
124
[{:keys [repository-system local-repo offline? transfer-listener mirror-selector]}]
125
(-> (MavenRepositorySystemSession.)
126
(.setLocalRepositoryManager (.newLocalRepositoryManager repository-system
127
(-> (io/file (or local-repo default-local-repo))
130
(.setMirrorSelector mirror-selector)
131
(.setOffline (boolean offline?))
132
(.setTransferListener (construct-transfer-listener transfer-listener))))
134
(def update-policies {:daily RepositoryPolicy/UPDATE_POLICY_DAILY
135
:always RepositoryPolicy/UPDATE_POLICY_ALWAYS
136
:never RepositoryPolicy/UPDATE_POLICY_NEVER})
138
(def checksum-policies {:fail RepositoryPolicy/CHECKSUM_POLICY_FAIL
139
:ignore RepositoryPolicy/CHECKSUM_POLICY_IGNORE
140
:warn RepositoryPolicy/CHECKSUM_POLICY_WARN})
143
[policy-settings enabled?]
146
(update-policies (:update policy-settings :daily))
147
(checksum-policies (:checksum policy-settings :fail))))
152
(.setPolicy true (policy settings (:snapshots settings true)))
153
(.setPolicy false (policy settings (:releases settings true)))))
155
(defn- set-authentication
156
"Calls the setAuthentication method on obj"
157
[obj {:keys [username password passphrase private-key-file] :as settings}]
158
(if (or username password private-key-file passphrase)
159
(.setAuthentication obj (Authentication. username password private-key-file passphrase))
163
[repo {:keys [type host port non-proxy-hosts ]
166
(if (and repo host port)
167
(let [prx-sel (doto (DefaultProxySelector.)
168
(.add (set-authentication (Proxy. type host port nil) proxy)
170
prx (.getProxy prx-sel repo)]
171
(.setProxy repo prx))
174
(defn- make-repository
175
[[id settings] proxy]
176
(let [settings-map (if (string? settings)
179
(doto (RemoteRepository. id
180
(:type settings-map "default")
181
(str (:url settings-map)))
182
(set-policies settings-map)
184
(set-authentication settings-map))))
188
(or (namespace group-artifact) (name group-artifact)))
191
(defn- coordinate-string
192
"Produces a coordinate string with a format of
193
<groupId>:<artifactId>[:<extension>[:<classifier>]]:<version>>
194
given a lein-style dependency spec. :extension defaults to jar."
195
[[group-artifact version & {:keys [classifier extension] :or {extension "jar"}}]]
196
(->> [(group group-artifact) (name group-artifact) extension classifier version]
202
[[group-artifact & {:as opts}]]
204
(group group-artifact)
205
(name group-artifact)
206
(:classifier opts "*")
207
(:extension opts "*")))
209
(defn- normalize-exclusion-spec [spec]
215
[[group-artifact version & {:keys [scope optional exclusions]
220
(Dependency. (DefaultArtifact. (coordinate-string dep-spec))
223
(map (comp exclusion normalize-exclusion-spec) exclusions)))
227
(defn- exclusion-spec
228
"Given an Aether Exclusion, returns a lein-style exclusion vector with the
229
:exclusion in its metadata."
231
(with-meta (-> ex bean dep-spec*) {:exclusion ex}))
234
"Given an Aether Dependency, returns a lein-style dependency vector with the
235
:dependency and its corresponding artifact's :file in its metadata."
237
(let [artifact (.getArtifact dep)]
238
(-> (merge (bean dep) (bean artifact))
240
(with-meta {:dependency dep :file (.getFile artifact)}))))
243
"Base function for producing lein-style dependency spec vectors for dependencies
245
[{:keys [groupId artifactId version classifier extension scope optional exclusions]
250
(let [group-artifact (apply symbol (if (= groupId artifactId)
252
[groupId artifactId]))]
253
(vec (concat [group-artifact]
254
(when version [version])
255
(when (and (seq classifier)
256
(not= "*" classifier))
257
[:classifier classifier])
258
(when (and (seq extension)
259
(not (#{"*" "jar"} extension)))
260
[:extension extension])
261
(when optional [:optional true])
262
(when (not= scope "compile")
264
(when (seq exclusions)
265
[:exclusions (vec (map exclusion-spec exclusions))])))))
267
(defn- create-artifact
269
(if-let [file (get files artifact)]
270
(-> (coordinate-string artifact)
272
(.setFile (io/file file)))
273
(throw (IllegalArgumentException. (str "No file provided for artifact " artifact)))))
275
(defn deploy-artifacts
276
"Deploy the artifacts kwarg to the repository kwarg.
278
:files - map from artifact vectors to file paths or java.io.File objects
279
where the file to be deployed for each artifact is to be found
280
An artifact vector is e.g.
281
'[group/artifact \"1.0.0\"] or
282
'[group/artifact \"1.0.0\" :extension \"pom\"].
283
All artifacts should have the same version and group and artifact IDs
284
:repository - {name url} | {name settings}
286
:url - URL of the repository
287
:snapshots - use snapshots versions? (default true)
288
:releases - use release versions? (default true)
289
:username - username to log in with
290
:password - password to log in with
291
:passphrase - passphrase to log in wth
292
:private-key-file - private key file to log in with
293
:update - :daily (default) | :always | :never
294
:checksum - :fail | :ignore | :warn (default)
295
:local-repo - path to the local repository (defaults to ~/.m2/repository)
296
:transfer-listener - same as provided to resolve-dependencies
298
:proxy - proxy configuration, can be nil, the host scheme and type must match
299
:host - proxy hostname
300
:type - http (default) | http | https
302
:non-proxy-hosts - The list of hosts to exclude from proxying, may be null
303
:username - username to log in with, may be null
304
:password - password to log in with, may be null
305
:passphrase - passphrase to log in wth, may be null
306
:private-key-file - private key file to log in with, may be null"
308
[& {:keys [files repository local-repo transfer-listener proxy repository-session-fn]}]
310
(throw (IllegalArgumentException. "Must provide valid :files to deploy-artifacts")))
311
(when (->> (keys files)
312
(map (fn [[ga v]] [(if (namespace ga) ga (symbol (str ga) (str ga))) v]))
316
(throw (IllegalArgumentException.
317
(str "Provided artifacts have varying version, group, or artifact IDs: " (keys files)))))
318
(let [system (repository-system)
319
session ((or repository-session-fn
321
{:repository-system system
322
:local-repo local-repo
324
:transfer-listener transfer-listener})]
325
(.deploy system session
326
(doto (DeployRequest.)
327
(.setArtifacts (vec (map (partial create-artifact files) (keys files))))
328
(.setRepository (first (map #(make-repository % proxy) repository)))))))
330
(defn install-artifacts
331
"Deploy the file kwarg using the coordinates kwarg to the repository kwarg.
333
:files - same as with deploy-artifacts
334
:local-repo - path to the local repository (defaults to ~/.m2/repository)
335
:transfer-listener - same as provided to resolve-dependencies"
336
[& {:keys [files local-repo transfer-listener repository-session-fn]}]
337
(let [system (repository-system)
338
session ((or repository-session-fn
340
{:repository-system system
341
:local-repo local-repo
343
:transfer-listener transfer-listener})]
344
(.install system session
345
(doto (InstallRequest.)
346
(.setArtifacts (vec (map (partial create-artifact files) (keys files))))))))
349
"Takes a coordinates map, an a map from partial coordinates to "
350
[coordinates file-map]
351
(zipmap (map (partial into coordinates) (keys file-map)) (vals file-map)))
353
(defn- optional-artifact
354
"Takes a coordinates map, an a map from partial coordinates to "
355
[artifact-coords path]
356
(when path {artifact-coords path}))
359
"Deploy the jar-file kwarg using the pom-file kwarg and coordinates
360
kwarg to the repository kwarg.
362
:coordinates - [group/name \"version\"]
363
:artifact-map - a map from partial coordinates to file path or File
364
:jar-file - a file pointing to the jar
365
:pom-file - a file pointing to the pom
366
:repository - {name url} | {name settings}
368
:url - URL of the repository
369
:snapshots - use snapshots versions? (default true)
370
:releases - use release versions? (default true)
371
:username - username to log in with
372
:password - password to log in with
373
:passphrase - passphrase to log in wth
374
:private-key-file - private key file to log in with
375
:update - :daily (default) | :always | :never
376
:checksum - :fail (default) | :ignore | :warn
378
:local-repo - path to the local repository (defaults to ~/.m2/repository)
379
:transfer-listener - same as provided to resolve-dependencies
381
:proxy - proxy configuration, can be nil, the host scheme and type must match
382
:host - proxy hostname
383
:type - http (default) | http | https
385
:non-proxy-hosts - The list of hosts to exclude from proxying, may be null
386
:username - username to log in with, may be null
387
:password - password to log in with, may be null
388
:passphrase - passphrase to log in wth, may be null
389
:private-key-file - private key file to log in with, may be null"
390
[& {:keys [coordinates artifact-map jar-file pom-file] :as opts}]
391
(when (empty? coordinates)
393
(IllegalArgumentException. "Must provide valid :coordinates to deploy")))
394
(apply deploy-artifacts
395
(apply concat (assoc opts
396
:files (artifacts-for
400
(optional-artifact [:extension "pom"] pom-file)
401
(optional-artifact [] jar-file)))))))
404
"Install the artifacts specified by the jar-file or file-map and pom-file
405
kwargs using the coordinates kwarg.
407
:coordinates - [group/name \"version\"]
408
:artifact-map - a map from partial coordinates to file path or File
409
:jar-file - a file pointing to the jar
410
:pom-file - a file pointing to the pom
411
:local-repo - path to the local repository (defaults to ~/.m2/repository)
412
:transfer-listener - same as provided to resolve-dependencies"
413
[& {:keys [coordinates artifact-map jar-file pom-file] :as opts}]
414
(when (empty? coordinates)
416
(IllegalArgumentException. "Must provide valid :coordinates to install")))
417
(apply install-artifacts
418
(apply concat (assoc opts
419
:files (artifacts-for
423
(optional-artifact [:extension "pom"] pom-file)
424
(optional-artifact [] jar-file)))))))
426
(defn- dependency-graph
428
(reduce (fn [g ^DependencyNode n]
429
(if-let [dep (.getDependency n)]
430
(update-in g [(dep-spec dep)]
432
(->> (.getChildren n)
433
(map #(.getDependency %))
438
(tree-seq (constantly true)
439
#(seq (.getChildren %))
442
(defn- mirror-selector-fn
443
"Default mirror selection function. The first argument should be a map
444
like that described as the :mirrors argument in resolve-dependencies.
445
The second argument should be a repository spec, also as described in
446
resolve-dependencies. Will return the mirror spec that matches the
447
provided repository spec."
448
[mirrors {:keys [name url snapshots releases]}]
449
(let [mirrors (filter (fn [[matcher mirror-spec]]
451
(and (string? matcher) (or (= matcher name) (= matcher url)))
452
(and (instance? java.util.regex.Pattern matcher)
453
(or (re-matches matcher name) (re-matches matcher url)))))
455
(case (count mirrors)
457
1 (-> mirrors first second)
458
(if (some nil? (map second mirrors))
461
(throw (IllegalArgumentException.
462
(str "Multiple mirrors configured to match repository " {name url} ": "
463
(into {} (map #(update-in % [1] select-keys [:name :url]) mirrors)))))))))
465
(defn- mirror-selector
466
"Returns a MirrorSelector that delegates matching of mirrors to given remote repositories
467
to the provided function. Any returned repository specifications are turned into
468
RemoteRepository instances, and configured to use the provided proxy."
469
[mirror-selector-fn proxy]
470
(reify MirrorSelector
472
(let [repo-spec {:name (.getId repo)
474
:snapshots (-> repo (.getPolicy true) .isEnabled)
475
:releases (-> repo (.getPolicy false) .isEnabled)}
477
{:keys [name repo-manager content-type] :as mirror-spec}
478
(mirror-selector-fn repo-spec)]
479
(when-let [mirror (and mirror-spec (make-repository [name mirror-spec] proxy))]
480
(-> (.setMirroredRepositories mirror [repo])
481
(.setRepositoryManager (boolean repo-manager))
482
(.setContentType (or content-type "default"))))))))
484
(defn resolve-dependencies*
485
"Collects dependencies for the coordinates kwarg, using repositories from the
486
`:repositories` kwarg.
487
Retrieval of dependencies can be disabled by providing `:retrieve false` as a kwarg.
488
Returns an instance of either `org.sonatype.aether.collection.CollectResult` if
489
`:retrieve false` or `org.sonatype.aether.resolution.DependencyResult` if
490
`:retrieve true` (the default). If you don't want to mess with the Aether
491
implmeentation classes, then use `resolve-dependencies` instead.
493
:coordinates - [[group/name \"version\" & settings] ..]
495
:extension - the maven extension (type) to require
496
:classifier - the maven classifier to require
497
:scope - the maven scope for the dependency (default \"compile\")
498
:optional - is the dependency optional? (default \"false\")
499
:exclusions - which sub-dependencies to skip : [group/name & settings]
501
:classifier (default \"*\")
502
:extension (default \"*\")
504
:repositories - {name url ..} | {name settings ..}
505
(defaults to {\"central\" \"http://repo1.maven.org/maven2/\"}
507
:url - URL of the repository
508
:snapshots - use snapshots versions? (default true)
509
:releases - use release versions? (default true)
510
:username - username to log in with
511
:password - password to log in with
512
:passphrase - passphrase to log in wth
513
:private-key-file - private key file to log in with
514
:update - :daily (default) | :always | :never
515
:checksum - :fail (default) | :ignore | :warn
517
:local-repo - path to the local repository (defaults to ~/.m2/repository)
518
:offline? - if true, no remote repositories will be contacted
519
:transfer-listener - the transfer listener that will be notifed of dependency
520
resolution and deployment events.
522
- nil (the default), i.e. no notification of events
523
- :stdout, corresponding to a default listener implementation that writes
524
notifications and progress indicators to stdout, suitable for an
525
interactive console program
526
- a function of one argument, which will be called with a map derived from
528
- an instance of org.sonatype.aether.transfer.TransferListener
530
:proxy - proxy configuration, can be nil, the host scheme and type must match
531
:host - proxy hostname
532
:type - http (default) | http | https
534
:non-proxy-hosts - The list of hosts to exclude from proxying, may be null
535
:username - username to log in with, may be null
536
:password - password to log in with, may be null
537
:passphrase - passphrase to log in wth, may be null
538
:private-key-file - private key file to log in with, may be null
540
:mirrors - {matches settings ..}
541
matches - a string or regex that will be used to match the mirror to
542
candidate repositories. Attempts will be made to match the
543
string/regex to repository names and URLs, with exact string
544
matches preferred. Wildcard mirrors can be specified with
545
a match-all regex such as #\".+\". Excluding a repository
546
from mirroring can be done by mapping a string or regex matching
547
the repository in question to nil.
548
settings include these keys, and all those supported by :repositories:
549
:name - name/id of the mirror
550
:repo-manager - whether the mirror is a repository manager"
552
[& {:keys [repositories coordinates files retrieve local-repo
553
transfer-listener offline? proxy mirrors repository-session-fn]
554
:or {retrieve true}}]
555
(let [repositories (or repositories maven-central)
556
system (repository-system)
557
mirror-selector-fn (memoize (partial mirror-selector-fn mirrors))
558
mirror-selector (mirror-selector mirror-selector-fn proxy)
559
session ((or repository-session-fn
561
{:repository-system system
562
:local-repo local-repo
564
:transfer-listener transfer-listener
565
:mirror-selector mirror-selector})
566
deps (->> coordinates
567
(map #(if-let [local-file (get files %)]
568
(.setArtifact (dependency %)
571
(.setProperties {ArtifactProperties/LOCAL_PATH
572
(.getPath (io/file local-file))})))
575
collect-request (doto (CollectRequest. deps
577
(vec (map #(let [repo (make-repository % proxy)]
583
(.setRequestContext "runtime"))]
585
(.resolveDependencies system session (DependencyRequest. collect-request nil))
586
(.collectDependencies system session collect-request))))
588
(defn resolve-dependencies
589
"Same as `resolve-dependencies*`, but returns a graph of dependencies; each
590
dependency's metadata contains the source Aether Dependency object, and
591
the dependency's :file on disk. Please refer to `resolve-dependencies*` for details
592
on usage, or use it if you need access to Aether dependency resolution objects."
594
(-> (apply resolve-dependencies* args)
598
(defn dependency-files
599
"Given a dependency graph obtained from `resolve-dependencies`, returns a seq of
600
files from the dependencies' metadata."
602
(->> graph keys (map (comp :file meta)) (remove nil?)))
604
(defn- exclusion= [spec1 spec2]
605
(let [[dep & opts] (normalize-exclusion-spec spec1)
606
[sdep & sopts] (normalize-exclusion-spec spec2)
607
om (apply hash-map opts)
608
som (apply hash-map sopts)]
613
(= (:extension om "*")
614
(:extension som "*"))
615
(= (:classifier om "*")
616
(:classifier som "*"))
619
(defn- exclusions-match? [excs sexcs]
620
(if-let [ex (first excs)]
621
(if-let [match (some (partial exclusion= ex) sexcs)]
622
(recur (next excs) (remove #{match} sexcs))
627
"Determines if the first coordinate would be a version in the second
628
coordinate. The first coordinate is not allowed to contain a
630
[[dep version & opts] [sdep sversion & sopts]]
631
(let [om (apply hash-map opts)
632
som (apply hash-map sopts)]
637
(= (:extension om "jar")
638
(:extension som "jar"))
641
(= (:scope om "compile")
642
(:scope som "compile"))
643
(= (:optional om false)
644
(:optional som false))
645
(exclusions-match? (:exclusions om) (:exclusions som))
646
(or (= version sversion)
647
(if-let [[_ ver] (re-find #"^(.*)-SNAPSHOT$" sversion)]
648
(re-find (re-pattern (str "^" ver "-\\d+\\.\\d+-\\d+$"))
650
(let [gsv (GenericVersionScheme.)
651
vc (.parseVersionConstraint gsv sversion)
652
v (.parseVersion gsv version)]
653
(.containsVersion vc v)))))))
655
(defn dependency-hierarchy
656
"Returns a dependency hierarchy based on the provided dependency graph
657
(as returned by `resolve-dependencies`) and the coordinates that should
658
be the root(s) of the hierarchy. Siblings are sorted alphabetically."
659
[root-coordinates dep-graph]
660
(let [root-specs (map (comp dep-spec dependency) root-coordinates)
661
hierarchy (for [root (filter
662
#(some (fn [root] (within? % root)) root-specs)
664
[root (dependency-hierarchy (dep-graph root) dep-graph)])]
665
(when (seq hierarchy)
666
(into (sorted-map-by #(apply compare (map coordinate-string %&))) hierarchy))))