← Back to branch summary

~ubuntu-branches/ubuntu/wily/dovecot/wily

« back to all changes in this revision

Viewing changes to src/lib-http/http-client-host.c

  • Committer: Package Import Robot
  • Author(s): Jaldhar H. Vyas
  • Date: 2013-09-09 00:57:32 UTC
  • mfrom: (1.13.11)
  • mto: (4.8.5 experimental) (1.16.1)
  • mto: This revision was merged to the branch mainline in revision 97.
  • Revision ID: package-import@ubuntu.com-20130909005732-dn1eell8srqbhh0e
Tags: upstream-2.2.5
Import upstream version 2.2.5

Show diffs side-by-side

added added

removed removed

Lines of Context:
src/lib-http/http-client-host.c
 
1
/* Copyright (c) 2013 Dovecot authors, see the included COPYING file */
 
2
 
 
3
#include "lib.h"
 
4
#include "net.h"
 
5
#include "str.h"
 
6
#include "hash.h"
 
7
#include "array.h"
 
8
#include "llist.h"
 
9
#include "ioloop.h"
 
10
#include "istream.h"
 
11
#include "ostream.h"
 
12
#include "dns-lookup.h"
 
13
#include "http-response-parser.h"
 
14
 
 
15
#include "http-client-private.h"
 
16
 
 
17
/*
 
18
 * Logging
 
19
 */
 
20
 
 
21
static inline void
 
22
http_client_host_debug(struct http_client_host *host,
 
23
        const char *format, ...) ATTR_FORMAT(2, 3);
 
24
 
 
25
static inline void
 
26
http_client_host_debug(struct http_client_host *host,
 
27
        const char *format, ...)
 
28
{
 
29
        va_list args;
 
30
 
 
31
        if (host->client->set.debug) {
 
32
 
 
33
                va_start(args, format); 
 
34
                i_debug("http-client: host %s: %s", 
 
35
                        host->name, t_strdup_vprintf(format, args));
 
36
                va_end(args);
 
37
        }
 
38
}
 
39
 
 
40
/*
 
41
 * Host:port
 
42
 */
 
43
 
 
44
static void
 
45
http_client_host_port_connection_setup(struct http_client_host_port *hport);
 
46
 
 
47
static struct http_client_host_port *
 
48
http_client_host_port_find(struct http_client_host *host,
 
49
        unsigned int port, const char *https_name)
 
50
{
 
51
        struct http_client_host_port *hport;
 
52
 
 
53
        array_foreach_modifiable(&host->ports, hport) {
 
54
                if (hport->port == port &&
 
55
                    null_strcmp(hport->https_name, https_name) == 0)
 
56
                        return hport;
 
57
        }
 
58
 
 
59
        return NULL;
 
60
}
 
61
 
 
62
static struct http_client_host_port *
 
63
http_client_host_port_init(struct http_client_host *host,
 
64
        unsigned int port, const char *https_name)
 
65
{
 
66
        struct http_client_host_port *hport;
 
67
 
 
68
        hport = http_client_host_port_find(host, port, https_name);
 
69
        if (hport == NULL) {
 
70
                hport = array_append_space(&host->ports);
 
71
                hport->host = host;
 
72
                hport->port = port;
 
73
                hport->https_name = i_strdup(https_name);
 
74
                hport->ips_connect_idx = 0;
 
75
                i_array_init(&hport->request_queue, 16);
 
76
        }
 
77
 
 
78
        return hport;
 
79
}
 
80
 
 
81
static void http_client_host_port_error(struct http_client_host_port *hport,
 
82
        unsigned int status, const char *error)
 
83
{
 
84
        struct http_client_request **req;
 
85
 
 
86
        /* abort all pending requests */
 
87
        array_foreach_modifiable(&hport->request_queue, req) {
 
88
                http_client_request_error(*req, status, error);
 
89
        }
 
90
        array_clear(&hport->request_queue);
 
91
}
 
92
 
 
93
static void http_client_host_port_deinit(struct http_client_host_port *hport)
 
94
{
 
95
        http_client_host_port_error
 
96
                (hport, HTTP_CLIENT_REQUEST_ERROR_ABORTED, "Aborted");
 
97
        i_free(hport->https_name);
 
98
        array_free(&hport->request_queue);
 
99
}
 
100
 
 
101
static void
 
102
http_client_host_port_drop_request(struct http_client_host_port *hport,
 
103
        struct http_client_request *req)
 
104
{
 
105
        struct http_client_request **req_idx;
 
106
        unsigned int idx;
 
107
 
 
108
        array_foreach_modifiable(&hport->request_queue, req_idx) {
 
109
                if (*req_idx == req) {
 
110
                        idx = array_foreach_idx(&hport->request_queue, req_idx);
 
111
                        array_delete(&hport->request_queue, idx, 1);
 
112
                        break;
 
113
                }
 
114
        }
 
115
}
 
116
 
 
117
static bool
 
118
http_client_hport_is_last_connect_ip(struct http_client_host_port *hport)
 
119
{
 
120
        i_assert(hport->ips_connect_idx < hport->host->ips_count);
 
121
        i_assert(hport->ips_connect_start_idx < hport->host->ips_count);
 
122
 
 
123
        /* we'll always go through all the IPs. we don't necessarily start
 
124
           connecting from the first IP, so we'll need to treat the IPs as
 
125
           a ring buffer where we automatically wrap back to the first IP
 
126
           when necessary. */
 
127
        return (hport->ips_connect_idx + 1) % hport->host->ips_count ==
 
128
                hport->ips_connect_start_idx;
 
129
}
 
130
 
 
131
static void
 
132
http_client_host_port_soft_connect_timeout(struct http_client_host_port *hport)
 
133
{
 
134
        struct http_client_host *host = hport->host;
 
135
 
 
136
        if (hport->to_connect != NULL)
 
137
                timeout_remove(&hport->to_connect);
 
138
 
 
139
        if (http_client_hport_is_last_connect_ip(hport))
 
140
                return;
 
141
 
 
142
        /* if our our previous connection attempt takes longer than the
 
143
           soft_connect_timeout we start a connection attempt to the next IP in
 
144
           parallel */
 
145
 
 
146
        http_client_host_debug(host, "Connection to %s:%u%s is taking a long time; "
 
147
                "starting parallel connection attempt to next IP",
 
148
                net_ip2addr(&host->ips[hport->ips_connect_idx]), hport->port,
 
149
                hport->https_name == NULL ? "" :
 
150
                        t_strdup_printf(" (SSL=%s)", hport->https_name));
 
151
 
 
152
        hport->ips_connect_idx = (hport->ips_connect_idx + 1) % host->ips_count;
 
153
        http_client_host_port_connection_setup(hport);
 
154
}
 
155
 
 
156
static void
 
157
http_client_host_port_connection_setup(struct http_client_host_port *hport)
 
158
{
 
159
        struct http_client_host *host = hport->host;
 
160
        struct http_client_peer *peer = NULL;
 
161
        struct http_client_peer_addr addr;
 
162
        unsigned int msecs;
 
163
 
 
164
        addr.ip = host->ips[hport->ips_connect_idx];
 
165
        addr.port = hport->port;
 
166
        addr.https_name = hport->https_name;
 
167
 
 
168
        http_client_host_debug(host, "Setting up connection to %s:%u%s",
 
169
                net_ip2addr(&addr.ip), addr.port, addr.https_name == NULL ? "" :
 
170
                t_strdup_printf(" (SSL=%s)", addr.https_name));
 
171
 
 
172
        peer = http_client_peer_get(host->client, &addr);
 
173
        http_client_peer_add_host(peer, host);
 
174
        if (http_client_peer_handle_requests(peer))
 
175
                hport->pending_connection_count++;
 
176
 
 
177
        /* start soft connect time-out (but only if we have another IP left) */
 
178
        msecs = host->client->set.soft_connect_timeout_msecs;
 
179
        if (!http_client_hport_is_last_connect_ip(hport) && msecs > 0 &&
 
180
            hport->to_connect == NULL) {
 
181
                hport->to_connect =
 
182
                        timeout_add(msecs, http_client_host_port_soft_connect_timeout, hport);
 
183
        }
 
184
}
 
185
 
 
186
static void
 
187
http_client_host_drop_pending_connections(struct http_client_host_port *hport,
 
188
                                          const struct http_client_peer_addr *addr)
 
189
{
 
190
        struct http_client_peer *peer;
 
191
        struct http_client_connection *const *conns, *conn;
 
192
        unsigned int i, count;
 
193
 
 
194
        for (peer = hport->host->client->peers_list; peer != NULL; peer = peer->next) {
 
195
                if (http_client_peer_addr_cmp(&peer->addr, addr) == 0) {
 
196
                        /* don't drop any connections to the successfully
 
197
                           connected peer, even if some of the connections
 
198
                           are pending. they may be intended for urgent
 
199
                           requests. */
 
200
                        continue;
 
201
                }
 
202
                if (!http_client_peer_have_host(peer, hport->host))
 
203
                        continue;
 
204
 
 
205
                conns = array_get(&peer->conns, &count);
 
206
                for (i = count; i > 0; i--) {
 
207
                        conn = conns[i-1];
 
208
                        if (!conn->connected) {
 
209
                                i_assert(conn->refcount == 1);
 
210
                                /* avoid recreating the connection */
 
211
                                peer->last_connect_failed = TRUE;
 
212
                                http_client_connection_unref(&conn);
 
213
                        }
 
214
                }
 
215
        }
 
216
}
 
217
 
 
218
static unsigned int
 
219
http_client_host_get_ip_idx(struct http_client_host *host,
 
220
                            const struct ip_addr *ip)
 
221
{
 
222
        unsigned int i;
 
223
 
 
224
        for (i = 0; i < host->ips_count; i++) {
 
225
                if (net_ip_compare(&host->ips[i], ip))
 
226
                        return i;
 
227
        }
 
228
        i_unreached();
 
229
}
 
230
 
 
231
static void
 
232
http_client_host_port_connection_success(struct http_client_host_port *hport,
 
233
                                         const struct http_client_peer_addr *addr)
 
234
{
 
235
        /* we achieved at least one connection the the addr->ip */
 
236
        hport->ips_connect_start_idx =
 
237
                http_client_host_get_ip_idx(hport->host, &addr->ip);
 
238
 
 
239
        /* stop soft connect time-out */
 
240
        if (hport->to_connect != NULL)
 
241
                timeout_remove(&hport->to_connect);
 
242
 
 
243
        /* drop all other attempts to the hport. note that we get here whenever
 
244
           a connection is successfully created, so pending_connection_count
 
245
           may be 0. */
 
246
        if (hport->pending_connection_count > 1)
 
247
                http_client_host_drop_pending_connections(hport, addr);
 
248
        /* since this hport is now successfully connected, we won't be
 
249
           getting any connection failures to it anymore. so we need
 
250
           to reset the pending_connection_count count here. */
 
251
        hport->pending_connection_count = 0;
 
252
}
 
253
 
 
254
static bool
 
255
http_client_host_port_connection_failure(struct http_client_host_port *hport,
 
256
        const char *reason)
 
257
{
 
258
        struct http_client_host *host = hport->host;
 
259
 
 
260
        if (hport->pending_connection_count > 0) {
 
261
                /* we're still doing the initial connections to this hport. if
 
262
                   we're also doing parallel connections with soft timeouts
 
263
                   (pending_connection_count>1), wait for them to finish
 
264
                   first. */
 
265
                if (--hport->pending_connection_count > 0)
 
266
                        return TRUE;
 
267
        }
 
268
 
 
269
        /* one of the connections failed. if we're not using soft timeouts,
 
270
           we need to try to connect to the next IP. if we are using soft
 
271
           timeouts, we've already tried all of the IPs by now. */
 
272
        if (hport->to_connect != NULL)
 
273
                timeout_remove(&hport->to_connect);
 
274
 
 
275
        if (http_client_hport_is_last_connect_ip(hport)) {
 
276
                /* all IPs failed, but retry all of them again on the
 
277
                   next request. */
 
278
                hport->ips_connect_idx = hport->ips_connect_start_idx =
 
279
                        (hport->ips_connect_idx + 1) % host->ips_count;
 
280
                http_client_host_port_error(hport,
 
281
                        HTTP_CLIENT_REQUEST_ERROR_CONNECT_FAILED, reason);
 
282
                return FALSE;
 
283
        }
 
284
        hport->ips_connect_idx = (hport->ips_connect_idx + 1) % host->ips_count;
 
285
 
 
286
        http_client_host_port_connection_setup(hport);
 
287
        return TRUE;
 
288
}
 
289
 
 
290
/*
 
291
 * Host
 
292
 */
 
293
 
 
294
void http_client_host_connection_success(struct http_client_host *host,
 
295
        const struct http_client_peer_addr *addr)
 
296
{
 
297
        struct http_client_host_port *hport;
 
298
 
 
299
        http_client_host_debug(host, "Successfully connected to %s:%u",
 
300
                net_ip2addr(&addr->ip), addr->port);
 
301
 
 
302
        hport = http_client_host_port_find(host, addr->port, addr->https_name);
 
303
        if (hport == NULL)
 
304
                return;
 
305
 
 
306
        http_client_host_port_connection_success(hport, addr);
 
307
}
 
308
 
 
309
void http_client_host_connection_failure(struct http_client_host *host,
 
310
        const struct http_client_peer_addr *addr, const char *reason)
 
311
{
 
312
        struct http_client_host_port *hport;
 
313
 
 
314
        http_client_host_debug(host, "Failed to connect to %s:%u: %s",
 
315
                net_ip2addr(&addr->ip), addr->port, reason);
 
316
 
 
317
        hport = http_client_host_port_find(host, addr->port, addr->https_name);
 
318
        if (hport == NULL)
 
319
                return;
 
320
 
 
321
        if (!http_client_host_port_connection_failure(hport, reason)) {
 
322
                /* failed definitively for currently queued requests */
 
323
                if (host->client->ioloop != NULL)
 
324
                        io_loop_stop(host->client->ioloop);
 
325
        }
 
326
}
 
327
 
 
328
static void
 
329
http_client_host_lookup_failure(struct http_client_host *host, const char *error)
 
330
{
 
331
        struct http_client_host_port *hport;
 
332
 
 
333
        error = t_strdup_printf("Failed to lookup host %s: %s",
 
334
                                host->name, error);
 
335
        array_foreach_modifiable(&host->ports, hport) {
 
336
                http_client_host_port_error(hport,
 
337
                        HTTP_CLIENT_REQUEST_ERROR_HOST_LOOKUP_FAILED, error);
 
338
        }
 
339
}
 
340
 
 
341
static void
 
342
http_client_host_dns_callback(const struct dns_lookup_result *result,
 
343
                              struct http_client_host *host)
 
344
{
 
345
        struct http_client_host_port *hport;
 
346
        unsigned int requests = 0;
 
347
 
 
348
        host->dns_lookup = NULL;
 
349
 
 
350
        if (result->ret != 0) {
 
351
                http_client_host_lookup_failure(host, result->error);
 
352
                return;
 
353
        }
 
354
 
 
355
        http_client_host_debug(host,
 
356
                "DNS lookup successful; got %d IPs", result->ips_count);
 
357
 
 
358
        i_assert(result->ips_count > 0);
 
359
        host->ips_count = result->ips_count;
 
360
        host->ips = i_new(struct ip_addr, host->ips_count);
 
361
        memcpy(host->ips, result->ips, sizeof(*host->ips) * host->ips_count);
 
362
 
 
363
        // FIXME: make DNS result expire 
 
364
 
 
365
        /* make connections to requested ports */
 
366
        array_foreach_modifiable(&host->ports, hport) {
 
367
                unsigned int count = array_count(&hport->request_queue);
 
368
                hport->ips_connect_idx = hport->ips_connect_start_idx = 0;
 
369
                if (count > 0)
 
370
                        http_client_host_port_connection_setup(hport);
 
371
                requests += count;
 
372
        }
 
373
 
 
374
        if (requests == 0 && host->client->ioloop != NULL)
 
375
                io_loop_stop(host->client->ioloop);
 
376
}
 
377
 
 
378
static void http_client_host_lookup
 
379
(struct http_client_host *host)
 
380
{
 
381
        struct http_client *client = host->client;
 
382
        struct dns_lookup_settings dns_set;
 
383
        struct ip_addr ip, *ips;
 
384
        unsigned int ips_count;
 
385
        int ret;
 
386
 
 
387
        memset(&dns_set, 0, sizeof(dns_set));
 
388
        dns_set.dns_client_socket_path =
 
389
                client->set.dns_client_socket_path;
 
390
        dns_set.timeout_msecs = HTTP_CLIENT_DNS_LOOKUP_TIMEOUT_MSECS;
 
391
 
 
392
        if (host->ips_count == 0 &&
 
393
            net_addr2ip(host->name, &ip) == 0) { // FIXME: remove this?
 
394
                host->ips_count = 1;
 
395
                host->ips = i_new(struct ip_addr, host->ips_count);
 
396
                host->ips[0] = ip;
 
397
        } else if (dns_set.dns_client_socket_path == NULL) {
 
398
                ret = net_gethostbyname(host->name,     &ips, &ips_count);
 
399
                if (ret != 0) {
 
400
                        http_client_host_lookup_failure(host, net_gethosterror(ret));
 
401
                        return;
 
402
                }
 
403
 
 
404
                http_client_host_debug(host,
 
405
                        "DNS lookup successful; got %d IPs", ips_count);
 
406
 
 
407
                host->ips_count = ips_count;
 
408
                host->ips = i_new(struct ip_addr, ips_count);
 
409
                memcpy(host->ips, ips, ips_count * sizeof(*ips));
 
410
        }
 
411
 
 
412
        if (host->ips_count == 0) {
 
413
                http_client_host_debug(host,
 
414
                        "Performing asynchronous DNS lookup");
 
415
                (void)dns_lookup(host->name, &dns_set,
 
416
                                 http_client_host_dns_callback, host, &host->dns_lookup);
 
417
        }
 
418
}
 
419
 
 
420
struct http_client_host *http_client_host_get
 
421
(struct http_client *client, const char *hostname)
 
422
{
 
423
        struct http_client_host *host;
 
424
 
 
425
        host = hash_table_lookup(client->hosts, hostname);
 
426
        if (host == NULL) {
 
427
                // FIXME: limit the maximum number of inactive cached hosts
 
428
                host = i_new(struct http_client_host, 1);
 
429
                host->client = client;
 
430
                host->name = i_strdup(hostname);
 
431
                i_array_init(&host->ports, 4);
 
432
                i_array_init(&host->delayed_failing_requests, 1);
 
433
 
 
434
                hostname = host->name;
 
435
                hash_table_insert(client->hosts, hostname, host);
 
436
                DLLIST_PREPEND(&client->hosts_list, host);
 
437
 
 
438
                http_client_host_debug(host, "Host created");
 
439
        }
 
440
        return host;
 
441
}
 
442
 
 
443
void http_client_host_submit_request(struct http_client_host *host,
 
444
        struct http_client_request *req)
 
445
{
 
446
        struct http_client_host_port *hport;
 
447
        const char *https_name = req->ssl ? req->hostname : NULL;
 
448
        const char *error;
 
449
 
 
450
        req->host = host;
 
451
 
 
452
        if (req->ssl && host->client->ssl_ctx == NULL) {
 
453
                if (http_client_init_ssl_ctx(host->client, &error) < 0) {
 
454
                        http_client_request_error(req,
 
455
                                HTTP_CLIENT_REQUEST_ERROR_CONNECT_FAILED, error);
 
456
                        return;
 
457
                }
 
458
        }
 
459
 
 
460
        /* add request to host (grouped by tcp port) */
 
461
        hport = http_client_host_port_init(host, req->port, https_name);
 
462
        if (req->urgent)
 
463
                array_insert(&hport->request_queue, 0, &req, 1);
 
464
        else
 
465
                array_append(&hport->request_queue, &req, 1);
 
466
 
 
467
        /* start DNS lookup if necessary */
 
468
        if (host->ips_count == 0 && host->dns_lookup == NULL)   
 
469
                http_client_host_lookup(host);
 
470
 
 
471
        /* make a connection if we have an IP already */
 
472
        if (host->ips_count == 0)
 
473
                return;
 
474
        i_assert(hport->ips_connect_idx < host->ips_count);
 
475
        http_client_host_port_connection_setup(hport);
 
476
}
 
477
 
 
478
struct http_client_request *
 
479
http_client_host_claim_request(struct http_client_host *host,
 
480
        const struct http_client_peer_addr *addr, bool no_urgent)
 
481
{
 
482
        struct http_client_host_port *hport;
 
483
        struct http_client_request *const *requests;
 
484
        struct http_client_request *req;
 
485
        unsigned int i, count;
 
486
 
 
487
        hport = http_client_host_port_find(host, addr->port, addr->https_name);
 
488
        if (hport == NULL)
 
489
                return NULL;
 
490
 
 
491
        requests = array_get(&hport->request_queue, &count);
 
492
        if (count == 0)
 
493
                return NULL;
 
494
        i = 0;
 
495
        if (requests[0]->urgent && no_urgent) {
 
496
                for (; requests[i]->urgent; i++) {
 
497
                        if (i == count)
 
498
                                return NULL;
 
499
                }
 
500
        }
 
501
        req = requests[i];
 
502
        array_delete(&hport->request_queue, i, 1);
 
503
 
 
504
        http_client_host_debug(host,
 
505
                "Connection to peer %s:%u claimed request %s %s",
 
506
                net_ip2addr(&addr->ip), addr->port, http_client_request_label(req),
 
507
                (req->urgent ? "(urgent)" : ""));
 
508
 
 
509
        return req;
 
510
}
 
511
 
 
512
unsigned int http_client_host_requests_pending(struct http_client_host *host,
 
513
        const struct http_client_peer_addr *addr, unsigned int *num_urgent_r)
 
514
{
 
515
        struct http_client_host_port *hport;
 
516
        struct http_client_request *const *requests;
 
517
        unsigned int count, i;
 
518
 
 
519
        *num_urgent_r = 0;
 
520
 
 
521
        hport = http_client_host_port_find(host, addr->port, addr->https_name);
 
522
        if (hport == NULL)
 
523
                return 0;
 
524
 
 
525
        requests = array_get(&hport->request_queue, &count);
 
526
        for (i = 0; i < count && requests[i]->urgent; i++)
 
527
                (*num_urgent_r)++;
 
528
        return count;
 
529
}
 
530
 
 
531
void http_client_host_drop_request(struct http_client_host *host,
 
532
        struct http_client_request *req)
 
533
{
 
534
        struct http_client_host_port *hport;
 
535
        const char *https_name = req->ssl ? req->hostname : NULL;
 
536
 
 
537
        hport = http_client_host_port_find(host, req->port, https_name);
 
538
        if (hport == NULL)
 
539
                return;
 
540
 
 
541
        http_client_host_port_drop_request(hport, req);
 
542
}
 
543
 
 
544
void http_client_host_free(struct http_client_host **_host)
 
545
{
 
546
        struct http_client_host *host = *_host;
 
547
        struct http_client_host_port *hport;
 
548
        struct http_client_request *req, *const *reqp;
 
549
        const char *hostname = host->name;
 
550
 
 
551
        http_client_host_debug(host, "Host destroy");
 
552
 
 
553
        DLLIST_REMOVE(&host->client->hosts_list, host);
 
554
        hash_table_remove(host->client->hosts, hostname);
 
555
 
 
556
        if (host->dns_lookup != NULL)
 
557
                dns_lookup_abort(&host->dns_lookup);
 
558
 
 
559
        /* drop request queues */
 
560
        array_foreach_modifiable(&host->ports, hport) {
 
561
                http_client_host_port_deinit(hport);
 
562
        }
 
563
        array_free(&host->ports);
 
564
 
 
565
        while (array_count(&host->delayed_failing_requests) > 0) {
 
566
                reqp = array_idx(&host->delayed_failing_requests, 0);
 
567
                req = *reqp;
 
568
 
 
569
                i_assert(req->refcount == 1);
 
570
                http_client_request_unref(&req);
 
571
        }
 
572
        array_free(&host->delayed_failing_requests);
 
573
 
 
574
        i_free(host->ips);
 
575
        i_free(host->name);
 
576
        i_free(host);
 
577
}
 
578
 
 
579
void http_client_host_switch_ioloop(struct http_client_host *host)
 
580
{
 
581
        struct http_client_request **req;
 
582
 
 
583
        if (host->dns_lookup != NULL)
 
584
                dns_lookup_switch_ioloop(host->dns_lookup);
 
585
        array_foreach_modifiable(&host->delayed_failing_requests, req) {
 
586
                (*req)->to_delayed_error =
 
587
                        io_loop_move_timeout(&(*req)->to_delayed_error);
 
588
        }
 
589
 
 
590
}