← Back to branch summary

~ubuntu-branches/ubuntu/utopic/dovecot/utopic-proposed

« back to all changes in this revision

Viewing changes to src/lib-http/http-client-peer.c

  • Committer: Package Import Robot
  • Author(s): James Page
  • Date: 2014-01-08 09:35:49 UTC
  • mfrom: (4.1.35 sid)
  • Revision ID: package-import@ubuntu.com-20140108093549-i72o93pux8p0dlaf
Tags: 1:2.2.9-1ubuntu1
* Merge from Debian unstable, remaining changes:
  + Add mail-stack-delivery package:
    - Update d/rules
    - d/control: convert existing dovecot-postfix package to a dummy
      package and add new mail-stack-delivery package.
    - Update maintainer scripts.
    - Rename d/dovecot-postfix.* to debian/mail-stack-delivery.*
    - d/mail-stack-delivery.preinst: Move previously installed backups and
      config files to a new package namespace.
    - d/mail-stack-delivery.prerm: Added to handle downgrades.
  + Use Snakeoil SSL certificates by default:
    - d/control: Depend on ssl-cert.
    - d/dovecot-core.postinst: Relax grep for SSL_* a bit.
  + Add autopkgtest to debian/tests/*.
  + Add ufw integration:
    - d/dovecot-core.ufw.profile: new ufw profile.
    - d/rules: install profile in dovecot-core.
    - d/control: dovecot-core - suggest ufw.
  + d/dovecot-core.dirs: Added usr/share/doc/dovecot-core
  + Add apport hook:
    - d/rules, d/source_dovecot.py
  + Add upstart job:
    - d/rules, d/dovecot-core.dovecot.upstart, d/control,
      d/dovecot-core.dirs, dovecot-imapd.{postrm, postinst, prerm},
      d/dovecot-pop3d.{postinst, postrm, prerm}.
      d/mail-stack-deliver.postinst: Convert init script to upstart.
  + Use the autotools-dev dh addon to update config.guess/config.sub for
    arm64.
* Dropped changes, included in Debian:
  - Update Dovecot name to reflect distribution in login greeting.
  - Update Drac plugin for >= 2.0.0 support.
* d/control: Drop dovecot-postfix package as its no longer required.

Show diffs side-by-side

added added

removed removed

Lines of Context:
src/lib-http/http-client-peer.c
 
1
/* Copyright (c) 2013 Dovecot authors, see the included COPYING file */
 
2
 
 
3
#include "lib.h"
 
4
#include "net.h"
 
5
#include "str.h"
 
6
#include "hash.h"
 
7
#include "array.h"
 
8
#include "llist.h"
 
9
#include "istream.h"
 
10
#include "ostream.h"
 
11
#include "iostream-ssl.h"
 
12
#include "http-response-parser.h"
 
13
 
 
14
#include "http-client-private.h"
 
15
 
 
16
/*
 
17
 * Logging
 
18
 */
 
19
 
 
20
static inline void
 
21
http_client_peer_debug(struct http_client_peer *peer,
 
22
        const char *format, ...) ATTR_FORMAT(2, 3);
 
23
 
 
24
static inline void
 
25
http_client_peer_debug(struct http_client_peer *peer,
 
26
        const char *format, ...)
 
27
{
 
28
        va_list args;
 
29
 
 
30
        if (peer->client->set.debug) {
 
31
                va_start(args, format); 
 
32
                i_debug("http-client: peer %s: %s", 
 
33
                        http_client_peer_label(peer), t_strdup_vprintf(format, args));
 
34
                va_end(args);
 
35
        }
 
36
}
 
37
 
 
38
/*
 
39
 * Peer address
 
40
 */
 
41
 
 
42
unsigned int http_client_peer_addr_hash
 
43
(const struct http_client_peer_addr *peer)
 
44
{
 
45
        switch (peer->type) {
 
46
        case HTTP_CLIENT_PEER_ADDR_RAW:
 
47
                return net_ip_hash(&peer->ip) + peer->port + 1;
 
48
        case HTTP_CLIENT_PEER_ADDR_HTTP:
 
49
                return net_ip_hash(&peer->ip) + peer->port;
 
50
        case HTTP_CLIENT_PEER_ADDR_HTTPS:
 
51
        case HTTP_CLIENT_PEER_ADDR_HTTPS_TUNNEL:
 
52
                return net_ip_hash(&peer->ip) + peer->port +
 
53
                        (peer->https_name == NULL ? 0 : str_hash(peer->https_name));
 
54
        }
 
55
        i_unreached();
 
56
        return 0;
 
57
}
 
58
 
 
59
int http_client_peer_addr_cmp
 
60
(const struct http_client_peer_addr *peer1,
 
61
        const struct http_client_peer_addr *peer2)
 
62
{
 
63
        int ret;
 
64
 
 
65
        if (peer1->type != peer2->type)
 
66
                return (peer1->type > peer2->type ? 1 : -1);
 
67
        if ((ret=net_ip_cmp(&peer1->ip, &peer2->ip)) != 0)
 
68
                return ret;
 
69
        if (peer1->port != peer2->port)
 
70
                return (peer1->port > peer2->port ? 1 : -1);
 
71
        if (peer1->type != HTTP_CLIENT_PEER_ADDR_HTTPS)
 
72
                return 0;
 
73
        return null_strcmp(peer1->https_name, peer2->https_name);
 
74
}
 
75
 
 
76
/*
 
77
 * Peer
 
78
 */
 
79
 
 
80
static void
 
81
http_client_peer_connect(struct http_client_peer *peer, unsigned int count)
 
82
{
 
83
        unsigned int i;
 
84
 
 
85
        for (i = 0; i < count; i++) {
 
86
                http_client_peer_debug(peer,
 
87
                        "Making new connection %u of %u", i+1, count);
 
88
                (void)http_client_connection_create(peer);
 
89
        }
 
90
}
 
91
 
 
92
bool http_client_peer_is_connected(struct http_client_peer *peer)
 
93
{
 
94
        struct http_client_connection *const *conn_idx;
 
95
 
 
96
        array_foreach(&peer->conns, conn_idx) {
 
97
                if ((*conn_idx)->connected)
 
98
                        return TRUE;
 
99
        }
 
100
 
 
101
        return FALSE;
 
102
}
 
103
 
 
104
static void http_client_peer_check_idle(struct http_client_peer *peer)
 
105
{
 
106
        struct http_client_connection *const *conn_idx;
 
107
 
 
108
        array_foreach(&peer->conns, conn_idx) {
 
109
                http_client_connection_check_idle(*conn_idx);
 
110
        }
 
111
}
 
112
 
 
113
static unsigned int
 
114
http_client_peer_requests_pending(struct http_client_peer *peer,
 
115
                                  unsigned int *num_urgent_r)
 
116
{
 
117
        struct http_client_queue *const *queue;
 
118
        unsigned int num_requests = 0, num_urgent = 0, requests, urgent;
 
119
 
 
120
        array_foreach(&peer->queues, queue) {
 
121
                requests = http_client_queue_requests_pending(*queue, &urgent);
 
122
 
 
123
                num_requests += requests;
 
124
                num_urgent += urgent;
 
125
        }
 
126
        *num_urgent_r = num_urgent;
 
127
        return num_requests;
 
128
}
 
129
 
 
130
static void
 
131
http_client_peer_handle_requests_real(struct http_client_peer *peer)
 
132
{
 
133
        struct _conn_available {
 
134
                struct http_client_connection *conn;
 
135
                unsigned int pending_requests;
 
136
        };
 
137
        struct http_client_connection *const *conn_idx;
 
138
        ARRAY(struct _conn_available) conns_avail;
 
139
        struct _conn_available *conn_avail_idx;
 
140
        unsigned int connecting, closing, idle;
 
141
        unsigned int num_pending, num_urgent, new_connections,  working_conn_count;
 
142
        bool statistics_dirty = TRUE;
 
143
 
 
144
        /* FIXME: limit the number of requests handled in one run to prevent
 
145
           I/O starvation. */
 
146
 
 
147
        /* don't do anything unless we have pending requests */
 
148
        num_pending = http_client_peer_requests_pending(peer, &num_urgent);
 
149
        if (num_pending == 0) {
 
150
                http_client_peer_check_idle(peer);
 
151
                return;
 
152
        }
 
153
 
 
154
        t_array_init(&conns_avail, array_count(&peer->conns));
 
155
        do {
 
156
                array_clear(&conns_avail);
 
157
                connecting = closing = idle = 0;
 
158
 
 
159
                /* gather connection statistics */
 
160
                array_foreach(&peer->conns, conn_idx) {
 
161
                        if (http_client_connection_is_ready(*conn_idx)) {                       
 
162
                                struct _conn_available *conn_avail;
 
163
                                unsigned int insert_idx, pending_requests;
 
164
 
 
165
                                /* compile sorted availability list */
 
166
                                pending_requests = http_client_connection_count_pending(*conn_idx);
 
167
                                if (array_count(&conns_avail) == 0) {
 
168
                                        insert_idx = 0;
 
169
                                } else {
 
170
                                        insert_idx = array_count(&conns_avail);
 
171
                                        array_foreach_modifiable(&conns_avail, conn_avail_idx) {
 
172
                                                if (conn_avail_idx->pending_requests > pending_requests) {
 
173
                                                        insert_idx = array_foreach_idx(&conns_avail, conn_avail_idx);
 
174
                                                        break;
 
175
                                                }
 
176
                                        }
 
177
                                }
 
178
                                conn_avail = array_insert_space(&conns_avail, insert_idx);
 
179
                                conn_avail->conn = *conn_idx;
 
180
                                conn_avail->pending_requests = pending_requests;
 
181
                                if (pending_requests == 0)
 
182
                                        idle++;
 
183
                        }
 
184
                        /* count the number of connecting and closing connections */
 
185
                        if ((*conn_idx)->closing)
 
186
                                closing++;
 
187
                        else if (!(*conn_idx)->connected)
 
188
                                connecting++;
 
189
                }
 
190
 
 
191
                working_conn_count = array_count(&peer->conns) - closing;
 
192
                statistics_dirty = FALSE;
 
193
 
 
194
                /* use idle connections right away */
 
195
                if (idle > 0) {
 
196
                        http_client_peer_debug(peer,
 
197
                                "Using %u idle connections to handle %u requests "
 
198
                                "(%u total connections ready)",
 
199
                                idle, num_pending > idle ? idle : num_pending,
 
200
                                array_count(&conns_avail));
 
201
 
 
202
                        array_foreach_modifiable(&conns_avail, conn_avail_idx) {
 
203
                                if (num_pending == 0 || conn_avail_idx->pending_requests > 0)
 
204
                                        break;
 
205
                                idle--;
 
206
                                if (http_client_connection_next_request(conn_avail_idx->conn) <= 0) {
 
207
                                        /* no longer available (probably connection error/closed) */
 
208
                                        statistics_dirty = TRUE;
 
209
                                        conn_avail_idx->conn = NULL;
 
210
                                } else {
 
211
                                        /* update statistics */
 
212
                                        conn_avail_idx->pending_requests++;
 
213
                                        if (num_urgent > 0)
 
214
                                                num_urgent--;
 
215
                                        num_pending--;
 
216
                                }
 
217
                        }
 
218
                }
 
219
        
 
220
                /* don't continue unless we have more pending requests */
 
221
                num_pending = http_client_peer_requests_pending(peer, &num_urgent);
 
222
                if (num_pending == 0) {
 
223
                        http_client_peer_check_idle(peer);
 
224
                        return;
 
225
                }
 
226
        } while (statistics_dirty);
 
227
 
 
228
        i_assert(idle == 0);
 
229
 
 
230
        /* determine how many new connections we can set up */
 
231
        if (peer->last_connect_failed && working_conn_count > 0 &&
 
232
            working_conn_count == connecting) {
 
233
                /* don't create new connections until the existing ones have
 
234
                   finished connecting successfully. */
 
235
                new_connections = 0;
 
236
        } else {
 
237
                if (working_conn_count - connecting + num_urgent >=
 
238
                        peer->client->set.max_parallel_connections) {
 
239
                        /* only create connections for urgent requests */
 
240
                        new_connections = (num_urgent > connecting ? num_urgent - connecting : 0);
 
241
                } else if (num_pending <= connecting) {
 
242
                        /* there are already enough connections being made */
 
243
                        new_connections = 0;
 
244
                } else if (working_conn_count == connecting) {
 
245
                        /* no connections succeeded so far, don't hammer the server with more
 
246
                           than one connection attempt unless its urgent */
 
247
                        if (num_urgent > 0) {
 
248
                                new_connections =
 
249
                                        (num_urgent > connecting ? num_urgent - connecting : 0);
 
250
                        } else {
 
251
                                new_connections = (connecting == 0 ? 1 : 0);
 
252
                        }
 
253
                } else if (num_pending - connecting >
 
254
                        peer->client->set.max_parallel_connections - working_conn_count) {
 
255
                        /* create maximum allowed connections */
 
256
                        new_connections =
 
257
                                peer->client->set.max_parallel_connections - working_conn_count;
 
258
                } else {
 
259
                        /* create as many connections as we need */
 
260
                        new_connections = num_pending - connecting;
 
261
                }
 
262
        }
 
263
 
 
264
        /* create connections */
 
265
        if (new_connections > 0) {
 
266
                http_client_peer_debug(peer,
 
267
                        "Creating %u new connections to handle requests "
 
268
                        "(already %u usable, connecting to %u, closing %u)",
 
269
                        new_connections, working_conn_count - connecting,
 
270
                        connecting, closing);
 
271
                http_client_peer_connect(peer, new_connections);
 
272
                return;
 
273
        }
 
274
 
 
275
        /* cannot create new connections for normal request; attempt pipelining */
 
276
        if (working_conn_count - connecting >=
 
277
                peer->client->set.max_parallel_connections) {
 
278
                unsigned int pipeline_level = 0, total_handled = 0, handled;
 
279
 
 
280
                if (!peer->allows_pipelining) {
 
281
                        http_client_peer_debug(peer,
 
282
                                "Will not pipeline until peer has shown support");
 
283
                        return;
 
284
                }
 
285
 
 
286
                /* fill pipelines */
 
287
                do {
 
288
                        handled = 0;
 
289
                        /* fill smallest pipelines first,
 
290
                           until all pipelines are filled to the same level */
 
291
                        array_foreach_modifiable(&conns_avail, conn_avail_idx) {
 
292
                                if (conn_avail_idx->conn == NULL)
 
293
                                        continue;
 
294
                                if (pipeline_level == 0) {
 
295
                                        pipeline_level = conn_avail_idx->pending_requests;
 
296
                                } else if (conn_avail_idx->pending_requests > pipeline_level) {
 
297
                                        pipeline_level = conn_avail_idx->pending_requests;
 
298
                                        break; /* restart from least busy connection */
 
299
                                }
 
300
                                /* pipeline it */
 
301
                                if (http_client_connection_next_request(conn_avail_idx->conn) <= 0) {
 
302
                                        /* connection now unavailable */
 
303
                                        conn_avail_idx->conn = NULL;
 
304
                                } else {
 
305
                                        /* successfully pipelined */
 
306
                                        conn_avail_idx->pending_requests++;
 
307
                                        num_pending--;
 
308
                                        handled++;
 
309
                                }
 
310
                        }
 
311
                        
 
312
                        total_handled += handled;
 
313
                } while (num_pending > num_urgent && handled > 0);
 
314
 
 
315
                http_client_peer_debug(peer,
 
316
                        "Pipelined %u requests (filled pipelines up to %u requests)",
 
317
                        total_handled, pipeline_level);
 
318
                return;
 
319
        }
 
320
 
 
321
        /* still waiting for connections to finish */
 
322
        http_client_peer_debug(peer,
 
323
                "No request handled; waiting for new connections");
 
324
        return;
 
325
}
 
326
 
 
327
static void http_client_peer_handle_requests(struct http_client_peer *peer)
 
328
{
 
329
        if (peer->to_req_handling != NULL)
 
330
                timeout_remove(&peer->to_req_handling);
 
331
        
 
332
        T_BEGIN {
 
333
                http_client_peer_handle_requests_real(peer);
 
334
        } T_END;
 
335
}
 
336
 
 
337
void http_client_peer_trigger_request_handler(struct http_client_peer *peer)
 
338
{
 
339
        /* trigger request handling through timeout */
 
340
        if (peer->to_req_handling == NULL) {
 
341
                peer->to_req_handling =
 
342
                        timeout_add_short(0, http_client_peer_handle_requests, peer);
 
343
        }
 
344
}
 
345
 
 
346
static struct http_client_peer *
 
347
http_client_peer_create(struct http_client *client,
 
348
                              const struct http_client_peer_addr *addr)
 
349
{
 
350
        struct http_client_peer *peer;
 
351
 
 
352
        i_assert(addr->https_name == NULL || client->ssl_ctx != NULL);
 
353
 
 
354
        peer = i_new(struct http_client_peer, 1);
 
355
        peer->client = client;
 
356
        peer->addr = *addr;
 
357
        peer->https_name = i_strdup(addr->https_name);
 
358
        peer->addr.https_name = peer->https_name;
 
359
        i_array_init(&peer->queues, 16);
 
360
        i_array_init(&peer->conns, 16);
 
361
 
 
362
        hash_table_insert
 
363
                (client->peers, (const struct http_client_peer_addr *)&peer->addr, peer);
 
364
        DLLIST_PREPEND(&client->peers_list, peer);
 
365
 
 
366
        http_client_peer_debug(peer, "Peer created");
 
367
        return peer;
 
368
}
 
369
 
 
370
void http_client_peer_free(struct http_client_peer **_peer)
 
371
{
 
372
        struct http_client_peer *peer = *_peer;
 
373
        struct http_client_connection **conn;
 
374
        ARRAY_TYPE(http_client_connection) conns;
 
375
 
 
376
        if (peer->destroyed)
 
377
                return;
 
378
        peer->destroyed = TRUE;
 
379
 
 
380
        http_client_peer_debug(peer, "Peer destroy");
 
381
 
 
382
        if (peer->to_req_handling != NULL)
 
383
                timeout_remove(&peer->to_req_handling);
 
384
 
 
385
        /* make a copy of the connection array; freed connections modify it */
 
386
        t_array_init(&conns, array_count(&peer->conns));
 
387
        array_copy(&conns.arr, 0, &peer->conns.arr, 0, array_count(&peer->conns));
 
388
        array_foreach_modifiable(&conns, conn) {
 
389
                http_client_connection_unref(conn);
 
390
        }
 
391
 
 
392
        i_assert(array_count(&peer->conns) == 0);
 
393
        array_free(&peer->conns);
 
394
        array_free(&peer->queues);
 
395
 
 
396
        hash_table_remove
 
397
                (peer->client->peers, (const struct http_client_peer_addr *)&peer->addr);
 
398
        DLLIST_REMOVE(&peer->client->peers_list, peer);
 
399
 
 
400
        i_free(peer->https_name);
 
401
        i_free(peer);
 
402
        *_peer = NULL;
 
403
}
 
404
 
 
405
struct http_client_peer *
 
406
http_client_peer_get(struct http_client *client,
 
407
                           const struct http_client_peer_addr *addr)
 
408
{
 
409
        struct http_client_peer *peer;
 
410
 
 
411
        peer = hash_table_lookup(client->peers, addr);
 
412
        if (peer == NULL)
 
413
                peer = http_client_peer_create(client, addr);
 
414
 
 
415
        return peer;
 
416
}
 
417
 
 
418
bool http_client_peer_have_queue(struct http_client_peer *peer,
 
419
                                struct http_client_queue *queue)
 
420
{
 
421
        struct http_client_queue *const *queue_idx;
 
422
 
 
423
        array_foreach(&peer->queues, queue_idx) {
 
424
                if (*queue_idx == queue)
 
425
                        return TRUE;
 
426
        }
 
427
        return FALSE;
 
428
}
 
429
 
 
430
void http_client_peer_link_queue(struct http_client_peer *peer,
 
431
                               struct http_client_queue *queue)
 
432
{
 
433
        if (!http_client_peer_have_queue(peer, queue))
 
434
                array_append(&peer->queues, &queue, 1);
 
435
}
 
436
 
 
437
void http_client_peer_unlink_queue(struct http_client_peer *peer,
 
438
                                struct http_client_queue *queue)
 
439
{
 
440
        struct http_client_queue *const *queue_idx;
 
441
 
 
442
        array_foreach(&peer->queues, queue_idx) {
 
443
                if (*queue_idx == queue) {
 
444
                        array_delete(&peer->queues,
 
445
                                array_foreach_idx(&peer->queues, queue_idx), 1);
 
446
                        if (array_count(&peer->queues) == 0)
 
447
                                http_client_peer_free(&peer);
 
448
                        return;
 
449
                }
 
450
        }
 
451
}
 
452
 
 
453
struct http_client_request *
 
454
http_client_peer_claim_request(struct http_client_peer *peer, bool no_urgent)
 
455
{
 
456
        struct http_client_queue *const *queue_idx;
 
457
        struct http_client_request *req;
 
458
 
 
459
        array_foreach(&peer->queues, queue_idx) {
 
460
                if ((req=http_client_queue_claim_request
 
461
                        (*queue_idx, &peer->addr, no_urgent)) != NULL) {
 
462
                        req->peer = peer;
 
463
                        return req;
 
464
                }
 
465
        }
 
466
 
 
467
        return NULL;
 
468
}
 
469
 
 
470
void http_client_peer_connection_success(struct http_client_peer *peer)
 
471
{
 
472
        struct http_client_queue *const *queue;
 
473
 
 
474
        peer->last_connect_failed = FALSE;
 
475
 
 
476
        array_foreach(&peer->queues, queue) {
 
477
                http_client_queue_connection_success(*queue, &peer->addr);
 
478
        }
 
479
 
 
480
        http_client_peer_trigger_request_handler(peer);
 
481
}
 
482
 
 
483
void http_client_peer_connection_failure(struct http_client_peer *peer,
 
484
                                         const char *reason)
 
485
{
 
486
        struct http_client_queue *const *queue;
 
487
        unsigned int num_urgent;
 
488
 
 
489
        i_assert(array_count(&peer->conns) > 0);
 
490
 
 
491
        http_client_peer_debug(peer, "Failed to make connection");
 
492
 
 
493
        peer->last_connect_failed = TRUE;
 
494
        if (array_count(&peer->conns) > 1) {
 
495
                /* if there are other connections attempting to connect, wait
 
496
                   for them before failing the requests. remember that we had
 
497
                   trouble with connecting so in future we don't try to create
 
498
                   more than one connection until connects work again. */
 
499
        } else {
 
500
                /* this was the only/last connection and connecting to it
 
501
                   failed. a second connect will probably also fail, so just
 
502
                   try another IP for the hosts(s) or abort all requests if this
 
503
                   was the only/last option. */
 
504
                array_foreach(&peer->queues, queue) {
 
505
                        http_client_queue_connection_failure(*queue, &peer->addr, reason);
 
506
                }
 
507
        }
 
508
        if (array_count(&peer->conns) == 0 &&
 
509
            http_client_peer_requests_pending(peer, &num_urgent) == 0)
 
510
                http_client_peer_free(&peer);
 
511
}
 
512
 
 
513
void http_client_peer_connection_lost(struct http_client_peer *peer)
 
514
{
 
515
        unsigned int num_urgent;
 
516
 
 
517
        /* we get here when an already connected connection fails. if the
 
518
           connect itself fails, http_client_peer_connection_failure() is
 
519
           called instead. */
 
520
 
 
521
        if (peer->destroyed)
 
522
                return;
 
523
 
 
524
        http_client_peer_debug(peer, "Lost a connection (%d connections left)",
 
525
                array_count(&peer->conns));
 
526
 
 
527
        /* if there are pending requests for this peer, create a new connection
 
528
           for them. */
 
529
        http_client_peer_trigger_request_handler(peer);
 
530
 
 
531
        if (array_count(&peer->conns) == 0 &&
 
532
            http_client_peer_requests_pending(peer, &num_urgent) == 0)
 
533
                http_client_peer_free(&peer);
 
534
}
 
535
 
 
536
unsigned int http_client_peer_idle_connections(struct http_client_peer *peer)
 
537
{
 
538
        struct http_client_connection *const *conn_idx;
 
539
        unsigned int idle = 0;
 
540
 
 
541
        /* find idle connections */
 
542
        array_foreach(&peer->conns, conn_idx) {
 
543
                if (http_client_connection_is_idle(*conn_idx))
 
544
                        idle++;
 
545
        }
 
546
 
 
547
        return idle;
 
548
}
 
549
 
 
550
void http_client_peer_switch_ioloop(struct http_client_peer *peer)
 
551
{
 
552
        if (peer->to_req_handling != NULL) {
 
553
                peer->to_req_handling =
 
554
                        io_loop_move_timeout(&peer->to_req_handling);
 
555
        }
 
556
}
 
557