1
/* $Id: ncbi_dispd.c,v 6.62 2004/08/19 15:48:15 lavr Exp $
2
* ===========================================================================
5
* National Center for Biotechnology Information
7
* This software/database is a "United States Government Work" under the
8
* terms of the United States Copyright Act. It was written as part of
9
* the author's official duties as a United States Government employee and
10
* thus cannot be copyrighted. This software/database is freely available
11
* to the public for use. The National Library of Medicine and the U.S.
12
* Government have not placed any restriction on its use or reproduction.
14
* Although all reasonable efforts have been taken to ensure the accuracy
15
* and reliability of the software and data, the NLM and the U.S.
16
* Government do not and cannot warrant the performance or results that
17
* may be obtained by using this software or data. The NLM and the U.S.
18
* Government disclaim all warranties, express or implied, including
19
* warranties of performance, merchantability or fitness for any particular
22
* Please cite the author in any work or product based on this material.
24
* ===========================================================================
26
* Author: Anton Lavrentiev
29
* Low-level API to resolve NCBI service name to the server meta-address
30
* with the use of NCBI network dispatcher (DISPD).
34
#include "ncbi_ansi_ext.h"
35
#include "ncbi_comm.h"
36
#include "ncbi_dispd.h"
37
#include "ncbi_priv.h"
38
#include <connect/ncbi_connection.h>
39
#include <connect/ncbi_http_connector.h>
45
/* Lower bound of up-to-date/out-of-date ratio */
46
#define SERV_DISPD_STALE_RATIO_OK 0.8
47
/* Default rate increase if svc runs locally */
48
#define SERV_DISPD_LOCAL_SVC_BONUS 1.2
51
/* Dispatcher messaging support */
52
static int s_MessageIssued = 0;
53
static FDISP_MessageHook s_MessageHook = 0;
59
static void s_Reset (SERV_ITER);
60
static SSERV_Info* s_GetNextInfo(SERV_ITER, HOST_INFO*);
61
static int/*bool*/ s_Update (SERV_ITER, TNCBI_Time, const char*);
62
static void s_Close (SERV_ITER);
64
static const SSERV_VTable s_op = {
65
s_Reset, s_GetNextInfo, s_Update, 0/*Penalize*/, s_Close, "DISPD"
72
static int s_RandomSeed = 0;
82
int/*bool*/ disp_fail;
83
SConnNetInfo* net_info;
90
static int/*bool*/ s_AddServerInfo(SDISPD_Data* data, SSERV_Info* info)
94
/* First check that the new server info updates an existing one */
95
for (i = 0; i < data->n_node; i++) {
96
if (SERV_EqualInfo(data->s_node[i].info, info)) {
97
/* Replace older version */
98
free(data->s_node[i].info);
99
data->s_node[i].info = info;
104
/* Next, add new service to the list */
105
if (data->n_node == data->n_max_node) {
106
size_t n = data->n_max_node + 10;
110
temp = (SDISPD_Node*) realloc(data->s_node, sizeof(*temp) * n);
112
temp = (SDISPD_Node*) malloc(sizeof(*temp) * n);
117
data->n_max_node = n;
120
data->s_node[data->n_node++].info = info;
127
static int s_ParseHeader(const char*, void*, int);
129
#endif /* __cplusplus */
132
static int/*bool*/ s_ParseHeader(const char* header, void *iter,
133
int/*ignored*/ server_error)
135
SERV_Update((SERV_ITER) iter, header);
136
return 1/*header parsed okay*/;
142
static int s_Adjust(SConnNetInfo*, void*, unsigned int);
144
#endif /* __cplusplus */
147
/* This callback is only for services called via direct HTTP */
148
static int/*bool*/ s_Adjust(SConnNetInfo* net_info,
152
SDISPD_Data* data = (SDISPD_Data*)((SERV_ITER) iter)->data;
153
return data->disp_fail ? 0/*failed*/ : 1/*try again*/;
157
static int/*bool*/ s_Resolve(SERV_ITER iter)
159
static const char service[] = "service";
160
static const char address[] = "address";
161
static const char platform[] = "platform";
162
SDISPD_Data* data = (SDISPD_Data*) iter->data;
163
SConnNetInfo *net_info = data->net_info;
171
/* Dispatcher CGI arguments (sacrifice some if they all do not fit) */
172
if ((arch = CORE_GetPlatform()) != 0 && *arch)
173
ConnNetInfo_PreOverrideArg(net_info, platform, arch);
174
if (*net_info->client_host && !strchr(net_info->client_host, '.') &&
175
(ip = SOCK_gethostbyname(net_info->client_host)) != 0 &&
176
SOCK_ntoa(ip, addr, sizeof(addr)) == 0) {
177
if ((s= malloc(strlen(net_info->client_host) + strlen(addr) + 3)) != 0)
178
sprintf(s, "%s(%s)", net_info->client_host, addr);
180
s = net_info->client_host;
182
s = net_info->client_host;
184
ConnNetInfo_PreOverrideArg(net_info, address, s);
185
if (s != net_info->client_host)
187
if (!ConnNetInfo_PreOverrideArg(net_info, service, iter->service)) {
188
ConnNetInfo_DeleteArg(net_info, platform);
189
if (!ConnNetInfo_PreOverrideArg(net_info, service, iter->service)) {
190
ConnNetInfo_DeleteArg(net_info, address);
191
if (!ConnNetInfo_PreOverrideArg(net_info, service, iter->service))
195
/* Reset request method to be GET ('cause no HTTP body will follow) */
196
net_info->req_method = eReqMethod_Get;
197
/* Obtain additional header information */
198
if ((!(s = SERV_Print(iter))
199
|| ConnNetInfo_OverrideUserHeader(net_info, s)) &&
200
ConnNetInfo_OverrideUserHeader(net_info, net_info->stateless
201
?"Client-Mode: STATELESS_ONLY\r\n"
202
:"Client-Mode: STATEFUL_CAPABLE\r\n") &&
203
ConnNetInfo_OverrideUserHeader(net_info,
204
"Dispatch-Mode: INFORMATION_ONLY\r\n")){
205
ConnNetInfo_OverrideUserHeader
206
(net_info, "User-Agent: NCBIServiceDispatcher/"
207
DISP_PROTOCOL_VERSION
208
#ifdef NCBI_CXX_TOOLKIT
215
/* All the rest in the net_info structure is fine with us */
216
conn = HTTP_CreateConnectorEx(net_info, fHCC_SureFlush, s_ParseHeader,
217
s_Adjust, iter/*data*/, 0/*cleanup*/);
220
ConnNetInfo_DeleteUserHeader(net_info, s);
223
if (!conn || CONN_Create(conn, &c) != eIO_Success) {
224
CORE_LOGF(eLOG_Error, ("[DISPATCHER] Unable to create aux. %s",
225
conn ? "connection" : "connector"));
229
/* This will also send all the HTTP data, and trigger header callback */
232
return ((SDISPD_Data*) iter->data)->n_node != 0;
236
static int/*bool*/ s_Update(SERV_ITER iter, TNCBI_Time now, const char* text)
238
static const char server_info[] = "Server-Info-";
239
SDISPD_Data* data = (SDISPD_Data*) iter->data;
240
size_t len = strlen(text);
242
if (len >= sizeof(server_info) &&
243
strncasecmp(text, server_info, sizeof(server_info) - 1) == 0) {
244
const char* p = text + sizeof(server_info) - 1;
249
if (sscanf(p, "%u: %n", &d1, &d2) < 1)
250
return 0/*not updated*/;
251
if ((info = SERV_ReadInfo(p + d2)) != 0) {
252
assert(info->rate != 0.0);
253
info->time += now; /* expiration time now */
254
if (s_AddServerInfo(data, info))
258
} else if (len >= sizeof(HTTP_DISP_FAILURES) &&
259
strncasecmp(text, HTTP_DISP_FAILURES,
260
sizeof(HTTP_DISP_FAILURES) - 1) == 0) {
261
#if defined(_DEBUG) && !defined(NDEBUG)
262
const char* p = text + sizeof(HTTP_DISP_FAILURES) - 1;
263
while (*p && isspace((unsigned char)(*p)))
265
if (data->net_info->debug_printout)
266
CORE_LOGF(eLOG_Warning, ("[DISPATCHER] %s", p));
270
} else if (len >= sizeof(HTTP_DISP_MESSAGE) &&
271
strncasecmp(text, HTTP_DISP_MESSAGE,
272
sizeof(HTTP_DISP_MESSAGE) - 1) == 0) {
273
const char* p = text + sizeof(HTTP_DISP_MESSAGE) - 1;
274
while (*p && isspace((unsigned char)(*p)))
277
if (s_MessageIssued <= 0) {
282
s_MessageIssued = -1;
283
CORE_LOGF(eLOG_Warning, ("[DISPATCHER] %s", p));
287
return 0/*not updated*/;
291
static int/*bool*/ s_IsUpdateNeeded(SDISPD_Data *data)
293
double status = 0.0, total = 0.0;
296
TNCBI_Time t = (TNCBI_Time) time(0);
298
while (i < data->n_node) {
299
SSERV_Info* info = data->s_node[i].info;
302
if (info->time < t) {
303
if (i < --data->n_node)
304
memmove(data->s_node + i, data->s_node + i + 1,
305
(data->n_node - i)*sizeof(*data->s_node));
308
status += info->rate;
314
return total == 0.0 ? 1 : (status/total < SERV_DISPD_STALE_RATIO_OK);
318
static SSERV_Info* s_GetNextInfo(SERV_ITER iter, HOST_INFO* host_info)
320
double total = 0.0, point = 0.0, access = 0.0, p = 0.0, status;
321
SDISPD_Data* data = (SDISPD_Data*) iter->data;
328
if (s_IsUpdateNeeded(data) && !s_Resolve(iter))
330
assert(data->n_node != 0);
332
for (i = 0; i < data->n_node; i++) {
333
info = data->s_node[i].info;
335
assert(status != 0.0);
337
if (info->host == iter->preferred_host) {
338
if (info->coef <= 0.0 || iter->preference) {
339
status *= SERV_DISPD_LOCAL_SVC_BONUS;
340
if (access < status &&
341
(iter->preference || info->coef < 0.0)) {
343
point = total + status; /* Latch this local server */
348
status *= info->coef;
351
data->s_node[i].status = total;
354
if (point > 0.0 && iter->preference) {
355
if (total != access) {
356
p = SERV_Preference(iter->preference, access/total, data->n_node);
358
p = total*(1.0 - p)/(total - access);
359
for (i = 0; i < data->n_node; i++) {
360
data->s_node[i].status *= p;
361
if (p*point <= data->s_node[i].status)
362
data->s_node[i].status += status - p*access;
367
/* We take pre-chosen local server only if its status is not less than
368
p% of the average remaining status; otherwise, we ignore the server,
369
and apply the generic procedure by seeding a random point. */
370
if (point <= 0.0 || access*(data->n_node - 1) < p*0.01*(total - access))
371
point = (total * rand()) / (double) RAND_MAX;
372
for (i = 0; i < data->n_node; i++) {
373
if (point <= data->s_node[i].status)
376
assert(i < data->n_node);
378
info = data->s_node[i].info;
379
info->rate = data->s_node[i].status - (i ? data->s_node[i-1].status : 0.0);
380
if (i < --data->n_node) {
381
memmove(data->s_node + i, data->s_node + i + 1,
382
(data->n_node - i)*sizeof(*data->s_node));
391
static void s_Reset(SERV_ITER iter)
393
SDISPD_Data* data = (SDISPD_Data*) iter->data;
394
if (data && data->s_node) {
396
assert(data->n_max_node);
397
for (i = 0; i < data->n_node; i++)
398
free(data->s_node[i].info);
404
static void s_Close(SERV_ITER iter)
406
SDISPD_Data* data = (SDISPD_Data*) iter->data;
407
assert(data->n_node == 0); /* s_Reset() had to be called before */
410
ConnNetInfo_Destroy(data->net_info);
416
/***********************************************************************
418
***********************************************************************/
421
const SSERV_VTable* SERV_DISPD_Open(SERV_ITER iter,
422
const SConnNetInfo* net_info,
423
SSERV_Info** info, HOST_INFO* u/*unused*/)
427
if (!(data = (SDISPD_Data*) calloc(1, sizeof(*data))))
430
s_RandomSeed = (int) time(0) + (int) SOCK_gethostbyname(0);
433
data->net_info = ConnNetInfo_Clone(net_info); /*called with non-NULL*/
434
if (iter->types & fSERV_StatelessOnly)
435
data->net_info->stateless = 1/*true*/;
436
if (iter->types & fSERV_Firewall)
437
data->net_info->firewall = 1/*true*/;
440
iter->op = &s_op; /* SERV_Update() - from HTTP callback - expects this */
441
if (!s_Resolve(iter)) {
448
/* call GetNextInfo if info is needed */
455
void DISP_SetMessageHook(FDISP_MessageHook hook)
458
if (hook != s_MessageHook)
459
s_MessageIssued = s_MessageIssued ? -1 : -2;
460
} else if (s_MessageIssued < -1)
462
s_MessageHook = hook;
467
* --------------------------------------------------------------------------
468
* $Log: ncbi_dispd.c,v $
469
* Revision 6.62 2004/08/19 15:48:15 lavr
470
* SERV_ITER::type renamed into SERV_ITER::types to reflect its bitmask nature
472
* Revision 6.61 2003/10/14 14:40:07 lavr
473
* Fix to avoid resolving empty client's host name
475
* Revision 6.60 2003/10/10 19:33:24 lavr
476
* Do not generate address CGI parameter if host address is unknown
478
* Revision 6.59 2003/08/11 19:07:03 lavr
479
* +DISP_SetMessageHook() and implementation of message delivery
481
* Revision 6.58 2003/05/31 05:14:38 lavr
482
* Add ARGSUSED where args are meant to be unused
484
* Revision 6.57 2003/05/22 20:31:40 lavr
487
* Revision 6.56 2003/05/14 15:43:31 lavr
488
* Add host address in dispatcher's CGI query
490
* Revision 6.55 2003/02/13 21:38:22 lavr
491
* Comply with new SERV_Preference() prototype
493
* Revision 6.54 2003/02/06 17:35:36 lavr
494
* Move reset of disp_fail to correct place in s_Resolve()
496
* Revision 6.53 2003/02/04 22:02:44 lavr
497
* Introduce adjustment routine and disp_fail member to avoid MAX_TRY retrying
499
* Revision 6.52 2003/01/31 21:17:37 lavr
500
* Implementation of perference for preferred host
502
* Revision 6.51 2002/12/10 22:11:50 lavr
503
* Stamp HTTP packets with "User-Agent:" header tag and DISP_PROTOCOL_VERSION
505
* Revision 6.50 2002/11/19 19:21:40 lavr
506
* Use client_host from net_info instead of obtaining it explicitly
508
* Revision 6.49 2002/11/01 20:14:07 lavr
509
* Expand hostname buffers to hold up to 256 chars
511
* Revision 6.48 2002/10/28 20:12:56 lavr
512
* Module renamed and host info API included
514
* Revision 6.47 2002/10/28 15:46:21 lavr
515
* Use "ncbi_ansi_ext.h" privately
517
* Revision 6.46 2002/10/21 18:32:35 lavr
518
* Append service arguments "address" and "platform" in dispatcher requests
520
* Revision 6.45 2002/10/11 19:55:20 lavr
521
* Append dispatcher request query with address and platform information
522
* (as the old dispatcher used to do). Also, take advantage of various new
523
* ConnNetInfo_*UserHeader() routines when preparing aux HTTP request.
525
* Revision 6.44 2002/09/24 15:08:50 lavr
526
* Change non-zero rate assertion into more readable (info->rate != 0.0)
528
* Revision 6.43 2002/09/18 16:31:38 lavr
529
* Temporary fix for precision loss removed & replaced with assert()
531
* Revision 6.42 2002/09/06 17:45:40 lavr
532
* Include <connect/ncbi_priv.h> unconditionally (reported by J.Kans)
534
* Revision 6.41 2002/09/06 15:44:19 lavr
535
* Use fHCC_SureFlush and CONN_Flush() instead of dummy read
537
* Revision 6.40 2002/08/12 15:13:50 lavr
538
* Temporary fix for precision loss in transmission of SERV_Info as text
540
* Revision 6.39 2002/08/07 16:33:43 lavr
541
* Changed EIO_ReadMethod enums accordingly; log moved to end
543
* Revision 6.38 2002/05/07 15:31:50 lavr
544
* +#include <stdio.h>: noticed by J.Kans
546
* Revision 6.37 2002/05/06 19:18:12 lavr
547
* Few changes to comply with the rest of API
549
* Revision 6.36 2002/04/13 06:40:05 lavr
550
* Few tweaks to reduce the number of syscalls made
552
* Revision 6.35 2002/03/11 22:01:47 lavr
553
* Threshold for choosing a local server explained better
555
* Revision 6.34 2001/12/04 15:57:05 lavr
556
* Change log correction
558
* Revision 6.33 2001/10/01 19:53:39 lavr
559
* -s_FreeData(), -s_ResetData() - do everything in s_Close()/s_Reset() instead
561
* Revision 6.32 2001/09/29 19:33:04 lavr
562
* BUGFIX: SERV_Update() requires VT bound (was not the case in constructor)
564
* Revision 6.31 2001/09/29 18:41:03 lavr
565
* "Server-Keyed-Info:" removed from protocol
567
* Revision 6.30 2001/09/28 20:52:16 lavr
568
* Update VT method revised as now called on a per-line basis
570
* Revision 6.29 2001/09/24 20:30:01 lavr
571
* Reset() VT method added and utilized
573
* Revision 6.28 2001/09/10 21:23:53 lavr
574
* "Relay-Mode:" tag eliminated from the dispatcher protocol
576
* Revision 6.27 2001/07/24 18:02:02 lavr
577
* Seed random generator at Open()
579
* Revision 6.26 2001/07/18 17:41:25 lavr
580
* BUGFIX: In code for selecting services by preferred host
582
* Revision 6.25 2001/07/03 20:49:44 lavr
583
* RAND_MAX included in the interval search
585
* Revision 6.24 2001/06/25 15:36:38 lavr
586
* s_GetNextInfo now takes one additional argument for host environment
588
* Revision 6.23 2001/06/20 17:27:49 kans
589
* include <time.h> for Mac compiler
591
* Revision 6.22 2001/06/19 19:12:01 lavr
592
* Type change: size_t -> TNCBI_Size; time_t -> TNCBI_Time
594
* Revision 6.21 2001/05/17 15:02:51 lavr
597
* Revision 6.20 2001/05/11 15:30:31 lavr
598
* Protocol change: REQUEST_FAILED -> DISP_FAILURES
600
* Revision 6.19 2001/05/03 16:58:16 lavr
601
* FIX: Percent is taken of local bonus coef instead of the value itself
603
* Revision 6.18 2001/05/03 16:35:53 lavr
604
* Local bonus coefficient modified: meaning of negative value changed
606
* Revision 6.17 2001/04/26 20:20:01 lavr
607
* Better way of choosing local server with a tiny (e.g. penalized) status
609
* Revision 6.16 2001/04/24 21:35:46 lavr
610
* Treatment of new bonus coefficient for local servers
612
* Revision 6.15 2001/03/21 21:24:11 lavr
613
* Type match (int) for %n in scanf
615
* Revision 6.14 2001/03/06 23:57:27 lavr
616
* SERV_DISPD_LOCAL_SVC_BONUS used for services running locally
618
* Revision 6.13 2001/03/05 23:10:46 lavr
619
* SERV_ReadInfo takes only one argument now
621
* Revision 6.12 2001/03/01 00:33:12 lavr
622
* FIXES: Empty update does not generate parse error
623
* Dispathing error is only logged in debug mode; milder severity
625
* Revision 6.11 2001/02/09 17:36:48 lavr
626
* Modified: fSERV_StatelessOnly overrides info->stateless
628
* Revision 6.10 2001/01/25 17:06:36 lavr
629
* s_FreeData now calls ConnNetInfo_Destroy() unconditionally
631
* Revision 6.9 2001/01/12 23:51:40 lavr
632
* Message logging modified for use LOG facility only
634
* Revision 6.8 2001/01/08 23:48:14 lavr
635
* (unsigned char) conversion in isspace
637
* Revision 6.7 2001/01/08 22:40:23 lavr
638
* Further development of service-mapping protocol: stateless/stateful
639
* is now separated from firewall/direct mode (see also in few more files)
641
* Revision 6.6 2000/12/29 18:05:46 lavr
642
* First working revision.
644
* Revision 6.5 2000/10/20 17:36:05 lavr
645
* Partially working dispd dispatcher client (service mapping works)
646
* Checkin for backup purposes; working code '#if 0'-ed out
648
* Revision 6.4 2000/10/05 22:43:30 lavr
649
* Another dummy revision: still in development
651
* Revision 6.3 2000/10/05 22:34:23 lavr
652
* Temporary (dummy) revision for compilation to go
654
* Revision 6.2 2000/05/22 16:53:12 lavr
655
* Rename service_info -> server_info everywhere (including
656
* file names) as the latter name is more relevant
658
* Revision 6.1 2000/05/12 18:43:59 lavr
661
* ==========================================================================