296
294
erts_smp_mtx_lock(&(PS)->mtx)
297
295
#define ERTS_POLLSET_UNLOCK(PS) \
298
296
erts_smp_mtx_unlock(&(PS)->mtx)
299
#define ERTS_POLLSET_SET_POLLED_CHK(PS) \
300
((int) erts_smp_atomic_xchg(&(PS)->polled, (long) 1))
301
#define ERTS_POLLSET_SET_POLLED(PS) \
302
erts_smp_atomic_set(&(PS)->polled, (long) 1)
303
#define ERTS_POLLSET_UNSET_POLLED(PS) \
304
erts_smp_atomic_set(&(PS)->polled, (long) 0)
305
#define ERTS_POLLSET_IS_POLLED(PS) \
306
((int) erts_smp_atomic_read(&(PS)->polled))
308
#define ERTS_POLLSET_SET_POLLER_WOKEN_CHK(PS) set_poller_woken_chk((PS))
309
#define ERTS_POLLSET_SET_POLLER_WOKEN(PS) \
311
ERTS_THR_MEMORY_BARRIER; \
312
erts_smp_atomic_set(&(PS)->woken, (long) 1); \
314
#define ERTS_POLLSET_UNSET_POLLER_WOKEN(PS) \
316
erts_smp_atomic_set(&(PS)->woken, (long) 0); \
317
ERTS_THR_MEMORY_BARRIER; \
319
#define ERTS_POLLSET_IS_POLLER_WOKEN(PS) \
320
((int) erts_smp_atomic_read(&(PS)->woken))
322
#define ERTS_POLLSET_UNSET_INTERRUPTED_CHK(PS) unset_interrupted_chk((PS))
323
#define ERTS_POLLSET_UNSET_INTERRUPTED(PS) \
325
erts_smp_atomic_set(&(PS)->interrupt, (long) 0); \
326
ERTS_THR_MEMORY_BARRIER; \
328
#define ERTS_POLLSET_SET_INTERRUPTED(PS) \
330
ERTS_THR_MEMORY_BARRIER; \
331
erts_smp_atomic_set(&(PS)->interrupt, (long) 1); \
333
#define ERTS_POLLSET_IS_INTERRUPTED(PS) \
334
((int) erts_smp_atomic_read(&(PS)->interrupt))
336
static ERTS_INLINE int
337
unset_interrupted_chk(ErtsPollSet ps)
339
int res = (int) erts_smp_atomic_xchg(&ps->interrupt, (long) 0);
340
ERTS_THR_MEMORY_BARRIER;
345
static ERTS_INLINE int
346
set_poller_woken_chk(ErtsPollSet ps)
348
ERTS_THR_MEMORY_BARRIER;
349
return (int) erts_smp_atomic_xchg(&ps->woken, (long) 1);
354
300
#define ERTS_POLLSET_LOCK(PS)
355
301
#define ERTS_POLLSET_UNLOCK(PS)
356
#define ERTS_POLLSET_SET_POLLED_CHK(PS) 0
357
#define ERTS_POLLSET_UNSET_POLLED(PS)
358
#define ERTS_POLLSET_IS_POLLED(PS) 0
359
#define ERTS_POLLSET_SET_POLLER_WOKEN_CHK(PS) 1
360
#define ERTS_POLLSET_SET_POLLER_WOKEN(PS)
361
#define ERTS_POLLSET_UNSET_POLLER_WOKEN(PS)
362
#define ERTS_POLLSET_IS_POLLER_WOKEN(PS) 1
368
* While atomics are not yet implemented for windows in the common library...
370
* MSDN doc states that SMP machines and old compilers require
371
* InterLockedExchange to properly read and write interlocked
372
* variables, otherwise the processors might reschedule
373
* the access and order of atomics access is destroyed...
374
* While they only mention it in white-papers, the problem
375
* in VS2003 is due to the IA64 arch, so we can still count
376
* on the CPU not rescheduling the access to volatile in X86 arch using
377
* even the slightly older compiler...
379
* So here's (hopefully) a subset of the generally working atomic
383
#if defined(__GNUC__)
384
# if defined(__i386__) || defined(__x86_64__)
385
# define VOLATILE_IN_SEQUENCE 1
387
# define VOLATILE_IN_SEQUENCE 0
389
#elif defined(_MSC_VER)
391
# define VOLATILE_IN_SEQUENCE 0 /* Dont trust really old compilers */
393
# if defined(_M_IX86)
394
# define VOLATILE_IN_SEQUENCE 1
395
# else /* I.e. IA64 */
396
# if _MSC_VER >= 1400
397
# define VOLATILE_IN_SEQUENCE 1
399
# define VOLATILE_IN_SEQUENCE 0
404
# define VOLATILE_IN_SEQUENCE 0
410
306
* Communication with sys_interrupt
414
extern erts_smp_atomic_t erts_break_requested;
310
extern erts_smp_atomic32_t erts_break_requested;
415
311
#define ERTS_SET_BREAK_REQUESTED \
416
erts_smp_atomic_set(&erts_break_requested, (long) 1)
312
erts_smp_atomic32_set_nob(&erts_break_requested, (erts_aint32_t) 1)
417
313
#define ERTS_UNSET_BREAK_REQUESTED \
418
erts_smp_atomic_set(&erts_break_requested, (long) 0)
314
erts_smp_atomic32_set_nob(&erts_break_requested, (erts_aint32_t) 0)
420
316
extern volatile int erts_break_requested;
421
317
#define ERTS_SET_BREAK_REQUESTED (erts_break_requested = 1)
467
363
wait_standby(PS); \
470
#if ERTS_POLL_ASYNC_INTERRUPT_SUPPORT && !defined(ERTS_SMP)
472
static ERTS_INLINE int
473
unset_interrupted_chk(ErtsPollSet ps)
475
/* This operation isn't atomic, but we have no need at all for an
476
atomic operation here... */
477
int res = ps->interrupt;
366
#define ERTS_POLL_NOT_WOKEN ((erts_aint32_t) 0)
367
#define ERTS_POLL_WOKEN_IO_READY ((erts_aint32_t) 1)
368
#define ERTS_POLL_WOKEN_INTR ((erts_aint32_t) 2)
369
#define ERTS_POLL_WOKEN_TIMEDOUT ((erts_aint32_t) 3)
371
static ERTS_INLINE int
372
is_io_ready(ErtsPollSet ps)
374
return erts_atomic32_read_nob(&ps->wakeup_state) == ERTS_POLL_WOKEN_IO_READY;
377
static ERTS_INLINE void
378
woke_up(ErtsPollSet ps)
380
if (erts_atomic32_read_nob(&ps->wakeup_state) == ERTS_POLL_NOT_WOKEN)
381
erts_atomic32_cmpxchg_nob(&ps->wakeup_state,
382
ERTS_POLL_WOKEN_TIMEDOUT,
383
ERTS_POLL_NOT_WOKEN);
386
erts_aint32_t wakeup_state = erts_atomic32_read_nob(&ps->wakeup_state);
387
switch (wakeup_state) {
388
case ERTS_POLL_WOKEN_IO_READY:
389
case ERTS_POLL_WOKEN_INTR:
390
case ERTS_POLL_WOKEN_TIMEDOUT:
400
static ERTS_INLINE int
401
wakeup_cause(ErtsPollSet ps)
404
erts_aint32_t wakeup_state = erts_atomic32_read_nob(&ps->wakeup_state);
405
switch (wakeup_state) {
406
case ERTS_POLL_WOKEN_IO_READY:
409
case ERTS_POLL_WOKEN_INTR:
412
case ERTS_POLL_WOKEN_TIMEDOUT:
417
erl_exit(ERTS_ABORT_EXIT,
418
"%s:%d: Internal error: Invalid wakeup_state=%d\n",
419
__FILE__, __LINE__, (int) wakeup_state);
424
static ERTS_INLINE DWORD
425
poll_wait_timeout(ErtsPollSet ps, SysTimeval *tvp)
427
time_t timeout = tvp->tv_sec * 1000 + tvp->tv_usec / 1000;
434
ResetEvent(ps->event_io_ready);
436
* Since we don't know the internals of ResetEvent() we issue
437
* a memory barrier as a safety precaution ensuring that
438
* the load of wakeup_state wont be reordered with stores made
441
ERTS_THR_MEMORY_BARRIER;
442
if (erts_atomic32_read_nob(&ps->wakeup_state) != ERTS_POLL_NOT_WOKEN)
445
if (timeout > ((time_t) ERTS_AINT32_T_MAX))
446
timeout = ERTS_AINT32_T_MAX; /* Also prevents DWORD overflow */
448
erts_smp_atomic32_set_relb(&ps->timeout, (erts_aint32_t) timeout);
449
return (DWORD) timeout;
485
452
static ERTS_INLINE void
486
wake_poller(ErtsPollSet ps)
453
wake_poller(ErtsPollSet ps, int io_ready)
488
if (!ERTS_POLLSET_SET_POLLER_WOKEN_CHK(ps)) {
455
erts_aint32_t wakeup_state;
457
/* We may set the event multiple times. This is, however, harmless. */
458
wakeup_state = erts_atomic32_read_nob(&ps->wakeup_state);
459
erts_atomic32_set_relb(&ps->wakeup_state, ERTS_POLL_WOKEN_IO_READY);
462
ERTS_THR_MEMORY_BARRIER;
463
wakeup_state = erts_atomic32_read_nob(&ps->wakeup_state);
464
while (wakeup_state != ERTS_POLL_WOKEN_IO_READY
465
&& wakeup_state != ERTS_POLL_WOKEN_INTR) {
466
erts_aint32_t act = erts_atomic32_cmpxchg_nob(&ps->wakeup_state,
467
ERTS_POLL_WOKEN_INTR,
469
if (act == wakeup_state) {
476
if (wakeup_state == ERTS_POLL_NOT_WOKEN) {
478
* Since we don't know the internals of SetEvent() we issue
479
* a memory barrier as a safety precaution ensuring that
480
* the store we just made to wakeup_state wont be reordered
481
* with loads in SetEvent().
483
ERTS_THR_MEMORY_BARRIER;
489
484
SetEvent(ps->event_io_ready);
488
static ERTS_INLINE void
489
reset_io_ready(ErtsPollSet ps)
491
erts_atomic32_set_nob(&ps->wakeup_state, ERTS_POLL_NOT_WOKEN);
494
static ERTS_INLINE void
495
restore_io_ready(ErtsPollSet ps)
497
erts_atomic32_set_nob(&ps->wakeup_state, ERTS_POLL_WOKEN_IO_READY);
501
* notify_io_ready() is used by threads waiting for events, when
502
* notifying a poller thread about I/O ready.
504
static ERTS_INLINE void
505
notify_io_ready(ErtsPollSet ps)
510
static ERTS_INLINE void
511
reset_interrupt(ErtsPollSet ps)
513
/* We need to keep io-ready if set */
514
erts_aint32_t wakeup_state = erts_atomic32_read_nob(&ps->wakeup_state);
515
while (wakeup_state != ERTS_POLL_WOKEN_IO_READY
516
&& wakeup_state != ERTS_POLL_NOT_WOKEN) {
517
erts_aint32_t act = erts_atomic32_cmpxchg_nob(&ps->wakeup_state,
520
if (wakeup_state == act)
524
ERTS_THR_MEMORY_BARRIER;
527
static ERTS_INLINE void
528
set_interrupt(ErtsPollSet ps)
494
533
static void setup_standby_wait(ErtsPollSet ps, int num_threads)
967
1003
void erts_poll_interrupt(ErtsPollSet ps, int set /* bool */)
969
1005
HARDTRACEF(("In erts_poll_interrupt(%d)",set));
972
ERTS_POLLSET_SET_INTERRUPTED(ps);
976
ERTS_POLLSET_UNSET_INTERRUPTED(ps);
1007
reset_interrupt(ps);
979
1010
HARDTRACEF(("Out erts_poll_interrupt(%d)",set));
982
1013
void erts_poll_interrupt_timed(ErtsPollSet ps,
983
1014
int set /* bool */,
1015
erts_short_time_t msec)
986
1017
HARDTRACEF(("In erts_poll_interrupt_timed(%d,%ld)",set,msec));
989
if (erts_smp_atomic_read(&ps->timeout) > msec) {
990
ERTS_POLLSET_SET_INTERRUPTED(ps);
995
ERTS_POLLSET_UNSET_INTERRUPTED(ps);
1019
reset_interrupt(ps);
1020
else if (erts_smp_atomic32_read_acqb(&ps->timeout) > (erts_aint32_t) msec)
998
1022
HARDTRACEF(("Out erts_poll_interrupt_timed"));
1127
1149
no_fds = ERTS_POLL_MAX_RES;
1131
ResetEvent(ps->event_io_ready);
1132
ERTS_POLLSET_UNSET_POLLER_WOKEN(ps);
1135
if (ERTS_POLLSET_IS_INTERRUPTED(ps)) {
1136
/* Interrupt use zero timeout */
1143
timeout = tvp->tv_sec * 1000 + tvp->tv_usec / 1000;
1152
timeout = poll_wait_timeout(ps, tvp);
1144
1154
/*HARDDEBUGF(("timeout = %ld",(long) timeout));*/
1145
erts_smp_atomic_set(&ps->timeout, timeout);
1147
if (timeout > 0 && ! erts_atomic_read(&ps->sys_io_ready) && ! erts_atomic_read(&break_waiter_state)) {
1156
if (timeout > 0 && !erts_atomic32_read_nob(&break_waiter_state)) {
1148
1157
HANDLE harr[2] = {ps->event_io_ready, break_happened_event};
1151
HARDDEBUGF(("Start waiting %d [%d]",num_h, (long) timeout));
1160
HARDDEBUGF(("Start waiting %d [%d]",num_h, (int) timeout));
1152
1161
ERTS_POLLSET_UNLOCK(ps);
1163
erts_thr_progress_prepare_wait(NULL);
1153
1165
WaitForMultipleObjects(num_h, harr, FALSE, timeout);
1167
erts_thr_progress_finalize_wait(NULL);
1154
1169
ERTS_POLLSET_LOCK(ps);
1155
HARDDEBUGF(("Stop waiting %d [%d]",num_h, (long) timeout));
1170
HARDDEBUGF(("Stop waiting %d [%d]",num_h, (int) timeout));
1158
1174
ERTS_UNSET_BREAK_REQUESTED;
1159
if(erts_atomic_read(&break_waiter_state)) {
1175
if(erts_atomic32_read_nob(&break_waiter_state)) {
1160
1176
erts_mtx_lock(&break_waiter_lock);
1161
break_state = erts_atomic_read(&break_waiter_state);
1162
erts_atomic_set(&break_waiter_state,0);
1177
break_state = erts_atomic32_read_nob(&break_waiter_state);
1178
erts_atomic32_set_nob(&break_waiter_state,0);
1163
1179
ResetEvent(break_happened_event);
1164
1180
erts_mtx_unlock(&break_waiter_lock);
1165
1181
switch (break_state) {
1306
1320
ps->standby_wait_counter = 0;
1307
1321
ps->event_io_ready = CreateManualEvent(FALSE);
1308
1322
ps->standby_wait_event = CreateManualEvent(FALSE);
1309
erts_atomic_init(&ps->sys_io_ready,0);
1310
1323
ps->restore_events = 0;
1325
erts_atomic32_init_nob(&ps->wakeup_state, ERTS_POLL_NOT_WOKEN);
1312
1326
#ifdef ERTS_SMP
1313
erts_smp_atomic_init(&ps->woken, 0);
1314
1327
erts_smp_mtx_init(&ps->mtx, "pollset");
1315
erts_smp_atomic_init(&ps->interrupt, 0);
1317
erts_smp_atomic_init(&ps->timeout, LONG_MAX);
1329
erts_smp_atomic32_init_nob(&ps->timeout, ERTS_AINT32_T_MAX);
1319
1331
HARDTRACEF(("Out erts_poll_create_pollset"));