|
P99
|
|
P99
|
00001 /* This may look like nonsense, but it really is -*- mode: C -*- */ 00002 /* */ 00003 /* Except for parts copied from previous work and as explicitly stated below, */ 00004 /* the authors and copyright holders for this work are as follows: */ 00005 /* (C) copyright 2011-2013 Jens Gustedt, INRIA, France */ 00006 /* (C) copyright 2012 William Morris */ 00007 /* */ 00008 /* This file is free software; it is part of the P99 project. */ 00009 /* You can redistribute it and/or modify it under the terms of the QPL as */ 00010 /* given in the file LICENSE. It is distributed without any warranty; */ 00011 /* without even the implied warranty of merchantability or fitness for a */ 00012 /* particular purpose. */ 00013 /* */ 00014 #ifndef P99_ATOMIC_H 00015 #define P99_ATOMIC_H 1 00016 00017 #include "p99_enum.h" 00018 #include "p99_generic.h" 00019 00059 #ifdef P99_TEST_ATOMIC 00060 # undef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_1 00061 # undef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_2 00062 # undef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_4 00063 # undef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_8 00064 #endif 00065 00066 #if defined(__arm__) 00067 # include "p99_atomic_arm.h" 00068 #elif defined(__x86_64__) || defined(__i386__) 00069 # include "p99_atomic_x86.h" 00070 #elif defined(__GNUC__) 00071 # include "p99_atomic_gcc.h" 00072 #else 00073 # warning "no support for atomic operations detected for this platform" 00074 # define P00_NO_ATOMICS 00075 #endif 00076 00081 #if !defined(P00_NO_ATOMICS) || defined(P00_DOXYGEN) 00082 00107 #ifndef ATOMIC_INT8_LOCK_FREE 00108 # if defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_1) 00109 # define ATOMIC_INT8_LOCK_FREE 2 00110 # define P00_TYPE_LIST_ELEM1 , (1, uint8_t) 00111 # else 00112 # define ATOMIC_INT8_LOCK_FREE 0 00113 # define P00_TYPE_LIST_ELEM1 00114 # endif 00115 #endif 00116 #ifndef ATOMIC_INT16_LOCK_FREE 00117 # if defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_2) 00118 # define ATOMIC_INT16_LOCK_FREE 2 00119 # define P00_TYPE_LIST_ELEM2 , (2, uint16_t) 00120 # else 00121 # define ATOMIC_INT16_LOCK_FREE 0 00122 # define P00_TYPE_LIST_ELEM2 00123 # endif 00124 #endif 00125 #ifndef ATOMIC_INT32_LOCK_FREE 00126 # if defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_4) 00127 # define ATOMIC_INT32_LOCK_FREE 2 00128 # define P00_TYPE_LIST_ELEM4 , (4, uint32_t) 00129 # else 00130 # define ATOMIC_INT32_LOCK_FREE 0 00131 # define P00_TYPE_LIST_ELEM4 00132 # endif 00133 #endif 00134 #ifndef ATOMIC_INT64_LOCK_FREE 00135 # if defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_8) 00136 # define ATOMIC_INT64_LOCK_FREE 2 00137 # define P00_TYPE_LIST_ELEM8 , (8, uint64_t) 00138 # else 00139 # define ATOMIC_INT64_LOCK_FREE 0 00140 # define P00_TYPE_LIST_ELEM8 00141 # endif 00142 #endif 00143 #ifndef ATOMIC_INT128_LOCK_FREE 00144 # if defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_16) && (defined(UINT128_MAX) || defined(p99x_uint128)) 00145 # define ATOMIC_INT128_LOCK_FREE 2 00146 # if defined(UINT128_MAX) 00147 # define P00_TYPE_LIST_ELEM16 , (16, uint128_t) 00148 # else 00149 # define P00_TYPE_LIST_ELEM16 , (16, p99x_uint128) 00150 # endif 00151 # else 00152 # define ATOMIC_INT128_LOCK_FREE 0 00153 # define P00_TYPE_LIST_ELEM16 00154 # endif 00155 #endif 00156 00157 00158 #ifndef ATOMIC_BOOL_LOCK_FREE 00159 # if defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_4) && (UINT_MAX <= UINT32_MAX) 00160 #define ATOMIC_BOOL_LOCK_FREE 2 00161 # else 00162 #define ATOMIC_BOOL_LOCK_FREE 0 00163 # endif 00164 #endif 00165 #ifndef ATOMIC_CHAR_LOCK_FREE 00166 # define ATOMIC_CHAR_LOCK_FREE ATOMIC_INT8_LOCK_FREE 00167 #endif 00168 #if USHRT_MAX == UINT8_MAX 00169 # define ATOMIC_SHORT_LOCK_FREE ATOMIC_INT8_LOCK_FREE 00170 #elif USHRT_MAX == UINT16_MAX 00171 # define ATOMIC_SHORT_LOCK_FREE ATOMIC_INT16_LOCK_FREE 00172 #elif USHRT_MAX == UINT32_MAX 00173 # define ATOMIC_SHORT_LOCK_FREE ATOMIC_INT32_LOCK_FREE 00174 #elif USHRT_MAX == UINT64_MAX 00175 # define ATOMIC_SHORT_LOCK_FREE ATOMIC_INT64_LOCK_FREE 00176 #elif USHRT_MAX == UINT128_MAX 00177 # define ATOMIC_SHORT_LOCK_FREE ATOMIC_INT128_LOCK_FREE 00178 #else 00179 # define ATOMIC_SHORT_LOCK_FREE 0 00180 #endif 00181 #if UINT_MAX == UINT8_MAX 00182 # define ATOMIC_INT_LOCK_FREE ATOMIC_INT8_LOCK_FREE 00183 #elif UINT_MAX == UINT16_MAX 00184 # define ATOMIC_INT_LOCK_FREE ATOMIC_INT16_LOCK_FREE 00185 #elif UINT_MAX == UINT32_MAX 00186 # define ATOMIC_INT_LOCK_FREE ATOMIC_INT32_LOCK_FREE 00187 #elif UINT_MAX == UINT64_MAX 00188 # define ATOMIC_INT_LOCK_FREE ATOMIC_INT64_LOCK_FREE 00189 #elif UINT_MAX == UINT128_MAX 00190 # define ATOMIC_INT_LOCK_FREE ATOMIC_INT128_LOCK_FREE 00191 #else 00192 # define ATOMIC_INT_LOCK_FREE 0 00193 #endif 00194 #if ULONG_MAX == UINT8_MAX 00195 # define ATOMIC_LONG_LOCK_FREE ATOMIC_INT8_LOCK_FREE 00196 #elif ULONG_MAX == UINT16_MAX 00197 # define ATOMIC_LONG_LOCK_FREE ATOMIC_INT16_LOCK_FREE 00198 #elif ULONG_MAX == UINT32_MAX 00199 # define ATOMIC_LONG_LOCK_FREE ATOMIC_INT32_LOCK_FREE 00200 #elif ULONG_MAX == UINT64_MAX 00201 # define ATOMIC_LONG_LOCK_FREE ATOMIC_INT64_LOCK_FREE 00202 #elif ULONG_MAX == UINT128_MAX 00203 # define ATOMIC_LONG_LOCK_FREE ATOMIC_INT128_LOCK_FREE 00204 #else 00205 # define ATOMIC_LONG_LOCK_FREE 0 00206 #endif 00207 #if ULLONG_MAX == UINT8_MAX 00208 # define ATOMIC_LLONG_LOCK_FREE ATOMIC_INT8_LOCK_FREE 00209 #elif ULLONG_MAX == UINT16_MAX 00210 # define ATOMIC_LLONG_LOCK_FREE ATOMIC_INT16_LOCK_FREE 00211 #elif ULLONG_MAX == UINT32_MAX 00212 # define ATOMIC_LLONG_LOCK_FREE ATOMIC_INT32_LOCK_FREE 00213 #elif ULLONG_MAX == UINT64_MAX 00214 # define ATOMIC_LLONG_LOCK_FREE ATOMIC_INT64_LOCK_FREE 00215 #elif ULLONG_MAX == UINT128_MAX 00216 # define ATOMIC_LLONG_LOCK_FREE ATOMIC_INT128_LOCK_FREE 00217 #else 00218 # define ATOMIC_LLONG_LOCK_FREE 0 00219 #endif 00220 #if UINTPTR_MAX == UINT8_MAX 00221 # define ATOMIC_INTPTR_LOCK_FREE ATOMIC_INT8_LOCK_FREE 00222 #elif UINTPTR_MAX == UINT16_MAX 00223 # define ATOMIC_INTPTR_LOCK_FREE ATOMIC_INT16_LOCK_FREE 00224 #elif UINTPTR_MAX == UINT32_MAX 00225 # define ATOMIC_INTPTR_LOCK_FREE ATOMIC_INT32_LOCK_FREE 00226 #elif UINTPTR_MAX == UINT64_MAX 00227 # define ATOMIC_INTPTR_LOCK_FREE ATOMIC_INT64_LOCK_FREE 00228 #elif UINTPTR_MAX == UINT128_MAX 00229 # define ATOMIC_INTPTR_LOCK_FREE ATOMIC_INT128_LOCK_FREE 00230 #else 00231 # define ATOMIC_INTPTR_LOCK_FREE 0 00232 #endif 00233 #ifndef ATOMIC_POINTER_LOCK_FREE 00234 #define ATOMIC_POINTER_LOCK_FREE ATOMIC_INTPTR_LOCK_FREE 00235 #endif 00236 #if ATOMIC_INTPTR_LOCK_FREE == 2 00237 #if UINTPTR_MAX == UINT8_MAX 00238 # define P00_UINT_TYPE_LIST \ 00239 (1, uintptr_t) \ 00240 P00_TYPE_LIST_ELEM2 \ 00241 P00_TYPE_LIST_ELEM4 \ 00242 P00_TYPE_LIST_ELEM8 \ 00243 P00_TYPE_LIST_ELEM16 00244 #elif UINTPTR_MAX == UINT16_MAX 00245 # define P00_UINT_TYPE_LIST \ 00246 (2, uintptr_t) \ 00247 P00_TYPE_LIST_ELEM1 \ 00248 P00_TYPE_LIST_ELEM4 \ 00249 P00_TYPE_LIST_ELEM8 \ 00250 P00_TYPE_LIST_ELEM16 00251 #elif UINTPTR_MAX == UINT32_MAX 00252 # define P00_UINT_TYPE_LIST \ 00253 (4, uintptr_t) \ 00254 P00_TYPE_LIST_ELEM1 \ 00255 P00_TYPE_LIST_ELEM2 \ 00256 P00_TYPE_LIST_ELEM8 \ 00257 P00_TYPE_LIST_ELEM16 00258 #elif UINTPTR_MAX == UINT64_MAX 00259 # define P00_UINT_TYPE_LIST \ 00260 (8, uintptr_t) \ 00261 P00_TYPE_LIST_ELEM1 \ 00262 P00_TYPE_LIST_ELEM2 \ 00263 P00_TYPE_LIST_ELEM4 \ 00264 P00_TYPE_LIST_ELEM16 00265 #elif UINTPTR_MAX == UINT128_MAX 00266 # define P00_UINT_TYPE_LIST \ 00267 (16, uintptr_t) \ 00268 P00_TYPE_LIST_ELEM1 \ 00269 P00_TYPE_LIST_ELEM2 \ 00270 P00_TYPE_LIST_ELEM4 \ 00271 P00_TYPE_LIST_ELEM8 00272 #else 00273 # define P00_UINT_TYPE_LIST \ 00274 (1, uint8_t) \ 00275 P00_TYPE_LIST_ELEM2 \ 00276 P00_TYPE_LIST_ELEM4 \ 00277 P00_TYPE_LIST_ELEM8 \ 00278 P00_TYPE_LIST_ELEM16 00279 #endif 00280 #endif 00281 #if UINT_LEAST16_MAX == UINT16_MAX 00282 # define ATOMIC_CHAR16_T_LOCK_FREE ATOMIC_INT16_LOCK_FREE 00283 #elif UINT_LEAST16_MAX == UINT32_MAX 00284 # define ATOMIC_CHAR16_T_LOCK_FREE ATOMIC_INT32_LOCK_FREE 00285 #elif UINT_LEAST16_MAX == UINT64_MAX 00286 # define ATOMIC_CHAR16_T_LOCK_FREE ATOMIC_INT64_LOCK_FREE 00287 #elif UINT_LEAST16_MAX == UINT128_MAX 00288 # define ATOMIC_CHAR16_T_LOCK_FREE ATOMIC_INT128_LOCK_FREE 00289 #else 00290 # define ATOMIC_CHAR16_T_LOCK_FREE 0 00291 #endif 00292 #if UINT_LEAST32_MAX == UINT32_MAX 00293 # define ATOMIC_CHAR32_T_LOCK_FREE ATOMIC_INT32_LOCK_FREE 00294 #elif UINT_LEAST32_MAX == UINT64_MAX 00295 # define ATOMIC_CHAR32_T_LOCK_FREE ATOMIC_INT64_LOCK_FREE 00296 #elif UINT_LEAST32_MAX == UINT128_MAX 00297 # define ATOMIC_CHAR32_T_LOCK_FREE ATOMIC_INT128_LOCK_FREE 00298 #else 00299 # define ATOMIC_CHAR32_T_LOCK_FREE 0 00300 #endif 00301 #if WCHAR_MAX == UINT8_MAX || (!WCHAR_MIN && (WCHAR_MAX == INT8_MAX)) 00302 # define ATOMIC_WCHAR_LOCK_FREE ATOMIC_INT8_LOCK_FREE 00303 #elif WCHAR_MAX == UINT16_MAX || (!WCHAR_MIN && (WCHAR_MAX == INT16_MAX)) 00304 # define ATOMIC_WCHAR_LOCK_FREE ATOMIC_INT16_LOCK_FREE 00305 #elif WCHAR_MAX == UINT32_MAX || (!WCHAR_MIN && (WCHAR_MAX == INT32_MAX)) 00306 # define ATOMIC_WCHAR_LOCK_FREE ATOMIC_INT32_LOCK_FREE 00307 #elif WCHAR_MAX == UINT64_MAX || (!WCHAR_MIN && (WCHAR_MAX == INT64_MAX)) 00308 # define ATOMIC_WCHAR_LOCK_FREE ATOMIC_INT64_LOCK_FREE 00309 #elif WCHAR_MAX == UINT128_MAX || (!WCHAR_MIN && (WCHAR_MAX == INT128_MAX)) 00310 # define ATOMIC_WCHAR_LOCK_FREE ATOMIC_INT128_LOCK_FREE 00311 #else 00312 # define ATOMIC_WCHAR_LOCK_FREE 0 00313 #endif 00314 00337 P00_DOCUMENT_TYPE_ARGUMENT(P99_UINT_DEFAULT, 0) 00338 #if defined(P00_UINT_TYPE_LIST) || defined(P00_DOXYGEN) 00339 # define P99_UINT_DEFAULT(T) \ 00340 __typeof__(P99_GENERIC_SIZE_LIT(sizeof(T), (uintptr_t){ 0 }, P00_UINT_TYPE_LIST)) 00341 #else 00342 # define P00_UINT_TYPE_LIST 00343 # define P99_UINT_DEFAULT(T) uintptr_t 00344 #endif 00345 00350 #define ATOMIC_FLAG_INIT P99_ENC_INIT(0) 00351 00356 #define ATOMIC_VAR_INIT(V) { .p00_xval = { .p00_t = (V), }, } 00357 00372 #ifndef P99_ATOMIC_LOCK_FREE_TYPES 00373 # define P99_ATOMIC_LOCK_FREE_TYPES P00_ATOMIC_LOCK_FREE_TYPES6_ 00374 00375 #if ATOMIC_BOOL_LOCK_FREE == 2 00376 # define P00_ATOMIC_LOCK_FREE_TYPES0 _Bool 00377 #endif 00378 #if ATOMIC_CHAR_LOCK_FREE == 2 00379 # define P00_ATOMIC_LOCK_FREE_TYPES1 char, signed char, unsigned char 00380 #endif 00381 #if ATOMIC_SHORT_LOCK_FREE == 2 00382 # define P00_ATOMIC_LOCK_FREE_TYPES2 short, unsigned short 00383 #endif 00384 #if ATOMIC_INT_LOCK_FREE == 2 00385 # define P00_ATOMIC_LOCK_FREE_TYPES3 int, unsigned int 00386 #endif 00387 #if ATOMIC_LONG_LOCK_FREE == 2 00388 # define P00_ATOMIC_LOCK_FREE_TYPES4 long, unsigned long 00389 #endif 00390 #if ATOMIC_LLONG_LOCK_FREE == 2 00391 # define P00_ATOMIC_LOCK_FREE_TYPES5 long long, unsigned long long 00392 #endif 00393 #if ATOMIC_POINTER_LOCK_FREE == 2 00394 # define P00_ATOMIC_LOCK_FREE_TYPES6 char*, signed char*, unsigned char* 00395 #endif 00396 00397 #ifdef P00_ATOMIC_LOCK_FREE_TYPES0 00398 # ifdef P00_ATOMIC_LOCK_FREE_TYPES1 00399 # define P00_ATOMIC_LOCK_FREE_TYPES1_ P00_ATOMIC_LOCK_FREE_TYPES0, P00_ATOMIC_LOCK_FREE_TYPES1 00400 # else 00401 # define P00_ATOMIC_LOCK_FREE_TYPES1_ P00_ATOMIC_LOCK_FREE_TYPES0 00402 # endif 00403 #else 00404 # ifdef P00_ATOMIC_LOCK_FREE_TYPES1 00405 # define P00_ATOMIC_LOCK_FREE_TYPES1_ P00_ATOMIC_LOCK_FREE_TYPES1 00406 # endif 00407 #endif 00408 00409 #ifdef P00_ATOMIC_LOCK_FREE_TYPES1_ 00410 # ifdef P00_ATOMIC_LOCK_FREE_TYPES2 00411 # define P00_ATOMIC_LOCK_FREE_TYPES2_ P00_ATOMIC_LOCK_FREE_TYPES1_, P00_ATOMIC_LOCK_FREE_TYPES2 00412 # else 00413 # define P00_ATOMIC_LOCK_FREE_TYPES2_ P00_ATOMIC_LOCK_FREE_TYPES1_ 00414 # endif 00415 #else 00416 # ifdef P00_ATOMIC_LOCK_FREE_TYPES2 00417 # define P00_ATOMIC_LOCK_FREE_TYPES2_ P00_ATOMIC_LOCK_FREE_TYPES2 00418 # endif 00419 #endif 00420 00421 #ifdef P00_ATOMIC_LOCK_FREE_TYPES2_ 00422 # ifdef P00_ATOMIC_LOCK_FREE_TYPES3 00423 # define P00_ATOMIC_LOCK_FREE_TYPES3_ P00_ATOMIC_LOCK_FREE_TYPES2_, P00_ATOMIC_LOCK_FREE_TYPES3 00424 # else 00425 # define P00_ATOMIC_LOCK_FREE_TYPES3_ P00_ATOMIC_LOCK_FREE_TYPES2_ 00426 # endif 00427 #else 00428 # ifdef P00_ATOMIC_LOCK_FREE_TYPES3 00429 # define P00_ATOMIC_LOCK_FREE_TYPES3_ P00_ATOMIC_LOCK_FREE_TYPES3 00430 # endif 00431 #endif 00432 00433 #ifdef P00_ATOMIC_LOCK_FREE_TYPES3_ 00434 # ifdef P00_ATOMIC_LOCK_FREE_TYPES4 00435 # define P00_ATOMIC_LOCK_FREE_TYPES4_ P00_ATOMIC_LOCK_FREE_TYPES3_, P00_ATOMIC_LOCK_FREE_TYPES4 00436 # else 00437 # define P00_ATOMIC_LOCK_FREE_TYPES4_ P00_ATOMIC_LOCK_FREE_TYPES3_ 00438 # endif 00439 #else 00440 # ifdef P00_ATOMIC_LOCK_FREE_TYPES4 00441 # define P00_ATOMIC_LOCK_FREE_TYPES4_ P00_ATOMIC_LOCK_FREE_TYPES4 00442 # endif 00443 #endif 00444 00445 #ifdef P00_ATOMIC_LOCK_FREE_TYPES4_ 00446 # ifdef P00_ATOMIC_LOCK_FREE_TYPES5 00447 # define P00_ATOMIC_LOCK_FREE_TYPES5_ P00_ATOMIC_LOCK_FREE_TYPES4_, P00_ATOMIC_LOCK_FREE_TYPES5 00448 # else 00449 # define P00_ATOMIC_LOCK_FREE_TYPES5_ P00_ATOMIC_LOCK_FREE_TYPES4_ 00450 # endif 00451 #else 00452 # ifdef P00_ATOMIC_LOCK_FREE_TYPES5 00453 # define P00_ATOMIC_LOCK_FREE_TYPES5_ P00_ATOMIC_LOCK_FREE_TYPES5 00454 # endif 00455 #endif 00456 00457 #ifdef P00_ATOMIC_LOCK_FREE_TYPES5_ 00458 # ifdef P00_ATOMIC_LOCK_FREE_TYPES6 00459 # define P00_ATOMIC_LOCK_FREE_TYPES6_ P00_ATOMIC_LOCK_FREE_TYPES5_, P00_ATOMIC_LOCK_FREE_TYPES6 00460 # else 00461 # define P00_ATOMIC_LOCK_FREE_TYPES6_ P00_ATOMIC_LOCK_FREE_TYPES5_ 00462 # endif 00463 #else 00464 # ifdef P00_ATOMIC_LOCK_FREE_TYPES6 00465 # define P00_ATOMIC_LOCK_FREE_TYPES6_ P00_ATOMIC_LOCK_FREE_TYPES6 00466 # endif 00467 #endif 00468 #endif 00469 00470 #ifndef P00_ATOMIC_LOCK_FREE_TYPES6_ 00471 # define P00_ATOMIC_LOCK_FREE_TYPES6_ 00472 #endif 00473 00484 P99_DECLARE_ENUM(memory_order, 00485 memory_order_relaxed, 00486 memory_order_consume, 00487 memory_order_acquire, 00488 memory_order_release, 00489 memory_order_acq_rel, 00490 memory_order_seq_cst 00491 ); 00492 00493 00518 P99_ENC_DECLARE(uint32_t volatile, atomic_flag); 00519 00520 #define P00_AX(OBJP) ((OBJP)->p00_xval) 00521 #define P00_AT(OBJP) (P00_AX(OBJP).p00_t) 00522 #define P00_AO(OBJP) (P00_AX(OBJP).p00_o) 00523 #define P00_AM(OBJP) (P00_AX(OBJP).p00_m) 00524 00525 P00_DOCUMENT_IDENTIFIER_ARGUMENT(P00_DECLARE_ATOMIC_TYPE, 0) 00526 P00_DOCUMENT_TYPE_ARGUMENT(P00_DECLARE_ATOMIC_TYPE, 1) 00527 P00_DOCUMENT_IDENTIFIER_ARGUMENT(P00_DECLARE_ATOMIC_TYPE, 2) 00528 #define P00_DECLARE_ATOMIC_TYPE(TAGGER, TI, T, NAME) \ 00529 TAGGER P99_PASTE3(NAME, _, TAGGER) { \ 00530 atomic_flag p00_lock; \ 00531 union { \ 00532 TI p00_o; \ 00533 P99_UINT_DEFAULT(T) p00_m; \ 00534 T p00_t; \ 00535 } p00_xval; \ 00536 }; \ 00537 typedef TAGGER P99_PASTE3(NAME, _, TAGGER) P99_PASTE3(NAME, _, TAGGER) 00538 00539 00547 P00_DOCUMENT_TYPE_ARGUMENT(P99_DECLARE_ATOMIC_LOCK_FREE, 0) 00548 P00_DOCUMENT_IDENTIFIER_ARGUMENT(P99_DECLARE_ATOMIC_LOCK_FREE, 1) 00549 #ifdef P00_DOXYGEN 00550 # define P99_DECLARE_ATOMIC_LOCK_FREE(T, NAME) \ 00551 \ 00552 typedef P99_PASTE3(p99_, NAME, _union) NAME 00553 #else 00554 #define P99_DECLARE_ATOMIC_LOCK_FREE(T, NAME) \ 00555 P00_DECLARE_ATOMIC_TYPE(union, T, T, P99_PASTE2(p99_, NAME)); \ 00556 P00_DECLARE_ATOMIC_TYPE(struct, uintptr_t, T, P99_PASTE2(p99_, NAME)); \ 00557 typedef P99_PASTE3(p99_, NAME, _union) NAME 00558 #endif 00559 00560 00561 #define P00_DECLARE_ATOMIC2(T, ...) \ 00562 P00_DECLARE_ATOMIC_TYPE(union, T, T, P99_PASTE(p99_, __VA_ARGS__)); \ 00563 P00_DECLARE_ATOMIC_TYPE(struct, uintptr_t, T, P99_PASTE(p99_, __VA_ARGS__)); \ 00564 \ 00565 typedef P99_PASTE(p99_, __VA_ARGS__, _struct) __VA_ARGS__ 00566 00586 P00_DOCUMENT_TYPE_ARGUMENT(P99_DECLARE_ATOMIC, 0) 00587 P00_DOCUMENT_IDENTIFIER_ARGUMENT(P99_DECLARE_ATOMIC, 1) 00588 #ifdef P00_DOXYGEN 00589 #define P99_DECLARE_ATOMIC(T, NAME) 00590 #else 00591 #define P99_DECLARE_ATOMIC(...) \ 00592 \ 00593 P99_IF_EQ_1(P99_NARG(__VA_ARGS__)) \ 00594 (P00_DECLARE_ATOMIC2(__VA_ARGS__, P99_PASTE2(atomic_, __VA_ARGS__))) \ 00595 (P00_DECLARE_ATOMIC2(__VA_ARGS__)) 00596 #endif 00597 00598 #define P00_DECLARE_ATOMIC(T, ...) \ 00599 \ 00600 typedef P00_ATOMIC_STRUCT(T, __VA_ARGS__) __VA_ARGS__ 00601 00602 #define P00_DECLARE_ATOMIC_CHOICE(MAC, T, NAME) \ 00603 P99_IF_EQ_2(MAC) \ 00604 (P99_DECLARE_ATOMIC_LOCK_FREE(T, NAME)) \ 00605 (P99_DECLARE_ATOMIC(T, NAME)) 00606 00607 00614 P00_DECLARE_ATOMIC_CHOICE(ATOMIC_INT_LOCK_FREE, int, atomic_int); 00615 00625 P00_DECLARE_ATOMIC_CHOICE(ATOMIC_BOOL_LOCK_FREE, _Bool, atomic_bool); 00626 00627 00628 P00_DECLARE_ATOMIC_CHOICE(ATOMIC_CHAR_LOCK_FREE, char, atomic_char); 00629 P00_DECLARE_ATOMIC_CHOICE(ATOMIC_CHAR_LOCK_FREE, signed char, atomic_schar); 00630 P00_DECLARE_ATOMIC_CHOICE(ATOMIC_CHAR_LOCK_FREE, unsigned char, atomic_uchar); 00631 P00_DECLARE_ATOMIC_CHOICE(ATOMIC_SHORT_LOCK_FREE, short, atomic_short); 00632 P00_DECLARE_ATOMIC_CHOICE(ATOMIC_SHORT_LOCK_FREE, unsigned short, atomic_ushort); 00633 P00_DECLARE_ATOMIC_CHOICE(ATOMIC_INT_LOCK_FREE, unsigned int, atomic_uint); 00634 P00_DECLARE_ATOMIC_CHOICE(ATOMIC_LONG_LOCK_FREE, long, atomic_long); 00635 P00_DECLARE_ATOMIC_CHOICE(ATOMIC_LONG_LOCK_FREE, unsigned long, atomic_ulong); 00636 P00_DECLARE_ATOMIC_CHOICE(ATOMIC_LLONG_LOCK_FREE, long long, atomic_llong); 00637 P00_DECLARE_ATOMIC_CHOICE(ATOMIC_LONG_LOCK_FREE, unsigned long long, atomic_ullong); 00638 00639 P00_DECLARE_ATOMIC_CHOICE(ATOMIC_POINTER_LOCK_FREE, char*, atomic_char_ptr); 00640 P00_DECLARE_ATOMIC_CHOICE(ATOMIC_POINTER_LOCK_FREE, signed char*, atomic_schar_ptr); 00641 P00_DECLARE_ATOMIC_CHOICE(ATOMIC_POINTER_LOCK_FREE, unsigned char*, atomic_uchar_ptr); 00642 00643 P99_DECLARE_ATOMIC(float, atomic_float); 00644 P99_DECLARE_ATOMIC(double, atomic_double); 00645 P99_DECLARE_ATOMIC(long double, atomic_ldouble); 00646 00647 #ifndef __STDC_NO_COMPLEX__ 00648 P99_DECLARE_ATOMIC(float _Complex, atomic_cfloat); 00649 P99_DECLARE_ATOMIC(double _Complex, atomic_cdouble); 00650 P99_DECLARE_ATOMIC(long double _Complex, atomic_cldouble); 00651 #define P00_ATOMIC_TYPES \ 00652 (_Bool, atomic_bool*), \ 00653 (char, atomic_char*), \ 00654 (signed char, atomic_schar*), \ 00655 (unsigned char, atomic_uchar*), \ 00656 (short int, atomic_short*), \ 00657 (unsigned short int, atomic_ushort*), \ 00658 (int, atomic_int*), \ 00659 (unsigned int, atomic_uint*), \ 00660 (long int, atomic_long*), \ 00661 (unsigned long int, atomic_ulong*), \ 00662 (long long int, atomic_llong*), \ 00663 (unsigned long long int, atomic_ullong*), \ 00664 (float, atomic_float*), \ 00665 (double, atomic_double*), \ 00666 (long double, atomic_ldouble*), \ 00667 (float _Complex, atomic_cfloat*), \ 00668 (double _Complex, atomic_cdouble*), \ 00669 (long double _Complex, atomic_cldouble*) 00670 #else 00671 #define P00_ATOMIC_TYPES \ 00672 (_Bool, atomic_bool*), \ 00673 (char, atomic_char*), \ 00674 (signed char, atomic_schar*), \ 00675 (unsigned char, atomic_uchar*), \ 00676 (short int, atomic_short*), \ 00677 (unsigned short int, atomic_ushort*), \ 00678 (int, atomic_int*), \ 00679 (unsigned int, atomic_uint*), \ 00680 (long int, atomic_long*), \ 00681 (unsigned long int, atomic_ulong*), \ 00682 (long long int, atomic_llong*), \ 00683 (unsigned long long int, atomic_ullong*), \ 00684 (float, atomic_float*), \ 00685 (double, atomic_double*), \ 00686 (long double, atomic_ldouble*) 00687 #endif 00688 00689 00690 P00_DOCUMENT_TYPE_ARGUMENT(P99_ATOMIC_INHERIT, 0) 00691 #define P99_ATOMIC_INHERIT(T) \ 00692 (*P99_GENERIC_LIT \ 00693 ((T){ 0 }, \ 00694 P99_GENERIC_SIZE_LIT \ 00695 (sizeof(T)+1, \ 00696 (struct P99_PASTE3(p99_atomic_, T, _struct)*){ 0 }, \ 00697 (1, union P99_PASTE3(p99_atomic_, T, _union)*) \ 00698 P99_IF_EQ_2(ATOMIC_INT8_LOCK_FREE)(,(2, union P99_PASTE3(p99_atomic_, T, _union)*))() \ 00699 P99_IF_EQ_2(ATOMIC_INT16_LOCK_FREE)(,(3, union P99_PASTE3(p99_atomic_, T, _union)*))() \ 00700 P99_IF_EQ_2(ATOMIC_INT32_LOCK_FREE)(,(5, union P99_PASTE3(p99_atomic_, T, _union)*))() \ 00701 P99_IF_EQ_2(ATOMIC_INT64_LOCK_FREE)(,(9, union P99_PASTE3(p99_atomic_, T, _union)*))() \ 00702 P99_IF_EQ_2(ATOMIC_INT128_LOCK_FREE)(,(17, union P99_PASTE3(p99_atomic_, T, _union)*))() \ 00703 ), \ 00704 P00_ATOMIC_TYPES)) 00705 00706 p99_inline 00707 P99_CONST_FUNCTION uintptr_t p00_exchange_ignore(void* p00_x, ...) { P99_UNUSED(p00_x); return 0; } 00708 00709 #define P00_EXCHANGE(X) \ 00710 P99_GENERIC_SIZE \ 00711 (sizeof(X), \ 00712 p00_exchange_ignore, \ 00713 (1, p00_atomic_exchange_1), \ 00714 (2, p00_atomic_exchange_2), \ 00715 (4, p00_atomic_exchange_4) \ 00716 P99_IF_EQ_2(ATOMIC_INT64_LOCK_FREE)(,(8, p00_atomic_exchange_8))() \ 00717 ) 00718 00809 #ifdef P00_DOXYGEN 00810 # define _Atomic(T) P99_PASTE2(atomic_, T) 00811 #else 00812 # define _Atomic(T) __typeof__(P99_ATOMIC_INHERIT(T)) 00813 #endif 00814 00815 typedef _Atomic(char16_t) atomic_char16_t; 00816 typedef _Atomic(char32_t) atomic_char32_t; 00817 typedef _Atomic(wchar_t) atomic_wchar_t; 00818 typedef _Atomic(int_least8_t) atomic_int_least8_t; 00819 typedef _Atomic(uint_least8_t) atomic_uint_least8_t; 00820 typedef _Atomic(int_least16_t) atomic_int_least16_t; 00821 typedef _Atomic(uint_least16_t) atomic_uint_least16_t; 00822 typedef _Atomic(int_least32_t) atomic_int_least32_t; 00823 typedef _Atomic(uint_least32_t) atomic_uint_least32_t; 00824 typedef _Atomic(int_least64_t) atomic_int_least64_t; 00825 typedef _Atomic(uint_least64_t) atomic_uint_least64_t; 00826 typedef _Atomic(int_fast8_t) atomic_int_fast8_t; 00827 typedef _Atomic(uint_fast8_t) atomic_uint_fast8_t; 00828 typedef _Atomic(int_fast16_t) atomic_int_fast16_t; 00829 typedef _Atomic(uint_fast16_t) atomic_uint_fast16_t; 00830 typedef _Atomic(int_fast32_t) atomic_int_fast32_t; 00831 typedef _Atomic(uint_fast32_t) atomic_uint_fast32_t; 00832 typedef _Atomic(int_fast64_t) atomic_int_fast64_t; 00833 typedef _Atomic(uint_fast64_t) atomic_uint_fast64_t; 00834 typedef _Atomic(intptr_t) atomic_intptr_t; 00835 typedef _Atomic(uintptr_t) atomic_uintptr_t; 00836 typedef _Atomic(size_t) atomic_size_t; 00837 typedef _Atomic(ptrdiff_t) atomic_ptrdiff_t; 00838 typedef _Atomic(intmax_t) atomic_intmax_t; 00839 typedef _Atomic(uintmax_t) atomic_uintmax_t; 00840 00854 p99_inline 00855 void atomic_thread_fence(memory_order p00_ord) { 00856 switch (p00_ord) { 00857 case memory_order_relaxed: break; 00858 default: p00_mfence(); break; 00859 } 00860 } 00861 00866 #define atomic_signal_fence atomic_thread_fence 00867 00884 p99_inline 00885 _Bool atomic_flag_test_and_set_explicit(volatile atomic_flag *p00_objp, memory_order p00_ord) { 00886 _Bool p00_ret; 00887 switch (p00_ord) { 00888 /* This case doesn't require any guarantee. */ 00889 case memory_order_relaxed: 00890 p00_ret = P99_ENCP(p00_objp); 00891 P99_ENCP(p00_objp) = 1; 00892 break; 00893 /* For these three the acquire semantics are not sufficient. */ 00894 case memory_order_release: ; 00895 case memory_order_acq_rel: ; 00896 case memory_order_seq_cst: 00897 atomic_thread_fence(p00_ord); 00898 p00_ret = p00_sync_lock_test_and_set(&P99_ENCP(p00_objp)); 00899 break; 00900 default: 00901 p00_ret = p00_sync_lock_test_and_set(&P99_ENCP(p00_objp)); 00902 break; 00903 } 00904 return p00_ret; 00905 } 00906 00913 p99_inline 00914 _Bool atomic_flag_test_and_set(volatile atomic_flag *p00_objp) { 00915 return atomic_flag_test_and_set_explicit(p00_objp, memory_order_seq_cst); 00916 } 00917 00929 p99_inline 00930 void atomic_flag_clear_explicit(volatile atomic_flag *p00_objp, memory_order p00_ord) { 00931 switch(p00_ord) { 00932 /* This case doesn't require any guarantee. */ 00933 case memory_order_relaxed: 00934 P99_ENCP(p00_objp) = 0; 00935 break; 00936 /* For these three the release semantics are not sufficient. */ 00937 case memory_order_acquire: ; 00938 case memory_order_acq_rel: ; 00939 case memory_order_seq_cst: 00940 p00_sync_lock_release(&P99_ENCP(p00_objp)); 00941 atomic_thread_fence(p00_ord); 00942 break; 00943 default: 00944 p00_sync_lock_release(&P99_ENCP(p00_objp)); 00945 break; 00946 } 00947 } 00948 00954 p99_inline 00955 void atomic_flag_clear(volatile atomic_flag *p00_objp) { 00956 atomic_flag_clear_explicit(p00_objp, memory_order_seq_cst); 00957 } 00958 00969 p99_inline 00970 void atomic_flag_lock(volatile atomic_flag *p00_objp) { 00971 while (atomic_flag_test_and_set_explicit(p00_objp, memory_order_acquire)); 00972 } 00973 00985 p99_inline 00986 _Bool atomic_flag_trylock(volatile atomic_flag *p00_objp) { 00987 return !atomic_flag_test_and_set_explicit(p00_objp, memory_order_acquire); 00988 } 00989 00997 p99_inline 00998 void atomic_flag_unlock(volatile atomic_flag *p00_objp) { 00999 atomic_flag_clear_explicit(p00_objp, memory_order_release); 01000 } 01001 01025 #define P99_SPIN_EXCLUDE(FLAGP) \ 01026 P00_BLK_START \ 01027 P00_BLK_DECL(register atomic_flag volatile*const, P99_FILEID(flg), (FLAGP)) \ 01028 P00_BLK_BEFAFT(atomic_flag_lock(P99_FILEID(flg)), \ 01029 atomic_flag_unlock(P99_FILEID(flg))) \ 01030 P00_BLK_END 01031 01046 #define atomic_is_lock_free(OBJP) (!offsetof(__typeof__(*OBJP), p00_xval)) 01047 01048 #define P00_ATOMIC_TERN(OBJP, VAL, ALT) P99_TYPED_TERN(atomic_is_lock_free(OBJP), (VAL), (ALT)) 01049 01060 #define atomic_init(OBJP, VAL) \ 01061 p99_extension \ 01062 (void)({ \ 01063 P99_MAC_ARGS((p00_objp, OBJP), (p00_val, VAL)); \ 01064 /* To take care of the atomic_flag and padding bytes. */ \ 01065 memset(p00_objp, 0, sizeof *p00_objp); \ 01066 P00_AT(p00_objp) = p00_val; \ 01067 }) 01068 01069 #ifdef P00_DOXYGEN 01070 01080 #define atomic_exchange(OBJP, DESIRED) 01081 #else 01082 #define atomic_exchange(OBJP, DESIRED) \ 01083 p99_extension \ 01084 ({ \ 01085 P99_MACRO_PVAR(p00_objp, (OBJP)); \ 01086 register __typeof__(P00_AT(p00_objp)) const p00_des = (DESIRED); \ 01087 register __typeof__(P00_AX(p00_objp)) p00_ret = P99_INIT; \ 01088 if (!atomic_is_lock_free(p00_objp)) \ 01089 P99_SPIN_EXCLUDE(&p00_objp->p00_lock) { \ 01090 p00_ret.p00_t = P00_AT(p00_objp); \ 01091 P00_AT(p00_objp) = p00_des; \ 01092 } \ 01093 P99_IF_EMPTY(P99_ATOMIC_LOCK_FREE_TYPES) \ 01094 () \ 01095 (else { \ 01096 register __typeof__(P00_AX(p00_objp)) const p00_desm = { .p00_t = p00_des }; \ 01097 switch (sizeof(P00_AT(p00_objp))) { \ 01098 case 1:; \ 01099 case 2:; \ 01100 case 4:; \ 01101 case 8:; \ 01102 p00_ret.p00_m = P00_EXCHANGE(P00_AT(p00_objp))(&P00_AM(p00_objp), p00_desm.p00_m); \ 01103 break; \ 01104 default: \ 01105 p00_ret.p00_m = P00_AM(p00_objp); \ 01106 for (;;) { \ 01107 P99_MACRO_VAR(p00_valm, \ 01108 __sync_val_compare_and_swap(&P00_AM(p00_objp), p00_ret.p00_m, p00_desm.p00_m)); \ 01109 if (P99_LIKELY(p00_valm == p00_ret.p00_m)) break; \ 01110 p00_ret.p00_m = p00_valm; \ 01111 } \ 01112 } \ 01113 }) \ 01114 p00_ret.p00_t; \ 01115 }) 01116 #endif 01117 01126 #define atomic_load(OBJP) \ 01127 p99_extension \ 01128 ({ \ 01129 P99_MACRO_PVAR(p00_objp, (OBJP)); \ 01130 ((!atomic_is_lock_free(p00_objp)) \ 01131 ? ({ \ 01132 register __typeof__(P00_AX(p00_objp).p00_t) p00_ret; \ 01133 P99_SPIN_EXCLUDE(&p00_objp->p00_lock) \ 01134 p00_ret = P00_AT(p00_objp); \ 01135 p00_ret; \ 01136 }) \ 01137 : (P99_IF_EMPTY(P99_ATOMIC_LOCK_FREE_TYPES) \ 01138 (P00_AT(p00_objp)) \ 01139 ((__typeof__(P00_AX(p00_objp))){ \ 01140 .p00_m = \ 01141 P00_ATOMIC_TERN(p00_objp, \ 01142 __sync_val_compare_and_swap(&P00_AM(p00_objp), 0, 0), \ 01143 P00_AM(p00_objp)), \ 01144 }).p00_t)); \ 01145 }) 01146 01147 #define P00_CVT(EXP) ((void const*)(((struct { void const volatile* a; }){ .a = (EXP) }).a)) 01148 01165 #define atomic_compare_exchange_weak(OBJP, EXPECTED, DESIRED) \ 01166 p99_extension \ 01167 ({ \ 01168 P99_MACRO_PVAR(p00_objp, (OBJP)); \ 01169 register __typeof__(P00_AT(p00_objp))*const p00_exp = (EXPECTED); \ 01170 register __typeof__(P00_AT(p00_objp)) const p00_des = (DESIRED); \ 01171 register _Bool p00_ret = false; \ 01172 if (!atomic_is_lock_free(p00_objp)) { \ 01173 P99_SPIN_EXCLUDE(&p00_objp->p00_lock) { \ 01174 p00_ret = !memcmp(P00_CVT(p00_exp), P00_CVT(&P00_AT(p00_objp)), sizeof P00_AT(p00_objp)); \ 01175 /* Both, *EXPECTED and DESIRED must be assignment compatible \ 01176 with the base type */ \ 01177 if (p00_ret) P00_AT(p00_objp) = p00_des; \ 01178 else *p00_exp = P00_AT(p00_objp); \ 01179 } \ 01180 } \ 01181 P99_IF_EMPTY(P99_ATOMIC_LOCK_FREE_TYPES) \ 01182 (else p00_ret = false;) \ 01183 (else { \ 01184 register __typeof__(P00_AM(p00_objp))*const p00_expm \ 01185 = (__typeof__(P00_AM(p00_objp))*)P00_ATOMIC_TERN(p00_objp, p00_exp, 0); \ 01186 register __typeof__(P00_AX(p00_objp)) const p00_desm = { .p00_t = p00_des }; \ 01187 register __typeof__(P00_AM(p00_objp)) const p00_valm \ 01188 = P00_ATOMIC_TERN \ 01189 (p00_objp, \ 01190 __sync_val_compare_and_swap(&P00_AM(p00_objp), *p00_expm, p00_desm.p00_m), \ 01191 0); \ 01192 p00_ret = (*p00_expm == p00_valm); \ 01193 if (!p00_ret) *p00_expm = p00_valm; \ 01194 }) \ 01195 p00_ret; \ 01196 }) 01197 01198 #ifdef P00_DOXYGEN 01199 01209 #define atomic_store(OBJP, DES) 01210 #else 01211 #define atomic_store(OBJP, DES) ((void)atomic_exchange(OBJP, DES)) 01212 #endif 01213 01214 #define P00_FETCH_OP(OBJP, OPERAND, BUILTIN, OPERATOR) \ 01215 p99_extension \ 01216 ({ \ 01217 P99_MACRO_PVAR(p00_objp, (OBJP)); \ 01218 P99_MACRO_VAR(p00_op, OPERAND); \ 01219 ((!atomic_is_lock_free(p00_objp)) \ 01220 ? ({ \ 01221 register __typeof__(P00_AT(p00_objp)) p00_ret; \ 01222 P99_SPIN_EXCLUDE(&p00_objp->p00_lock) { \ 01223 p00_ret = P00_AT(p00_objp); \ 01224 P00_AT(p00_objp) OPERATOR p00_op; \ 01225 } \ 01226 p00_ret; \ 01227 }) \ 01228 : (P99_IF_EMPTY(P99_ATOMIC_LOCK_FREE_TYPES) \ 01229 (P00_AT(p00_objp)) \ 01230 (P00_ATOMIC_TERN(p00_objp, \ 01231 BUILTIN(&P00_AO(p00_objp), \ 01232 P00_ATOMIC_TERN(p00_objp, p00_op, 0)), \ 01233 P00_AT(p00_objp))))); \ 01234 }) 01235 01248 #define atomic_fetch_add(OBJP, OPERAND) P00_FETCH_OP((OBJP), (OPERAND), __sync_fetch_and_add, +=) 01249 01250 01263 #define atomic_fetch_sub(OBJP, OPERAND) P00_FETCH_OP((OBJP), (OPERAND), __sync_fetch_and_sub, -=) 01264 01278 #define atomic_fetch_or(OBJP, OPERAND) P00_FETCH_OP((OBJP), (OPERAND), __sync_fetch_and_or, |=) 01279 01293 #define atomic_fetch_and(OBJP, OPERAND) P00_FETCH_OP((OBJP), (OPERAND), __sync_fetch_and_and, &=) 01294 01309 #define atomic_fetch_xor(OBJP, OPERAND) P00_FETCH_OP((OBJP), (OPERAND), __sync_fetch_and_xor, ^=) 01310 01311 #define atomic_fetch_add_conditional(OBJP, OPERAND) \ 01312 p99_extension \ 01313 ({ \ 01314 P99_MACRO_PVAR(p00_objp, (OBJP)); \ 01315 P99_MACRO_VAR(p00_op, (OPERAND)); \ 01316 P99_MACRO_VAR(p00_ret, atomic_load(p00_objp)); \ 01317 while (p00_ret) { \ 01318 P99_MACRO_VAR(p00_des, p00_ret + p00_op); \ 01319 if (atomic_compare_exchange_weak(p00_objp, &p00_ret, p00_des)) break; \ 01320 } \ 01321 /* be sure that the result can not be used as an lvalue */ \ 01322 register __typeof__(p00_ret = p00_ret) p00_r = p00_ret; \ 01323 p00_r; \ 01324 }) 01325 01326 #define atomic_fetch_max(OBJP, OPERAND) \ 01327 p99_extension \ 01328 ({ \ 01329 P99_MACRO_PVAR(p00_objp, (OBJP)); \ 01330 P99_MACRO_VAR(p00_op, (OPERAND)); \ 01331 P99_MACRO_VAR(p00_ret, atomic_load(p00_objp)); \ 01332 while (p00_ret <= p00_op) { \ 01333 if (atomic_compare_exchange_weak(p00_objp, &p00_ret, p00_op)) break; \ 01334 } \ 01335 /* be sure that the result can not be used as an lvalue */ \ 01336 register __typeof__(p00_ret = p00_ret) p00_r = p00_ret; \ 01337 p00_r; \ 01338 }) 01339 01372 P99_BLOCK_DOCUMENT 01373 #define P99_CRITICAL \ 01374 P00_BLK_START \ 01375 P00_BLK_DECL_STATIC(atomic_flag, P99_LINEID(crit), ATOMIC_FLAG_INIT) \ 01376 P99_SPIN_EXCLUDE(P99_LINEID(crit)) 01377 01382 # endif /* P00_NO_ATOMICS */ 01383 #undef P00_NO_ATOMICS 01384 01385 #endif
1.7.6.1
