/* vim:set sw=4 ts=8 fileencoding=cp1251::Кодировка:WINDOWS-1251[АБЁЪЯабёъя] */
#ifdef _WIN32
    #pragma setlocale("rus")
#endif
/*
 * Copyright(C) 2014 Проект ИОК
 *
 * Этот файл содержит информацию, являющуюся
 * собственностью компании Крипто Про.
 *
 * Любая часть этого файла не может быть скопирована,
 * исправлена, переведена на другие языки,
 * локализована или модифицирована любым способом,
 * откомпилирована, передана по сети с или на
 * любую компьютерную систему без предварительного
 * заключения соглашения с компанией Крипто Про.
 */

/*!
 * \file $RCSfile$
 * \version $Revision: 235201 $
 * \date $Date:: 2021-11-18 13:26:14 +0300#$
 * \author $Author: maxdm $
 *
 * \brief Атомарные inline функции
 */

#ifndef _CPCLOCKS_H_INCLUDED
#define _CPCLOCKS_H_INCLUDED

#if defined (UNIX) && defined(IOS)
    #include<libkern/OSAtomic.h>
#endif

#ifdef __cplusplus
extern "C" {
#endif

// На UNIX платформах обязательно определить обе переменные!
#if defined(UNIX) && !defined(PROC_TYPE_I386)
    #error Undefined PROC_TYPE_I386. Possibly common.h is not included
#endif
#if defined(UNIX) && !defined(PROCESSOR_TYPE)
    #error Undefined PROCESSOR_TYPE. Possibly common.h is not included
#endif

#if defined(SUP_INLINE)
    #define CPCL_INLINE SUP_INLINE
#elif defined(UNIX)
    #define CPCL_INLINE inline
#else
    #define CPCL_INLINE __inline
#endif

#if defined(SUP_STOP)
    #define CPCL_STOP SUP_STOP
#elif defined(_DEBUG) && !defined CSP_LITE
    #include <stdlib.h>
    #define CPCL_STOP abort()
#else
    #define CPCL_STOP 
#endif

#if defined(_DEBUG) && defined(WIN32)
    static CPCL_INLINE void
    support_check_alignment(const void volatile *ptr, size_t sz)
    {
	if(0 != ((size_t)(LONG_PTR)(PVOID)(ptr)) % sz) {
	    CPCL_STOP;
	}
    }
#else
    #define support_check_alignment( ptr, sz )
#endif

    // Общий код для NONSMP, как для общего понимания, 
    // так и для предоставления компилятору возможностей по
    // оптимизации в случае компиляции для одного процессора

#define support_nonSMP_compiler_fence()
#define support_nonSMP_fence_acq()	    support_nonSMP_compiler_fence()
#define support_nonSMP_fence_rel()	    support_nonSMP_compiler_fence()
#define support_nonSMP_fence_seq_cst()	    support_nonSMP_compiler_fence()

static CPCL_INLINE LONG 
support_interlocked_nonSMP_exchange_add(LONG volatile *dest, LONG incr)
{
    LONG res;
    
    support_nonSMP_fence_rel();
    res = (*dest);
    (*dest) += incr;
    support_nonSMP_fence_seq_cst();
    return res;
}

/* используется только в тестах */
static CPCL_INLINE LONGLONG
support_interlocked_nonSMP_exchange_add64(LONGLONG volatile *dest, LONGLONG incr)
{
    LONGLONG res;

    support_nonSMP_fence_rel();
    res = (*dest);
    (*dest) += incr;
    support_nonSMP_fence_seq_cst();
    return res;
}

/* используется только в тестах */
static CPCL_INLINE LONG
support_interlocked_nonSMP_increment(LONG volatile *dest)
{
    LONG res;
    
    support_nonSMP_fence_rel();
    res = ++(*dest);
    support_nonSMP_fence_seq_cst();
    return res;
}

/* используется только в тестах */
static CPCL_INLINE LONG
support_interlocked_nonSMP_decrement(LONG volatile *dest)
{
    LONG res;
    
    support_nonSMP_fence_rel();
    res = --(*dest);
    support_nonSMP_fence_seq_cst();
    return res;
}

static CPCL_INLINE int
support_interlocked_nonSMP_CASL(LONG volatile *dest, 
				LONG xchgbe, LONG compare)
{
    support_nonSMP_fence_rel();
    if(compare == *dest) {
	*dest = xchgbe;
	support_nonSMP_fence_seq_cst();
	return 1;
    }
    support_nonSMP_fence_seq_cst();
    return 0;
}

static CPCL_INLINE int
support_interlocked_nonSMP_CASLL(LONGLONG volatile *dest, 
				 LONGLONG xchgbe, LONGLONG compare)
{
    support_nonSMP_fence_rel();
    if(compare == *dest) {
	*dest = xchgbe;
	support_nonSMP_fence_seq_cst();
	return 1;
    }
    support_nonSMP_fence_seq_cst();
    return 0;
}

    // Атомарное чтение с семантикой захвата блокировки, т.е.
    // все последующие чтения из ОЗУ будут происходить не ранее этого чтения
/* используется только в тестах */
static CPCL_INLINE LONG
support_interlocked_nonSMP_load_acq(const LONG volatile *src)
{
    LONG res = *src;

    support_nonSMP_fence_acq();
    return res;
}

    // Атомарная запись с семантикой освобождения блокировки, т.е.
    // все предыдущие записи в ОЗУ завершатся не позднее этой записи
/* используется только в тестах */
static CPCL_INLINE void
support_interlocked_nonSMP_store_rel(LONG volatile *dest, LONG val)
{
    support_nonSMP_fence_rel();
    *dest = val;
}

    // Атомарное чтение с семантикой захвата блокировки, т.е.
    // все последующие чтения из ОЗУ будут происходить не ранее этого чтения
/* используется только в тестах */
static CPCL_INLINE LONGLONG
support_interlocked_nonSMP_load_acq64(const LONGLONG volatile *src)
{
    LONGLONG res = *src;

    support_nonSMP_fence_acq();
    return res;
}

    // Атомарная запись с семантикой освобождения блокировки, т.е.
    // все предыдущие записи в ОЗУ завершатся не позднее этой записи
/* используется только в тестах */
static CPCL_INLINE void
support_interlocked_nonSMP_store_rel64(LONGLONG volatile *dest, LONGLONG val)
{
    support_nonSMP_fence_rel();
    *dest = val;
}

#if defined(SOLARIS)
    extern volatile LONG support_interlocked_exchange_add (LONG volatile *dest, LONG incr);
    extern volatile int support_interlocked_CASL(LONG volatile *dest, LONG xchgbe, LONG compare);
    extern volatile int support_interlocked_CASLL(LONGLONG volatile *dest, LONGLONG xchgbe, LONGLONG compare);

    static inline LONG
    support_interlocked_increment(LONG volatile *dest)
    {
	return support_interlocked_exchange_add(dest, 1) + 1;
    }

    static inline LONG
    support_interlocked_decrement(LONG volatile *dest)
    {
	return support_interlocked_exchange_add(dest, -1) - 1;
    }

#elif defined (AIX) 
    //cross:FIX etu poka ostavil obschuu, ne znau na kakie velichini tam grishka zakladivalsya 
    static inline LONG
    support_interlocked_exchange_add(LONG volatile *dest, LONG incr)
    {
	LONG ret = 0;

	__asm__ __volatile__ (
	    "lwsync \n"
	    "0:    lwarx %0, 0, %1\n"
	    "      add %0, %2, %0\n"
	    "      stwcx. %0, 0, %1\n"
	    "      bne- 0b\n"
	    "lwsync \n"
	    : "=&r" (ret)
	    : "r"(dest), "r"(incr)
	    : "cr0", "memory", "r0"
	    );
	return ret - incr;
    }

    #if PROCESSOR_TYPE == PROC_TYPE_PPC32
	static inline int
	support_interlocked_CASL(LONG volatile *dest, LONG xchg, LONG compare)
	{
	    LONG store = 0;
	    LONG ret = 0;

	    __asm__ __volatile__ (
		"lwsync \n"
		"0:  lwarx %0,0,%2\n"
		"    xor. %1,%4,%0\n"
		"    bne 1f\n"
		"    stwcx. %3,0,%2\n"
		"    bne- 0b\n"
		"1:" 
		"lwsync \n"
		: "=&r"(store), "=&r"(ret)
		: "r"(dest), "r"(xchg), "r"(compare)
		: "cr0","memory","r0");
	    return (!ret);
	}

	/* 
	 * Powerpc32 processors don't implement the 64-bit (doubleword) forms of
	 * load and reserve (ldarx) and store conditional (stdcx.) instructions.  
	 * So for powerpc32 we stub out the 64-bit forms.
	 */
	static inline int
	support_interlocked_CASLL(LONGLONG volatile *dest, LONGLONG xchg, 
				LONGLONG compare)
	{
	    CPCL_STOP; //ibo nefig
	    return support_interlocked_nonSMP_CASLL(dest, xchg, compare);
	}

	#define SPECIFIC_SUPPORT_INTERLOCKED_EXCHANGE_ADD64 1
	    // Текущая реализация ограничена, она предназначена только для
	    // CPCSP-4605 (монотонное увеличение счётчика с независимой, 
	    // несвязанной между собой, асинхронной проверкой только, либо 
	    // старшей, либо младшей его частей)
	    //
	    // В частности, полным результатом 
	    // support_interlocked_exchange_add64() на AIX/PPC32 пользоваться 
	    // запрещается.
	static inline LONGLONG
	support_interlocked_exchange_add64(LONGLONG volatile * dest, LONGLONG incr)
	{
	    LONG volatile *pvlDest = (LONG volatile *)dest;
	    LONG *plIncr = (LONG *)&incr;
	    LONG rhi, rlo;

	    // plIncr[0] - старшие разряды, plIncr[1] - младшие
	    if(0 > plIncr[0]) {
		CPCL_STOP; // Реализация корректна только в случае монотонности
			   // изменения *dest
	    }
	    rhi = support_interlocked_exchange_add(&pvlDest[0], plIncr[0]);

	    if((uint32_t)plIncr[1] > 
		    (uint32_t)(plIncr[1] + 
				(rlo = support_interlocked_exchange_add(
						&pvlDest[1], plIncr[1])))) {
		support_interlocked_exchange_add(&pvlDest[0], 1L);
	    }
	    return ((LONGLONG)rhi << 32) | ((LONGLONG)rlo & 0xffffffffLL);
	}

	#define SPECIFIC_SUPPORT_INTERLOCKED_LOAD_ACQ64 1
	static CPCL_INLINE LONGLONG
	support_interlocked_load_acq64(const LONGLONG volatile *src)
	{
	    CPCL_STOP; //ibo nefig
	    return support_interlocked_exchange_add64(
				    (LONGLONG volatile *)src, 0LL);
	}

	// Не определяем SPECIFIC_SUPPORT_INTERLOCKED_STORE_REL64
	// Вызов support_interlocked_store_rel64() обратится к 
	// support_interlocked_CASLL() и приведёт к CPCL_STOP

#elif PROCESSOR_TYPE == PROC_TYPE_PPC64
	static inline int
	support_interlocked_CASL(LONG volatile *dest, LONG xchg, 
				LONG compare)
	{
	    LONG __tmp = -1;

	    __asm__ __volatile__ (
		"lwsync \n"
		"1:  lwarx   %0,0,%1\n"
		"    extsw   %0,%0\n"
		"    subf.   %0,%2,%0\n"
		"    bne 2f\n"
		"    stwcx.  %3,0,%1\n"
		"    bne-    1b\n"
		"2: "
		"lwsync \n"
		: "=&r" (__tmp)
		: "b" (dest), "r" (compare), "r" (xchg)
		: "cr0", "memory");        
	    return (__tmp == 0);
	}

	static inline int
	support_interlocked_CASLL(LONGLONG volatile *dest, LONGLONG xchg,
				  LONGLONG compare){
	    LONGLONG __tmp = -1;

	    __asm__ __volatile__ (
		"lwsync \n"
		"1:  ldarx   %0,0,%1\n"
		"    subf.   %0,%2,%0\n"
		"    bne 2f\n"
		"    stdcx.  %3,0,%1\n"
		"    bne-    1b\n"
		"2: "
		"lwsync \n"
		: "=&r" (__tmp)
		: "b" (dest), "r" (compare), "r" (xchg)
		: "cr0", "memory");
	    return (__tmp == 0);
	}

	#define SPECIFIC_SUPPORT_INTERLOCKED_EXCHANGE_ADD64 1
	static inline LONGLONG
	support_interlocked_exchange_add64(LONGLONG volatile *dest, LONGLONG incr)
	{
            LONGLONG ret = 0;
           __asm __volatile__ (
                "lwsync \n"
                "0:  ldarx   %0,0,%1\n"
                "    add %0,%2,%0\n"
                "    stdcx.  %0,0,%1\n"
                "    bne-    0b\n"
                "lwsync \n"
                : "=&r" (ret)
                : "r"(dest), "r"(incr)
                : "cr0", "memory", "r0"
                );
            return ret - incr;

	}
    #endif //PROC_TYPE_PPC32/PROC_TYPE_PPC64 

    static inline LONG
    support_interlocked_increment(LONG volatile *dest)
    {
	return support_interlocked_exchange_add(dest, 1) + 1;
    }

    static inline LONG
    support_interlocked_decrement(LONG volatile* dest)
    {
      return support_interlocked_exchange_add(dest, -1) - 1;
    }
#elif defined LINUX && defined __GNUC__ && __GNUC__ >= 7 && !(defined(SAILFISH) && defined(PROCESSOR_TYPE) && PROCESSOR_TYPE == PROC_TYPE_I386)
/* dim 2017-nov-30 согласно https://gcc.gnu.org/onlinedocs/
 * требования можно снизить до версии 4.6 и не требовать LINUX
 */
#define SPECIFIC_SUPPORT_INTERLOCKED_EXCHANGE_ADD64 1
#define SPECIFIC_SUPPORT_INTERLOCKED_LOAD_ACQ 1
#define SPECIFIC_SUPPORT_INTERLOCKED_LOAD_ACQ64 1
#define SPECIFIC_SUPPORT_INTERLOCKED_STORE_REL 1
#define SPECIFIC_SUPPORT_INTERLOCKED_STORE_REL64 1

    static inline LONG
    support_interlocked_load_acq(const LONG volatile *src)
    {
	return __atomic_load_4(src, __ATOMIC_SEQ_CST);
    }

    static inline LONGLONG
    support_interlocked_load_acq64(const LONGLONG volatile *src)
    {
	return __atomic_load_8(src, __ATOMIC_SEQ_CST);
    }

    static inline void
    support_interlocked_store_rel(LONG volatile *dest, LONG value)
    {
	__atomic_store_4(dest, value, __ATOMIC_SEQ_CST);
    }

    static inline void
    support_interlocked_store_rel64(LONGLONG volatile *dest, LONGLONG value)
    {
	__atomic_store_8(dest, value, __ATOMIC_SEQ_CST);
    }

    static inline LONG
    support_interlocked_exchange_add(LONG volatile *dest, LONG incr)
    {
      return __atomic_fetch_add(dest, incr, __ATOMIC_SEQ_CST);
    }

    static inline LONGLONG
    support_interlocked_exchange_add64(LONGLONG volatile *dest, LONGLONG incr)
    {
      return __atomic_fetch_add(dest, incr, __ATOMIC_SEQ_CST);
    }

    static inline LONG
    support_interlocked_increment(LONG volatile *dest)
    {
      return __atomic_add_fetch(dest, 1, __ATOMIC_SEQ_CST); 
    }

    static inline LONG
    support_interlocked_decrement(LONG volatile *dest)
    {
      return __atomic_sub_fetch(dest, 1, __ATOMIC_SEQ_CST);
    }

    static inline int
    support_interlocked_CASL(LONG volatile *dest, LONG xchgbe, LONG compare)
    {
	return __atomic_compare_exchange_4(dest, &compare, xchgbe,
					  0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);
    }

    static inline int
    support_interlocked_CASLL(LONGLONG volatile *dest, LONGLONG xchgbe, LONGLONG compare)
    {
	return __atomic_compare_exchange_8(dest, &compare, xchgbe,
					   0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);
    }
#elif defined (UNIX) && defined(IOS)
    static inline LONG
    support_interlocked_exchange_add(LONG volatile *dest, LONG incr)
    {
	return OSAtomicAdd32Barrier(incr, dest) - incr;
    }

    #define SPECIFIC_SUPPORT_INTERLOCKED_EXCHANGE_ADD64 1
    static inline LONGLONG
    support_interlocked_exchange_add64(LONGLONG volatile *dest, LONGLONG incr)
    {
	return OSAtomicAdd64Barrier(incr, dest) - incr;
    }

    static inline LONG
    support_interlocked_increment (LONG volatile* dest)
    {
	return OSAtomicIncrement32Barrier(dest);
    }

    static inline LONG
    support_interlocked_decrement (LONG volatile* dest)
    {
	return OSAtomicDecrement32Barrier(dest);
    }

    static inline int support_interlocked_CASL(LONG volatile *dest, LONG xchgbe, LONG compare)
    {
	return (int)OSAtomicCompareAndSwap32Barrier(compare, xchgbe, dest);
    }

    static inline int support_interlocked_CASLL(LONGLONG volatile *dest, LONGLONG xchgbe, LONGLONG compare)
    {
	return (int)OSAtomicCompareAndSwap64Barrier(compare, xchgbe, dest);
    }
#elif defined(UNIX) && (defined(ANDROID) || defined(PROCESSOR_TYPE) && (PROCESSOR_TYPE == PROC_TYPE_ARM || PROCESSOR_TYPE == PROC_TYPE_ARM64)) || (defined(PROCESSOR_TYPE) && PROCESSOR_TYPE == PROC_TYPE_PPC64 || defined(PROCESSOR_TYPE) && PROCESSOR_TYPE == PROC_TYPE_MIPS32) || (defined(PROCESSOR_TYPE) && (PROCESSOR_TYPE == PROC_TYPE_E2K32 || PROCESSOR_TYPE == PROC_TYPE_E2K64))

    // TODO: для ARM выяснить оптимальный способ обработки барьеров
    #define support_compiler_fence()	__asm __volatile(" " : : : "memory")
    #define support_fence_acq()		__sync_synchronize()
    #define support_fence_rel()		__sync_synchronize()
    #define support_fence_seq_cst()	__sync_synchronize()

    static inline LONG
    support_interlocked_exchange_add(LONG volatile *dest, LONG incr)
    {
	return __sync_fetch_and_add(dest, incr); 
    }

    static inline LONG
    support_interlocked_increment(LONG volatile *dest)
    {
	return __sync_add_and_fetch(dest, 1); 
    }

    static inline LONG
    support_interlocked_decrement(LONG volatile *dest)
    {
	return __sync_sub_and_fetch(dest, 1);
    }

    static inline int
    support_interlocked_CASL(LONG volatile *dest,
				    LONG xchgbe, LONG compare)
    {
	return __sync_bool_compare_and_swap(dest, compare, xchgbe);
    }
#if !defined(PROCESSOR_TYPE) || PROCESSOR_TYPE != PROC_TYPE_MIPS32
	// ARM LDREXB, LDREXH, LDREXD, STREXB, STREXD, and 
        // STREXH are available in ARMv6K and above.
	// All these 32-bit Thumb instructions are available in 
	// ARMv6T2 and above, except that LDREXD and STREXD are 
	// not available in the ARMv7-M architecture.
    static inline int
    support_interlocked_CASLL(LONGLONG volatile *dest, LONGLONG xchgbe, LONGLONG compare)
    {
	return __sync_bool_compare_and_swap(dest, compare, xchgbe);
    }
#else
    
#define SUP_BARRIER_LOCK 1
#define SUP_BARRIER_UNLOCK 0
    static inline void
    support_memory_barrier(LONG lock)
    {
        static LONG volatile barrier = 0;
        if(lock)
            while( !support_interlocked_CASL( &barrier, 1, 0 ) ){}
        else
            barrier = 0;

    }
    //FAKE atomic CASLL for MIPS32
    static inline int
    support_interlocked_CASLL(LONGLONG volatile *dest, LONGLONG xchgbe, LONGLONG compare)
    {
        int rc;
        support_memory_barrier(SUP_BARRIER_LOCK);
        if (*dest == compare) 
        {
            *dest = xchgbe;
            rc = 1;
        }
        else 
            rc = 0;

        support_memory_barrier(SUP_BARRIER_UNLOCK);
        return rc;
    }
#endif    

#if (defined(PROCESSOR_TYPE) && PROCESSOR_TYPE == PROC_TYPE_MIPS32)
	#define SPECIFIC_SUPPORT_INTERLOCKED_EXCHANGE_ADD64 1
	static inline LONGLONG
	support_interlocked_exchange_add64(LONGLONG volatile * dest, LONGLONG incr)
	{
        LONGLONG oll;
        do {
            oll = *dest;
        } while(!support_interlocked_CASLL(dest, oll+incr, oll));
        return oll;
	}

#else
    #define SPECIFIC_SUPPORT_INTERLOCKED_EXCHANGE_ADD64
    static CPCL_INLINE LONGLONG 
    support_interlocked_exchange_add64(LONGLONG volatile *dest, LONGLONG incr)
    {
	return __sync_fetch_and_add(dest, incr); 
    }
#endif    
    
#elif defined(UNIX) && !defined(SOLARIS) && !defined(AIX) && !defined(ANDROID) \
      && (PROCESSOR_TYPE == PROC_TYPE_I386 || PROCESSOR_TYPE == PROC_TYPE_X64)
    #define support_compiler_fence()	__asm __volatile(" " : : : "memory")
    #define support_fence_acq()		__asm __volatile("lfence" : : : "memory")
    #define support_fence_rel()		__asm __volatile("sfence" : : : "memory")
    #define support_fence_seq_cst()	__asm __volatile("mfence" : : : "memory")

    /* 
     * For contivity only static inline functions are allowed
     */
    static inline LONG
    support_interlocked_exchange_add(LONG volatile *dest, LONG incr)
    {
	int ret;

	__asm __volatile__(
	"lock\n\t"\
	    "xaddl %0,(%1)"
	    : "=r" (ret)
	    : "r" (dest), "0" (incr)
	    : "memory"
	    );
	return ret;
    }

    static inline LONG
    support_interlocked_increment(LONG volatile *dest)
    {
	return support_interlocked_exchange_add(dest, 1) + 1;
    }

    static inline LONG
    support_interlocked_decrement (LONG volatile *dest)
    {
	return support_interlocked_exchange_add(dest, -1) - 1;
    }

    static inline int
    support_interlocked_CASL(LONG volatile *dest, LONG xchg, LONG compare)
    {
	char s;

	__asm __volatile__(
	    "lock; cmpxchgl %3,%1\n\t"\
		"setz %0"\
		: "=a"(s)
		: "m"(*dest), "a" (compare), "r" (xchg)
		: "memory", "cc");
	return s;
    }

    #if PROCESSOR_TYPE == PROC_TYPE_X64 /* PROCESSOR/ASSEMBLER */
	static inline int
	support_interlocked_CASLL(LONGLONG volatile *dest, LONGLONG xchg,
				  LONGLONG compare)
	{
	    char s;

	    __asm __volatile__(
		"lock; cmpxchgq %3,%1\n\t"\
		    "setz %0"\
		    : "=a"(s)
		    : "m"(*dest), "a" (compare), "r" (xchg)
		    : "memory", "cc");
	    return s;
	}
    #elif PROCESSOR_TYPE == PROC_TYPE_I386 /* PROCESSOR */
	union LL_2L {
	    LONGLONG ll;
	    struct {
		LONG l[2];
	    } l2;
	};

	static inline int
	support_interlocked_CASLL(LONGLONG volatile *dest, LONGLONG xchg,
				  LONGLONG compare)
	{
	    char s;
	    union LL_2L v;

	    v.ll = xchg;
	    __asm __volatile__(
		"push %%ebx\n\t"
		"mov %3, %%ebx\n\t"
		"lock; cmpxchg8b (%%edi)\n\t"
		"setz %0\n\t"
		"pop %%ebx"
		: "=a"(s)
		: "D"(dest), "A" (compare), "r"(v.l2.l[0]), "c"(v.l2.l[1])
		: "memory", "cc");    // TODO: убрать union, push и pop
	    return s;
	}

	#define SPECIFIC_SUPPORT_INTERLOCKED_LOAD_ACQ64 1
	static __inline LONGLONG
	support_interlocked_load_acq64(const LONGLONG volatile *src)
	{
	    LONGLONG res;
#if 0
	    __asm __volatile__ (
		"movl	%%eax, %0	\n" : : "r" (src) ;
		"movl	%%ebx, %%eax	\n"
		"xor	%%ecx, %%ecx	\n"
		"xor	%%edx, %%edx	\n"
		"lock; cmpxchg8b %%eax" : "=&A" (res) : : "memory", "cc"
	    );
#else
	    res = *src;
#endif

	    return res;	    
	}
    #endif // PROC_TYPE_I386 
#elif defined (_WIN32) && (defined(_M_IA64) || defined(_M_AMD64) || defined(_M_IX86) || defined(_M_ARM64))
    // Вытащил куски из WinBase.h, из-за бардака, который здесь царит,
    // включить его здесь не представляется возможным.
    // DEVL - означает только для режима драйвера

    #define InterlockedIncrement _InterlockedIncrement
    #define InterlockedDecrement _InterlockedDecrement
    #define InterlockedExchangeAdd _InterlockedExchangeAdd
    #define InterlockedCompareExchange64 _InterlockedCompareExchange64
    #define InterlockedCompareExchange _InterlockedCompareExchange

    #if defined (DEVL)
	long InterlockedIncrement(long volatile *Addend);
	long InterlockedDecrement(long volatile *Addend);
	long InterlockedExchangeAdd(long volatile *Addend, long Value);
	long InterlockedCompareExchange(long volatile *Destination, long ExChange, long Comperand);
	__int64 InterlockedCompareExchange64(__int64 volatile *Destination, __int64 ExChange, __int64 Comperand);
    #endif

    #pragma intrinsic(_InterlockedIncrement)
    #pragma intrinsic(_InterlockedDecrement)
    #pragma intrinsic(_InterlockedExchangeAdd)
    #pragma intrinsic(_InterlockedCompareExchange)
    #pragma intrinsic(_InterlockedCompareExchange64)

    VOID _ReadWriteBarrier(VOID);   // Барьер в компиляторе
    #pragma intrinsic(_ReadWriteBarrier)

#if !defined( _M_ARM64EC )
    VOID _mm_lfence(VOID);	    // Барьер загрузки в конвейере процессора, 
				    // команда LFENCE практически не отличима 
				    // от NOP+_ReadWriteBarrier()
    VOID _mm_sfence(VOID);	    // Барьер сохранения в конвейере процессора, 
				    // команда SFENCE практически не отличима 
				    // от NOP+_ReadWriteBarrier()
    VOID _mm_mfence(VOID);	    // Барьер глобальной синхронизации в конвейере 
				    // процессора, команда MFENCE достаточно 
				    // медленая штука
#endif  /*!defined( _M_ARM64EC )*/

#if !defined(_M_ARM64) && !defined( _M_ARM64EC )
    #pragma intrinsic(_mm_lfence)
    #pragma intrinsic(_mm_sfence)
    #pragma intrinsic(_mm_mfence)
#endif  /*!defined(_M_ARM64) && !defined( _M_ARM64EC )*/ 

    #define support_compiler_fence()	_ReadWriteBarrier()
    #define support_fence_acq()		_mm_lfence(); _ReadWriteBarrier()
    #define support_fence_rel()		_mm_sfence(); _ReadWriteBarrier()
    #define support_fence_seq_cst()	_mm_mfence(); _ReadWriteBarrier()

    static volatile __inline LONG
    support_interlocked_exchange_add(LONG volatile *dest, LONG incr)
    {
	return InterlockedExchangeAdd((long volatile *)dest, incr);
    }

    static volatile __inline LONG
    support_interlocked_increment(LONG volatile *dest)
    {
	return InterlockedIncrement((long volatile *)dest);
    }

    static volatile __inline LONG
    support_interlocked_decrement(LONG volatile *dest)
    {
	return InterlockedDecrement((long volatile *)dest);
    }

    static volatile __inline int
    support_interlocked_CASL(LONG volatile *dest, LONG xchgbe, LONG compare)
    {
	LONG s = compare;

	return (InterlockedCompareExchange((long volatile *)dest, xchgbe, s) == s);
    }

    static volatile __inline int
    support_interlocked_CASLL(LONGLONG volatile *dest, LONGLONG xchgbe,
			       LONGLONG compare)
    {
	LONGLONG s = compare;

	return (InterlockedCompareExchange64(dest, xchgbe, s) == s);
    }
#endif /*CPU*/

#if SIZEOF_VOID_P == 8 || defined(_WIN64)
    #define support_interlocked_CASPTR(dst, xchg, cmp)                  \
                support_interlocked_CASLL((LONG_PTR *)(dst),            \
                                    (LONG_PTR)(xchg), (LONG_PTR)(cmp))
    #define support_interlocked_LOADPTR(src)                            \
                support_interlocked_load_acq64((const LONGLONG volatile *)(src))
#elif !defined(UNIX) || SIZEOF_VOID_P == 4
    #define support_interlocked_CASPTR(dst, xchg, cmp)                  \
                support_interlocked_CASL((LONG *)(dst),                 \
                                        (LONG)(LONG_PTR)(PVOID)(xchg),  \
                                        (LONG)(LONG_PTR)(PVOID)(cmp))
    #define support_interlocked_LOADPTR(src) \
                support_interlocked_load_acq((const LONG volatile *)(src))
#else
    #error "SIZEOF_VOID_P not defined"
#endif

#if !defined(SPECIFIC_SUPPORT_INTERLOCKED_EXCHANGE_ADD64)
    static CPCL_INLINE LONGLONG 
    support_interlocked_exchange_add64(LONGLONG volatile *dest, LONGLONG incr)
    {
	LONGLONG oll;

	do {
	    oll = *dest;
	} while (!support_interlocked_CASLL(dest, oll + incr, oll));
	return oll;
    }
#endif

#if !defined(SPECIFIC_SUPPORT_INTERLOCKED_LOAD_ACQ)
    #if !defined(support_fence_acq)
	static CPCL_INLINE LONG
	support_interlocked_load_acq(const LONG volatile *src)
	{
	    return support_interlocked_exchange_add((LONG volatile *)src, 0);
	}
    #else
	static CPCL_INLINE LONG
	support_interlocked_load_acq(const LONG volatile *src)
	{
	    LONG res;

	    support_check_alignment(src, sizeof(*src));
	    res = *src;
	    support_fence_acq(); 
	    return res;
	}
    #endif
#endif

#if !defined(SPECIFIC_SUPPORT_INTERLOCKED_STORE_REL)
    #if !defined(support_fence_rel)
	static CPCL_INLINE void
	support_interlocked_store_rel(LONG volatile *dest, LONG val)
	{
	    LONG ol;

	    do {
		ol = *dest;
	    } while(!support_interlocked_CASL(dest, val, ol));
	}
    #else
	static CPCL_INLINE void
	support_interlocked_store_rel(LONG volatile *dest, LONG val)
	{
	    support_check_alignment(dest, sizeof(*dest));
	    support_fence_rel();
	    *dest = val;
	}
    #endif
#endif

#if !defined(SPECIFIC_SUPPORT_INTERLOCKED_LOAD_ACQ64)
    #if !defined(support_fence_acq) || !(defined(_WIN64) || 8 == SIZEOF_VOID_P)
	static CPCL_INLINE LONGLONG
	support_interlocked_load_acq64(const LONGLONG volatile *src)
	{
	    return support_interlocked_exchange_add64(
				    (LONGLONG volatile *)src, 0LL);
	}
    #else
	static CPCL_INLINE LONGLONG
	support_interlocked_load_acq64(const LONGLONG volatile *src)
	{
	    LONGLONG res;

	    support_check_alignment(src, sizeof(*src));
	    res = *src;
	    support_fence_acq(); 
	    return res;
	}
    #endif
#endif

#if !defined(SPECIFIC_SUPPORT_INTERLOCKED_STORE_REL64)
    #if !defined(support_fence_rel) || !(defined(_WIN64) || 8 == SIZEOF_VOID_P)
	static CPCL_INLINE void
	support_interlocked_store_rel64(LONGLONG volatile *dest, LONGLONG val)
	{
	    LONGLONG oll;

	    do {
		oll = *dest;
	    } while(!support_interlocked_CASLL(dest, val, oll));
	}
    #else
	static CPCL_INLINE void
	support_interlocked_store_rel64(LONGLONG volatile *dest, LONGLONG val)
	{
	    support_check_alignment(dest, sizeof(*dest));
	    support_fence_rel();
	    *dest = val;
	}
    #endif
#endif

#if !defined(support_compiler_fence)
    #define support_compiler_fence()	(error_undefined_fence_for_this_platform)
#endif

#if !defined(support_fence_acq)
    #define support_fence_acq()		(error_undefined_fence_for_this_platform)
#endif

#if !defined(support_fence_rel)
    #define support_fence_rel()		(error_undefined_fence_for_this_platform)
#endif

#if !defined(support_fence_seq_cst)
    #define support_fence_seq_cst()	(error_undefined_fence_for_this_platform)
#endif


#ifdef __cplusplus
}
#endif
#endif /* _CPCLOCKS_H_INCLUDED */
