diff --git a/system/lib/llvm-libc/config/app.h b/system/lib/llvm-libc/config/app.h index 27f4141d80c4b..e9206c7242250 100644 --- a/system/lib/llvm-libc/config/app.h +++ b/system/lib/llvm-libc/config/app.h @@ -15,6 +15,8 @@ #include "gpu/app.h" #elif defined(__linux__) #include "linux/app.h" +#elif defined(__UEFI__) +#include "uefi/app.h" #endif #endif // LLVM_LIBC_CONFIG_APP_H diff --git a/system/lib/llvm-libc/config/baremetal/aarch64/entrypoints.txt b/system/lib/llvm-libc/config/baremetal/aarch64/entrypoints.txt index 694cd7b1993ca..a8e653fdd5159 100644 --- a/system/lib/llvm-libc/config/baremetal/aarch64/entrypoints.txt +++ b/system/lib/llvm-libc/config/baremetal/aarch64/entrypoints.txt @@ -2,6 +2,9 @@ set(TARGET_LIBC_ENTRYPOINTS # assert.h entrypoints libc.src.assert.__assert_fail + # compiler entrypoints (no corresponding header) + libc.src.compiler.__stack_chk_fail + # ctype.h entrypoints libc.src.ctype.isalnum libc.src.ctype.isalpha @@ -20,12 +23,34 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.ctype.tolower libc.src.ctype.toupper - # compiler entrypoints (no corresponding header) - libc.src.compiler.__stack_chk_fail + # ctype.h entrypoints + libc.src.ctype.isalnum_l + libc.src.ctype.isalpha_l + libc.src.ctype.isblank_l + libc.src.ctype.iscntrl_l + libc.src.ctype.isdigit_l + libc.src.ctype.isgraph_l + libc.src.ctype.islower_l + libc.src.ctype.isprint_l + libc.src.ctype.ispunct_l + libc.src.ctype.isspace_l + libc.src.ctype.isupper_l + libc.src.ctype.isxdigit_l + libc.src.ctype.tolower_l + libc.src.ctype.toupper_l # errno.h entrypoints libc.src.errno.errno + # locale.h entrypoints + libc.src.locale.localeconv + libc.src.locale.duplocale + libc.src.locale.freelocale + libc.src.locale.localeconv + libc.src.locale.newlocale + libc.src.locale.setlocale + libc.src.locale.uselocale + # setjmp.h entrypoints libc.src.setjmp.longjmp libc.src.setjmp.setjmp @@ -51,6 +76,7 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.string.strcoll libc.src.string.strcpy libc.src.string.strcspn + libc.src.string.strdup libc.src.string.strerror libc.src.string.strerror_r libc.src.string.strlcat @@ -59,6 +85,7 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.string.strncat libc.src.string.strncmp libc.src.string.strncpy + libc.src.string.strndup libc.src.string.strnlen libc.src.string.strpbrk libc.src.string.strrchr @@ -69,15 +96,26 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.string.strtok_r libc.src.string.strxfrm + # string.h entrypoints + libc.src.string.strcoll_l + libc.src.string.strxfrm_l + # strings.h entrypoints libc.src.strings.bcmp libc.src.strings.bcopy libc.src.strings.bzero + libc.src.strings.ffs + libc.src.strings.ffsl + libc.src.strings.ffsll libc.src.strings.index libc.src.strings.rindex libc.src.strings.strcasecmp libc.src.strings.strncasecmp + # strings.h entrypoints + libc.src.strings.strcasecmp_l + libc.src.strings.strncasecmp_l + # inttypes.h entrypoints libc.src.inttypes.imaxabs libc.src.inttypes.imaxdiv @@ -85,18 +123,22 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.inttypes.strtoumax # stdio.h entrypoints + libc.src.stdio.asprintf libc.src.stdio.getchar libc.src.stdio.printf libc.src.stdio.putchar libc.src.stdio.puts libc.src.stdio.remove + libc.src.stdio.scanf libc.src.stdio.snprintf libc.src.stdio.sprintf - libc.src.stdio.asprintf + libc.src.stdio.sscanf + libc.src.stdio.vasprintf libc.src.stdio.vprintf + libc.src.stdio.vscanf libc.src.stdio.vsnprintf libc.src.stdio.vsprintf - libc.src.stdio.vasprintf + libc.src.stdio.vsscanf # stdbit.h entrypoints libc.src.stdbit.stdc_bit_ceil_uc @@ -172,6 +214,7 @@ set(TARGET_LIBC_ENTRYPOINTS # stdlib.h entrypoints libc.src.stdlib._Exit + libc.src.stdlib.a64l libc.src.stdlib.abort libc.src.stdlib.abs libc.src.stdlib.aligned_alloc @@ -184,15 +227,21 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.stdlib.div libc.src.stdlib.exit libc.src.stdlib.free + libc.src.stdlib.l64a libc.src.stdlib.labs libc.src.stdlib.ldiv libc.src.stdlib.llabs libc.src.stdlib.lldiv libc.src.stdlib.malloc + libc.src.stdlib.memalignment libc.src.stdlib.qsort + libc.src.stdlib.qsort_r libc.src.stdlib.rand libc.src.stdlib.realloc libc.src.stdlib.srand + # libc.src.stdlib.strfromd + # libc.src.stdlib.strfromf + # libc.src.stdlib.strfroml libc.src.stdlib.strtod libc.src.stdlib.strtof libc.src.stdlib.strtol @@ -201,6 +250,15 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.stdlib.strtoul libc.src.stdlib.strtoull + # stdlib.h entrypoints + libc.src.stdlib.strtod_l + libc.src.stdlib.strtof_l + libc.src.stdlib.strtol_l + libc.src.stdlib.strtold_l + libc.src.stdlib.strtoll_l + libc.src.stdlib.strtoul_l + libc.src.stdlib.strtoull_l + # time.h entrypoints libc.src.time.asctime libc.src.time.asctime_r @@ -210,14 +268,35 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.time.gmtime libc.src.time.gmtime_r libc.src.time.mktime + libc.src.time.strftime + libc.src.time.strftime_l libc.src.time.timespec_get + # wchar.h entrypoints + libc.src.wchar.btowc + libc.src.wchar.wcslen + libc.src.wchar.wctob + # internal entrypoints libc.startup.baremetal.init libc.startup.baremetal.fini ) set(TARGET_LIBM_ENTRYPOINTS + # complex.h entrypoints + libc.src.complex.creal + libc.src.complex.crealf + libc.src.complex.creall + libc.src.complex.cimag + libc.src.complex.cimagf + libc.src.complex.cimagl + libc.src.complex.conj + libc.src.complex.conjf + libc.src.complex.conjl + libc.src.complex.cproj + libc.src.complex.cprojf + libc.src.complex.cprojl + # fenv.h entrypoints libc.src.fenv.feclearexcept libc.src.fenv.fedisableexcept @@ -243,6 +322,8 @@ set(TARGET_LIBM_ENTRYPOINTS libc.src.math.asinhf libc.src.math.atan2 libc.src.math.atan2f + # libc.src.math.atan2l + libc.src.math.atan libc.src.math.atanf libc.src.math.atanhf libc.src.math.canonicalize @@ -259,10 +340,18 @@ set(TARGET_LIBM_ENTRYPOINTS libc.src.math.cos libc.src.math.cosf libc.src.math.coshf + libc.src.math.cospif + libc.src.math.dfmal + libc.src.math.dmull + libc.src.math.dsqrtl + libc.src.math.daddl + libc.src.math.ddivl + libc.src.math.dsubl libc.src.math.erff libc.src.math.exp libc.src.math.exp10 libc.src.math.exp10f + libc.src.math.exp10m1f libc.src.math.exp2 libc.src.math.exp2f libc.src.math.exp2m1f @@ -272,9 +361,16 @@ set(TARGET_LIBM_ENTRYPOINTS libc.src.math.fabs libc.src.math.fabsf libc.src.math.fabsl + libc.src.math.fadd + libc.src.math.faddl + libc.src.math.fadd libc.src.math.fdim libc.src.math.fdimf libc.src.math.fdiml + libc.src.math.fdiv + libc.src.math.fdivl + libc.src.math.ffma + libc.src.math.ffmal libc.src.math.floor libc.src.math.floorf libc.src.math.floorl @@ -314,6 +410,7 @@ set(TARGET_LIBM_ENTRYPOINTS libc.src.math.fmodf libc.src.math.fmodl libc.src.math.fmul + libc.src.math.fmull libc.src.math.frexp libc.src.math.frexpf libc.src.math.frexpl @@ -323,14 +420,27 @@ set(TARGET_LIBM_ENTRYPOINTS libc.src.math.fromfpx libc.src.math.fromfpxf libc.src.math.fromfpxl + libc.src.math.fsqrt + libc.src.math.fsqrtl + libc.src.math.fsub + libc.src.math.fsubl + libc.src.math.getpayload + libc.src.math.getpayloadf + libc.src.math.getpayloadl libc.src.math.hypot libc.src.math.hypotf libc.src.math.ilogb libc.src.math.ilogbf libc.src.math.ilogbl + libc.src.math.iscanonical + libc.src.math.iscanonicalf + libc.src.math.iscanonicall libc.src.math.isnan libc.src.math.isnanf libc.src.math.isnanl + libc.src.math.issignaling + libc.src.math.issignalingf + libc.src.math.issignalingl libc.src.math.ldexp libc.src.math.ldexpf libc.src.math.ldexpl @@ -404,17 +514,30 @@ set(TARGET_LIBM_ENTRYPOINTS libc.src.math.scalbn libc.src.math.scalbnf libc.src.math.scalbnl + libc.src.math.setpayload + libc.src.math.setpayloadf + libc.src.math.setpayloadl + libc.src.math.setpayloadsig + libc.src.math.setpayloadsigf + libc.src.math.setpayloadsigl libc.src.math.sin libc.src.math.sincos libc.src.math.sincosf libc.src.math.sinf libc.src.math.sinhf + libc.src.math.sinpif libc.src.math.sqrt libc.src.math.sqrtf libc.src.math.sqrtl libc.src.math.tan libc.src.math.tanf libc.src.math.tanhf + libc.src.math.totalorder + libc.src.math.totalorderf + libc.src.math.totalorderl + libc.src.math.totalordermag + libc.src.math.totalordermagf + libc.src.math.totalordermagl libc.src.math.trunc libc.src.math.truncf libc.src.math.truncl @@ -426,6 +549,210 @@ set(TARGET_LIBM_ENTRYPOINTS libc.src.math.ufromfpxl ) +if(LIBC_TYPES_HAS_CFLOAT16) + list(APPEND TARGET_LIBM_ENTRYPOINTS + # complex.h C23 _Complex _Float16 entrypoints + libc.src.complex.crealf16 + libc.src.complex.cimagf16 + libc.src.complex.conjf16 + libc.src.complex.cprojf16 + ) +endif() + +if(LIBC_TYPES_HAS_FLOAT16) + list(APPEND TARGET_LIBM_ENTRYPOINTS + # math.h C23 _Float16 entrypoints + libc.src.math.acosf16 + libc.src.math.acoshf16 + libc.src.math.asinf16 + libc.src.math.asinhf16 + libc.src.math.canonicalizef16 + libc.src.math.ceilf16 + libc.src.math.copysignf16 + libc.src.math.cosf16 + libc.src.math.coshf16 + libc.src.math.cospif16 + libc.src.math.exp10f16 + libc.src.math.exp10m1f16 + libc.src.math.exp2f16 + libc.src.math.exp2m1f16 + libc.src.math.expf16 + libc.src.math.expm1f16 + libc.src.math.f16add + libc.src.math.f16addf + libc.src.math.f16addl + libc.src.math.f16div + libc.src.math.f16divf + libc.src.math.f16divl + libc.src.math.f16fma + libc.src.math.f16fmaf + libc.src.math.f16fmal + libc.src.math.f16mul + libc.src.math.f16mulf + libc.src.math.f16mull + libc.src.math.f16sqrt + libc.src.math.f16sqrtf + libc.src.math.f16sqrtl + libc.src.math.f16sub + libc.src.math.f16subf + libc.src.math.f16subl + libc.src.math.fabsf16 + libc.src.math.fdimf16 + libc.src.math.floorf16 + libc.src.math.fmaf16 + libc.src.math.fmaxf16 + libc.src.math.fmaximum_mag_numf16 + libc.src.math.fmaximum_magf16 + libc.src.math.fmaximum_numf16 + libc.src.math.fmaximumf16 + libc.src.math.fminf16 + libc.src.math.fminimum_mag_numf16 + libc.src.math.fminimum_magf16 + libc.src.math.fminimum_numf16 + libc.src.math.fminimumf16 + libc.src.math.fmodf16 + libc.src.math.frexpf16 + libc.src.math.fromfpf16 + libc.src.math.fromfpxf16 + libc.src.math.getpayloadf16 + libc.src.math.hypotf16 + libc.src.math.ilogbf16 + libc.src.math.iscanonicalf16 + libc.src.math.issignalingf16 + libc.src.math.ldexpf16 + libc.src.math.llogbf16 + libc.src.math.llrintf16 + libc.src.math.llroundf16 + libc.src.math.log10f16 + libc.src.math.log2f16 + libc.src.math.logbf16 + libc.src.math.logf16 + libc.src.math.lrintf16 + libc.src.math.lroundf16 + libc.src.math.modff16 + libc.src.math.nanf16 + libc.src.math.nearbyintf16 + libc.src.math.nextafterf16 + libc.src.math.nextdownf16 + libc.src.math.nexttowardf16 + libc.src.math.nextupf16 + libc.src.math.remainderf16 + libc.src.math.remquof16 + libc.src.math.rintf16 + libc.src.math.roundevenf16 + libc.src.math.roundf16 + libc.src.math.scalblnf16 + libc.src.math.scalbnf16 + libc.src.math.setpayloadf16 + libc.src.math.setpayloadsigf16 + libc.src.math.sinf16 + libc.src.math.sinhf16 + libc.src.math.sinpif16 + libc.src.math.sqrtf16 + libc.src.math.tanf16 + libc.src.math.tanhf16 + libc.src.math.tanpif16 + libc.src.math.totalorderf16 + libc.src.math.totalordermagf16 + libc.src.math.truncf16 + libc.src.math.ufromfpf16 + libc.src.math.ufromfpxf16 + ) + + if(LIBC_TYPES_HAS_FLOAT128) + list(APPEND TARGET_LIBM_ENTRYPOINTS + # math.h C23 mixed _Float16 and _Float128 entrypoints + libc.src.math.f16addf128 + libc.src.math.f16divf128 + libc.src.math.f16fmaf128 + libc.src.math.f16mulf128 + libc.src.math.f16sqrtf128 + libc.src.math.f16subf128 + ) + endif() +endif() + +if(LIBC_TYPES_HAS_CFLOAT128) + list(APPEND TARGET_LIBM_ENTRYPOINTS + # complex.h C23 _Complex _Float128 entrypoints + libc.src.complex.crealf128 + libc.src.complex.cimagf128 + libc.src.complex.conjf128 + libc.src.complex.cprojf128 + ) +endif() + +if(LIBC_TYPES_HAS_FLOAT128) + list(APPEND TARGET_LIBM_ENTRYPOINTS + # math.h C23 _Float128 entrypoints + libc.src.math.atan2f128 + libc.src.math.canonicalizef128 + libc.src.math.ceilf128 + libc.src.math.copysignf128 + libc.src.math.daddf128 + libc.src.math.ddivf128 + libc.src.math.dfmaf128 + libc.src.math.dmulf128 + libc.src.math.dsqrtf128 + libc.src.math.dsubf128 + libc.src.math.fabsf128 + libc.src.math.faddf128 + libc.src.math.fdimf128 + libc.src.math.fdivf128 + libc.src.math.ffmaf128 + libc.src.math.floorf128 + libc.src.math.fmaxf128 + libc.src.math.fmaximum_mag_numf128 + libc.src.math.fmaximum_magf128 + libc.src.math.fmaximum_numf128 + libc.src.math.fmaximumf128 + libc.src.math.fminf128 + libc.src.math.fminimum_mag_numf128 + libc.src.math.fminimum_magf128 + libc.src.math.fminimum_numf128 + libc.src.math.fminimumf128 + libc.src.math.fmodf128 + libc.src.math.fmulf128 + libc.src.math.frexpf128 + libc.src.math.fromfpf128 + libc.src.math.fromfpxf128 + libc.src.math.fsqrtf128 + libc.src.math.fsubf128 + libc.src.math.getpayloadf128 + libc.src.math.ilogbf128 + libc.src.math.iscanonicalf128 + libc.src.math.issignalingf128 + libc.src.math.ldexpf128 + libc.src.math.llogbf128 + libc.src.math.llrintf128 + libc.src.math.llroundf128 + libc.src.math.logbf128 + libc.src.math.lrintf128 + libc.src.math.lroundf128 + libc.src.math.modff128 + libc.src.math.nanf128 + libc.src.math.nearbyintf128 + libc.src.math.nextafterf128 + libc.src.math.nextdownf128 + libc.src.math.nextupf128 + libc.src.math.remainderf128 + libc.src.math.remquof128 + libc.src.math.rintf128 + libc.src.math.roundevenf128 + libc.src.math.roundf128 + libc.src.math.scalblnf128 + libc.src.math.scalbnf128 + libc.src.math.setpayloadf128 + libc.src.math.setpayloadsigf128 + libc.src.math.sqrtf128 + libc.src.math.totalorderf128 + libc.src.math.totalordermagf128 + libc.src.math.truncf128 + libc.src.math.ufromfpf128 + libc.src.math.ufromfpxf128 + ) +endif() + if(LIBC_COMPILER_HAS_FIXED_POINT) list(APPEND TARGET_LIBM_ENTRYPOINTS # stdfix.h _Fract and _Accum entrypoints @@ -469,6 +796,30 @@ if(LIBC_COMPILER_HAS_FIXED_POINT) libc.src.stdfix.ukbits libc.src.stdfix.lkbits libc.src.stdfix.ulkbits + libc.src.stdfix.bitshr + libc.src.stdfix.bitsr + libc.src.stdfix.bitslr + libc.src.stdfix.bitshk + libc.src.stdfix.bitsk + libc.src.stdfix.bitslk + libc.src.stdfix.bitsuhr + libc.src.stdfix.bitsur + libc.src.stdfix.bitsulr + libc.src.stdfix.bitsuhk + libc.src.stdfix.bitsuk + libc.src.stdfix.bitsulk + libc.src.stdfix.countlshr + libc.src.stdfix.countlsr + libc.src.stdfix.countlslr + libc.src.stdfix.countlshk + libc.src.stdfix.countlsk + libc.src.stdfix.countlslk + libc.src.stdfix.countlsuhr + libc.src.stdfix.countlsur + libc.src.stdfix.countlsulr + libc.src.stdfix.countlsuhk + libc.src.stdfix.countlsuk + libc.src.stdfix.countlsulk ) endif() diff --git a/system/lib/llvm-libc/config/baremetal/aarch64/headers.txt b/system/lib/llvm-libc/config/baremetal/aarch64/headers.txt index 6cc8a80455d3a..5666ef7e0012d 100644 --- a/system/lib/llvm-libc/config/baremetal/aarch64/headers.txt +++ b/system/lib/llvm-libc/config/baremetal/aarch64/headers.txt @@ -1,13 +1,18 @@ set(TARGET_PUBLIC_HEADERS libc.include.assert + libc.include.complex libc.include.ctype libc.include.errno libc.include.features libc.include.fenv libc.include.float libc.include.inttypes + libc.include.limits + libc.include.locale libc.include.math libc.include.setjmp + libc.include.stdbit + libc.include.stdckdint libc.include.stdfix libc.include.stdint libc.include.stdio @@ -17,4 +22,5 @@ set(TARGET_PUBLIC_HEADERS libc.include.sys_queue libc.include.time libc.include.uchar + libc.include.wchar ) diff --git a/system/lib/llvm-libc/config/baremetal/arm/entrypoints.txt b/system/lib/llvm-libc/config/baremetal/arm/entrypoints.txt index 694cd7b1993ca..acafef17fa5d1 100644 --- a/system/lib/llvm-libc/config/baremetal/arm/entrypoints.txt +++ b/system/lib/llvm-libc/config/baremetal/arm/entrypoints.txt @@ -2,6 +2,9 @@ set(TARGET_LIBC_ENTRYPOINTS # assert.h entrypoints libc.src.assert.__assert_fail + # compiler entrypoints (no corresponding header) + libc.src.compiler.__stack_chk_fail + # ctype.h entrypoints libc.src.ctype.isalnum libc.src.ctype.isalpha @@ -20,12 +23,34 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.ctype.tolower libc.src.ctype.toupper - # compiler entrypoints (no corresponding header) - libc.src.compiler.__stack_chk_fail + # ctype.h entrypoints + libc.src.ctype.isalnum_l + libc.src.ctype.isalpha_l + libc.src.ctype.isblank_l + libc.src.ctype.iscntrl_l + libc.src.ctype.isdigit_l + libc.src.ctype.isgraph_l + libc.src.ctype.islower_l + libc.src.ctype.isprint_l + libc.src.ctype.ispunct_l + libc.src.ctype.isspace_l + libc.src.ctype.isupper_l + libc.src.ctype.isxdigit_l + libc.src.ctype.tolower_l + libc.src.ctype.toupper_l # errno.h entrypoints libc.src.errno.errno + # locale.h entrypoints + libc.src.locale.localeconv + libc.src.locale.duplocale + libc.src.locale.freelocale + libc.src.locale.localeconv + libc.src.locale.newlocale + libc.src.locale.setlocale + libc.src.locale.uselocale + # setjmp.h entrypoints libc.src.setjmp.longjmp libc.src.setjmp.setjmp @@ -51,6 +76,7 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.string.strcoll libc.src.string.strcpy libc.src.string.strcspn + libc.src.string.strdup libc.src.string.strerror libc.src.string.strerror_r libc.src.string.strlcat @@ -59,6 +85,7 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.string.strncat libc.src.string.strncmp libc.src.string.strncpy + libc.src.string.strndup libc.src.string.strnlen libc.src.string.strpbrk libc.src.string.strrchr @@ -69,15 +96,26 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.string.strtok_r libc.src.string.strxfrm + # string.h entrypoints + libc.src.string.strcoll_l + libc.src.string.strxfrm_l + # strings.h entrypoints libc.src.strings.bcmp libc.src.strings.bcopy libc.src.strings.bzero + libc.src.strings.ffs + libc.src.strings.ffsl + libc.src.strings.ffsll libc.src.strings.index libc.src.strings.rindex libc.src.strings.strcasecmp libc.src.strings.strncasecmp + # strings.h entrypoints + libc.src.strings.strcasecmp_l + libc.src.strings.strncasecmp_l + # inttypes.h entrypoints libc.src.inttypes.imaxabs libc.src.inttypes.imaxdiv @@ -85,18 +123,22 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.inttypes.strtoumax # stdio.h entrypoints + libc.src.stdio.asprintf libc.src.stdio.getchar libc.src.stdio.printf libc.src.stdio.putchar libc.src.stdio.puts libc.src.stdio.remove + libc.src.stdio.scanf libc.src.stdio.snprintf libc.src.stdio.sprintf - libc.src.stdio.asprintf + libc.src.stdio.sscanf + libc.src.stdio.vasprintf libc.src.stdio.vprintf + libc.src.stdio.vscanf libc.src.stdio.vsnprintf libc.src.stdio.vsprintf - libc.src.stdio.vasprintf + libc.src.stdio.vsscanf # stdbit.h entrypoints libc.src.stdbit.stdc_bit_ceil_uc @@ -172,6 +214,7 @@ set(TARGET_LIBC_ENTRYPOINTS # stdlib.h entrypoints libc.src.stdlib._Exit + libc.src.stdlib.a64l libc.src.stdlib.abort libc.src.stdlib.abs libc.src.stdlib.aligned_alloc @@ -184,15 +227,21 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.stdlib.div libc.src.stdlib.exit libc.src.stdlib.free + libc.src.stdlib.l64a libc.src.stdlib.labs libc.src.stdlib.ldiv libc.src.stdlib.llabs libc.src.stdlib.lldiv libc.src.stdlib.malloc + libc.src.stdlib.memalignment libc.src.stdlib.qsort + libc.src.stdlib.qsort_r libc.src.stdlib.rand libc.src.stdlib.realloc libc.src.stdlib.srand + # libc.src.stdlib.strfromd + # libc.src.stdlib.strfromf + # libc.src.stdlib.strfroml libc.src.stdlib.strtod libc.src.stdlib.strtof libc.src.stdlib.strtol @@ -201,6 +250,15 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.stdlib.strtoul libc.src.stdlib.strtoull + # stdlib.h entrypoints + libc.src.stdlib.strtod_l + libc.src.stdlib.strtof_l + libc.src.stdlib.strtol_l + libc.src.stdlib.strtold_l + libc.src.stdlib.strtoll_l + libc.src.stdlib.strtoul_l + libc.src.stdlib.strtoull_l + # time.h entrypoints libc.src.time.asctime libc.src.time.asctime_r @@ -210,14 +268,35 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.time.gmtime libc.src.time.gmtime_r libc.src.time.mktime + libc.src.time.strftime + libc.src.time.strftime_l libc.src.time.timespec_get + # wchar.h entrypoints + libc.src.wchar.btowc + libc.src.wchar.wcslen + libc.src.wchar.wctob + # internal entrypoints libc.startup.baremetal.init libc.startup.baremetal.fini ) set(TARGET_LIBM_ENTRYPOINTS + # complex.h entrypoints + libc.src.complex.creal + libc.src.complex.crealf + libc.src.complex.creall + libc.src.complex.cimag + libc.src.complex.cimagf + libc.src.complex.cimagl + libc.src.complex.conj + libc.src.complex.conjf + libc.src.complex.conjl + libc.src.complex.cproj + libc.src.complex.cprojf + libc.src.complex.cprojl + # fenv.h entrypoints libc.src.fenv.feclearexcept libc.src.fenv.fedisableexcept @@ -243,6 +322,8 @@ set(TARGET_LIBM_ENTRYPOINTS libc.src.math.asinhf libc.src.math.atan2 libc.src.math.atan2f + libc.src.math.atan2l + libc.src.math.atan libc.src.math.atanf libc.src.math.atanhf libc.src.math.canonicalize @@ -259,10 +340,18 @@ set(TARGET_LIBM_ENTRYPOINTS libc.src.math.cos libc.src.math.cosf libc.src.math.coshf + libc.src.math.cospif + libc.src.math.dfmal + libc.src.math.dmull + libc.src.math.dsqrtl + libc.src.math.daddl + libc.src.math.ddivl + libc.src.math.dsubl libc.src.math.erff libc.src.math.exp libc.src.math.exp10 libc.src.math.exp10f + libc.src.math.exp10m1f libc.src.math.exp2 libc.src.math.exp2f libc.src.math.exp2m1f @@ -272,9 +361,16 @@ set(TARGET_LIBM_ENTRYPOINTS libc.src.math.fabs libc.src.math.fabsf libc.src.math.fabsl + libc.src.math.fadd + libc.src.math.faddl + libc.src.math.fadd libc.src.math.fdim libc.src.math.fdimf libc.src.math.fdiml + libc.src.math.fdiv + libc.src.math.fdivl + libc.src.math.ffma + libc.src.math.ffmal libc.src.math.floor libc.src.math.floorf libc.src.math.floorl @@ -314,6 +410,7 @@ set(TARGET_LIBM_ENTRYPOINTS libc.src.math.fmodf libc.src.math.fmodl libc.src.math.fmul + libc.src.math.fmull libc.src.math.frexp libc.src.math.frexpf libc.src.math.frexpl @@ -323,14 +420,27 @@ set(TARGET_LIBM_ENTRYPOINTS libc.src.math.fromfpx libc.src.math.fromfpxf libc.src.math.fromfpxl + libc.src.math.fsqrt + libc.src.math.fsqrtl + libc.src.math.fsub + libc.src.math.fsubl + libc.src.math.getpayload + libc.src.math.getpayloadf + libc.src.math.getpayloadl libc.src.math.hypot libc.src.math.hypotf libc.src.math.ilogb libc.src.math.ilogbf libc.src.math.ilogbl + libc.src.math.iscanonical + libc.src.math.iscanonicalf + libc.src.math.iscanonicall libc.src.math.isnan libc.src.math.isnanf libc.src.math.isnanl + libc.src.math.issignaling + libc.src.math.issignalingf + libc.src.math.issignalingl libc.src.math.ldexp libc.src.math.ldexpf libc.src.math.ldexpl @@ -404,17 +514,30 @@ set(TARGET_LIBM_ENTRYPOINTS libc.src.math.scalbn libc.src.math.scalbnf libc.src.math.scalbnl + libc.src.math.setpayload + libc.src.math.setpayloadf + libc.src.math.setpayloadl + libc.src.math.setpayloadsig + libc.src.math.setpayloadsigf + libc.src.math.setpayloadsigl libc.src.math.sin libc.src.math.sincos libc.src.math.sincosf libc.src.math.sinf libc.src.math.sinhf + libc.src.math.sinpif libc.src.math.sqrt libc.src.math.sqrtf libc.src.math.sqrtl libc.src.math.tan libc.src.math.tanf libc.src.math.tanhf + libc.src.math.totalorder + libc.src.math.totalorderf + libc.src.math.totalorderl + libc.src.math.totalordermag + libc.src.math.totalordermagf + libc.src.math.totalordermagl libc.src.math.trunc libc.src.math.truncf libc.src.math.truncl @@ -426,6 +549,210 @@ set(TARGET_LIBM_ENTRYPOINTS libc.src.math.ufromfpxl ) +if(LIBC_TYPES_HAS_CFLOAT16) + list(APPEND TARGET_LIBM_ENTRYPOINTS + # complex.h C23 _Complex _Float16 entrypoints + libc.src.complex.crealf16 + libc.src.complex.cimagf16 + libc.src.complex.conjf16 + libc.src.complex.cprojf16 + ) +endif() + +if(LIBC_TYPES_HAS_FLOAT16) + list(APPEND TARGET_LIBM_ENTRYPOINTS + # math.h C23 _Float16 entrypoints + libc.src.math.acosf16 + libc.src.math.acoshf16 + libc.src.math.asinf16 + libc.src.math.asinhf16 + libc.src.math.canonicalizef16 + libc.src.math.ceilf16 + libc.src.math.copysignf16 + libc.src.math.cosf16 + libc.src.math.coshf16 + libc.src.math.cospif16 + libc.src.math.exp10f16 + libc.src.math.exp10m1f16 + libc.src.math.exp2f16 + libc.src.math.exp2m1f16 + libc.src.math.expf16 + libc.src.math.expm1f16 + libc.src.math.f16add + libc.src.math.f16addf + libc.src.math.f16addl + libc.src.math.f16div + libc.src.math.f16divf + libc.src.math.f16divl + libc.src.math.f16fma + libc.src.math.f16fmaf + libc.src.math.f16fmal + libc.src.math.f16mul + libc.src.math.f16mulf + libc.src.math.f16mull + libc.src.math.f16sqrt + libc.src.math.f16sqrtf + libc.src.math.f16sqrtl + libc.src.math.f16sub + libc.src.math.f16subf + libc.src.math.f16subl + libc.src.math.fabsf16 + libc.src.math.fdimf16 + libc.src.math.floorf16 + libc.src.math.fmaf16 + libc.src.math.fmaxf16 + libc.src.math.fmaximum_mag_numf16 + libc.src.math.fmaximum_magf16 + libc.src.math.fmaximum_numf16 + libc.src.math.fmaximumf16 + libc.src.math.fminf16 + libc.src.math.fminimum_mag_numf16 + libc.src.math.fminimum_magf16 + libc.src.math.fminimum_numf16 + libc.src.math.fminimumf16 + libc.src.math.fmodf16 + libc.src.math.frexpf16 + libc.src.math.fromfpf16 + libc.src.math.fromfpxf16 + libc.src.math.getpayloadf16 + libc.src.math.hypotf16 + libc.src.math.ilogbf16 + libc.src.math.iscanonicalf16 + libc.src.math.issignalingf16 + libc.src.math.ldexpf16 + libc.src.math.llogbf16 + libc.src.math.llrintf16 + libc.src.math.llroundf16 + libc.src.math.log10f16 + libc.src.math.log2f16 + libc.src.math.logbf16 + libc.src.math.logf16 + libc.src.math.lrintf16 + libc.src.math.lroundf16 + libc.src.math.modff16 + libc.src.math.nanf16 + libc.src.math.nearbyintf16 + libc.src.math.nextafterf16 + libc.src.math.nextdownf16 + libc.src.math.nexttowardf16 + libc.src.math.nextupf16 + libc.src.math.remainderf16 + libc.src.math.remquof16 + libc.src.math.rintf16 + libc.src.math.roundevenf16 + libc.src.math.roundf16 + libc.src.math.scalblnf16 + libc.src.math.scalbnf16 + libc.src.math.setpayloadf16 + libc.src.math.setpayloadsigf16 + libc.src.math.sinf16 + libc.src.math.sinhf16 + libc.src.math.sinpif16 + libc.src.math.sqrtf16 + libc.src.math.tanf16 + libc.src.math.tanhf16 + libc.src.math.tanpif16 + libc.src.math.totalorderf16 + libc.src.math.totalordermagf16 + libc.src.math.truncf16 + libc.src.math.ufromfpf16 + libc.src.math.ufromfpxf16 + ) + + if(LIBC_TYPES_HAS_FLOAT128) + list(APPEND TARGET_LIBM_ENTRYPOINTS + # math.h C23 mixed _Float16 and _Float128 entrypoints + libc.src.math.f16addf128 + libc.src.math.f16divf128 + libc.src.math.f16fmaf128 + libc.src.math.f16mulf128 + libc.src.math.f16sqrtf128 + libc.src.math.f16subf128 + ) + endif() +endif() + +if(LIBC_TYPES_HAS_CFLOAT128) + list(APPEND TARGET_LIBM_ENTRYPOINTS + # complex.h C23 _Complex _Float128 entrypoints + libc.src.complex.crealf128 + libc.src.complex.cimagf128 + libc.src.complex.conjf128 + libc.src.complex.cprojf128 + ) +endif() + +if(LIBC_TYPES_HAS_FLOAT128) + list(APPEND TARGET_LIBM_ENTRYPOINTS + # math.h C23 _Float128 entrypoints + libc.src.math.atan2f128 + libc.src.math.canonicalizef128 + libc.src.math.ceilf128 + libc.src.math.copysignf128 + libc.src.math.daddf128 + libc.src.math.ddivf128 + libc.src.math.dfmaf128 + libc.src.math.dmulf128 + libc.src.math.dsqrtf128 + libc.src.math.dsubf128 + libc.src.math.fabsf128 + libc.src.math.faddf128 + libc.src.math.fdimf128 + libc.src.math.fdivf128 + libc.src.math.ffmaf128 + libc.src.math.floorf128 + libc.src.math.fmaxf128 + libc.src.math.fmaximum_mag_numf128 + libc.src.math.fmaximum_magf128 + libc.src.math.fmaximum_numf128 + libc.src.math.fmaximumf128 + libc.src.math.fminf128 + libc.src.math.fminimum_mag_numf128 + libc.src.math.fminimum_magf128 + libc.src.math.fminimum_numf128 + libc.src.math.fminimumf128 + libc.src.math.fmodf128 + libc.src.math.fmulf128 + libc.src.math.frexpf128 + libc.src.math.fromfpf128 + libc.src.math.fromfpxf128 + libc.src.math.fsqrtf128 + libc.src.math.fsubf128 + libc.src.math.getpayloadf128 + libc.src.math.ilogbf128 + libc.src.math.iscanonicalf128 + libc.src.math.issignalingf128 + libc.src.math.ldexpf128 + libc.src.math.llogbf128 + libc.src.math.llrintf128 + libc.src.math.llroundf128 + libc.src.math.logbf128 + libc.src.math.lrintf128 + libc.src.math.lroundf128 + libc.src.math.modff128 + libc.src.math.nanf128 + libc.src.math.nearbyintf128 + libc.src.math.nextafterf128 + libc.src.math.nextdownf128 + libc.src.math.nextupf128 + libc.src.math.remainderf128 + libc.src.math.remquof128 + libc.src.math.rintf128 + libc.src.math.roundevenf128 + libc.src.math.roundf128 + libc.src.math.scalblnf128 + libc.src.math.scalbnf128 + libc.src.math.setpayloadf128 + libc.src.math.setpayloadsigf128 + libc.src.math.sqrtf128 + libc.src.math.totalorderf128 + libc.src.math.totalordermagf128 + libc.src.math.truncf128 + libc.src.math.ufromfpf128 + libc.src.math.ufromfpxf128 + ) +endif() + if(LIBC_COMPILER_HAS_FIXED_POINT) list(APPEND TARGET_LIBM_ENTRYPOINTS # stdfix.h _Fract and _Accum entrypoints @@ -469,6 +796,38 @@ if(LIBC_COMPILER_HAS_FIXED_POINT) libc.src.stdfix.ukbits libc.src.stdfix.lkbits libc.src.stdfix.ulkbits + libc.src.stdfix.bitshr + libc.src.stdfix.bitsr + libc.src.stdfix.bitslr + libc.src.stdfix.bitshk + libc.src.stdfix.bitsk + libc.src.stdfix.bitslk + libc.src.stdfix.bitsuhr + libc.src.stdfix.bitsur + libc.src.stdfix.bitsulr + libc.src.stdfix.bitsuhk + libc.src.stdfix.bitsuk + libc.src.stdfix.bitsulk + libc.src.stdfix.countlshr + libc.src.stdfix.countlsr + libc.src.stdfix.countlslr + libc.src.stdfix.countlshk + libc.src.stdfix.countlsk + libc.src.stdfix.countlslk + libc.src.stdfix.countlsuhr + libc.src.stdfix.countlsur + libc.src.stdfix.countlsulr + libc.src.stdfix.countlsuhk + libc.src.stdfix.countlsuk + libc.src.stdfix.countlsulk + libc.src.stdfix.idivr + libc.src.stdfix.idivlr + libc.src.stdfix.idivk + libc.src.stdfix.idivlk + libc.src.stdfix.idivur + libc.src.stdfix.idivulr + libc.src.stdfix.idivuk + libc.src.stdfix.idivulk ) endif() diff --git a/system/lib/llvm-libc/config/baremetal/arm/headers.txt b/system/lib/llvm-libc/config/baremetal/arm/headers.txt index 6cc8a80455d3a..5666ef7e0012d 100644 --- a/system/lib/llvm-libc/config/baremetal/arm/headers.txt +++ b/system/lib/llvm-libc/config/baremetal/arm/headers.txt @@ -1,13 +1,18 @@ set(TARGET_PUBLIC_HEADERS libc.include.assert + libc.include.complex libc.include.ctype libc.include.errno libc.include.features libc.include.fenv libc.include.float libc.include.inttypes + libc.include.limits + libc.include.locale libc.include.math libc.include.setjmp + libc.include.stdbit + libc.include.stdckdint libc.include.stdfix libc.include.stdint libc.include.stdio @@ -17,4 +22,5 @@ set(TARGET_PUBLIC_HEADERS libc.include.sys_queue libc.include.time libc.include.uchar + libc.include.wchar ) diff --git a/system/lib/llvm-libc/config/baremetal/config.json b/system/lib/llvm-libc/config/baremetal/config.json index 08c581d1c6822..105e417d165f0 100644 --- a/system/lib/llvm-libc/config/baremetal/config.json +++ b/system/lib/llvm-libc/config/baremetal/config.json @@ -5,6 +5,9 @@ } }, "printf": { + "LIBC_CONF_PRINTF_DISABLE_FIXED_POINT": { + "value": true + }, "LIBC_CONF_PRINTF_DISABLE_FLOAT": { "value": true }, diff --git a/system/lib/llvm-libc/config/baremetal/riscv/entrypoints.txt b/system/lib/llvm-libc/config/baremetal/riscv/entrypoints.txt index 667ab40dca999..023826f12d723 100644 --- a/system/lib/llvm-libc/config/baremetal/riscv/entrypoints.txt +++ b/system/lib/llvm-libc/config/baremetal/riscv/entrypoints.txt @@ -2,6 +2,9 @@ set(TARGET_LIBC_ENTRYPOINTS # assert.h entrypoints libc.src.assert.__assert_fail + # compiler entrypoints (no corresponding header) + libc.src.compiler.__stack_chk_fail + # ctype.h entrypoints libc.src.ctype.isalnum libc.src.ctype.isalpha @@ -20,12 +23,38 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.ctype.tolower libc.src.ctype.toupper - # compiler entrypoints (no corresponding header) - libc.src.compiler.__stack_chk_fail + # ctype.h entrypoints + libc.src.ctype.isalnum_l + libc.src.ctype.isalpha_l + libc.src.ctype.isblank_l + libc.src.ctype.iscntrl_l + libc.src.ctype.isdigit_l + libc.src.ctype.isgraph_l + libc.src.ctype.islower_l + libc.src.ctype.isprint_l + libc.src.ctype.ispunct_l + libc.src.ctype.isspace_l + libc.src.ctype.isupper_l + libc.src.ctype.isxdigit_l + libc.src.ctype.tolower_l + libc.src.ctype.toupper_l # errno.h entrypoints libc.src.errno.errno + # locale.h entrypoints + libc.src.locale.localeconv + libc.src.locale.duplocale + libc.src.locale.freelocale + libc.src.locale.localeconv + libc.src.locale.newlocale + libc.src.locale.setlocale + libc.src.locale.uselocale + + # setjmp.h entrypoints + # libc.src.setjmp.longjmp + # libc.src.setjmp.setjmp + # string.h entrypoints libc.src.string.memccpy libc.src.string.memchr @@ -47,6 +76,7 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.string.strcoll libc.src.string.strcpy libc.src.string.strcspn + libc.src.string.strdup libc.src.string.strerror libc.src.string.strerror_r libc.src.string.strlcat @@ -55,6 +85,7 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.string.strncat libc.src.string.strncmp libc.src.string.strncpy + libc.src.string.strndup libc.src.string.strnlen libc.src.string.strpbrk libc.src.string.strrchr @@ -65,15 +96,26 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.string.strtok_r libc.src.string.strxfrm + # string.h entrypoints + libc.src.string.strcoll_l + libc.src.string.strxfrm_l + # strings.h entrypoints libc.src.strings.bcmp libc.src.strings.bcopy libc.src.strings.bzero + libc.src.strings.ffs + libc.src.strings.ffsl + libc.src.strings.ffsll libc.src.strings.index libc.src.strings.rindex libc.src.strings.strcasecmp libc.src.strings.strncasecmp + # strings.h entrypoints + libc.src.strings.strcasecmp_l + libc.src.strings.strncasecmp_l + # inttypes.h entrypoints libc.src.inttypes.imaxabs libc.src.inttypes.imaxdiv @@ -81,18 +123,22 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.inttypes.strtoumax # stdio.h entrypoints + libc.src.stdio.asprintf libc.src.stdio.getchar libc.src.stdio.printf libc.src.stdio.putchar libc.src.stdio.puts libc.src.stdio.remove + libc.src.stdio.scanf libc.src.stdio.snprintf libc.src.stdio.sprintf - libc.src.stdio.asprintf + libc.src.stdio.sscanf + libc.src.stdio.vasprintf libc.src.stdio.vprintf + libc.src.stdio.vscanf libc.src.stdio.vsnprintf libc.src.stdio.vsprintf - libc.src.stdio.vasprintf + libc.src.stdio.vsscanf # stdbit.h entrypoints libc.src.stdbit.stdc_bit_ceil_uc @@ -168,6 +214,7 @@ set(TARGET_LIBC_ENTRYPOINTS # stdlib.h entrypoints libc.src.stdlib._Exit + libc.src.stdlib.a64l libc.src.stdlib.abort libc.src.stdlib.abs libc.src.stdlib.aligned_alloc @@ -180,15 +227,21 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.stdlib.div libc.src.stdlib.exit libc.src.stdlib.free + libc.src.stdlib.l64a libc.src.stdlib.labs libc.src.stdlib.ldiv libc.src.stdlib.llabs libc.src.stdlib.lldiv libc.src.stdlib.malloc + libc.src.stdlib.memalignment libc.src.stdlib.qsort + libc.src.stdlib.qsort_r libc.src.stdlib.rand libc.src.stdlib.realloc libc.src.stdlib.srand + # libc.src.stdlib.strfromd + # libc.src.stdlib.strfromf + # libc.src.stdlib.strfroml libc.src.stdlib.strtod libc.src.stdlib.strtof libc.src.stdlib.strtol @@ -197,6 +250,15 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.stdlib.strtoul libc.src.stdlib.strtoull + # stdlib.h entrypoints + libc.src.stdlib.strtod_l + libc.src.stdlib.strtof_l + libc.src.stdlib.strtol_l + libc.src.stdlib.strtold_l + libc.src.stdlib.strtoll_l + libc.src.stdlib.strtoul_l + libc.src.stdlib.strtoull_l + # time.h entrypoints libc.src.time.asctime libc.src.time.asctime_r @@ -206,14 +268,35 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.time.gmtime libc.src.time.gmtime_r libc.src.time.mktime + libc.src.time.strftime + libc.src.time.strftime_l libc.src.time.timespec_get + # wchar.h entrypoints + libc.src.wchar.btowc + libc.src.wchar.wcslen + libc.src.wchar.wctob + # internal entrypoints libc.startup.baremetal.init libc.startup.baremetal.fini ) set(TARGET_LIBM_ENTRYPOINTS + # complex.h entrypoints + libc.src.complex.creal + libc.src.complex.crealf + libc.src.complex.creall + libc.src.complex.cimag + libc.src.complex.cimagf + libc.src.complex.cimagl + libc.src.complex.conj + libc.src.complex.conjf + libc.src.complex.conjl + libc.src.complex.cproj + libc.src.complex.cprojf + libc.src.complex.cprojl + # fenv.h entrypoints libc.src.fenv.feclearexcept libc.src.fenv.fedisableexcept @@ -239,6 +322,8 @@ set(TARGET_LIBM_ENTRYPOINTS libc.src.math.asinhf libc.src.math.atan2 libc.src.math.atan2f + # libc.src.math.atan2l + libc.src.math.atan libc.src.math.atanf libc.src.math.atanhf libc.src.math.canonicalize @@ -255,10 +340,18 @@ set(TARGET_LIBM_ENTRYPOINTS libc.src.math.cos libc.src.math.cosf libc.src.math.coshf + libc.src.math.cospif + libc.src.math.dfmal + libc.src.math.dmull + libc.src.math.dsqrtl + libc.src.math.daddl + libc.src.math.ddivl + libc.src.math.dsubl libc.src.math.erff libc.src.math.exp libc.src.math.exp10 libc.src.math.exp10f + libc.src.math.exp10m1f libc.src.math.exp2 libc.src.math.exp2f libc.src.math.exp2m1f @@ -268,9 +361,16 @@ set(TARGET_LIBM_ENTRYPOINTS libc.src.math.fabs libc.src.math.fabsf libc.src.math.fabsl + libc.src.math.fadd + libc.src.math.faddl + libc.src.math.fadd libc.src.math.fdim libc.src.math.fdimf libc.src.math.fdiml + libc.src.math.fdiv + libc.src.math.fdivl + libc.src.math.ffma + libc.src.math.ffmal libc.src.math.floor libc.src.math.floorf libc.src.math.floorl @@ -308,7 +408,9 @@ set(TARGET_LIBM_ENTRYPOINTS libc.src.math.fminl libc.src.math.fmod libc.src.math.fmodf + # libc.src.math.fmodl libc.src.math.fmul + libc.src.math.fmull libc.src.math.frexp libc.src.math.frexpf libc.src.math.frexpl @@ -318,14 +420,27 @@ set(TARGET_LIBM_ENTRYPOINTS libc.src.math.fromfpx libc.src.math.fromfpxf libc.src.math.fromfpxl + libc.src.math.fsqrt + libc.src.math.fsqrtl + libc.src.math.fsub + libc.src.math.fsubl + libc.src.math.getpayload + libc.src.math.getpayloadf + libc.src.math.getpayloadl libc.src.math.hypot libc.src.math.hypotf libc.src.math.ilogb libc.src.math.ilogbf libc.src.math.ilogbl + libc.src.math.iscanonical + libc.src.math.iscanonicalf + libc.src.math.iscanonicall libc.src.math.isnan libc.src.math.isnanf libc.src.math.isnanl + libc.src.math.issignaling + libc.src.math.issignalingf + libc.src.math.issignalingl libc.src.math.ldexp libc.src.math.ldexpf libc.src.math.ldexpl @@ -399,17 +514,30 @@ set(TARGET_LIBM_ENTRYPOINTS libc.src.math.scalbn libc.src.math.scalbnf libc.src.math.scalbnl + libc.src.math.setpayload + libc.src.math.setpayloadf + libc.src.math.setpayloadl + libc.src.math.setpayloadsig + libc.src.math.setpayloadsigf + libc.src.math.setpayloadsigl libc.src.math.sin libc.src.math.sincos libc.src.math.sincosf libc.src.math.sinf libc.src.math.sinhf + libc.src.math.sinpif libc.src.math.sqrt libc.src.math.sqrtf libc.src.math.sqrtl libc.src.math.tan libc.src.math.tanf libc.src.math.tanhf + libc.src.math.totalorder + libc.src.math.totalorderf + libc.src.math.totalorderl + libc.src.math.totalordermag + libc.src.math.totalordermagf + libc.src.math.totalordermagl libc.src.math.trunc libc.src.math.truncf libc.src.math.truncl @@ -421,6 +549,210 @@ set(TARGET_LIBM_ENTRYPOINTS libc.src.math.ufromfpxl ) +if(LIBC_TYPES_HAS_CFLOAT16) + list(APPEND TARGET_LIBM_ENTRYPOINTS + # complex.h C23 _Complex _Float16 entrypoints + libc.src.complex.crealf16 + libc.src.complex.cimagf16 + libc.src.complex.conjf16 + libc.src.complex.cprojf16 + ) +endif() + +if(LIBC_TYPES_HAS_FLOAT16) + list(APPEND TARGET_LIBM_ENTRYPOINTS + # math.h C23 _Float16 entrypoints + libc.src.math.acosf16 + libc.src.math.acoshf16 + libc.src.math.asinf16 + libc.src.math.asinhf16 + libc.src.math.canonicalizef16 + libc.src.math.ceilf16 + libc.src.math.copysignf16 + libc.src.math.cosf16 + libc.src.math.coshf16 + libc.src.math.cospif16 + libc.src.math.exp10f16 + libc.src.math.exp10m1f16 + libc.src.math.exp2f16 + libc.src.math.exp2m1f16 + libc.src.math.expf16 + libc.src.math.expm1f16 + libc.src.math.f16add + libc.src.math.f16addf + libc.src.math.f16addl + libc.src.math.f16div + libc.src.math.f16divf + libc.src.math.f16divl + libc.src.math.f16fma + libc.src.math.f16fmaf + libc.src.math.f16fmal + libc.src.math.f16mul + libc.src.math.f16mulf + libc.src.math.f16mull + libc.src.math.f16sqrt + libc.src.math.f16sqrtf + libc.src.math.f16sqrtl + libc.src.math.f16sub + libc.src.math.f16subf + libc.src.math.f16subl + libc.src.math.fabsf16 + libc.src.math.fdimf16 + libc.src.math.floorf16 + libc.src.math.fmaf16 + libc.src.math.fmaxf16 + libc.src.math.fmaximum_mag_numf16 + libc.src.math.fmaximum_magf16 + libc.src.math.fmaximum_numf16 + libc.src.math.fmaximumf16 + libc.src.math.fminf16 + libc.src.math.fminimum_mag_numf16 + libc.src.math.fminimum_magf16 + libc.src.math.fminimum_numf16 + libc.src.math.fminimumf16 + libc.src.math.fmodf16 + libc.src.math.frexpf16 + libc.src.math.fromfpf16 + libc.src.math.fromfpxf16 + libc.src.math.getpayloadf16 + libc.src.math.hypotf16 + libc.src.math.ilogbf16 + libc.src.math.iscanonicalf16 + libc.src.math.issignalingf16 + libc.src.math.ldexpf16 + libc.src.math.llogbf16 + libc.src.math.llrintf16 + libc.src.math.llroundf16 + libc.src.math.log10f16 + libc.src.math.log2f16 + libc.src.math.logbf16 + libc.src.math.logf16 + libc.src.math.lrintf16 + libc.src.math.lroundf16 + libc.src.math.modff16 + libc.src.math.nanf16 + libc.src.math.nearbyintf16 + libc.src.math.nextafterf16 + libc.src.math.nextdownf16 + libc.src.math.nexttowardf16 + libc.src.math.nextupf16 + libc.src.math.remainderf16 + libc.src.math.remquof16 + libc.src.math.rintf16 + libc.src.math.roundevenf16 + libc.src.math.roundf16 + libc.src.math.scalblnf16 + libc.src.math.scalbnf16 + libc.src.math.setpayloadf16 + libc.src.math.setpayloadsigf16 + libc.src.math.sinf16 + libc.src.math.sinhf16 + libc.src.math.sinpif16 + libc.src.math.sqrtf16 + libc.src.math.tanf16 + libc.src.math.tanhf16 + libc.src.math.tanpif16 + libc.src.math.totalorderf16 + libc.src.math.totalordermagf16 + libc.src.math.truncf16 + libc.src.math.ufromfpf16 + libc.src.math.ufromfpxf16 + ) + + if(LIBC_TYPES_HAS_FLOAT128) + list(APPEND TARGET_LIBM_ENTRYPOINTS + # math.h C23 mixed _Float16 and _Float128 entrypoints + libc.src.math.f16addf128 + libc.src.math.f16divf128 + libc.src.math.f16fmaf128 + libc.src.math.f16mulf128 + libc.src.math.f16sqrtf128 + libc.src.math.f16subf128 + ) + endif() +endif() + +if(LIBC_TYPES_HAS_CFLOAT128) + list(APPEND TARGET_LIBM_ENTRYPOINTS + # complex.h C23 _Complex _Float128 entrypoints + libc.src.complex.crealf128 + libc.src.complex.cimagf128 + libc.src.complex.conjf128 + libc.src.complex.cprojf128 + ) +endif() + +if(LIBC_TYPES_HAS_FLOAT128) + list(APPEND TARGET_LIBM_ENTRYPOINTS + # math.h C23 _Float128 entrypoints + libc.src.math.atan2f128 + libc.src.math.canonicalizef128 + libc.src.math.ceilf128 + libc.src.math.copysignf128 + libc.src.math.daddf128 + libc.src.math.ddivf128 + libc.src.math.dfmaf128 + libc.src.math.dmulf128 + libc.src.math.dsqrtf128 + libc.src.math.dsubf128 + libc.src.math.fabsf128 + libc.src.math.faddf128 + libc.src.math.fdimf128 + libc.src.math.fdivf128 + libc.src.math.ffmaf128 + libc.src.math.floorf128 + libc.src.math.fmaxf128 + libc.src.math.fmaximum_mag_numf128 + libc.src.math.fmaximum_magf128 + libc.src.math.fmaximum_numf128 + libc.src.math.fmaximumf128 + libc.src.math.fminf128 + libc.src.math.fminimum_mag_numf128 + libc.src.math.fminimum_magf128 + libc.src.math.fminimum_numf128 + libc.src.math.fminimumf128 + # libc.src.math.fmodf128 + libc.src.math.fmulf128 + libc.src.math.frexpf128 + libc.src.math.fromfpf128 + libc.src.math.fromfpxf128 + libc.src.math.fsqrtf128 + libc.src.math.fsubf128 + libc.src.math.getpayloadf128 + libc.src.math.ilogbf128 + libc.src.math.iscanonicalf128 + libc.src.math.issignalingf128 + libc.src.math.ldexpf128 + libc.src.math.llogbf128 + libc.src.math.llrintf128 + libc.src.math.llroundf128 + libc.src.math.logbf128 + libc.src.math.lrintf128 + libc.src.math.lroundf128 + libc.src.math.modff128 + libc.src.math.nanf128 + libc.src.math.nearbyintf128 + libc.src.math.nextafterf128 + libc.src.math.nextdownf128 + libc.src.math.nextupf128 + libc.src.math.remainderf128 + libc.src.math.remquof128 + libc.src.math.rintf128 + libc.src.math.roundevenf128 + libc.src.math.roundf128 + libc.src.math.scalblnf128 + libc.src.math.scalbnf128 + libc.src.math.setpayloadf128 + libc.src.math.setpayloadsigf128 + libc.src.math.sqrtf128 + libc.src.math.totalorderf128 + libc.src.math.totalordermagf128 + libc.src.math.truncf128 + libc.src.math.ufromfpf128 + libc.src.math.ufromfpxf128 + ) +endif() + if(LIBC_COMPILER_HAS_FIXED_POINT) list(APPEND TARGET_LIBM_ENTRYPOINTS # stdfix.h _Fract and _Accum entrypoints @@ -464,6 +796,38 @@ if(LIBC_COMPILER_HAS_FIXED_POINT) libc.src.stdfix.ukbits libc.src.stdfix.lkbits libc.src.stdfix.ulkbits + libc.src.stdfix.bitshr + libc.src.stdfix.bitsr + libc.src.stdfix.bitslr + libc.src.stdfix.bitshk + libc.src.stdfix.bitsk + libc.src.stdfix.bitslk + libc.src.stdfix.bitsuhr + libc.src.stdfix.bitsur + libc.src.stdfix.bitsulr + libc.src.stdfix.bitsuhk + libc.src.stdfix.bitsuk + libc.src.stdfix.bitsulk + libc.src.stdfix.countlshr + libc.src.stdfix.countlsr + libc.src.stdfix.countlslr + libc.src.stdfix.countlshk + libc.src.stdfix.countlsk + libc.src.stdfix.countlslk + libc.src.stdfix.countlsuhr + libc.src.stdfix.countlsur + libc.src.stdfix.countlsulr + libc.src.stdfix.countlsuhk + libc.src.stdfix.countlsuk + libc.src.stdfix.countlsulk + libc.src.stdfix.idivr + libc.src.stdfix.idivlr + libc.src.stdfix.idivk + libc.src.stdfix.idivlk + libc.src.stdfix.idivur + libc.src.stdfix.idivulr + libc.src.stdfix.idivuk + libc.src.stdfix.idivulk ) endif() diff --git a/system/lib/llvm-libc/config/baremetal/riscv/headers.txt b/system/lib/llvm-libc/config/baremetal/riscv/headers.txt index c4fa24f025a71..5666ef7e0012d 100644 --- a/system/lib/llvm-libc/config/baremetal/riscv/headers.txt +++ b/system/lib/llvm-libc/config/baremetal/riscv/headers.txt @@ -1,12 +1,18 @@ set(TARGET_PUBLIC_HEADERS libc.include.assert + libc.include.complex libc.include.ctype libc.include.errno libc.include.features libc.include.fenv libc.include.float libc.include.inttypes + libc.include.limits + libc.include.locale libc.include.math + libc.include.setjmp + libc.include.stdbit + libc.include.stdckdint libc.include.stdfix libc.include.stdint libc.include.stdio @@ -16,4 +22,5 @@ set(TARGET_PUBLIC_HEADERS libc.include.sys_queue libc.include.time libc.include.uchar + libc.include.wchar ) diff --git a/system/lib/llvm-libc/config/config.json b/system/lib/llvm-libc/config/config.json index 9a5d5c3c68da6..d53b2936edb07 100644 --- a/system/lib/llvm-libc/config/config.json +++ b/system/lib/llvm-libc/config/config.json @@ -2,7 +2,7 @@ "errno": { "LIBC_CONF_ERRNO_MODE": { "value": "LIBC_ERRNO_MODE_DEFAULT", - "doc": "The implementation used for errno, acceptable values are LIBC_ERRNO_MODE_DEFAULT, LIBC_ERRNO_MODE_UNDEFINED, LIBC_ERRNO_MODE_THREAD_LOCAL, LIBC_ERRNO_MODE_SHARED, LIBC_ERRNO_MODE_EXTERNAL, and LIBC_ERRNO_MODE_SYSTEM." + "doc": "The implementation used for errno, acceptable values are LIBC_ERRNO_MODE_DEFAULT, LIBC_ERRNO_MODE_UNDEFINED, LIBC_ERRNO_MODE_THREAD_LOCAL, LIBC_ERRNO_MODE_SHARED, LIBC_ERRNO_MODE_EXTERNAL, LIBC_ERRNO_MODE_SYSTEM, and LIBC_ERRNO_MODE_SYSTEM_INLINE." } }, "printf": { @@ -30,6 +30,10 @@ "value": false, "doc": "Use the same mode for double and long double in printf." }, + "LIBC_CONF_PRINTF_FLOAT_TO_STR_USE_FLOAT320": { + "value": false, + "doc": "Use an alternative printf float implementation based on 320-bit floats" + }, "LIBC_CONF_PRINTF_DISABLE_FIXED_POINT": { "value": false, "doc": "Disable printing fixed point values in printf and friends." @@ -37,6 +41,10 @@ "LIBC_CONF_PRINTF_DISABLE_STRERROR": { "value": false, "doc": "Disable handling of %m to print strerror in printf and friends." + }, + "LIBC_CONF_PRINTF_RUNTIME_DISPATCH": { + "value": true, + "doc": "Use dynamic dispatch for the output mechanism to reduce code size." } }, "scanf": { @@ -90,7 +98,7 @@ }, "LIBC_CONF_FREXP_INF_NAN_EXPONENT": { "value": "", - "doc": "The value written back to the second parameter when calling frexp/frexpf/frexpl` with `+/-Inf`/`NaN` is unspecified. Configue an explicit exp value for Inf/NaN inputs." + "doc": "The value written back to the second parameter when calling frexp/frexpf/frexpl` with `+/-Inf`/`NaN` is unspecified. Configure an explicit exp value for Inf/NaN inputs." } }, "qsort": { diff --git a/system/lib/llvm-libc/config/darwin/aarch64/entrypoints.txt b/system/lib/llvm-libc/config/darwin/aarch64/entrypoints.txt new file mode 100644 index 0000000000000..308fc49d681d7 --- /dev/null +++ b/system/lib/llvm-libc/config/darwin/aarch64/entrypoints.txt @@ -0,0 +1,649 @@ +set(TARGET_LIBC_ENTRYPOINTS + # ctype.h entrypoints + libc.src.ctype.isalnum + libc.src.ctype.isalpha + libc.src.ctype.isascii + libc.src.ctype.isblank + libc.src.ctype.iscntrl + libc.src.ctype.isdigit + libc.src.ctype.isgraph + libc.src.ctype.islower + libc.src.ctype.isprint + libc.src.ctype.ispunct + libc.src.ctype.isspace + libc.src.ctype.isupper + libc.src.ctype.isxdigit + libc.src.ctype.toascii + libc.src.ctype.tolower + libc.src.ctype.toupper + + # errno.h entrypoints + libc.src.errno.errno + + # string.h entrypoints + libc.src.string.memccpy + libc.src.string.memchr + libc.src.string.memcmp + libc.src.string.memcpy + libc.src.string.memmem + libc.src.string.memmove + libc.src.string.mempcpy + libc.src.string.memrchr + libc.src.string.memset + libc.src.string.stpcpy + libc.src.string.stpncpy + libc.src.string.strcasestr + libc.src.string.strcat + libc.src.string.strchr + libc.src.string.strchrnul + libc.src.string.strcmp + libc.src.string.strcpy + libc.src.string.strcspn + libc.src.string.strlcat + libc.src.string.strlcpy + libc.src.string.strlen + libc.src.string.strncat + libc.src.string.strncmp + libc.src.string.strncpy + libc.src.string.strnlen + libc.src.string.strpbrk + libc.src.string.strrchr + libc.src.string.strspn + libc.src.string.strstr + libc.src.string.strtok + libc.src.string.strtok_r + + # string.h entrypoints that depend on malloc + libc.src.string.strdup + libc.src.string.strndup + + # strings.h entrypoints + libc.src.strings.bcmp + libc.src.strings.bcopy + libc.src.strings.bzero + libc.src.strings.strcasecmp + libc.src.strings.strncasecmp + + # inttypes.h entrypoints + libc.src.inttypes.imaxabs + libc.src.inttypes.imaxdiv + libc.src.inttypes.strtoimax + libc.src.inttypes.strtoumax + + # stdlib.h entrypoints + libc.src.stdlib.abs + libc.src.stdlib.atoi + libc.src.stdlib.atof + libc.src.stdlib.atol + libc.src.stdlib.atoll + libc.src.stdlib.bsearch + libc.src.stdlib.div + libc.src.stdlib.labs + libc.src.stdlib.ldiv + libc.src.stdlib.llabs + libc.src.stdlib.lldiv + libc.src.stdlib.memalignment + libc.src.stdlib.qsort + libc.src.stdlib.rand + libc.src.stdlib.srand + libc.src.stdlib.strtod + libc.src.stdlib.strtof + libc.src.stdlib.strtol + libc.src.stdlib.strtold + libc.src.stdlib.strtoll + libc.src.stdlib.strtoul + libc.src.stdlib.strtoull + + # stdlib.h external entrypoints + libc.src.stdlib.malloc + libc.src.stdlib.calloc + libc.src.stdlib.realloc + libc.src.stdlib.free +) + +set(TARGET_LIBM_ENTRYPOINTS + # complex.h entrypoints + libc.src.complex.creal + libc.src.complex.crealf + libc.src.complex.creall + libc.src.complex.cimag + libc.src.complex.cimagf + libc.src.complex.cimagl + libc.src.complex.conj + libc.src.complex.conjf + libc.src.complex.conjl + libc.src.complex.cproj + libc.src.complex.cprojf + libc.src.complex.cprojl + + # fenv.h entrypoints + libc.src.fenv.feclearexcept + libc.src.fenv.fedisableexcept + libc.src.fenv.feenableexcept + libc.src.fenv.fegetenv + libc.src.fenv.fegetexcept + libc.src.fenv.fegetexceptflag + libc.src.fenv.fegetround + libc.src.fenv.feholdexcept + libc.src.fenv.feraiseexcept + libc.src.fenv.fesetenv + libc.src.fenv.fesetexcept + libc.src.fenv.fesetexceptflag + libc.src.fenv.fesetround + libc.src.fenv.fetestexcept + libc.src.fenv.fetestexceptflag + libc.src.fenv.feupdateenv + + # math.h entrypoints + libc.src.math.acos + libc.src.math.acosf + libc.src.math.acoshf + libc.src.math.asin + libc.src.math.asinf + libc.src.math.asinhf + libc.src.math.atan2 + libc.src.math.atan2f + libc.src.math.atan + libc.src.math.atanf + libc.src.math.atanhf + libc.src.math.canonicalize + libc.src.math.canonicalizef + libc.src.math.canonicalizel + libc.src.math.cbrt + libc.src.math.cbrtf + libc.src.math.ceil + libc.src.math.ceilf + libc.src.math.ceill + libc.src.math.copysign + libc.src.math.copysignf + libc.src.math.copysignl + libc.src.math.cos + libc.src.math.cosf + libc.src.math.coshf + libc.src.math.cospif + libc.src.math.dfmal + libc.src.math.dmull + libc.src.math.dsqrtl + libc.src.math.daddl + libc.src.math.ddivl + libc.src.math.dsubl + libc.src.math.erff + libc.src.math.exp + libc.src.math.exp10 + libc.src.math.exp10f + libc.src.math.exp10m1f + libc.src.math.exp2 + libc.src.math.exp2f + libc.src.math.exp2m1f + libc.src.math.expf + libc.src.math.expm1 + libc.src.math.expm1f + libc.src.math.fabs + libc.src.math.fabsf + libc.src.math.fabsl + libc.src.math.fadd + libc.src.math.faddl + libc.src.math.fadd + libc.src.math.fdim + libc.src.math.fdimf + libc.src.math.fdiml + libc.src.math.fdiv + libc.src.math.fdivl + libc.src.math.ffma + libc.src.math.ffmal + libc.src.math.floor + libc.src.math.floorf + libc.src.math.floorl + libc.src.math.fma + libc.src.math.fmaf + libc.src.math.fmax + libc.src.math.fmaxf + libc.src.math.fmaximum + libc.src.math.fmaximum_mag + libc.src.math.fmaximum_mag_num + libc.src.math.fmaximum_mag_numf + libc.src.math.fmaximum_mag_numl + libc.src.math.fmaximum_magf + libc.src.math.fmaximum_magl + libc.src.math.fmaximum_num + libc.src.math.fmaximum_numf + libc.src.math.fmaximum_numl + libc.src.math.fmaximumf + libc.src.math.fmaximuml + libc.src.math.fmaxl + libc.src.math.fmin + libc.src.math.fminf + libc.src.math.fminimum + libc.src.math.fminimum_mag + libc.src.math.fminimum_mag_num + libc.src.math.fminimum_mag_numf + libc.src.math.fminimum_mag_numl + libc.src.math.fminimum_magf + libc.src.math.fminimum_magl + libc.src.math.fminimum_num + libc.src.math.fminimum_numf + libc.src.math.fminimum_numl + libc.src.math.fminimumf + libc.src.math.fminimuml + libc.src.math.fminl + libc.src.math.fmod + libc.src.math.fmodf + libc.src.math.fmodl + libc.src.math.fmul + libc.src.math.fmull + libc.src.math.frexp + libc.src.math.frexpf + libc.src.math.frexpl + libc.src.math.fromfp + libc.src.math.fromfpf + libc.src.math.fromfpl + libc.src.math.fromfpx + libc.src.math.fromfpxf + libc.src.math.fromfpxl + libc.src.math.fsqrt + libc.src.math.fsqrtl + libc.src.math.fsub + libc.src.math.fsubl + libc.src.math.getpayload + libc.src.math.getpayloadf + libc.src.math.getpayloadl + libc.src.math.hypot + libc.src.math.hypotf + libc.src.math.ilogb + libc.src.math.ilogbf + libc.src.math.ilogbl + libc.src.math.iscanonical + libc.src.math.iscanonicalf + libc.src.math.iscanonicall + libc.src.math.isnan + libc.src.math.isnanf + libc.src.math.isnanl + libc.src.math.issignaling + libc.src.math.issignalingf + libc.src.math.issignalingl + libc.src.math.ldexp + libc.src.math.ldexpf + libc.src.math.ldexpl + libc.src.math.llogb + libc.src.math.llogbf + libc.src.math.llogbl + libc.src.math.llrint + libc.src.math.llrintf + libc.src.math.llrintl + libc.src.math.llround + libc.src.math.llroundf + libc.src.math.llroundl + libc.src.math.log + libc.src.math.log10 + libc.src.math.log10f + libc.src.math.log1p + libc.src.math.log1pf + libc.src.math.log2 + libc.src.math.log2f + libc.src.math.logb + libc.src.math.logbf + libc.src.math.logbl + libc.src.math.logf + libc.src.math.lrint + libc.src.math.lrintf + libc.src.math.lrintl + libc.src.math.lround + libc.src.math.lroundf + libc.src.math.lroundl + libc.src.math.modf + libc.src.math.modff + libc.src.math.modfl + libc.src.math.nan + libc.src.math.nanf + libc.src.math.nanl + libc.src.math.nearbyint + libc.src.math.nearbyintf + libc.src.math.nearbyintl + libc.src.math.nextafter + libc.src.math.nextafterf + libc.src.math.nextafterl + libc.src.math.nextdown + libc.src.math.nextdownf + libc.src.math.nextdownl + libc.src.math.nexttoward + libc.src.math.nexttowardf + libc.src.math.nexttowardl + libc.src.math.nextup + libc.src.math.nextupf + libc.src.math.nextupl + libc.src.math.pow + libc.src.math.powf + libc.src.math.remainder + libc.src.math.remainderf + libc.src.math.remainderl + libc.src.math.remquo + libc.src.math.remquof + libc.src.math.remquol + libc.src.math.rint + libc.src.math.rintf + libc.src.math.rintl + libc.src.math.round + libc.src.math.roundeven + libc.src.math.roundevenf + libc.src.math.roundevenl + libc.src.math.roundf + libc.src.math.roundl + libc.src.math.scalbln + libc.src.math.scalblnf + libc.src.math.scalblnl + libc.src.math.scalbn + libc.src.math.scalbnf + libc.src.math.scalbnl + libc.src.math.setpayload + libc.src.math.setpayloadf + libc.src.math.setpayloadl + libc.src.math.setpayloadsig + libc.src.math.setpayloadsigf + libc.src.math.setpayloadsigl + libc.src.math.sin + libc.src.math.sincos + libc.src.math.sincosf + libc.src.math.sinf + libc.src.math.sinhf + libc.src.math.sinpif + libc.src.math.sqrt + libc.src.math.sqrtf + libc.src.math.sqrtl + libc.src.math.tan + libc.src.math.tanf + libc.src.math.tanhf + libc.src.math.totalorder + libc.src.math.totalorderf + libc.src.math.totalorderl + libc.src.math.totalordermag + libc.src.math.totalordermagf + libc.src.math.totalordermagl + libc.src.math.trunc + libc.src.math.truncf + libc.src.math.truncl + libc.src.math.ufromfp + libc.src.math.ufromfpf + libc.src.math.ufromfpl + libc.src.math.ufromfpx + libc.src.math.ufromfpxf + libc.src.math.ufromfpxl +) + +if(LIBC_TYPES_HAS_CFLOAT16) + list(APPEND TARGET_LIBM_ENTRYPOINTS + # complex.h C23 _Complex _Float16 entrypoints + libc.src.complex.crealf16 + libc.src.complex.cimagf16 + libc.src.complex.conjf16 + libc.src.complex.cprojf16 + ) +endif() + +if(LIBC_TYPES_HAS_FLOAT16) + list(APPEND TARGET_LIBM_ENTRYPOINTS + # math.h C23 _Float16 entrypoints + libc.src.math.acosf16 + libc.src.math.acoshf16 + libc.src.math.asinf16 + libc.src.math.asinhf16 + libc.src.math.canonicalizef16 + libc.src.math.ceilf16 + libc.src.math.copysignf16 + libc.src.math.cosf16 + libc.src.math.coshf16 + libc.src.math.cospif16 + libc.src.math.exp10f16 + libc.src.math.exp10m1f16 + libc.src.math.exp2f16 + libc.src.math.exp2m1f16 + libc.src.math.expf16 + libc.src.math.expm1f16 + libc.src.math.f16add + libc.src.math.f16addf + libc.src.math.f16addl + libc.src.math.f16div + libc.src.math.f16divf + libc.src.math.f16divl + libc.src.math.f16fma + libc.src.math.f16fmaf + libc.src.math.f16fmal + libc.src.math.f16mul + libc.src.math.f16mulf + libc.src.math.f16mull + libc.src.math.f16sqrt + libc.src.math.f16sqrtf + libc.src.math.f16sqrtl + libc.src.math.f16sub + libc.src.math.f16subf + libc.src.math.f16subl + libc.src.math.fabsf16 + libc.src.math.fdimf16 + libc.src.math.floorf16 + libc.src.math.fmaf16 + libc.src.math.fmaxf16 + libc.src.math.fmaximum_mag_numf16 + libc.src.math.fmaximum_magf16 + libc.src.math.fmaximum_numf16 + libc.src.math.fmaximumf16 + libc.src.math.fminf16 + libc.src.math.fminimum_mag_numf16 + libc.src.math.fminimum_magf16 + libc.src.math.fminimum_numf16 + libc.src.math.fminimumf16 + libc.src.math.fmodf16 + libc.src.math.frexpf16 + libc.src.math.fromfpf16 + libc.src.math.fromfpxf16 + libc.src.math.getpayloadf16 + libc.src.math.hypotf16 + libc.src.math.ilogbf16 + libc.src.math.iscanonicalf16 + libc.src.math.issignalingf16 + libc.src.math.ldexpf16 + libc.src.math.llogbf16 + libc.src.math.llrintf16 + libc.src.math.llroundf16 + libc.src.math.log10f16 + libc.src.math.log2f16 + libc.src.math.logbf16 + libc.src.math.logf16 + libc.src.math.lrintf16 + libc.src.math.lroundf16 + libc.src.math.modff16 + libc.src.math.nanf16 + libc.src.math.nearbyintf16 + libc.src.math.nextafterf16 + libc.src.math.nextdownf16 + libc.src.math.nexttowardf16 + libc.src.math.nextupf16 + libc.src.math.remainderf16 + libc.src.math.remquof16 + libc.src.math.rintf16 + libc.src.math.roundevenf16 + libc.src.math.roundf16 + libc.src.math.scalblnf16 + libc.src.math.scalbnf16 + libc.src.math.setpayloadf16 + libc.src.math.setpayloadsigf16 + libc.src.math.sinf16 + libc.src.math.sinhf16 + libc.src.math.sinpif16 + libc.src.math.sqrtf16 + libc.src.math.tanf16 + libc.src.math.tanhf16 + libc.src.math.tanpif16 + libc.src.math.totalorderf16 + libc.src.math.totalordermagf16 + libc.src.math.truncf16 + libc.src.math.ufromfpf16 + libc.src.math.ufromfpxf16 + ) + + if(LIBC_TYPES_HAS_FLOAT128) + list(APPEND TARGET_LIBM_ENTRYPOINTS + # math.h C23 mixed _Float16 and _Float128 entrypoints + libc.src.math.f16addf128 + libc.src.math.f16divf128 + libc.src.math.f16fmaf128 + libc.src.math.f16mulf128 + libc.src.math.f16sqrtf128 + libc.src.math.f16subf128 + ) + endif() +endif() + +if(LIBC_TYPES_HAS_CFLOAT128) + list(APPEND TARGET_LIBM_ENTRYPOINTS + # complex.h C23 _Complex _Float128 entrypoints + libc.src.complex.crealf128 + libc.src.complex.cimagf128 + libc.src.complex.conjf128 + libc.src.complex.cprojf128 + ) +endif() + +if(LIBC_TYPES_HAS_FLOAT128) + list(APPEND TARGET_LIBM_ENTRYPOINTS + # math.h C23 _Float128 entrypoints + libc.src.math.atan2f128 + libc.src.math.canonicalizef128 + libc.src.math.ceilf128 + libc.src.math.copysignf128 + libc.src.math.daddf128 + libc.src.math.ddivf128 + libc.src.math.dfmaf128 + libc.src.math.dmulf128 + libc.src.math.dsqrtf128 + libc.src.math.dsubf128 + libc.src.math.fabsf128 + libc.src.math.faddf128 + libc.src.math.fdimf128 + libc.src.math.fdivf128 + libc.src.math.ffmaf128 + libc.src.math.floorf128 + libc.src.math.fmaxf128 + libc.src.math.fmaximum_mag_numf128 + libc.src.math.fmaximum_magf128 + libc.src.math.fmaximum_numf128 + libc.src.math.fmaximumf128 + libc.src.math.fminf128 + libc.src.math.fminimum_mag_numf128 + libc.src.math.fminimum_magf128 + libc.src.math.fminimum_numf128 + libc.src.math.fminimumf128 + libc.src.math.fmodf128 + libc.src.math.fmulf128 + libc.src.math.frexpf128 + libc.src.math.fromfpf128 + libc.src.math.fromfpxf128 + libc.src.math.fsqrtf128 + libc.src.math.fsubf128 + libc.src.math.getpayloadf128 + libc.src.math.ilogbf128 + libc.src.math.iscanonicalf128 + libc.src.math.issignalingf128 + libc.src.math.ldexpf128 + libc.src.math.llogbf128 + libc.src.math.llrintf128 + libc.src.math.llroundf128 + libc.src.math.logbf128 + libc.src.math.lrintf128 + libc.src.math.lroundf128 + libc.src.math.modff128 + libc.src.math.nanf128 + libc.src.math.nearbyintf128 + libc.src.math.nextafterf128 + libc.src.math.nextdownf128 + libc.src.math.nextupf128 + libc.src.math.remainderf128 + libc.src.math.remquof128 + libc.src.math.rintf128 + libc.src.math.roundevenf128 + libc.src.math.roundf128 + libc.src.math.scalblnf128 + libc.src.math.scalbnf128 + libc.src.math.setpayloadf128 + libc.src.math.setpayloadsigf128 + libc.src.math.sqrtf128 + libc.src.math.totalorderf128 + libc.src.math.totalordermagf128 + libc.src.math.truncf128 + libc.src.math.ufromfpf128 + libc.src.math.ufromfpxf128 + ) +endif() + +if(LIBC_COMPILER_HAS_FIXED_POINT) + list(APPEND TARGET_LIBM_ENTRYPOINTS + # stdfix.h _Fract and _Accum entrypoints + libc.src.stdfix.abshk + libc.src.stdfix.abshr + libc.src.stdfix.absk + libc.src.stdfix.abslk + libc.src.stdfix.abslr + libc.src.stdfix.absr + libc.src.stdfix.exphk + libc.src.stdfix.expk + libc.src.stdfix.roundhk + libc.src.stdfix.roundhr + libc.src.stdfix.roundk + libc.src.stdfix.roundlk + libc.src.stdfix.roundlr + libc.src.stdfix.roundr + libc.src.stdfix.rounduhk + libc.src.stdfix.rounduhr + libc.src.stdfix.rounduk + libc.src.stdfix.roundulk + libc.src.stdfix.roundulr + libc.src.stdfix.roundur + libc.src.stdfix.sqrtuhk + libc.src.stdfix.sqrtuhr + libc.src.stdfix.sqrtuk + libc.src.stdfix.sqrtur + # libc.src.stdfix.sqrtulk + libc.src.stdfix.sqrtulr + libc.src.stdfix.uhksqrtus + libc.src.stdfix.uksqrtui + libc.src.stdfix.hrbits + libc.src.stdfix.uhrbits + libc.src.stdfix.rbits + libc.src.stdfix.urbits + libc.src.stdfix.lrbits + libc.src.stdfix.ulrbits + libc.src.stdfix.hkbits + libc.src.stdfix.uhkbits + libc.src.stdfix.kbits + libc.src.stdfix.ukbits + libc.src.stdfix.lkbits + libc.src.stdfix.ulkbits + libc.src.stdfix.bitshr + libc.src.stdfix.bitsr + libc.src.stdfix.bitslr + libc.src.stdfix.bitshk + libc.src.stdfix.bitsk + libc.src.stdfix.bitslk + libc.src.stdfix.bitsuhr + libc.src.stdfix.bitsur + libc.src.stdfix.bitsulr + libc.src.stdfix.bitsuhk + libc.src.stdfix.bitsuk + libc.src.stdfix.bitsulk + libc.src.stdfix.countlshr + libc.src.stdfix.countlsr + libc.src.stdfix.countlslr + libc.src.stdfix.countlshk + libc.src.stdfix.countlsk + libc.src.stdfix.countlslk + libc.src.stdfix.countlsuhr + libc.src.stdfix.countlsur + libc.src.stdfix.countlsulr + libc.src.stdfix.countlsuhk + libc.src.stdfix.countlsuk + libc.src.stdfix.countlsulk + ) +endif() + +set(TARGET_LLVMLIBC_ENTRYPOINTS + ${TARGET_LIBC_ENTRYPOINTS} + ${TARGET_LIBM_ENTRYPOINTS} +) diff --git a/system/lib/llvm-libc/config/darwin/arm/headers.txt b/system/lib/llvm-libc/config/darwin/aarch64/headers.txt similarity index 100% rename from system/lib/llvm-libc/config/darwin/arm/headers.txt rename to system/lib/llvm-libc/config/darwin/aarch64/headers.txt diff --git a/system/lib/llvm-libc/config/darwin/arm/entrypoints.txt b/system/lib/llvm-libc/config/darwin/arm/entrypoints.txt deleted file mode 100644 index 7972d285f963a..0000000000000 --- a/system/lib/llvm-libc/config/darwin/arm/entrypoints.txt +++ /dev/null @@ -1,273 +0,0 @@ -set(TARGET_LIBC_ENTRYPOINTS - # ctype.h entrypoints - libc.src.ctype.isalnum - libc.src.ctype.isalpha - libc.src.ctype.isascii - libc.src.ctype.isblank - libc.src.ctype.iscntrl - libc.src.ctype.isdigit - libc.src.ctype.isgraph - libc.src.ctype.islower - libc.src.ctype.isprint - libc.src.ctype.ispunct - libc.src.ctype.isspace - libc.src.ctype.isupper - libc.src.ctype.isxdigit - libc.src.ctype.toascii - libc.src.ctype.tolower - libc.src.ctype.toupper - - # errno.h entrypoints - libc.src.errno.errno - - # string.h entrypoints - libc.src.string.memccpy - libc.src.string.memchr - libc.src.string.memcmp - libc.src.string.memcpy - libc.src.string.memmem - libc.src.string.memmove - libc.src.string.mempcpy - libc.src.string.memrchr - libc.src.string.memset - libc.src.string.stpcpy - libc.src.string.stpncpy - libc.src.string.strcasestr - libc.src.string.strcat - libc.src.string.strchr - libc.src.string.strchrnul - libc.src.string.strcmp - libc.src.string.strcpy - libc.src.string.strcspn - libc.src.string.strlcat - libc.src.string.strlcpy - libc.src.string.strlen - libc.src.string.strncat - libc.src.string.strncmp - libc.src.string.strncpy - libc.src.string.strnlen - libc.src.string.strpbrk - libc.src.string.strrchr - libc.src.string.strspn - libc.src.string.strstr - libc.src.string.strtok - libc.src.string.strtok_r - - # string.h entrypoints that depend on malloc - libc.src.string.strdup - libc.src.string.strndup - - # strings.h entrypoints - libc.src.strings.bcmp - libc.src.strings.bcopy - libc.src.strings.bzero - libc.src.strings.strcasecmp - libc.src.strings.strncasecmp - - # inttypes.h entrypoints - libc.src.inttypes.imaxabs - libc.src.inttypes.imaxdiv - libc.src.inttypes.strtoimax - libc.src.inttypes.strtoumax - - # stdlib.h entrypoints - libc.src.stdlib.abs - libc.src.stdlib.atoi - libc.src.stdlib.atof - libc.src.stdlib.atol - libc.src.stdlib.atoll - libc.src.stdlib.bsearch - libc.src.stdlib.div - libc.src.stdlib.labs - libc.src.stdlib.ldiv - libc.src.stdlib.llabs - libc.src.stdlib.lldiv - libc.src.stdlib.qsort - libc.src.stdlib.rand - libc.src.stdlib.srand - libc.src.stdlib.strtod - libc.src.stdlib.strtof - libc.src.stdlib.strtol - libc.src.stdlib.strtold - libc.src.stdlib.strtoll - libc.src.stdlib.strtoul - libc.src.stdlib.strtoull - - # stdlib.h external entrypoints - libc.src.stdlib.malloc - libc.src.stdlib.calloc - libc.src.stdlib.realloc - libc.src.stdlib.free -) - -set(TARGET_LIBM_ENTRYPOINTS - # fenv.h entrypoints - libc.src.fenv.feclearexcept - libc.src.fenv.fedisableexcept - libc.src.fenv.feenableexcept - libc.src.fenv.fegetenv - libc.src.fenv.fegetexcept - libc.src.fenv.fegetexceptflag - libc.src.fenv.fegetround - libc.src.fenv.feholdexcept - libc.src.fenv.fesetenv - libc.src.fenv.fesetexcept - libc.src.fenv.fesetexceptflag - libc.src.fenv.fesetround - libc.src.fenv.feraiseexcept - libc.src.fenv.fetestexcept - libc.src.fenv.fetestexceptflag - libc.src.fenv.feupdateenv - - # math.h entrypoints - libc.src.math.acosf - libc.src.math.acoshf - libc.src.math.asinf - libc.src.math.asinhf - libc.src.math.atan2 - libc.src.math.atan2f - libc.src.math.atanf - libc.src.math.atanhf - libc.src.math.cbrt - libc.src.math.cbrtf - libc.src.math.copysign - libc.src.math.copysignf - libc.src.math.copysignl - libc.src.math.ceil - libc.src.math.ceilf - libc.src.math.ceill - libc.src.math.coshf - libc.src.math.cos - libc.src.math.cosf - libc.src.math.cospif - libc.src.math.dfmal - libc.src.math.dsqrtl - libc.src.math.ddivl - libc.src.math.erff - libc.src.math.exp - libc.src.math.expf - libc.src.math.exp10 - libc.src.math.exp10f - libc.src.math.exp2 - libc.src.math.exp2f - libc.src.math.expm1 - libc.src.math.expm1f - libc.src.math.fabs - libc.src.math.fabsf - libc.src.math.fabsl - libc.src.math.fdim - libc.src.math.fdimf - libc.src.math.fdiml - libc.src.math.fdiv - libc.src.math.fdivl - libc.src.math.ffma - libc.src.math.ffmal - libc.src.math.floor - libc.src.math.floorf - libc.src.math.floorl - libc.src.math.fma - libc.src.math.fmaf - libc.src.math.fmax - libc.src.math.fmaxf - libc.src.math.fmaxl - libc.src.math.fmin - libc.src.math.fminf - libc.src.math.fminl - libc.src.math.fmod - libc.src.math.fmodf - libc.src.math.fmul - libc.src.math.frexp - libc.src.math.frexpf - libc.src.math.frexpl - libc.src.math.fsub - libc.src.math.fsubl - libc.src.math.hypot - libc.src.math.hypotf - libc.src.math.ilogb - libc.src.math.ilogbf - libc.src.math.ilogbl - libc.src.math.llrint - libc.src.math.llrintf - libc.src.math.llrintl - libc.src.math.llround - libc.src.math.llroundf - libc.src.math.llroundl - libc.src.math.lrint - libc.src.math.lrintf - libc.src.math.lrintl - libc.src.math.lround - libc.src.math.lroundf - libc.src.math.lroundl - libc.src.math.ldexp - libc.src.math.ldexpf - libc.src.math.ldexpl - libc.src.math.log10 - libc.src.math.log10f - libc.src.math.log1p - libc.src.math.log1pf - libc.src.math.log2 - libc.src.math.log2f - libc.src.math.log - libc.src.math.logf - libc.src.math.logb - libc.src.math.logbf - libc.src.math.logbl - libc.src.math.modf - libc.src.math.modff - libc.src.math.modfl - libc.src.math.nan - libc.src.math.nanf - libc.src.math.nanl - libc.src.math.nearbyint - libc.src.math.nearbyintf - libc.src.math.nearbyintl - libc.src.math.nextafter - libc.src.math.nextafterf - libc.src.math.nextafterl - libc.src.math.nexttoward - libc.src.math.nexttowardf - libc.src.math.nexttowardl - libc.src.math.pow - libc.src.math.powf - libc.src.math.remainderf - libc.src.math.remainder - libc.src.math.remainderl - libc.src.math.remquof - libc.src.math.remquo - libc.src.math.remquol - libc.src.math.rint - libc.src.math.rintf - libc.src.math.rintl - libc.src.math.round - libc.src.math.roundf - libc.src.math.roundl - libc.src.math.scalbln - libc.src.math.scalblnf - libc.src.math.scalblnl - libc.src.math.scalbn - libc.src.math.scalbnf - libc.src.math.scalbnl - libc.src.math.sincos - libc.src.math.sincosf - libc.src.math.sinhf - libc.src.math.sin - libc.src.math.sinf - libc.src.math.sinpif - libc.src.math.sqrt - libc.src.math.sqrtf - libc.src.math.sqrtl - libc.src.math.tan - libc.src.math.tanf - libc.src.math.tanhf - libc.src.math.totalordermag - libc.src.math.totalordermagf - libc.src.math.totalordermagl - libc.src.math.trunc - libc.src.math.truncf - libc.src.math.truncl -) - -set(TARGET_LLVMLIBC_ENTRYPOINTS - ${TARGET_LIBC_ENTRYPOINTS} - ${TARGET_LIBM_ENTRYPOINTS} -) diff --git a/system/lib/llvm-libc/config/darwin/x86_64/entrypoints.txt b/system/lib/llvm-libc/config/darwin/x86_64/entrypoints.txt index 19230cde47198..c55b6aa275690 100644 --- a/system/lib/llvm-libc/config/darwin/x86_64/entrypoints.txt +++ b/system/lib/llvm-libc/config/darwin/x86_64/entrypoints.txt @@ -78,6 +78,7 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.stdlib.ldiv libc.src.stdlib.llabs libc.src.stdlib.lldiv + libc.src.stdlib.memalignment libc.src.stdlib.qsort libc.src.stdlib.strtod libc.src.stdlib.strtof diff --git a/system/lib/llvm-libc/config/gpu/amdgpu/config.json b/system/lib/llvm-libc/config/gpu/amdgpu/config.json index d99f48ecbede1..30ae10e2cfd61 100644 --- a/system/lib/llvm-libc/config/gpu/amdgpu/config.json +++ b/system/lib/llvm-libc/config/gpu/amdgpu/config.json @@ -19,6 +19,9 @@ }, "LIBC_CONF_PRINTF_DISABLE_STRERROR": { "value": true + }, + "LIBC_CONF_PRINTF_RUNTIME_DISPATCH": { + "value": false } }, "scanf": { diff --git a/system/lib/llvm-libc/config/gpu/amdgpu/entrypoints.txt b/system/lib/llvm-libc/config/gpu/amdgpu/entrypoints.txt index 756b2cdc7496e..f41ebdc59f6b2 100644 --- a/system/lib/llvm-libc/config/gpu/amdgpu/entrypoints.txt +++ b/system/lib/llvm-libc/config/gpu/amdgpu/entrypoints.txt @@ -172,6 +172,7 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.stdlib.ldiv libc.src.stdlib.llabs libc.src.stdlib.lldiv + libc.src.stdlib.memalignment libc.src.stdlib.qsort libc.src.stdlib.qsort_r libc.src.stdlib.rand @@ -259,6 +260,9 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.time.clock_gettime libc.src.time.timespec_get libc.src.time.nanosleep + libc.src.time.strftime + libc.src.time.strftime_l + libc.src.time.mktime # wchar.h entrypoints libc.src.wchar.wcslen @@ -278,18 +282,14 @@ set(TARGET_LIBM_ENTRYPOINTS # math.h entrypoints libc.src.math.acos libc.src.math.acosf - libc.src.math.acosh - libc.src.math.acoshf libc.src.math.asin libc.src.math.asinf - libc.src.math.asinh libc.src.math.asinhf libc.src.math.atan libc.src.math.atan2 libc.src.math.atan2f libc.src.math.atan2l libc.src.math.atanf - libc.src.math.atanh libc.src.math.atanhf libc.src.math.canonicalize libc.src.math.canonicalizef @@ -304,14 +304,12 @@ set(TARGET_LIBM_ENTRYPOINTS libc.src.math.copysignl libc.src.math.cos libc.src.math.cosf - libc.src.math.cosh libc.src.math.coshf libc.src.math.cospif libc.src.math.ddivl libc.src.math.dfmal libc.src.math.dmull libc.src.math.dsqrtl - libc.src.math.erf libc.src.math.erff libc.src.math.exp libc.src.math.exp10 @@ -452,8 +450,6 @@ set(TARGET_LIBM_ENTRYPOINTS libc.src.math.nextupl libc.src.math.pow libc.src.math.powf - libc.src.math.powi - libc.src.math.powif libc.src.math.remainder libc.src.math.remainderf libc.src.math.remainderl @@ -485,7 +481,6 @@ set(TARGET_LIBM_ENTRYPOINTS libc.src.math.sincos libc.src.math.sincosf libc.src.math.sinf - libc.src.math.sinh libc.src.math.sinhf libc.src.math.sinpif libc.src.math.sqrt @@ -493,7 +488,6 @@ set(TARGET_LIBM_ENTRYPOINTS libc.src.math.sqrtl libc.src.math.tan libc.src.math.tanf - libc.src.math.tanh libc.src.math.tanhf libc.src.math.tgamma libc.src.math.tgammaf @@ -546,6 +540,7 @@ if(LIBC_TYPES_HAS_FLOAT16) libc.src.math.fabsf16 libc.src.math.fdimf16 libc.src.math.floorf16 + libc.src.math.fmaf16 libc.src.math.fmaxf16 libc.src.math.fmaximum_mag_numf16 libc.src.math.fmaximum_magf16 diff --git a/system/lib/llvm-libc/config/gpu/nvptx/config.json b/system/lib/llvm-libc/config/gpu/nvptx/config.json index d99f48ecbede1..30ae10e2cfd61 100644 --- a/system/lib/llvm-libc/config/gpu/nvptx/config.json +++ b/system/lib/llvm-libc/config/gpu/nvptx/config.json @@ -19,6 +19,9 @@ }, "LIBC_CONF_PRINTF_DISABLE_STRERROR": { "value": true + }, + "LIBC_CONF_PRINTF_RUNTIME_DISPATCH": { + "value": false } }, "scanf": { diff --git a/system/lib/llvm-libc/config/gpu/nvptx/entrypoints.txt b/system/lib/llvm-libc/config/gpu/nvptx/entrypoints.txt index 6b25dae158cc9..2b8c29fda3dc8 100644 --- a/system/lib/llvm-libc/config/gpu/nvptx/entrypoints.txt +++ b/system/lib/llvm-libc/config/gpu/nvptx/entrypoints.txt @@ -172,6 +172,7 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.stdlib.ldiv libc.src.stdlib.llabs libc.src.stdlib.lldiv + libc.src.stdlib.memalignment libc.src.stdlib.qsort libc.src.stdlib.qsort_r libc.src.stdlib.rand @@ -259,6 +260,9 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.time.clock_gettime libc.src.time.timespec_get libc.src.time.nanosleep + libc.src.time.strftime + libc.src.time.strftime_l + libc.src.time.mktime # wchar.h entrypoints libc.src.wchar.wcslen @@ -276,20 +280,16 @@ set(TARGET_LIBC_ENTRYPOINTS set(TARGET_LIBM_ENTRYPOINTS # math.h entrypoints - libc.src.math.acos libc.src.math.acosf - libc.src.math.acosh libc.src.math.acoshf libc.src.math.asin libc.src.math.asinf - libc.src.math.asinh libc.src.math.asinhf libc.src.math.atan libc.src.math.atan2 libc.src.math.atan2f libc.src.math.atan2l libc.src.math.atanf - libc.src.math.atanh libc.src.math.atanhf libc.src.math.canonicalize libc.src.math.canonicalizef @@ -304,14 +304,12 @@ set(TARGET_LIBM_ENTRYPOINTS libc.src.math.copysignl libc.src.math.cos libc.src.math.cosf - libc.src.math.cosh libc.src.math.coshf libc.src.math.cospif libc.src.math.ddivl libc.src.math.dfmal libc.src.math.dmull libc.src.math.dsqrtl - libc.src.math.erf libc.src.math.erff libc.src.math.exp libc.src.math.exp10 @@ -453,8 +451,6 @@ set(TARGET_LIBM_ENTRYPOINTS libc.src.math.nextupl libc.src.math.pow libc.src.math.powf - libc.src.math.powi - libc.src.math.powif libc.src.math.remainder libc.src.math.remainderf libc.src.math.remainderl @@ -486,7 +482,6 @@ set(TARGET_LIBM_ENTRYPOINTS libc.src.math.sincos libc.src.math.sincosf libc.src.math.sinf - libc.src.math.sinh libc.src.math.sinhf libc.src.math.sinpif libc.src.math.sqrt @@ -494,7 +489,6 @@ set(TARGET_LIBM_ENTRYPOINTS libc.src.math.sqrtl libc.src.math.tan libc.src.math.tanf - libc.src.math.tanh libc.src.math.tanhf libc.src.math.tgamma libc.src.math.tgammaf @@ -548,6 +542,7 @@ if(LIBC_TYPES_HAS_FLOAT16) libc.src.math.fabsf16 libc.src.math.fdimf16 libc.src.math.floorf16 + libc.src.math.fmaf16 libc.src.math.fmaxf16 libc.src.math.fmaximum_mag_numf16 libc.src.math.fmaximum_magf16 diff --git a/system/lib/llvm-libc/config/linux/aarch64/entrypoints.txt b/system/lib/llvm-libc/config/linux/aarch64/entrypoints.txt index 09eb51a3f8fc6..9e042cd4a8acb 100644 --- a/system/lib/llvm-libc/config/linux/aarch64/entrypoints.txt +++ b/system/lib/llvm-libc/config/linux/aarch64/entrypoints.txt @@ -32,6 +32,9 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.fcntl.open libc.src.fcntl.openat + # poll.h entrypoints + libc.src.poll.poll + # sched.h entrypoints libc.src.sched.sched_get_priority_max libc.src.sched.sched_get_priority_min @@ -90,6 +93,9 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.strings.bcmp libc.src.strings.bcopy libc.src.strings.bzero + libc.src.strings.ffs + libc.src.strings.ffsl + libc.src.strings.ffsll libc.src.strings.index libc.src.strings.rindex libc.src.strings.strcasecmp @@ -185,6 +191,7 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.stdlib.ldiv libc.src.stdlib.llabs libc.src.stdlib.lldiv + libc.src.stdlib.memalignment libc.src.stdlib.qsort libc.src.stdlib.qsort_r libc.src.stdlib.rand @@ -209,8 +216,6 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.stdlib.realloc # stdio.h entrypoints - libc.src.stdio.fdopen - libc.src.stdio.fileno libc.src.stdio.fprintf libc.src.stdio.fscanf libc.src.stdio.vfscanf @@ -240,6 +245,9 @@ set(TARGET_LIBC_ENTRYPOINTS # https://github.com/llvm/llvm-project/issues/80060 # libc.src.sys.epoll.epoll_pwait2 + # sys/ioctl.h entrypoints + libc.src.sys.ioctl.ioctl + # sys/mman.h entrypoints libc.src.sys.mman.madvise libc.src.sys.mman.mincore @@ -326,6 +334,7 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.unistd.geteuid libc.src.unistd.getpid libc.src.unistd.getppid + libc.src.unistd.getsid libc.src.unistd.gettid libc.src.unistd.getuid libc.src.unistd.isatty @@ -341,6 +350,7 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.unistd.readlink libc.src.unistd.readlinkat libc.src.unistd.rmdir + libc.src.unistd.setsid libc.src.unistd.symlink libc.src.unistd.symlinkat libc.src.unistd.sysconf @@ -355,6 +365,11 @@ set(TARGET_LIBC_ENTRYPOINTS # sys/uio.h entrypoints libc.src.sys.uio.writev + libc.src.sys.uio.readv + + # sys/time.h entrypoints + libc.src.sys.time.setitimer + libc.src.sys.time.getitimer ) if(LLVM_LIBC_INCLUDE_SCUDO) @@ -398,12 +413,15 @@ set(TARGET_LIBM_ENTRYPOINTS libc.src.fenv.feupdateenv # math.h entrypoints + libc.src.math.acos libc.src.math.acosf libc.src.math.acoshf + libc.src.math.asin libc.src.math.asinf libc.src.math.asinhf libc.src.math.atan2 libc.src.math.atan2f + libc.src.math.atan libc.src.math.atanf libc.src.math.atanhf libc.src.math.canonicalize @@ -637,10 +655,12 @@ endif() if(LIBC_TYPES_HAS_FLOAT16) list(APPEND TARGET_LIBM_ENTRYPOINTS # math.h C23 _Float16 entrypoints + # libc.src.math.acoshf16 libc.src.math.canonicalizef16 libc.src.math.ceilf16 libc.src.math.copysignf16 - libc.src.math.cospif16 + #TODO: Aarch64 bug https://github.com/llvm/llvm-project/issues/134917 + # libc.src.math.cospif16 # TODO: aarch64 bug # Please see https://github.com/llvm/llvm-project/pull/100632#issuecomment-2258772681 # libc.src.math.expf16 @@ -669,6 +689,7 @@ if(LIBC_TYPES_HAS_FLOAT16) libc.src.math.ffma libc.src.math.ffmal libc.src.math.floorf16 + libc.src.math.fmaf16 libc.src.math.fmaxf16 libc.src.math.fmaximum_mag_numf16 libc.src.math.fmaximum_magf16 @@ -713,7 +734,8 @@ if(LIBC_TYPES_HAS_FLOAT16) libc.src.math.scalbnf16 libc.src.math.setpayloadf16 libc.src.math.setpayloadsigf16 - libc.src.math.sinpif16 + #TODO: Aarch64 bug https://github.com/llvm/llvm-project/issues/134917 + # libc.src.math.sinpif16 libc.src.math.sqrtf16 libc.src.math.totalorderf16 libc.src.math.totalordermagf16 @@ -748,6 +770,7 @@ endif() if(LIBC_TYPES_HAS_FLOAT128) list(APPEND TARGET_LIBM_ENTRYPOINTS # math.h C23 _Float128 entrypoints + libc.src.math.atan2f128 libc.src.math.canonicalizef128 libc.src.math.ceilf128 libc.src.math.copysignf128 @@ -903,15 +926,25 @@ if(LLVM_LIBC_FULL_BUILD) # sched.h entrypoints libc.src.sched.__sched_getcpucount + libc.src.sched.__sched_setcpuzero + libc.src.sched.__sched_setcpuset + libc.src.sched.__sched_getcpuisset + + # strings.h entrypoints + libc.src.strings.strcasecmp_l + libc.src.strings.strncasecmp_l # setjmp.h entrypoints libc.src.setjmp.longjmp libc.src.setjmp.setjmp + libc.src.setjmp.siglongjmp + libc.src.setjmp.sigsetjmp # stdio.h entrypoints libc.src.stdio.clearerr libc.src.stdio.clearerr_unlocked libc.src.stdio.fclose + libc.src.stdio.fdopen libc.src.stdio.feof libc.src.stdio.feof_unlocked libc.src.stdio.ferror @@ -920,6 +953,7 @@ if(LLVM_LIBC_FULL_BUILD) libc.src.stdio.fgetc libc.src.stdio.fgetc_unlocked libc.src.stdio.fgets + libc.src.stdio.fileno libc.src.stdio.flockfile libc.src.stdio.fopen libc.src.stdio.fopencookie @@ -938,6 +972,7 @@ if(LLVM_LIBC_FULL_BUILD) libc.src.stdio.getc_unlocked libc.src.stdio.getchar libc.src.stdio.getchar_unlocked + libc.src.stdio.perror libc.src.stdio.putc libc.src.stdio.putchar libc.src.stdio.puts @@ -986,6 +1021,7 @@ if(LLVM_LIBC_FULL_BUILD) libc.src.search.hsearch_r libc.src.search.insque libc.src.search.lfind + libc.src.search.lsearch libc.src.search.remque # threads.h entrypoints diff --git a/system/lib/llvm-libc/config/linux/aarch64/headers.txt b/system/lib/llvm-libc/config/linux/aarch64/headers.txt index 05f15a0e4e5cb..01b0bf36498ce 100644 --- a/system/lib/llvm-libc/config/linux/aarch64/headers.txt +++ b/system/lib/llvm-libc/config/linux/aarch64/headers.txt @@ -1,39 +1,60 @@ set(TARGET_PUBLIC_HEADERS + libc.include.arpa_inet libc.include.assert libc.include.complex libc.include.ctype + libc.include.dirent libc.include.dlfcn libc.include.elf + libc.include.endian libc.include.errno + libc.include.fcntl libc.include.features libc.include.fenv libc.include.float - libc.include.stdint libc.include.inttypes libc.include.limits libc.include.link + libc.include.locale libc.include.malloc libc.include.math + libc.include.poll libc.include.pthread + libc.include.sched + libc.include.search + libc.include.setjmp libc.include.signal - libc.include.stdckdint + libc.include.spawn libc.include.stdbit + libc.include.stdckdint + libc.include.stdfix + libc.include.stdint libc.include.stdio libc.include.stdlib libc.include.string libc.include.strings - libc.include.search + libc.include.sys_auxv + libc.include.sys_epoll + libc.include.sys_ioctl libc.include.sys_mman + libc.include.sys_prctl + libc.include.sys_queue + libc.include.sys_random + libc.include.sys_resource + libc.include.sys_select libc.include.sys_socket + libc.include.sys_stat + libc.include.sys_statvfs libc.include.sys_syscall libc.include.sys_time + libc.include.sys_types + libc.include.sys_utsname + libc.include.sys_wait + libc.include.sysexits + libc.include.termios libc.include.threads libc.include.time + libc.include.uchar libc.include.unistd libc.include.wchar - libc.include.uchar - - libc.include.sys_ioctl - # Disabled due to epoll_wait syscalls not being available on this platform. - # libc.include.sys_epoll ) diff --git a/system/lib/llvm-libc/config/linux/arm/entrypoints.txt b/system/lib/llvm-libc/config/linux/arm/entrypoints.txt index 42ea803baac04..1161ae260be2e 100644 --- a/system/lib/llvm-libc/config/linux/arm/entrypoints.txt +++ b/system/lib/llvm-libc/config/linux/arm/entrypoints.txt @@ -20,6 +20,9 @@ set(TARGET_LIBC_ENTRYPOINTS # errno.h entrypoints libc.src.errno.errno + # poll.h entrypoints + libc.src.poll.poll + # string.h entrypoints libc.src.string.memccpy libc.src.string.memchr @@ -153,6 +156,7 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.stdlib.ldiv libc.src.stdlib.llabs libc.src.stdlib.lldiv + libc.src.stdlib.memalignment libc.src.stdlib.qsort libc.src.stdlib.qsort_r libc.src.stdlib.strtod @@ -168,6 +172,9 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.stdlib.free libc.src.stdlib.malloc + # sys/ioctl.h entrypoints + libc.src.sys.ioctl.ioctl + # sys/mman.h entrypoints libc.src.sys.mman.mmap libc.src.sys.mman.munmap @@ -181,6 +188,9 @@ set(TARGET_LIBC_ENTRYPOINTS # libc.src.sys.epoll.epoll_pwait # libc.src.sys.epoll.epoll_pwait2 + # sys/time.h entrypoints + libc.src.sys.time.setitimer + libc.src.sys.time.getitimer ) if(LLVM_LIBC_FULL_BUILD) @@ -235,12 +245,15 @@ set(TARGET_LIBM_ENTRYPOINTS libc.src.fenv.feupdateenv # math.h entrypoints + libc.src.math.acos libc.src.math.acosf libc.src.math.acoshf + libc.src.math.asin libc.src.math.asinf libc.src.math.asinhf libc.src.math.atan2 libc.src.math.atan2f + libc.src.math.atan libc.src.math.atanf libc.src.math.atanhf libc.src.math.cbrt diff --git a/system/lib/llvm-libc/config/linux/riscv/entrypoints.txt b/system/lib/llvm-libc/config/linux/riscv/entrypoints.txt index 14a05a2f3fbf2..db8f8a7cf0b74 100644 --- a/system/lib/llvm-libc/config/linux/riscv/entrypoints.txt +++ b/system/lib/llvm-libc/config/linux/riscv/entrypoints.txt @@ -32,6 +32,9 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.fcntl.open libc.src.fcntl.openat + # poll.h entrypoints + libc.src.poll.poll + # sched.h entrypoints libc.src.sched.sched_get_priority_max libc.src.sched.sched_get_priority_min @@ -87,12 +90,16 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.string.strxfrm # strings.h entrypoints - libc.src.strings.index - libc.src.strings.rindex libc.src.strings.bcmp libc.src.strings.bcopy libc.src.strings.bzero + libc.src.strings.ffs + libc.src.strings.ffsl + libc.src.strings.ffsll + libc.src.strings.index + libc.src.strings.rindex libc.src.strings.strcasecmp + libc.src.strings.strncasecmp # inttypes.h entrypoints libc.src.inttypes.imaxabs @@ -173,6 +180,7 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.stdbit.stdc_trailing_zeros_us # stdlib.h entrypoints + libc.src.stdlib.a64l libc.src.stdlib.abs libc.src.stdlib.atof libc.src.stdlib.atoi @@ -180,10 +188,12 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.stdlib.atoll libc.src.stdlib.bsearch libc.src.stdlib.div + libc.src.stdlib.l64a libc.src.stdlib.labs libc.src.stdlib.ldiv libc.src.stdlib.llabs libc.src.stdlib.lldiv + libc.src.stdlib.memalignment libc.src.stdlib.qsort libc.src.stdlib.qsort_r libc.src.stdlib.rand @@ -207,8 +217,6 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.stdlib.realloc # stdio.h entrypoints - libc.src.stdio.fdopen - libc.src.stdio.fileno libc.src.stdio.fprintf libc.src.stdio.fscanf libc.src.stdio.vfscanf @@ -238,6 +246,9 @@ set(TARGET_LIBC_ENTRYPOINTS # https://github.com/llvm/llvm-project/issues/80060 # libc.src.sys.epoll.epoll_pwait2 + # sys/ioctl.h entrypoints + libc.src.sys.ioctl.ioctl + # sys/mman.h entrypoints libc.src.sys.mman.madvise libc.src.sys.mman.mincore @@ -245,6 +256,7 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.sys.mman.mlock2 libc.src.sys.mman.mlockall libc.src.sys.mman.mmap + libc.src.sys.mman.mremap libc.src.sys.mman.mprotect libc.src.sys.mman.msync libc.src.sys.mman.munlock @@ -279,6 +291,9 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.sys.statvfs.fstatvfs libc.src.sys.statvfs.statvfs + # sys/utimes.h entrypoints + libc.src.sys.time.utimes + # sys/utsname.h entrypoints libc.src.sys.utsname.uname @@ -319,9 +334,11 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.unistd.fsync libc.src.unistd.ftruncate libc.src.unistd.getcwd + libc.src.unistd.getentropy libc.src.unistd.geteuid libc.src.unistd.getpid libc.src.unistd.getppid + libc.src.unistd.getsid libc.src.unistd.gettid libc.src.unistd.getuid libc.src.unistd.isatty @@ -337,6 +354,7 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.unistd.readlink libc.src.unistd.readlinkat libc.src.unistd.rmdir + libc.src.unistd.setsid libc.src.unistd.symlink libc.src.unistd.symlinkat libc.src.unistd.sysconf @@ -346,8 +364,17 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.unistd.write # wchar.h entrypoints + libc.src.wchar.btowc libc.src.wchar.wcslen libc.src.wchar.wctob + + # sys/uio.h entrypoints + libc.src.sys.uio.writev + libc.src.sys.uio.readv + + # sys/time.h entrypoints + libc.src.sys.time.setitimer + libc.src.sys.time.getitimer ) if(LLVM_LIBC_INCLUDE_SCUDO) @@ -391,12 +418,15 @@ set(TARGET_LIBM_ENTRYPOINTS libc.src.fenv.feupdateenv # math.h entrypoints + libc.src.math.acos libc.src.math.acosf libc.src.math.acoshf + libc.src.math.asin libc.src.math.asinf libc.src.math.asinhf libc.src.math.atan2 libc.src.math.atan2f + libc.src.math.atan libc.src.math.atanf libc.src.math.atanhf libc.src.math.canonicalize @@ -424,6 +454,7 @@ set(TARGET_LIBM_ENTRYPOINTS libc.src.math.exp libc.src.math.exp10 libc.src.math.exp10f + libc.src.math.exp10m1f libc.src.math.exp2 libc.src.math.exp2f libc.src.math.exp2m1f @@ -620,6 +651,130 @@ set(TARGET_LIBM_ENTRYPOINTS libc.src.math.ufromfpxf libc.src.math.ufromfpxl ) +if(LIBC_TYPES_HAS_CFLOAT16) + list(APPEND TARGET_LIBM_ENTRYPOINTS + # complex.h C23 _Complex _Float16 entrypoints + libc.src.complex.crealf16 + libc.src.complex.cimagf16 + libc.src.complex.conjf16 + libc.src.complex.cprojf16 + ) +endif() + +if(LIBC_TYPES_HAS_FLOAT16) + list(APPEND TARGET_LIBM_ENTRYPOINTS + # math.h C23 _Float16 entrypoints + libc.src.math.acosf16 + libc.src.math.acoshf16 + libc.src.math.acospif16 + libc.src.math.asinf16 + libc.src.math.asinhf16 + libc.src.math.atanhf16 + libc.src.math.canonicalizef16 + libc.src.math.ceilf16 + libc.src.math.copysignf16 + libc.src.math.cosf16 + libc.src.math.coshf16 + libc.src.math.cospif16 + libc.src.math.exp10f16 + libc.src.math.exp10m1f16 + libc.src.math.exp2f16 + libc.src.math.exp2m1f16 + libc.src.math.expf16 + libc.src.math.expm1f16 + libc.src.math.f16add + libc.src.math.f16addf + libc.src.math.f16addl + libc.src.math.f16div + libc.src.math.f16divf + libc.src.math.f16divl + libc.src.math.f16fma + libc.src.math.f16fmaf + libc.src.math.f16fmal + libc.src.math.f16mul + libc.src.math.f16mulf + libc.src.math.f16mull + libc.src.math.f16sqrt + libc.src.math.f16sqrtf + libc.src.math.f16sqrtl + libc.src.math.f16sub + libc.src.math.f16subf + libc.src.math.f16subl + libc.src.math.fabsf16 + libc.src.math.fdimf16 + libc.src.math.floorf16 + libc.src.math.fmaf16 + libc.src.math.fmaxf16 + libc.src.math.fmaximum_mag_numf16 + libc.src.math.fmaximum_magf16 + libc.src.math.fmaximum_numf16 + libc.src.math.fmaximumf16 + libc.src.math.fminf16 + libc.src.math.fminimum_mag_numf16 + libc.src.math.fminimum_magf16 + libc.src.math.fminimum_numf16 + libc.src.math.fminimumf16 + libc.src.math.fmodf16 + libc.src.math.frexpf16 + libc.src.math.fromfpf16 + libc.src.math.fromfpxf16 + libc.src.math.getpayloadf16 + libc.src.math.hypotf16 + libc.src.math.ilogbf16 + libc.src.math.iscanonicalf16 + libc.src.math.issignalingf16 + libc.src.math.ldexpf16 + libc.src.math.llogbf16 + libc.src.math.llrintf16 + libc.src.math.llroundf16 + libc.src.math.log10f16 + libc.src.math.log2f16 + libc.src.math.logbf16 + libc.src.math.logf16 + libc.src.math.lrintf16 + libc.src.math.lroundf16 + libc.src.math.modff16 + libc.src.math.nanf16 + libc.src.math.nearbyintf16 + libc.src.math.nextafterf16 + libc.src.math.nextdownf16 + libc.src.math.nexttowardf16 + libc.src.math.nextupf16 + libc.src.math.remainderf16 + libc.src.math.remquof16 + libc.src.math.rintf16 + libc.src.math.roundevenf16 + libc.src.math.roundf16 + libc.src.math.scalblnf16 + libc.src.math.scalbnf16 + libc.src.math.setpayloadf16 + libc.src.math.setpayloadsigf16 + libc.src.math.sinf16 + libc.src.math.sinhf16 + libc.src.math.sinpif16 + libc.src.math.sqrtf16 + libc.src.math.tanf16 + libc.src.math.tanhf16 + libc.src.math.tanpif16 + libc.src.math.totalorderf16 + libc.src.math.totalordermagf16 + libc.src.math.truncf16 + libc.src.math.ufromfpf16 + libc.src.math.ufromfpxf16 + ) + + if(LIBC_TYPES_HAS_FLOAT128) + list(APPEND TARGET_LIBM_ENTRYPOINTS + # math.h C23 mixed _Float16 and _Float128 entrypoints + libc.src.math.f16addf128 + libc.src.math.f16divf128 + libc.src.math.f16fmaf128 + libc.src.math.f16mulf128 + libc.src.math.f16sqrtf128 + libc.src.math.f16subf128 + ) + endif() +endif() if(LIBC_TYPES_HAS_CFLOAT128) list(APPEND TARGET_LIBM_ENTRYPOINTS @@ -634,6 +789,7 @@ endif() if(LIBC_TYPES_HAS_FLOAT128) list(APPEND TARGET_LIBM_ENTRYPOINTS # math.h C23 _Float128 entrypoints + libc.src.math.atan2f128 libc.src.math.canonicalizef128 libc.src.math.ceilf128 libc.src.math.copysignf128 @@ -728,6 +884,7 @@ if(LIBC_COMPILER_HAS_FIXED_POINT) libc.src.stdfix.sqrtuhr libc.src.stdfix.sqrtuk libc.src.stdfix.sqrtur + # libc.src.stdfix.sqrtulk libc.src.stdfix.sqrtulr libc.src.stdfix.uhksqrtus libc.src.stdfix.uksqrtui @@ -741,14 +898,78 @@ if(LIBC_COMPILER_HAS_FIXED_POINT) libc.src.stdfix.uhkbits libc.src.stdfix.kbits libc.src.stdfix.ukbits - # TODO: https://github.com/llvm/llvm-project/issues/115778 libc.src.stdfix.lkbits libc.src.stdfix.ulkbits + libc.src.stdfix.bitshr + libc.src.stdfix.bitsr + libc.src.stdfix.bitslr + libc.src.stdfix.bitshk + libc.src.stdfix.bitsk + libc.src.stdfix.bitslk + libc.src.stdfix.bitsuhr + libc.src.stdfix.bitsur + libc.src.stdfix.bitsulr + libc.src.stdfix.bitsuhk + libc.src.stdfix.bitsuk + libc.src.stdfix.bitsulk + libc.src.stdfix.countlshr + libc.src.stdfix.countlsr + libc.src.stdfix.countlslr + libc.src.stdfix.countlshk + libc.src.stdfix.countlsk + libc.src.stdfix.countlslk + libc.src.stdfix.countlsuhr + libc.src.stdfix.countlsur + libc.src.stdfix.countlsulr + libc.src.stdfix.countlsuhk + libc.src.stdfix.countlsuk + libc.src.stdfix.countlsulk + libc.src.stdfix.idivr + libc.src.stdfix.idivlr + libc.src.stdfix.idivk + libc.src.stdfix.idivlk + libc.src.stdfix.idivur + libc.src.stdfix.idivulr + libc.src.stdfix.idivuk + libc.src.stdfix.idivulk ) endif() if(LLVM_LIBC_FULL_BUILD) list(APPEND TARGET_LIBC_ENTRYPOINTS + # ctype.h entrypoints + libc.src.ctype.isalnum_l + libc.src.ctype.isalpha_l + libc.src.ctype.isblank_l + libc.src.ctype.iscntrl_l + libc.src.ctype.isdigit_l + libc.src.ctype.isgraph_l + libc.src.ctype.islower_l + libc.src.ctype.isprint_l + libc.src.ctype.ispunct_l + libc.src.ctype.isspace_l + libc.src.ctype.isupper_l + libc.src.ctype.isxdigit_l + libc.src.ctype.tolower_l + libc.src.ctype.toupper_l + + # stdlib.h entrypoints + libc.src.stdlib.strtod_l + libc.src.stdlib.strtof_l + libc.src.stdlib.strtol_l + libc.src.stdlib.strtold_l + libc.src.stdlib.strtoll_l + libc.src.stdlib.strtoul_l + libc.src.stdlib.strtoull_l + + # string.h entrypoints + libc.src.string.strcoll_l + libc.src.string.strxfrm_l + + # strings.h entrypoints + libc.src.strings.strcasecmp_l + libc.src.strings.strncasecmp_l + # assert.h entrypoints libc.src.assert.__assert_fail @@ -824,21 +1045,32 @@ if(LLVM_LIBC_FULL_BUILD) libc.src.pthread.pthread_rwlockattr_init libc.src.pthread.pthread_rwlockattr_setkind_np libc.src.pthread.pthread_rwlockattr_setpshared + libc.src.pthread.pthread_spin_destroy + libc.src.pthread.pthread_spin_init + libc.src.pthread.pthread_spin_lock + libc.src.pthread.pthread_spin_trylock + libc.src.pthread.pthread_spin_unlock libc.src.pthread.pthread_self libc.src.pthread.pthread_setname_np libc.src.pthread.pthread_setspecific # sched.h entrypoints libc.src.sched.__sched_getcpucount + libc.src.sched.__sched_setcpuzero + libc.src.sched.__sched_setcpuset + libc.src.sched.__sched_getcpuisset # setjmp.h entrypoints libc.src.setjmp.longjmp libc.src.setjmp.setjmp + libc.src.setjmp.siglongjmp + libc.src.setjmp.sigsetjmp # stdio.h entrypoints libc.src.stdio.clearerr libc.src.stdio.clearerr_unlocked libc.src.stdio.fclose + libc.src.stdio.fdopen libc.src.stdio.feof libc.src.stdio.feof_unlocked libc.src.stdio.ferror @@ -847,6 +1079,7 @@ if(LLVM_LIBC_FULL_BUILD) libc.src.stdio.fgetc libc.src.stdio.fgetc_unlocked libc.src.stdio.fgets + libc.src.stdio.fileno libc.src.stdio.flockfile libc.src.stdio.fopen libc.src.stdio.fopencookie @@ -865,6 +1098,7 @@ if(LLVM_LIBC_FULL_BUILD) libc.src.stdio.getc_unlocked libc.src.stdio.getchar libc.src.stdio.getchar_unlocked + libc.src.stdio.perror libc.src.stdio.putc libc.src.stdio.putchar libc.src.stdio.puts @@ -913,6 +1147,7 @@ if(LLVM_LIBC_FULL_BUILD) libc.src.search.hsearch_r libc.src.search.insque libc.src.search.lfind + libc.src.search.lsearch libc.src.search.remque # threads.h entrypoints @@ -950,9 +1185,20 @@ if(LLVM_LIBC_FULL_BUILD) libc.src.time.gmtime_r libc.src.time.mktime libc.src.time.nanosleep + libc.src.time.strftime + libc.src.time.strftime_l libc.src.time.time libc.src.time.timespec_get + # locale.h entrypoints + libc.src.locale.localeconv + libc.src.locale.duplocale + libc.src.locale.freelocale + libc.src.locale.localeconv + libc.src.locale.newlocale + libc.src.locale.setlocale + libc.src.locale.uselocale + # unistd.h entrypoints libc.src.unistd.__llvm_libc_syscall libc.src.unistd._exit @@ -972,6 +1218,13 @@ if(LLVM_LIBC_FULL_BUILD) # sys/socket.h entrypoints libc.src.sys.socket.bind libc.src.sys.socket.socket + libc.src.sys.socket.socketpair + libc.src.sys.socket.send + libc.src.sys.socket.sendto + libc.src.sys.socket.sendmsg + libc.src.sys.socket.recv + libc.src.sys.socket.recvfrom + libc.src.sys.socket.recvmsg ) endif() diff --git a/system/lib/llvm-libc/config/linux/riscv/headers.txt b/system/lib/llvm-libc/config/linux/riscv/headers.txt index b38659e0b8daf..01b0bf36498ce 100644 --- a/system/lib/llvm-libc/config/linux/riscv/headers.txt +++ b/system/lib/llvm-libc/config/linux/riscv/headers.txt @@ -1,41 +1,38 @@ set(TARGET_PUBLIC_HEADERS + libc.include.arpa_inet libc.include.assert libc.include.complex libc.include.ctype libc.include.dirent libc.include.dlfcn + libc.include.elf + libc.include.endian libc.include.errno libc.include.fcntl libc.include.features libc.include.fenv libc.include.float - libc.include.stdint libc.include.inttypes libc.include.limits + libc.include.link + libc.include.locale libc.include.malloc libc.include.math + libc.include.poll libc.include.pthread libc.include.sched + libc.include.search + libc.include.setjmp libc.include.signal libc.include.spawn - libc.include.setjmp - libc.include.stdckdint libc.include.stdbit + libc.include.stdckdint libc.include.stdfix + libc.include.stdint libc.include.stdio libc.include.stdlib libc.include.string libc.include.strings - libc.include.search - libc.include.termios - libc.include.threads - libc.include.time - libc.include.unistd - libc.include.wchar - libc.include.uchar - - libc.include.arpa_inet - libc.include.sys_auxv libc.include.sys_epoll libc.include.sys_ioctl @@ -53,4 +50,11 @@ set(TARGET_PUBLIC_HEADERS libc.include.sys_types libc.include.sys_utsname libc.include.sys_wait + libc.include.sysexits + libc.include.termios + libc.include.threads + libc.include.time + libc.include.uchar + libc.include.unistd + libc.include.wchar ) diff --git a/system/lib/llvm-libc/config/linux/x86_64/entrypoints.txt b/system/lib/llvm-libc/config/linux/x86_64/entrypoints.txt index 366e4d34294d1..aa2079faed409 100644 --- a/system/lib/llvm-libc/config/linux/x86_64/entrypoints.txt +++ b/system/lib/llvm-libc/config/linux/x86_64/entrypoints.txt @@ -32,6 +32,9 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.fcntl.open libc.src.fcntl.openat + # poll.h entrypoints + libc.src.poll.poll + # sched.h entrypoints libc.src.sched.sched_get_priority_max libc.src.sched.sched_get_priority_min @@ -90,6 +93,9 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.strings.bcmp libc.src.strings.bcopy libc.src.strings.bzero + libc.src.strings.ffs + libc.src.strings.ffsl + libc.src.strings.ffsll libc.src.strings.index libc.src.strings.rindex libc.src.strings.strcasecmp @@ -174,6 +180,7 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.stdbit.stdc_trailing_zeros_us # stdlib.h entrypoints + libc.src.stdlib.a64l libc.src.stdlib.abs libc.src.stdlib.atof libc.src.stdlib.atoi @@ -181,10 +188,12 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.stdlib.atoll libc.src.stdlib.bsearch libc.src.stdlib.div + libc.src.stdlib.l64a libc.src.stdlib.labs libc.src.stdlib.ldiv libc.src.stdlib.llabs libc.src.stdlib.lldiv + libc.src.stdlib.memalignment libc.src.stdlib.qsort libc.src.stdlib.qsort_r libc.src.stdlib.rand @@ -208,8 +217,6 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.stdlib.realloc # stdio.h entrypoints - libc.src.stdio.fdopen - libc.src.stdio.fileno libc.src.stdio.fprintf libc.src.stdio.fscanf libc.src.stdio.vfscanf @@ -239,6 +246,9 @@ set(TARGET_LIBC_ENTRYPOINTS # https://github.com/llvm/llvm-project/issues/80060 # libc.src.sys.epoll.epoll_pwait2 + # sys/ioctl.h entrypoints + libc.src.sys.ioctl.ioctl + # sys/mman.h entrypoints libc.src.sys.mman.madvise libc.src.sys.mman.mincore @@ -281,6 +291,9 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.sys.statvfs.fstatvfs libc.src.sys.statvfs.statvfs + # sys/utimes.h entrypoints + libc.src.sys.time.utimes + # sys/utsname.h entrypoints libc.src.sys.utsname.uname @@ -325,6 +338,7 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.unistd.geteuid libc.src.unistd.getpid libc.src.unistd.getppid + libc.src.unistd.getsid libc.src.unistd.gettid libc.src.unistd.getuid libc.src.unistd.isatty @@ -340,6 +354,7 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.unistd.readlink libc.src.unistd.readlinkat libc.src.unistd.rmdir + libc.src.unistd.setsid libc.src.unistd.symlink libc.src.unistd.symlinkat libc.src.unistd.sysconf @@ -352,9 +367,31 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.wchar.btowc libc.src.wchar.wcslen libc.src.wchar.wctob + libc.src.wchar.wmemmove + libc.src.wchar.wmemset + libc.src.wchar.wcschr + libc.src.wchar.wcsncmp + libc.src.wchar.wcscmp + libc.src.wchar.wcspbrk + libc.src.wchar.wcsrchr + libc.src.wchar.wcsspn + libc.src.wchar.wmemcmp + libc.src.wchar.wmempcpy + libc.src.wchar.wmemcpy + libc.src.wchar.wcsncpy + libc.src.wchar.wcscat + libc.src.wchar.wcsstr + libc.src.wchar.wcsncat + libc.src.wchar.wcscpy + libc.src.wchar.wmemchr # sys/uio.h entrypoints libc.src.sys.uio.writev + libc.src.sys.uio.readv + + # sys/time.h entrypoints + libc.src.sys.time.setitimer + libc.src.sys.time.getitimer ) if(LLVM_LIBC_INCLUDE_SCUDO) @@ -398,12 +435,15 @@ set(TARGET_LIBM_ENTRYPOINTS libc.src.fenv.feupdateenv # math.h entrypoints + libc.src.math.acos libc.src.math.acosf libc.src.math.acoshf + libc.src.math.asin libc.src.math.asinf libc.src.math.asinhf libc.src.math.atan2 libc.src.math.atan2f + libc.src.math.atan libc.src.math.atanf libc.src.math.atanhf libc.src.math.canonicalize @@ -642,6 +682,13 @@ endif() if(LIBC_TYPES_HAS_FLOAT16) list(APPEND TARGET_LIBM_ENTRYPOINTS # math.h C23 _Float16 entrypoints + libc.src.math.acosf16 + libc.src.math.acoshf16 + libc.src.math.acospif16 + libc.src.math.asinf16 + libc.src.math.asinhf16 + libc.src.math.atanf16 + libc.src.math.atanhf16 libc.src.math.canonicalizef16 libc.src.math.ceilf16 libc.src.math.copysignf16 @@ -675,6 +722,7 @@ if(LIBC_TYPES_HAS_FLOAT16) libc.src.math.fabsf16 libc.src.math.fdimf16 libc.src.math.floorf16 + libc.src.math.fmaf16 libc.src.math.fmaxf16 libc.src.math.fmaximum_mag_numf16 libc.src.math.fmaximum_magf16 @@ -690,6 +738,7 @@ if(LIBC_TYPES_HAS_FLOAT16) libc.src.math.fromfpf16 libc.src.math.fromfpxf16 libc.src.math.getpayloadf16 + libc.src.math.hypotf16 libc.src.math.ilogbf16 libc.src.math.iscanonicalf16 libc.src.math.issignalingf16 @@ -759,6 +808,7 @@ endif() if(LIBC_TYPES_HAS_FLOAT128) list(APPEND TARGET_LIBM_ENTRYPOINTS # math.h C23 _Float128 entrypoints + libc.src.math.atan2f128 libc.src.math.canonicalizef128 libc.src.math.ceilf128 libc.src.math.copysignf128 @@ -869,6 +919,38 @@ if(LIBC_COMPILER_HAS_FIXED_POINT) libc.src.stdfix.ukbits libc.src.stdfix.lkbits libc.src.stdfix.ulkbits + libc.src.stdfix.bitshr + libc.src.stdfix.bitsr + libc.src.stdfix.bitslr + libc.src.stdfix.bitshk + libc.src.stdfix.bitsk + libc.src.stdfix.bitslk + libc.src.stdfix.bitsuhr + libc.src.stdfix.bitsur + libc.src.stdfix.bitsulr + libc.src.stdfix.bitsuhk + libc.src.stdfix.bitsuk + libc.src.stdfix.bitsulk + libc.src.stdfix.countlshr + libc.src.stdfix.countlsr + libc.src.stdfix.countlslr + libc.src.stdfix.countlshk + libc.src.stdfix.countlsk + libc.src.stdfix.countlslk + libc.src.stdfix.countlsuhr + libc.src.stdfix.countlsur + libc.src.stdfix.countlsulr + libc.src.stdfix.countlsuhk + libc.src.stdfix.countlsuk + libc.src.stdfix.countlsulk + libc.src.stdfix.idivr + libc.src.stdfix.idivlr + libc.src.stdfix.idivk + libc.src.stdfix.idivlk + libc.src.stdfix.idivur + libc.src.stdfix.idivulr + libc.src.stdfix.idivuk + libc.src.stdfix.idivulk ) endif() @@ -903,6 +985,10 @@ if(LLVM_LIBC_FULL_BUILD) libc.src.string.strcoll_l libc.src.string.strxfrm_l + # strings.h entrypoints + libc.src.strings.strcasecmp_l + libc.src.strings.strncasecmp_l + # assert.h entrypoints libc.src.assert.__assert_fail @@ -989,15 +1075,21 @@ if(LLVM_LIBC_FULL_BUILD) # sched.h entrypoints libc.src.sched.__sched_getcpucount + libc.src.sched.__sched_setcpuzero + libc.src.sched.__sched_setcpuset + libc.src.sched.__sched_getcpuisset # setjmp.h entrypoints libc.src.setjmp.longjmp libc.src.setjmp.setjmp + libc.src.setjmp.siglongjmp + libc.src.setjmp.sigsetjmp # stdio.h entrypoints libc.src.stdio.clearerr libc.src.stdio.clearerr_unlocked libc.src.stdio.fclose + libc.src.stdio.fdopen libc.src.stdio.feof libc.src.stdio.feof_unlocked libc.src.stdio.ferror @@ -1006,6 +1098,7 @@ if(LLVM_LIBC_FULL_BUILD) libc.src.stdio.fgetc libc.src.stdio.fgetc_unlocked libc.src.stdio.fgets + libc.src.stdio.fileno libc.src.stdio.flockfile libc.src.stdio.fopen libc.src.stdio.fopencookie @@ -1024,6 +1117,7 @@ if(LLVM_LIBC_FULL_BUILD) libc.src.stdio.getc_unlocked libc.src.stdio.getchar libc.src.stdio.getchar_unlocked + libc.src.stdio.perror libc.src.stdio.putc libc.src.stdio.putchar libc.src.stdio.puts @@ -1072,6 +1166,7 @@ if(LLVM_LIBC_FULL_BUILD) libc.src.search.hsearch_r libc.src.search.insque libc.src.search.lfind + libc.src.search.lsearch libc.src.search.remque # threads.h entrypoints @@ -1109,6 +1204,8 @@ if(LLVM_LIBC_FULL_BUILD) libc.src.time.gmtime_r libc.src.time.mktime libc.src.time.nanosleep + libc.src.time.strftime + libc.src.time.strftime_l libc.src.time.time libc.src.time.timespec_get diff --git a/system/lib/llvm-libc/config/linux/x86_64/headers.txt b/system/lib/llvm-libc/config/linux/x86_64/headers.txt index 8750100302ea7..01b0bf36498ce 100644 --- a/system/lib/llvm-libc/config/linux/x86_64/headers.txt +++ b/system/lib/llvm-libc/config/linux/x86_64/headers.txt @@ -1,44 +1,38 @@ set(TARGET_PUBLIC_HEADERS + libc.include.arpa_inet libc.include.assert libc.include.complex libc.include.ctype libc.include.dirent libc.include.dlfcn libc.include.elf + libc.include.endian libc.include.errno libc.include.fcntl libc.include.features libc.include.fenv libc.include.float - libc.include.stdint libc.include.inttypes libc.include.limits libc.include.link + libc.include.locale libc.include.malloc libc.include.math + libc.include.poll libc.include.pthread libc.include.sched + libc.include.search + libc.include.setjmp libc.include.signal libc.include.spawn - libc.include.setjmp - libc.include.stdckdint libc.include.stdbit + libc.include.stdckdint libc.include.stdfix + libc.include.stdint libc.include.stdio libc.include.stdlib libc.include.string libc.include.strings - libc.include.search - libc.include.termios - libc.include.threads - libc.include.time - libc.include.unistd - libc.include.wchar - libc.include.uchar - libc.include.locale - - libc.include.arpa_inet - libc.include.sys_auxv libc.include.sys_epoll libc.include.sys_ioctl @@ -56,4 +50,11 @@ set(TARGET_PUBLIC_HEADERS libc.include.sys_types libc.include.sys_utsname libc.include.sys_wait + libc.include.sysexits + libc.include.termios + libc.include.threads + libc.include.time + libc.include.uchar + libc.include.unistd + libc.include.wchar ) diff --git a/system/lib/llvm-libc/config/uefi/app.h b/system/lib/llvm-libc/config/uefi/app.h new file mode 100644 index 0000000000000..0374a47ba3402 --- /dev/null +++ b/system/lib/llvm-libc/config/uefi/app.h @@ -0,0 +1,34 @@ +//===-- Classes to capture properites of UEFI applications ------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_CONFIG_UEFI_APP_H +#define LLVM_LIBC_CONFIG_UEFI_APP_H + +#include "include/llvm-libc-types/EFI_HANDLE.h" +#include "include/llvm-libc-types/EFI_SYSTEM_TABLE.h" +#include "src/__support/macros/config.h" +#include "src/__support/macros/properties/architectures.h" + +#include + +namespace LIBC_NAMESPACE_DECL { + +// Data structure which captures properties of a UEFI application. +struct AppProperties { + // UEFI system table + EFI_SYSTEM_TABLE *system_table; + + // UEFI image handle + EFI_HANDLE image_handle; +}; + +[[gnu::weak]] extern AppProperties app; + +} // namespace LIBC_NAMESPACE_DECL + +#endif // LLVM_LIBC_CONFIG_UEFI_APP_H diff --git a/system/lib/llvm-libc/config/uefi/config.json b/system/lib/llvm-libc/config/uefi/config.json new file mode 100644 index 0000000000000..51aa8fecf8b30 --- /dev/null +++ b/system/lib/llvm-libc/config/uefi/config.json @@ -0,0 +1,7 @@ +{ + "errno": { + "LIBC_CONF_ERRNO_MODE": { + "value": "LIBC_ERRNO_MODE_SHARED" + } + } +} diff --git a/system/lib/llvm-libc/config/uefi/entrypoints.txt b/system/lib/llvm-libc/config/uefi/entrypoints.txt new file mode 100644 index 0000000000000..2e11c534a4f3b --- /dev/null +++ b/system/lib/llvm-libc/config/uefi/entrypoints.txt @@ -0,0 +1,11 @@ +set(TARGET_LIBC_ENTRYPOINTS + # errno.h entrypoints + libc.src.errno.errno +) + +set(TARGET_LIBM_ENTRYPOINTS) + +set(TARGET_LLVMLIBC_ENTRYPOINTS + ${TARGET_LIBC_ENTRYPOINTS} + ${TARGET_LIBM_ENTRYPOINTS} +) diff --git a/system/lib/llvm-libc/config/uefi/headers.txt b/system/lib/llvm-libc/config/uefi/headers.txt new file mode 100644 index 0000000000000..a8e7b5bc5e37b --- /dev/null +++ b/system/lib/llvm-libc/config/uefi/headers.txt @@ -0,0 +1,4 @@ +set(TARGET_PUBLIC_HEADERS + libc.include.errno + libc.include.uefi +) diff --git a/system/lib/llvm-libc/config/windows/entrypoints.txt b/system/lib/llvm-libc/config/windows/entrypoints.txt index aad320995d339..09021a08cf731 100644 --- a/system/lib/llvm-libc/config/windows/entrypoints.txt +++ b/system/lib/llvm-libc/config/windows/entrypoints.txt @@ -79,6 +79,7 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.stdlib.ldiv libc.src.stdlib.llabs libc.src.stdlib.lldiv + libc.src.stdlib.memalignment libc.src.stdlib.qsort libc.src.stdlib.strtod libc.src.stdlib.strtof @@ -126,12 +127,15 @@ set(TARGET_LIBM_ENTRYPOINTS libc.src.fenv.feupdateenv # math.h entrypoints + libc.src.math.acos libc.src.math.acosf libc.src.math.acoshf + libc.src.math.asin libc.src.math.asinf libc.src.math.asinhf libc.src.math.atan2 libc.src.math.atan2f + libc.src.math.atan libc.src.math.atanf libc.src.math.atanhf libc.src.math.cbrt diff --git a/system/lib/llvm-libc/hdr/locale_macros.h b/system/lib/llvm-libc/hdr/locale_macros.h new file mode 100644 index 0000000000000..7d94f6ddabe41 --- /dev/null +++ b/system/lib/llvm-libc/hdr/locale_macros.h @@ -0,0 +1,22 @@ +//===-- Definition of macros from locale.h --------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_HDR_LOCALE_MACROS_H +#define LLVM_LIBC_HDR_LOCALE_MACROS_H + +#ifdef LIBC_FULL_BUILD + +#include "include/llvm-libc-macros/locale-macros.h" + +#else // Overlay mode + +#error "macros not available in overlay mode" + +#endif // LLVM_LIBC_FULL_BUILD + +#endif // LLVM_LIBC_HDR_LOCALE_MACROS_H diff --git a/system/lib/llvm-libc/hdr/offsetof_macros.h b/system/lib/llvm-libc/hdr/offsetof_macros.h new file mode 100644 index 0000000000000..42e853ffa92e5 --- /dev/null +++ b/system/lib/llvm-libc/hdr/offsetof_macros.h @@ -0,0 +1,23 @@ +//===-- Definition of macros for offsetof ---------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_HDR_OFFSETOF_MACROS_H +#define LLVM_LIBC_HDR_OFFSETOF_MACROS_H + +#ifdef LIBC_FULL_BUILD + +#include "include/llvm-libc-macros/offsetof-macro.h" + +#else // Overlay mode + +#define __need_offsetof +#include + +#endif // LLVM_LIBC_FULL_BUILD + +#endif // LLVM_LIBC_HDR_OFFSETOF_MACROS_H diff --git a/system/lib/llvm-libc/hdr/sched_macros.h b/system/lib/llvm-libc/hdr/sched_macros.h new file mode 100644 index 0000000000000..cfeaa99796786 --- /dev/null +++ b/system/lib/llvm-libc/hdr/sched_macros.h @@ -0,0 +1,22 @@ +//===-- Definition of macros from sched.h ---------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_HDR_SCHED_MACROS_H +#define LLVM_LIBC_HDR_SCHED_MACROS_H + +#ifdef LIBC_FULL_BUILD + +#include "include/llvm-libc-macros/sched-macros.h" + +#else // Overlay mode + +#include + +#endif // LLVM_LIBC_FULL_BUILD + +#endif // LLVM_LIBC_HDR_SCHED_MACROS_H diff --git a/system/lib/llvm-libc/hdr/stdlib_overlay.h b/system/lib/llvm-libc/hdr/stdlib_overlay.h index f095cafe5e0bc..53c32ec0ae332 100644 --- a/system/lib/llvm-libc/hdr/stdlib_overlay.h +++ b/system/lib/llvm-libc/hdr/stdlib_overlay.h @@ -19,6 +19,11 @@ // functions, causing external alias errors. They are guarded by // `__USE_FORTIFY_LEVEL`, which will be temporarily disabled. +#ifdef _FORTIFY_SOURCE +#define LIBC_OLD_FORTIFY_SOURCE _FORTIFY_SOURCE +#undef _FORTIFY_SOURCE +#endif + #ifdef __USE_FORTIFY_LEVEL #define LIBC_OLD_USE_FORTIFY_LEVEL __USE_FORTIFY_LEVEL #undef __USE_FORTIFY_LEVEL @@ -27,6 +32,11 @@ #include +#ifdef LIBC_OLD_FORTIFY_SOURCE +#define _FORTIFY_SOURCE LIBC_OLD_FORTIFY_SOURCE +#undef LIBC_OLD_FORTIFY_SOURCE +#endif + #ifdef LIBC_OLD_USE_FORTIFY_LEVEL #undef __USE_FORTIFY_LEVEL #define __USE_FORTIFY_LEVEL LIBC_OLD_USE_FORTIFY_LEVEL diff --git a/system/lib/llvm-libc/hdr/sys_ioctl_macros.h b/system/lib/llvm-libc/hdr/sys_ioctl_macros.h new file mode 100644 index 0000000000000..935d436273465 --- /dev/null +++ b/system/lib/llvm-libc/hdr/sys_ioctl_macros.h @@ -0,0 +1,22 @@ +//===-- Definition of macros from sys/ioctl.h -----------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_HDR_SYS_IOCTL_MACROS_H +#define LLVM_LIBC_HDR_SYS_IOCTL_MACROS_H + +#ifdef LIBC_FULL_BUILD + +#include "include/llvm-libc-macros/sys-ioctl-macros.h" + +#else // Overlay mode + +#include + +#endif // LLVM_LIBC_FULL_BUILD + +#endif // LLVM_LIBC_HDR_SYS_IOCTL_MACROS_H diff --git a/system/lib/llvm-libc/hdr/types/ACTION.h b/system/lib/llvm-libc/hdr/types/ACTION.h new file mode 100644 index 0000000000000..0b63521dff64d --- /dev/null +++ b/system/lib/llvm-libc/hdr/types/ACTION.h @@ -0,0 +1,22 @@ +//===-- Proxy header for ACTION -------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_HDR_TYPES_ACTION_H +#define LLVM_LIBC_HDR_TYPES_ACTION_H + +#ifdef LIBC_FULL_BUILD + +#include "include/llvm-libc-types/ACTION.h" + +#else // Overlay mode + +#include + +#endif // LLVM_LIBC_FULL_BUILD + +#endif // LLVM_LIBC_HDR_TYPES_ACTION_H diff --git a/system/lib/llvm-libc/hdr/types/ENTRY.h b/system/lib/llvm-libc/hdr/types/ENTRY.h new file mode 100644 index 0000000000000..5f4aee4b30fe2 --- /dev/null +++ b/system/lib/llvm-libc/hdr/types/ENTRY.h @@ -0,0 +1,22 @@ +//===-- Proxy header for ENTRY --------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_HDR_TYPES_ENTRY_H +#define LLVM_LIBC_HDR_TYPES_ENTRY_H + +#ifdef LIBC_FULL_BUILD + +#include "include/llvm-libc-types/ENTRY.h" + +#else // Overlay mode + +#include + +#endif // LLVM_LIBC_FULL_BUILD + +#endif // LLVM_LIBC_HDR_TYPES_ENTRY_H diff --git a/system/lib/llvm-libc/hdr/types/sighandler_t.h b/system/lib/llvm-libc/hdr/types/char32_t.h similarity index 55% rename from system/lib/llvm-libc/hdr/types/sighandler_t.h rename to system/lib/llvm-libc/hdr/types/char32_t.h index bc40dd8b4c8f4..94fe5747d3415 100644 --- a/system/lib/llvm-libc/hdr/types/sighandler_t.h +++ b/system/lib/llvm-libc/hdr/types/char32_t.h @@ -1,4 +1,4 @@ -//===-- Definition of macros from __sighandler_t.h ------------------------===// +//===-- Definition of char32_t.h ------------------------------------------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -6,19 +6,17 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_LIBC_HDR_TYPES_SIGHANDLER_T_H -#define LLVM_LIBC_HDR_TYPES_SIGHANDLER_T_H +#ifndef LLVM_LIBC_HDR_TYPES_CHAR32_T_H +#define LLVM_LIBC_HDR_TYPES_CHAR32_T_H #ifdef LIBC_FULL_BUILD -#include "include/llvm-libc-types/__sighandler_t.h" - -using sighandler_t = __sighandler_t; +#include "include/llvm-libc-types/char32_t.h" #else // overlay mode -#include +#include "hdr/uchar_overlay.h" #endif // LLVM_LIBC_FULL_BUILD -#endif // LLVM_LIBC_HDR_TYPES_SIGHANDLER_T_H +#endif // LLVM_LIBC_HDR_TYPES_CHAR32_T_H diff --git a/system/lib/llvm-libc/include/llvm-libc-types/__sighandler_t.h b/system/lib/llvm-libc/hdr/types/char8_t.h similarity index 57% rename from system/lib/llvm-libc/include/llvm-libc-types/__sighandler_t.h rename to system/lib/llvm-libc/hdr/types/char8_t.h index 9c1ac997fc4ee..4d71e3dd89098 100644 --- a/system/lib/llvm-libc/include/llvm-libc-types/__sighandler_t.h +++ b/system/lib/llvm-libc/hdr/types/char8_t.h @@ -1,4 +1,4 @@ -//===-- Definition of struct __sighandler_t -------------------------------===// +//===-- Definition of char8_t.h -------------------------------------------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -6,9 +6,9 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_LIBC_TYPES___SIGHANDLER_T_H -#define LLVM_LIBC_TYPES___SIGHANDLER_T_H +#ifndef LLVM_LIBC_HDR_TYPES_CHAR8_T_H +#define LLVM_LIBC_HDR_TYPES_CHAR8_T_H -typedef void (*__sighandler_t)(int); +#include "include/llvm-libc-types/char8_t.h" -#endif // LLVM_LIBC_TYPES___SIGHANDLER_T_H +#endif // LLVM_LIBC_HDR_TYPES_CHAR8_T_H diff --git a/system/lib/llvm-libc/hdr/types/cpu_set_t.h b/system/lib/llvm-libc/hdr/types/cpu_set_t.h new file mode 100644 index 0000000000000..26aed7592fa2c --- /dev/null +++ b/system/lib/llvm-libc/hdr/types/cpu_set_t.h @@ -0,0 +1,22 @@ +//===-- Proxy for cpu_set_t -----------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_HDR_TYPES_CPU_SET_T_H +#define LLVM_LIBC_HDR_TYPES_CPU_SET_T_H + +#ifdef LIBC_FULL_BUILD + +#include "include/llvm-libc-types/cpu_set_t.h" + +#else // Overlay mode + +#include + +#endif // LLVM_LIBC_FULL_BUILD + +#endif // LLVM_LIBC_HDR_TYPES_CPU_SET_T_H diff --git a/system/lib/llvm-libc/hdr/types/nfds_t.h b/system/lib/llvm-libc/hdr/types/nfds_t.h new file mode 100644 index 0000000000000..9143564c2333e --- /dev/null +++ b/system/lib/llvm-libc/hdr/types/nfds_t.h @@ -0,0 +1,23 @@ +//===-- Definition of nfds_t ----------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_HDR_TYPES_NFDS_T_H +#define LLVM_LIBC_HDR_TYPES_NFDS_T_H + +#ifdef LIBC_FULL_BUILD + +#include "include/llvm-libc-types/nfds_t.h" + +#else // overlay mode + +#include + +#endif // LLVM_LIBC_FULL_BUILD + +#endif // LLVM_LIBC_HDR_TYPES_NFDS_T_H diff --git a/system/lib/llvm-libc/hdr/types/struct_dl_phdr_info.h b/system/lib/llvm-libc/hdr/types/struct_dl_phdr_info.h new file mode 100644 index 0000000000000..0cfb3c1309bde --- /dev/null +++ b/system/lib/llvm-libc/hdr/types/struct_dl_phdr_info.h @@ -0,0 +1,21 @@ +//===-- Proxy for struct dl_phdr_info -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===---------------------------------------------------------------------===// +#ifndef LLVM_LIBC_HDR_TYPES_STRUCT_DL_PHDR_INFO_H +#define LLVM_LIBC_HDR_TYPES_STRUCT_DL_PHDR_INFO_H + +#ifdef LIBC_FULL_BUILD + +#include "include/llvm-libc-types/struct_dl_phdr_info.h" + +#else + +#include + +#endif // LIBC_FULL_BUILD + +#endif // LLVM_LIBC_HDR_TYPES_STRUCT_DL_PHDR_INFO_H diff --git a/system/lib/llvm-libc/hdr/types/struct_itimerval.h b/system/lib/llvm-libc/hdr/types/struct_itimerval.h new file mode 100644 index 0000000000000..b2281675b8023 --- /dev/null +++ b/system/lib/llvm-libc/hdr/types/struct_itimerval.h @@ -0,0 +1,21 @@ +//===-- Proxy for struct itimerval ----------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +#ifndef LLVM_LIBC_HDR_TYPES_STRUCT_ITIMERVAL_H +#define LLVM_LIBC_HDR_TYPES_STRUCT_ITIMERVAL_H + +#ifdef LIBC_FULL_BUILD + +#include "include/llvm-libc-types/struct_itimerval.h" + +#else + +#include + +#endif // LIBC_FULL_BUILD + +#endif // LLVM_LIBC_HDR_TYPES_STRUCT_ITIMERVAL_H diff --git a/system/lib/llvm-libc/hdr/types/struct_pollfd.h b/system/lib/llvm-libc/hdr/types/struct_pollfd.h new file mode 100644 index 0000000000000..efec6fc80ac10 --- /dev/null +++ b/system/lib/llvm-libc/hdr/types/struct_pollfd.h @@ -0,0 +1,23 @@ +//===-- Definition of struct pollfd ---------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_HDR_TYPES_STRUCT_POLLFD_H +#define LLVM_LIBC_HDR_TYPES_STRUCT_POLLFD_H + +#ifdef LIBC_FULL_BUILD + +#include "include/llvm-libc-types/struct_pollfd.h" + +#else // overlay mode + +#include + +#endif // LLVM_LIBC_FULL_BUILD + +#endif // LLVM_LIBC_HDR_TYPES_STRUCT_POLLFD_H diff --git a/system/lib/llvm-libc/hdr/uchar_overlay.h b/system/lib/llvm-libc/hdr/uchar_overlay.h new file mode 100644 index 0000000000000..44ed3d48c6c1d --- /dev/null +++ b/system/lib/llvm-libc/hdr/uchar_overlay.h @@ -0,0 +1,69 @@ +//===-- Including uchar.h in overlay mode ---------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_HDR_UCHAR_OVERLAY_H +#define LLVM_LIBC_HDR_UCHAR_OVERLAY_H + +#ifdef LIBC_FULL_BUILD +#error "This header should only be included in overlay mode" +#endif + +// Overlay mode + +// glibc header might provide extern inline definitions for few +// functions, causing external alias errors. They are guarded by +// `__USE_EXTERN_INLINES` macro. We temporarily disable `__USE_EXTERN_INLINES` +// macro by defining `__NO_INLINE__` before including . +// And the same with `__USE_FORTIFY_LEVEL`, which will be temporarily disabled +// with `_FORTIFY_SOURCE`. + +#ifdef _FORTIFY_SOURCE +#define LIBC_OLD_FORTIFY_SOURCE _FORTIFY_SOURCE +#undef _FORTIFY_SOURCE +#endif + +#ifndef __NO_INLINE__ +#define __NO_INLINE__ 1 +#define LIBC_SET_NO_INLINE +#endif + +#ifdef __USE_EXTERN_INLINES +#define LIBC_OLD_USE_EXTERN_INLINES +#undef __USE_EXTERN_INLINES +#endif + +#ifdef __USE_FORTIFY_LEVEL +#define LIBC_OLD_USE_FORTIFY_LEVEL __USE_FORTIFY_LEVEL +#undef __USE_FORTIFY_LEVEL +#define __USE_FORTIFY_LEVEL 0 +#endif + +#include + +#ifdef LIBC_OLD_FORTIFY_SOURCE +#define _FORTIFY_SOURCE LIBC_OLD_FORTIFY_SOURCE +#undef LIBC_OLD_FORTIFY_SOURCE +#endif + +#ifdef LIBC_SET_NO_INLINE +#undef __NO_INLINE__ +#undef LIBC_SET_NO_INLINE +#endif + +#ifdef LIBC_OLD_USE_FORTIFY_LEVEL +#undef __USE_FORTIFY_LEVEL +#define __USE_FORTIFY_LEVEL LIBC_OLD_USE_FORTIFY_LEVEL +#undef LIBC_OLD_USE_FORTIFY_LEVEL +#endif + +#ifdef LIBC_OLD_USE_EXTERN_INLINES +#define __USE_EXTERN_INLINES +#undef LIBC_OLD_USE_EXTERN_INLINES +#endif + +#endif // LLVM_LIBC_HDR_UCHAR_OVERLAY_H diff --git a/system/lib/llvm-libc/hdr/wchar_overlay.h b/system/lib/llvm-libc/hdr/wchar_overlay.h index 99a70899779e7..52d3a0cc65963 100644 --- a/system/lib/llvm-libc/hdr/wchar_overlay.h +++ b/system/lib/llvm-libc/hdr/wchar_overlay.h @@ -15,7 +15,7 @@ // Overlay mode -// glibc header might provide extern inline definitions for few // functions, causing external alias errors. They are guarded by // `__USE_EXTERN_INLINES` macro. We temporarily disable `__USE_EXTERN_INLINES` // macro by defining `__NO_INLINE__` before including . diff --git a/system/lib/llvm-libc/include/llvm-libc-macros/EFIAPI-macros.h b/system/lib/llvm-libc/include/llvm-libc-macros/EFIAPI-macros.h new file mode 100644 index 0000000000000..cb854928d0ab7 --- /dev/null +++ b/system/lib/llvm-libc/include/llvm-libc-macros/EFIAPI-macros.h @@ -0,0 +1,18 @@ +//===-- Definition of EFIAPI macro ------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_MACROS_EFIAPI_MACROS_H +#define LLVM_LIBC_MACROS_EFIAPI_MACROS_H + +#if defined(__x86_64__) && !defined(__ILP32__) +#define EFIAPI __attribute__((ms_abi)) +#else +#define EFIAPI +#endif + +#endif // LLVM_LIBC_MACROS_EFIAPI_MACROS_H diff --git a/system/lib/llvm-libc/include/llvm-libc-macros/baremetal/time-macros.h b/system/lib/llvm-libc/include/llvm-libc-macros/baremetal/time-macros.h new file mode 100644 index 0000000000000..3537376c4bca8 --- /dev/null +++ b/system/lib/llvm-libc/include/llvm-libc-macros/baremetal/time-macros.h @@ -0,0 +1,26 @@ +//===-- Definition of macros from time.h ---------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_MACROS_BAREMETAL_TIME_MACROS_H +#define LLVM_LIBC_MACROS_BAREMETAL_TIME_MACROS_H + +#ifdef __CLK_TCK +#define CLOCKS_PER_SEC __CLK_TCK +#else +#if defined(__arm__) || defined(_M_ARM) || defined(__aarch64__) || \ + defined(__arm64__) || defined(_M_ARM64) +// This default implementation of this function shall use semihosting +// Semihosting measures time in centiseconds +// https://github.com/ARM-software/abi-aa/blob/main/semihosting/semihosting.rst#sys-clock-0x10 +#define CLOCKS_PER_SEC 100 +#else +#define CLOCKS_PER_SEC 1000000 +#endif +#endif + +#endif // LLVM_LIBC_MACROS_BAREMETAL_TIME_MACROS_H diff --git a/system/lib/llvm-libc/include/llvm-libc-macros/endian-macros.h b/system/lib/llvm-libc/include/llvm-libc-macros/endian-macros.h new file mode 100644 index 0000000000000..52d95dc01cd83 --- /dev/null +++ b/system/lib/llvm-libc/include/llvm-libc-macros/endian-macros.h @@ -0,0 +1,50 @@ +//===-- Definition of macros from endian.h --------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_MACROS_ENDIAN_MACROS_H +#define LLVM_LIBC_MACROS_ENDIAN_MACROS_H + +#include "stdint-macros.h" + +#define LITTLE_ENDIAN __ORDER_LITTLE_ENDIAN__ +#define BIG_ENDIAN __ORDER_BIG_ENDIAN__ +#define BYTE_ORDER __BYTE_ORDER__ + +#if BYTE_ORDER == LITTLE_ENDIAN + +#define htobe16(x) __builtin_bswap16((x)) +#define htobe32(x) __builtin_bswap32((x)) +#define htobe64(x) __builtin_bswap64((x)) +#define htole16(x) __LLVM_LIBC_CAST(static_cast, uint16_t, x) +#define htole32(x) __LLVM_LIBC_CAST(static_cast, uint32_t, x) +#define htole64(x) __LLVM_LIBC_CAST(static_cast, uint64_t, x) +#define be16toh(x) __builtin_bswap16((x)) +#define be32toh(x) __builtin_bswap32((x)) +#define be64toh(x) __builtin_bswap64((x)) +#define le16toh(x) __LLVM_LIBC_CAST(static_cast, uint16_t, x) +#define le32toh(x) __LLVM_LIBC_CAST(static_cast, uint32_t, x) +#define le64toh(x) __LLVM_LIBC_CAST(static_cast, uint64_t, x) + +#else + +#define htobe16(x) __LLVM_LIBC_CAST(static_cast, uint16_t, x) +#define htobe32(x) __LLVM_LIBC_CAST(static_cast, uint32_t, x) +#define htobe64(x) __LLVM_LIBC_CAST(static_cast, uint64_t, x) +#define htole16(x) __builtin_bswap16((x)) +#define htole32(x) __builtin_bswap32((x)) +#define htole64(x) __builtin_bswap64((x)) +#define be16toh(x) __LLVM_LIBC_CAST(static_cast, uint16_t, x) +#define be32toh(x) __LLVM_LIBC_CAST(static_cast, uint32_t, x) +#define be64toh(x) __LLVM_LIBC_CAST(static_cast, uint64_t, x) +#define le16toh(x) __builtin_bswap16((x)) +#define le32toh(x) __builtin_bswap32((x)) +#define le64toh(x) __builtin_bswap64((x)) + +#endif + +#endif // LLVM_LIBC_MACROS_ENDIAN_MACROS_H diff --git a/system/lib/llvm-libc/include/llvm-libc-macros/gpu/signal-macros.h b/system/lib/llvm-libc/include/llvm-libc-macros/gpu/signal-macros.h index 2d8159240de8b..f0d49ea34fe0e 100644 --- a/system/lib/llvm-libc/include/llvm-libc-macros/gpu/signal-macros.h +++ b/system/lib/llvm-libc/include/llvm-libc-macros/gpu/signal-macros.h @@ -16,9 +16,9 @@ #define SIGSEGV 11 #define SIGTERM 15 -#define SIG_DFL ((__sighandler_t)(0)) -#define SIG_IGN ((__sighandler_t)(1)) -#define SIG_ERR ((__sighandler_t)(-1)) +#define SIG_DFL ((void (*)(int))(0)) +#define SIG_IGN ((void (*)(int))(1)) +#define SIG_ERR ((void (*)(int))(-1)) // Max signal number #define NSIG 64 diff --git a/system/lib/llvm-libc/include/llvm-libc-macros/limits-macros.h b/system/lib/llvm-libc/include/llvm-libc-macros/limits-macros.h index d4aa7ae539e8a..79bbbe401eb02 100644 --- a/system/lib/llvm-libc/include/llvm-libc-macros/limits-macros.h +++ b/system/lib/llvm-libc/include/llvm-libc-macros/limits-macros.h @@ -235,4 +235,12 @@ #define _POSIX_PATH_MAX 256 #endif +#ifndef _POSIX_ARG_MAX +#define _POSIX_ARG_MAX 4096 +#endif + +#ifndef IOV_MAX +#define IOV_MAX 1024 +#endif // IOV_MAX + #endif // LLVM_LIBC_MACROS_LIMITS_MACROS_H diff --git a/system/lib/llvm-libc/include/llvm-libc-macros/linux/poll-macros.h b/system/lib/llvm-libc/include/llvm-libc-macros/linux/poll-macros.h new file mode 100644 index 0000000000000..d6724c5ee3211 --- /dev/null +++ b/system/lib/llvm-libc/include/llvm-libc-macros/linux/poll-macros.h @@ -0,0 +1,65 @@ +//===-- Macros defined in poll.h header file ------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_MACROS_LINUX_POLL_MACROS_H +#define LLVM_LIBC_MACROS_LINUX_POLL_MACROS_H + +// From asm-generic/poll.h, redefined here to avoid redeclaring struct pollfd. +#ifndef POLLIN +#define POLLIN 0x0001 +#endif + +#ifndef POLLPRI +#define POLLPRI 0x0002 +#endif + +#ifndef POLLOUT +#define POLLOUT 0x0004 +#endif + +#ifndef POLLERR +#define POLLERR 0x0008 +#endif + +#ifndef POLLHUP +#define POLLHUP 0x0010 +#endif + +#ifndef POLLNVAL +#define POLLNVAL 0x0020 +#endif + +#ifndef POLLRDNORM +#define POLLRDNORM 0x0040 +#endif + +#ifndef POLLRDBAND +#define POLLRDBAND 0x0080 +#endif + +#ifndef POLLWRNORM +#define POLLWRNORM 0x0100 +#endif + +#ifndef POLLWRBAND +#define POLLWRBAND 0x0200 +#endif + +#ifndef POLLMSG +#define POLLMSG 0x0400 +#endif + +#ifndef POLLREMOVE +#define POLLREMOVE 0x1000 +#endif + +#ifndef POLLRDHUP +#define POLLRDHUP 0x2000 +#endif + +#endif // LLVM_LIBC_MACROS_LINUX_POLL_MACROS_H diff --git a/system/lib/llvm-libc/include/llvm-libc-macros/linux/sched-macros.h b/system/lib/llvm-libc/include/llvm-libc-macros/linux/sched-macros.h index ace620049ca0c..597789bb4ce29 100644 --- a/system/lib/llvm-libc/include/llvm-libc-macros/linux/sched-macros.h +++ b/system/lib/llvm-libc/include/llvm-libc-macros/linux/sched-macros.h @@ -23,7 +23,15 @@ #define SCHED_IDLE 5 #define SCHED_DEADLINE 6 +#define CPU_SETSIZE __CPU_SETSIZE +#define NCPUBITS __NCPUBITS #define CPU_COUNT_S(setsize, set) __sched_getcpucount(setsize, set) #define CPU_COUNT(set) CPU_COUNT_S(sizeof(cpu_set_t), set) +#define CPU_ZERO_S(setsize, set) __sched_setcpuzero(setsize, set) +#define CPU_ZERO(set) CPU_ZERO_S(sizeof(cpu_set_t), set) +#define CPU_SET_S(cpu, setsize, set) __sched_setcpuset(cpu, setsize, set) +#define CPU_SET(cpu, setsize, set) CPU_SET_S(cpu, sizeof(cpt_set_t), set) +#define CPU_ISSET_S(cpu, setsize, set) __sched_getcpuisset(cpu, setsize, set) +#define CPU_ISSET(cpu, setsize, set) CPU_ISSET_S(cpu, sizeof(cpt_set_t), set) #endif // LLVM_LIBC_MACROS_LINUX_SCHED_MACROS_H diff --git a/system/lib/llvm-libc/include/llvm-libc-macros/linux/signal-macros.h b/system/lib/llvm-libc/include/llvm-libc-macros/linux/signal-macros.h index 0b7317ebc9b80..d220241a38206 100644 --- a/system/lib/llvm-libc/include/llvm-libc-macros/linux/signal-macros.h +++ b/system/lib/llvm-libc/include/llvm-libc-macros/linux/signal-macros.h @@ -86,9 +86,9 @@ #error "Signal stack sizes not defined for your platform." #endif -#define SIG_DFL ((__sighandler_t)0) -#define SIG_IGN ((__sighandler_t)1) -#define SIG_ERR ((__sighandler_t)-1) +#define SIG_DFL ((void (*)(int))0) +#define SIG_IGN ((void (*)(int))1) +#define SIG_ERR ((void (*)(int))(-1)) // SIGCHLD si_codes #define CLD_EXITED 1 // child has exited diff --git a/system/lib/llvm-libc/include/llvm-libc-macros/linux/sys-ioctl-macros.h b/system/lib/llvm-libc/include/llvm-libc-macros/linux/sys-ioctl-macros.h index 5eb779aeeca56..41226080084c3 100644 --- a/system/lib/llvm-libc/include/llvm-libc-macros/linux/sys-ioctl-macros.h +++ b/system/lib/llvm-libc/include/llvm-libc-macros/linux/sys-ioctl-macros.h @@ -15,5 +15,6 @@ // around the definitions of macros like _IO, _IOR, _IOW, and _IOWR that I don't // think is worth digging into right now. #define TIOCGETD 0x5424 +#define FIONREAD 0x541B #endif // LLVM_LIBC_MACROS_LINUX_SYS_IOCTL_MACROS_H diff --git a/system/lib/llvm-libc/include/llvm-libc-macros/linux/sys-wait-macros.h b/system/lib/llvm-libc/include/llvm-libc-macros/linux/sys-wait-macros.h index c101638fdae34..d01cfa71ba390 100644 --- a/system/lib/llvm-libc/include/llvm-libc-macros/linux/sys-wait-macros.h +++ b/system/lib/llvm-libc/include/llvm-libc-macros/linux/sys-wait-macros.h @@ -9,36 +9,19 @@ #ifndef LLVM_LIBC_MACROS_LINUX_SYS_WAIT_MACROS_H #define LLVM_LIBC_MACROS_LINUX_SYS_WAIT_MACROS_H -// Wait flags -#define WNOHANG 1 // Do not block -#define WUNTRACED 2 // Report is a child has stopped even if untraced -#define WEXITED 4 // Report dead child -#define WCONTINUED 8 // Report if a stopped child has been resumed by SIGCONT -#define WSTOPPED WUNTRACED - -// Wait status info macros -#define __WEXITSTATUS(status) (((status)&0xff00) >> 8) -#define __WTERMSIG(status) ((status)&0x7f) -#define __WIFEXITED(status) (__WTERMSIG(status) == 0) - -// Macros for constructing status values. -#define __W_EXITCODE(ret, sig) ((ret) << 8 | (sig)) -#define __W_STOPCODE(sig) ((sig) << 8 | 0x7f) -#define __W_CONTINUED 0xffff -#define __WCOREFLAG 0x80 - -#define WEXITSTATUS(status) __WEXITSTATUS(status) -#define WTERMSIG(status) __WTERMSIG(status) -#define WIFEXITED(status) __WIFEXITED(status) - -#define WCOREFLAG __WCOREFLAG -#define W_EXITCODE(ret, sig) __W_EXITCODE(ret, sig) -#define W_STOPCODE(sig) __W_STOPCODE(sig) - -// First argument to waitid: -#define P_ALL 0 -#define P_PID 1 -#define P_PGID 2 -#define P_PIDFD 3 +#include + +#define WCOREDUMP(status) ((status) & WCOREFLAG) +#define WEXITSTATUS(status) (((status) & 0xff00) >> 8) +#define WIFCONTINUED(status) ((status) == 0xffff) +#define WIFEXITED(status) (WTERMSIG(status) == 0) +#define WIFSIGNALED(status) ((WTERMSIG(status) + 1) >= 2) +#define WIFSTOPPED(status) (WTERMSIG(status) == 0x7f) +#define WSTOPSIG(status) WEXITSTATUS(status) +#define WTERMSIG(status) ((status) & 0x7f) + +#define WCOREFLAG 0x80 +#define W_EXITCODE(ret, sig) ((ret) << 8 | (sig)) +#define W_STOPCODE(sig) ((sig) << 8 | 0x7f) #endif // LLVM_LIBC_MACROS_LINUX_SYS_WAIT_MACROS_H diff --git a/system/lib/llvm-libc/include/llvm-libc-types/__bsearchcompare_t.h b/system/lib/llvm-libc/include/llvm-libc-macros/poll-macros.h similarity index 53% rename from system/lib/llvm-libc/include/llvm-libc-types/__bsearchcompare_t.h rename to system/lib/llvm-libc/include/llvm-libc-macros/poll-macros.h index 0b1987be1fdd6..52b59a978a21f 100644 --- a/system/lib/llvm-libc/include/llvm-libc-types/__bsearchcompare_t.h +++ b/system/lib/llvm-libc/include/llvm-libc-macros/poll-macros.h @@ -1,4 +1,4 @@ -//===-- Definition of type __bsearchcompare_t -----------------------------===// +//===-- Macros defined in poll.h header file ------------------------------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -6,9 +6,11 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_LIBC_TYPES___BSEARCHCOMPARE_T_H -#define LLVM_LIBC_TYPES___BSEARCHCOMPARE_T_H +#ifndef LLVM_LIBC_MACROS_POLL_MACROS_H +#define LLVM_LIBC_MACROS_POLL_MACROS_H -typedef int (*__bsearchcompare_t)(const void *, const void *); +#ifdef __linux__ +#include "linux/poll-macros.h" +#endif -#endif // LLVM_LIBC_TYPES___BSEARCHCOMPARE_T_H +#endif // LLVM_LIBC_MACROS_POLL_MACROS_H diff --git a/system/lib/llvm-libc/include/llvm-libc-macros/pthread-macros.h b/system/lib/llvm-libc/include/llvm-libc-macros/pthread-macros.h index 8a144dbd2e611..fcc6ef925e3f4 100644 --- a/system/lib/llvm-libc/include/llvm-libc-macros/pthread-macros.h +++ b/system/lib/llvm-libc/include/llvm-libc-macros/pthread-macros.h @@ -9,6 +9,8 @@ #ifndef LLVM_LIBC_MACROS_PTHREAD_MACRO_H #define LLVM_LIBC_MACROS_PTHREAD_MACRO_H +#include "null-macro.h" + #define PTHREAD_CREATE_JOINABLE 0 #define PTHREAD_CREATE_DETACHED 1 @@ -25,8 +27,34 @@ #define PTHREAD_PROCESS_PRIVATE 0 #define PTHREAD_PROCESS_SHARED 1 -#define PTHREAD_MUTEX_INITIALIZER {0} -#define PTHREAD_RWLOCK_INITIALIZER {0} +#ifdef __linux__ +#define PTHREAD_MUTEX_INITIALIZER \ + { \ + /* .__timed = */ 0, /* .__recursive = */ 0, \ + /* .__robust = */ 0, /* .__owner = */ NULL, \ + /* .__lock_count = */ 0, /* .__futex_word = */ {0}, \ + } +#else +#define PTHREAD_MUTEX_INITIALIZER \ + { \ + /* .__timed = */ 0, /* .__recursive = */ 0, \ + /* .__robust = */ 0, /* .__owner = */ NULL, \ + /* .__lock_count = */ 0, \ + } +#endif + +#define PTHREAD_RWLOCK_INITIALIZER \ + { \ + /* .__is_pshared = */ 0, \ + /* .__preference = */ 0, \ + /* .__state = */ 0, \ + /* .__write_tid = */ 0, \ + /* .__wait_queue_mutex = */ {0}, \ + /* .__pending_readers = */ {0}, \ + /* .__pending_writers = */ {0}, \ + /* .__reader_serialization = */ {0}, \ + /* .__writer_serialization = */ {0}, \ + } // glibc extensions #define PTHREAD_STACK_MIN (1 << 14) // 16KB diff --git a/system/lib/llvm-libc/include/llvm-libc-macros/stdfix-macros.h b/system/lib/llvm-libc/include/llvm-libc-macros/stdfix-macros.h index 554ebe544a42e..04097e14e9747 100644 --- a/system/lib/llvm-libc/include/llvm-libc-macros/stdfix-macros.h +++ b/system/lib/llvm-libc/include/llvm-libc-macros/stdfix-macros.h @@ -323,6 +323,45 @@ #define ULACCUM_EPSILON 0x1.0p-32ULK #endif // ULACCUM_EPSILON +#define absfx(x) \ + _Generic((x), \ + fract: absr, \ + short fract: abshr, \ + long fract: abslr, \ + accum: absk, \ + short accum: abshk, \ + long accum: abslk)(x) + +#define countlsfx(x) \ + _Generic((x), \ + fract: countlsr, \ + short fract: countlshr, \ + long fract: countlslr, \ + accum: countlsk, \ + short accum: countlshk, \ + long accum: countlslk, \ + unsigned fract: countlsur, \ + unsigned short fract: countlsuhr, \ + unsigned long fract: countlsulr, \ + unsigned accum: countlsuk, \ + unsigned short accum: countlsuhk, \ + unsigned long accum: countlsulk)(x) + +#define roundfx(x, y) \ + _Generic((x), \ + fract: roundr, \ + short fract: roundhr, \ + long fract: roundlr, \ + accum: roundk, \ + short accum: roundhk, \ + long accum: roundlk, \ + unsigned fract: roundur, \ + unsigned short fract: rounduhr, \ + unsigned long fract: roundulr, \ + unsigned accum: rounduk, \ + unsigned short accum: rounduhk, \ + unsigned long accum: roundulk)(x, y) + #endif // LIBC_COMPILER_HAS_FIXED_POINT #endif // LLVM_LIBC_MACROS_STDFIX_MACROS_H diff --git a/system/lib/llvm-libc/include/llvm-libc-macros/sysexits-macros.h b/system/lib/llvm-libc/include/llvm-libc-macros/sysexits-macros.h new file mode 100644 index 0000000000000..52f838788632b --- /dev/null +++ b/system/lib/llvm-libc/include/llvm-libc-macros/sysexits-macros.h @@ -0,0 +1,29 @@ +//===-- Macros defined in sysexits.h header file --------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef SYSEXITS_MACROS_H +#define SYSEXITS_MACROS_H + +#define EX_OK 0 // Successful termination +#define EX_USAGE 64 // Command line usage error +#define EX_DATAERR 65 // Data format error +#define EX_NOINPUT 66 // Cannot open input +#define EX_NOUSER 67 // Addressee unknown +#define EX_NOHOST 68 // Host name unknown +#define EX_UNAVAILABLE 69 // Service unavailable +#define EX_SOFTWARE 70 // Internal software error +#define EX_OSERR 71 // Operating system error +#define EX_OSFILE 72 // System file error +#define EX_CANTCREAT 73 // Cannot create (user) output file +#define EX_IOERR 74 // Input/output error +#define EX_TEMPFAIL 75 // Temporary failure, try again +#define EX_PROTOCOL 76 // Remote protocol error +#define EX_NOPERM 77 // Permission denied +#define EX_CONFIG 78 // Configuration error + +#endif // SYSEXITS_MACROS_H diff --git a/system/lib/llvm-libc/include/llvm-libc-macros/time-macros.h b/system/lib/llvm-libc/include/llvm-libc-macros/time-macros.h index 445d8b3e837ed..30e0a310a5485 100644 --- a/system/lib/llvm-libc/include/llvm-libc-macros/time-macros.h +++ b/system/lib/llvm-libc/include/llvm-libc-macros/time-macros.h @@ -5,8 +5,14 @@ #include "gpu/time-macros.h" #elif defined(__linux__) #include "linux/time-macros.h" +#elif defined(__ELF__) +#include "baremetal/time-macros.h" +#else +#define CLOCKS_PER_SEC 1000000 #endif +#define CLK_TCK CLOCKS_PER_SEC + #define TIME_UTC 1 #define TIME_MONOTONIC 2 #define TIME_ACTIVE 3 diff --git a/system/lib/llvm-libc/include/llvm-libc-types/EFI_ALLOCATE_TYPE.h b/system/lib/llvm-libc/include/llvm-libc-types/EFI_ALLOCATE_TYPE.h new file mode 100644 index 0000000000000..90f23969678f4 --- /dev/null +++ b/system/lib/llvm-libc/include/llvm-libc-types/EFI_ALLOCATE_TYPE.h @@ -0,0 +1,19 @@ +//===-- Definition of EFI_ALLOCATE_TYPE type ------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_TYPES_EFI_ALLOCATE_TYPE_H +#define LLVM_LIBC_TYPES_EFI_ALLOCATE_TYPE_H + +typedef enum { + AllocateAnyPages, + AllocateMaxAddress, + AllocateAddress, + MaxAllocateType +} EFI_ALLOCATE_TYPE; + +#endif // LLVM_LIBC_TYPES_EFI_ALLOCATE_TYPE_H diff --git a/system/lib/llvm-libc/include/llvm-libc-types/EFI_BOOT_SERVICES.h b/system/lib/llvm-libc/include/llvm-libc-types/EFI_BOOT_SERVICES.h new file mode 100644 index 0000000000000..8b7a6aadd7a24 --- /dev/null +++ b/system/lib/llvm-libc/include/llvm-libc-types/EFI_BOOT_SERVICES.h @@ -0,0 +1,250 @@ +//===-- Definition of EFI_BOOT_SERVICES type ------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_TYPES_EFI_BOOT_SERVICES_H +#define LLVM_LIBC_TYPES_EFI_BOOT_SERVICES_H + +#include "../llvm-libc-macros/EFIAPI-macros.h" +#include "EFI_ALLOCATE_TYPE.h" +#include "EFI_DEVICE_PATH_PROTOCOL.h" +#include "EFI_EVENT.h" +#include "EFI_GUID.h" +#include "EFI_INTERFACE_TYPE.h" +#include "EFI_LOCATE_SEARCH_TYPE.h" +#include "EFI_MEMORY_DESCRIPTOR.h" +#include "EFI_MEMORY_TYPE.h" +#include "EFI_OPEN_PROTOCOL_INFORMATION_ENTRY.h" +#include "EFI_PHYSICAL_ADDRESS.h" +#include "EFI_STATUS.h" +#include "EFI_TABLE_HEADER.h" +#include "EFI_TIMER_DELAY.h" +#include "EFI_TPL.h" +#include "char16_t.h" +#include "size_t.h" + +#define EFI_BOOT_SERVICES_SIGNATURE 0x56524553544f4f42 +#define EFI_BOOT_SERVICES_REVISION EFI_SPECIFICATION_VERSION + +typedef EFI_TPL(EFIAPI *EFI_RAISE_TPL)(EFI_TPL NewTpl); +typedef void(EFIAPI *EFI_RESTORE_TPL)(EFI_TPL OldTpl); + +typedef EFI_STATUS(EFIAPI *EFI_ALLOCATE_PAGES)(EFI_ALLOCATE_TYPE Type, + EFI_MEMORY_TYPE MemoryType, + size_t Pages, + EFI_PHYSICAL_ADDRESS *Memory); +typedef EFI_STATUS(EFIAPI *EFI_FREE_PAGES)(EFI_PHYSICAL_ADDRESS Memory, + size_t Pages); +typedef EFI_STATUS(EFIAPI *EFI_GET_MEMORY_MAP)(size_t *MemoryMapSize, + EFI_MEMORY_DESCRIPTOR *MemoryMap, + size_t *MapKey, + size_t *DescriptorSize, + uint32_t *DescriptorVersion); + +typedef EFI_STATUS(EFIAPI *EFI_ALLOCATE_POOL)(EFI_MEMORY_TYPE PoolType, + size_t Size, void **Buffer); +typedef EFI_STATUS(EFIAPI *EFI_FREE_POOL)(void *Buffer); + +typedef void(EFIAPI *EFI_EVENT_NOTIFY)(EFI_EVENT Event, void *Context); + +typedef EFI_STATUS(EFIAPI *EFI_CREATE_EVENT)(uint32_t Type, EFI_TPL NotifyTpl, + EFI_EVENT_NOTIFY NotifyFunction, + void *NotifyContext, + EFI_EVENT *Event); +typedef EFI_STATUS(EFIAPI *EFI_SET_TIMER)(EFI_EVENT Event, EFI_TIMER_DELAY Type, + uint64_t TriggerTime); +typedef EFI_STATUS(EFIAPI *EFI_WAIT_FOR_EVENT)(size_t NumberOfEvents, + EFI_EVENT *Event, size_t *Index); +typedef EFI_STATUS(EFIAPI *EFI_SIGNAL_EVENT)(EFI_EVENT Event); +typedef EFI_STATUS(EFIAPI *EFI_CLOSE_EVENT)(EFI_EVENT Event); +typedef EFI_STATUS(EFIAPI *EFI_CHECK_EVENT)(EFI_EVENT Event); + +typedef EFI_STATUS(EFIAPI *EFI_INSTALL_PROTOCOL_INTERFACE)( + EFI_HANDLE *Handle, EFI_GUID *Protocol, EFI_INTERFACE_TYPE InterfaceType, + void *Interface); +typedef EFI_STATUS(EFIAPI *EFI_REINSTALL_PROTOCOL_INTERFACE)( + EFI_HANDLE Handle, EFI_GUID *Protocol, void *OldInterface, + void *NewInterface); +typedef EFI_STATUS(EFIAPI *EFI_UNINSTALL_PROTOCOL_INTERFACE)(EFI_HANDLE Handle, + EFI_GUID *Protocol, + void *Interface); + +typedef EFI_STATUS(EFIAPI *EFI_HANDLE_PROTOCOL)(EFI_HANDLE Handle, + EFI_GUID *Protocol, + void **Interface); +typedef EFI_STATUS(EFIAPI *EFI_REGISTER_PROTOCOL_NOTIFY)(EFI_GUID *Protocol, + EFI_EVENT Event, + void **Registration); + +typedef EFI_STATUS(EFIAPI *EFI_LOCATE_HANDLE)(EFI_LOCATE_SEARCH_TYPE SearchType, + EFI_GUID *Protocol, + void *SearchKey, + size_t *BufferSize, + EFI_HANDLE *Buffer); +typedef EFI_STATUS(EFIAPI *EFI_LOCATE_DEVICE_PATH)( + EFI_GUID *Protocol, EFI_DEVICE_PATH_PROTOCOL **DevicePath, + EFI_HANDLE *Device); + +typedef EFI_STATUS(EFIAPI *EFI_INSTALL_CONFIGURATION_TABLE)(EFI_GUID *Guid, + void *Table); +typedef EFI_STATUS(EFIAPI *EFI_IMAGE_UNLOAD)(EFI_HANDLE ImageHandle); +typedef EFI_STATUS(EFIAPI *EFI_IMAGE_START)(EFI_HANDLE ImageHandle, + size_t *ExitDataSize, + char16_t **ExitData); + +typedef EFI_STATUS(EFIAPI *EFI_EXIT)(EFI_HANDLE ImageHandle, + EFI_STATUS ExitStatus, size_t ExitDataSize, + char16_t *ExitData); +typedef EFI_STATUS(EFIAPI *EFI_EXIT_BOOT_SERVICES)(EFI_HANDLE ImageHandle, + size_t MapKey); +typedef EFI_STATUS(EFIAPI *EFI_GET_NEXT_MONOTONIC_COUNT)(uint64_t *Count); +typedef EFI_STATUS(EFIAPI *EFI_STALL)(size_t Microseconds); +typedef EFI_STATUS(EFIAPI *EFI_SET_WATCHDOG_TIMER)(size_t Timeout, + uint64_t WatchdogCode, + size_t DataSize, + char16_t *WatchdogData); + +typedef EFI_STATUS(EFIAPI *EFI_CONNECT_CONTROLLER)( + EFI_HANDLE ControllerHandle, EFI_HANDLE *DriverImageHandle, + EFI_DEVICE_PATH_PROTOCOL *RemainingDevicePath, bool Recursive); + +typedef EFI_STATUS(EFIAPI *EFI_DISCONNECT_CONTROLLER)( + EFI_HANDLE ControllerHandle, EFI_HANDLE DriverImageHandle, + EFI_HANDLE ChildHandle); + +typedef EFI_STATUS(EFIAPI *EFI_OPEN_PROTOCOL)( + EFI_HANDLE Handle, EFI_GUID *Protocol, void **Interface, + EFI_HANDLE AgentHandle, EFI_HANDLE ControllerHandle, uint32_t Attributes); + +typedef EFI_STATUS(EFIAPI *EFI_CLOSE_PROTOCOL)(EFI_HANDLE Handle, + EFI_GUID *Protocol, + EFI_HANDLE AgentHandle, + EFI_HANDLE ControllerHandle); + +typedef EFI_STATUS(EFIAPI *EFI_OPEN_PROTOCOL_INFORMATION)( + EFI_HANDLE Handle, EFI_GUID *Protocol, + EFI_OPEN_PROTOCOL_INFORMATION_ENTRY **EntryBuffer, size_t *EntryCount); + +typedef EFI_STATUS(EFIAPI *EFI_PROTOCOLS_PER_HANDLE)( + EFI_HANDLE Handle, EFI_GUID ***ProtocolBuffer, size_t *ProtocolBufferCount); + +typedef EFI_STATUS(EFIAPI *EFI_LOCATE_HANDLE_BUFFER)( + EFI_LOCATE_SEARCH_TYPE SearchType, EFI_GUID *Protocol, void *SearchKey, + size_t *NoHandles, EFI_HANDLE **Buffer); + +typedef EFI_STATUS(EFIAPI *EFI_LOCATE_PROTOCOL)(EFI_GUID *Protocol, + void *Registration, + void **Interface); + +typedef EFI_STATUS(EFIAPI *EFI_UNINSTALL_MULTIPLE_PROTOCOL_INTERFACES)( + EFI_HANDLE Handle, ...); +typedef EFI_STATUS(EFIAPI *EFI_CALCULATE_CRC32)(void *Data, size_t DataSize, + uint32_t *Crc32); + +typedef void(EFIAPI *EFI_COPY_MEM)(void *Destination, void *Source, + size_t Length); +typedef void(EFIAPI *EFI_SET_MEM)(void *Buffer, size_t Size, uint8_t Value); + +typedef EFI_STATUS(EFIAPI *EFI_CREATE_EVENT_EX)( + uint32_t Type, EFI_TPL NotifyTpl, EFI_EVENT_NOTIFY NotifyFunction, + const void *NotifyContext, const EFI_GUID *EventGroup, EFI_EVENT *Event); + +typedef struct { + EFI_TABLE_HEADER Hdr; + + // + // Task Priority Services + // + EFI_RAISE_TPL RaiseTPL; // EFI 1.0+ + EFI_RESTORE_TPL RestoreTPL; // EFI 1.0+ + + // + // Memory Services + // + EFI_ALLOCATE_PAGES AllocatePages; // EFI 1.0+ + EFI_FREE_PAGES FreePages; // EFI 1.0+ + EFI_GET_MEMORY_MAP GetMemoryMap; // EFI 1.0+ + EFI_ALLOCATE_POOL AllocatePool; // EFI 1.0+ + EFI_FREE_POOL FreePool; // EFI 1.0+ + + // + // Event & Timer Services + // + EFI_CREATE_EVENT CreateEvent; // EFI 1.0+ + EFI_SET_TIMER SetTimer; // EFI 1.0+ + EFI_WAIT_FOR_EVENT WaitForEvent; // EFI 1.0+ + EFI_SIGNAL_EVENT SignalEvent; // EFI 1.0+ + EFI_CLOSE_EVENT CloseEvent; // EFI 1.0+ + EFI_CHECK_EVENT CheckEvent; // EFI 1.0+ + + // + // Protocol Handler Services + // + EFI_INSTALL_PROTOCOL_INTERFACE InstallProtocolInterface; // EFI 1.0+ + EFI_REINSTALL_PROTOCOL_INTERFACE ReinstallProtocolInterface; // EFI 1.0+ + EFI_UNINSTALL_PROTOCOL_INTERFACE UninstallProtocolInterface; // EFI 1.0+ + EFI_HANDLE_PROTOCOL HandleProtocol; // EFI 1.0+ + void *Reserved; // EFI 1.0+ + EFI_REGISTER_PROTOCOL_NOTIFY RegisterProtocolNotify; // EFI 1.0+ + EFI_LOCATE_HANDLE LocateHandle; // EFI 1.+ + EFI_LOCATE_DEVICE_PATH LocateDevicePath; // EFI 1.0+ + EFI_INSTALL_CONFIGURATION_TABLE InstallConfigurationTable; // EFI 1.0+ + + // + // Image Services + // + EFI_IMAGE_UNLOAD LoadImage; // EFI 1.0+ + EFI_IMAGE_START StartImage; // EFI 1.0+ + EFI_EXIT Exit; // EFI 1.0+ + EFI_IMAGE_UNLOAD UnloadImage; // EFI 1.0+ + EFI_EXIT_BOOT_SERVICES ExitBootServices; // EFI 1.0+ + + // + // Miscellaneous Services + // + EFI_GET_NEXT_MONOTONIC_COUNT GetNextMonotonicCount; // EFI 1.0+ + EFI_STALL Stall; // EFI 1.0+ + EFI_SET_WATCHDOG_TIMER SetWatchdogTimer; // EFI 1.0+ + + // + // DriverSupport Services + // + EFI_CONNECT_CONTROLLER ConnectController; // EFI 1.1 + EFI_DISCONNECT_CONTROLLER DisconnectController; // EFI 1.1+ + + // + // Open and Close Protocol Services + // + EFI_OPEN_PROTOCOL OpenProtocol; // EFI 1.1+ + EFI_CLOSE_PROTOCOL CloseProtocol; // EFI 1.1+ + EFI_OPEN_PROTOCOL_INFORMATION OpenProtocolInformation; // EFI 1.1+ + + // + // Library Services + // + EFI_PROTOCOLS_PER_HANDLE ProtocolsPerHandle; // EFI 1.1+ + EFI_LOCATE_HANDLE_BUFFER LocateHandleBuffer; // EFI 1.1+ + EFI_LOCATE_PROTOCOL LocateProtocol; // EFI 1.1+ + EFI_UNINSTALL_MULTIPLE_PROTOCOL_INTERFACES + InstallMultipleProtocolInterfaces; // EFI 1.1+ + EFI_UNINSTALL_MULTIPLE_PROTOCOL_INTERFACES + UninstallMultipleProtocolInterfaces; // EFI 1.1+* + + // + // 32-bit CRC Services + // + EFI_CALCULATE_CRC32 CalculateCrc32; // EFI 1.1+ + + // + // Miscellaneous Services + // + EFI_COPY_MEM CopyMem; // EFI 1.1+ + EFI_SET_MEM SetMem; // EFI 1.1+ + EFI_CREATE_EVENT_EX CreateEventEx; // UEFI 2.0+ +} EFI_BOOT_SERVICES; + +#endif // LLVM_LIBC_TYPES_EFI_BOOT_SERVICES_H diff --git a/system/lib/llvm-libc/include/llvm-libc-types/EFI_CAPSULE.h b/system/lib/llvm-libc/include/llvm-libc-types/EFI_CAPSULE.h new file mode 100644 index 0000000000000..c7440c9b03b75 --- /dev/null +++ b/system/lib/llvm-libc/include/llvm-libc-types/EFI_CAPSULE.h @@ -0,0 +1,26 @@ +//===-- Definition of EFI_CAPSULE type ------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_TYPES_EFI_CAPSULE_H +#define LLVM_LIBC_TYPES_EFI_CAPSULE_H + +#include "../llvm-libc-macros/stdint-macros.h" +#include "EFI_GUID.h" + +typedef struct { + EFI_GUID CapsuleGuid; + uint32_t HeaderSize; + uint32_t Flags; + uint32_t CapsuleImageSize; +} EFI_CAPSULE_HEADER; + +#define CAPSULE_FLAGS_PERSIST_ACROSS_RESET 0x00010000 +#define CAPSULE_FLAGS_POPULATE_SYSTEM_TABLE 0x00020000 +#define CAPSULE_FLAGS_INITIATE_RESET 0x00040000 + +#endif // LLVM_LIBC_TYPES_EFI_CAPSULE_H diff --git a/system/lib/llvm-libc/include/llvm-libc-types/EFI_CONFIGURATION_TABLE.h b/system/lib/llvm-libc/include/llvm-libc-types/EFI_CONFIGURATION_TABLE.h new file mode 100644 index 0000000000000..56cd3e4fbb587 --- /dev/null +++ b/system/lib/llvm-libc/include/llvm-libc-types/EFI_CONFIGURATION_TABLE.h @@ -0,0 +1,19 @@ +//===-- Definition of EFI_CONFIGURATION_TABLE type ------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===---------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_TYPES_EFI_CONFIGURATION_TABLE_H +#define LLVM_LIBC_TYPES_EFI_CONFIGURATION_TABLE_H + +#include "EFI_GUID.h" + +typedef struct { + EFI_GUID VendorGuid; + void *VendorTable; +} EFI_CONFIGURATION_TABLE; + +#endif // LLVM_LIBC_TYPES_EFI_CONFIGURATION_TABLE_H diff --git a/system/lib/llvm-libc/include/llvm-libc-types/EFI_DEVICE_PATH_PROTOCOL.h b/system/lib/llvm-libc/include/llvm-libc-types/EFI_DEVICE_PATH_PROTOCOL.h new file mode 100644 index 0000000000000..f6a0b2e1f45c0 --- /dev/null +++ b/system/lib/llvm-libc/include/llvm-libc-types/EFI_DEVICE_PATH_PROTOCOL.h @@ -0,0 +1,23 @@ +//===-- Definition of EFI_DEVICE_PATH_PROTOCOL type -----------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_TYPES_EFI_DEVICE_PATH_PROTOCOL_H +#define LLVM_LIBC_TYPES_EFI_DEVICE_PATH_PROTOCOL_H + +#include "../llvm-libc-macros/stdint-macros.h" + +#define EFI_DEVICE_PATH_PROTOCOL_GUID \ + {0x09576e91, 0x6d3f, 0x11d2, {0x8e, 0x39, 0x00, 0xa0, 0xc9, 0x69, 0x72, 0x3b}} + +typedef struct _EFI_DEVICE_PATH_PROTOCOL { + uint8_t Type; + uint8_t SubType; + uint8_t Length[2]; +} EFI_DEVICE_PATH_PROTOCOL; + +#endif // LLVM_LIBC_TYPES_EFI_DEVICE_PATH_PROTOCOL_H diff --git a/system/lib/llvm-libc/include/llvm-libc-types/EFI_EVENT.h b/system/lib/llvm-libc/include/llvm-libc-types/EFI_EVENT.h new file mode 100644 index 0000000000000..938856b8e791e --- /dev/null +++ b/system/lib/llvm-libc/include/llvm-libc-types/EFI_EVENT.h @@ -0,0 +1,21 @@ +//===-- Definition of EFI_EVENT type --------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_TYPES_EFI_EVENT_H +#define LLVM_LIBC_TYPES_EFI_EVENT_H + +typedef void *EFI_EVENT; + +#define EVT_TIMER 0x80000000 +#define EVT_RUNTIME 0x40000000 +#define EVT_NOTIFY_WAIT 0x00000100 +#define EVT_NOTIFY_SIGNAL 0x00000200 +#define EVT_SIGNAL_EXIT_BOOT_SERVICES 0x00000201 +#define EVT_SIGNAL_VIRTUAL_ADDRESS_CHANGE 0x60000202 + +#endif // LLVM_LIBC_TYPES_EFI_EVENT_H diff --git a/system/lib/llvm-libc/include/llvm-libc-types/EFI_GUID.h b/system/lib/llvm-libc/include/llvm-libc-types/EFI_GUID.h new file mode 100644 index 0000000000000..b3530008384dd --- /dev/null +++ b/system/lib/llvm-libc/include/llvm-libc-types/EFI_GUID.h @@ -0,0 +1,21 @@ +//===-- Definition of EFI_GUID type -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_TYPES_EFI_GUID_H +#define LLVM_LIBC_TYPES_EFI_GUID_H + +#include "../llvm-libc-macros/stdint-macros.h" + +typedef struct { + uint32_t Data1; + uint16_t Data2; + uint16_t Data3; + uint8_t Data4[8]; +} EFI_GUID; + +#endif // LLVM_LIBC_TYPES_EFI_GUID_H diff --git a/system/lib/llvm-libc/include/llvm-libc-types/EFI_HANDLE.h b/system/lib/llvm-libc/include/llvm-libc-types/EFI_HANDLE.h new file mode 100644 index 0000000000000..d4376dd247533 --- /dev/null +++ b/system/lib/llvm-libc/include/llvm-libc-types/EFI_HANDLE.h @@ -0,0 +1,14 @@ +//===-- Definition of EFI_HANDLE type ---------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_TYPES_EFI_HANDLE_H +#define LLVM_LIBC_TYPES_EFI_HANDLE_H + +typedef void *EFI_HANDLE; + +#endif // LLVM_LIBC_TYPES_EFI_HANDLE_H diff --git a/system/lib/llvm-libc/include/llvm-libc-types/__lsearchcompare_t.h b/system/lib/llvm-libc/include/llvm-libc-types/EFI_INTERFACE_TYPE.h similarity index 53% rename from system/lib/llvm-libc/include/llvm-libc-types/__lsearchcompare_t.h rename to system/lib/llvm-libc/include/llvm-libc-types/EFI_INTERFACE_TYPE.h index 08dc2db274d0c..d463c5381b3f0 100644 --- a/system/lib/llvm-libc/include/llvm-libc-types/__lsearchcompare_t.h +++ b/system/lib/llvm-libc/include/llvm-libc-types/EFI_INTERFACE_TYPE.h @@ -1,4 +1,4 @@ -//===-- Definition of type __lsearchcompare_t -----------------------------===// +//===-- Definition of EFI_INTERFACE_TYPE type -----------------------------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -6,9 +6,11 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_LIBC_TYPES___LSEARCHCOMPARE_T_H -#define LLVM_LIBC_TYPES___LSEARCHCOMPARE_T_H +#ifndef LLVM_LIBC_TYPES_EFI_INTERFACE_TYPE_H +#define LLVM_LIBC_TYPES_EFI_INTERFACE_TYPE_H -typedef int (*__lsearchcompare_t)(const void *, const void *); +typedef enum { + EFI_NATIVE_INTERFACE, +} EFI_INTERFACE_TYPE; -#endif // LLVM_LIBC_TYPES___LSEARCHCOMPARE_T_H +#endif // LLVM_LIBC_TYPES_EFI_INTERFACE_TYPE_H diff --git a/system/lib/llvm-libc/include/llvm-libc-types/EFI_LOCATE_SEARCH_TYPE.h b/system/lib/llvm-libc/include/llvm-libc-types/EFI_LOCATE_SEARCH_TYPE.h new file mode 100644 index 0000000000000..3a8fd7bc3e776 --- /dev/null +++ b/system/lib/llvm-libc/include/llvm-libc-types/EFI_LOCATE_SEARCH_TYPE.h @@ -0,0 +1,18 @@ +//===-- Definition of EFI_LOCATE_SEARCH_TYPE type -------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_TYPES_EFI_LOCATE_SEARCH_TYPE_H +#define LLVM_LIBC_TYPES_EFI_LOCATE_SEARCH_TYPE_H + +typedef enum { + AllHandles, + ByRegisterNotify, + ByProtocol, +} EFI_LOCATE_SEARCH_TYPE; + +#endif // LLVM_LIBC_TYPES_EFI_LOCATE_SEARCH_TYPE_H diff --git a/system/lib/llvm-libc/include/llvm-libc-types/EFI_MEMORY_DESCRIPTOR.h b/system/lib/llvm-libc/include/llvm-libc-types/EFI_MEMORY_DESCRIPTOR.h new file mode 100644 index 0000000000000..72d0579aef76c --- /dev/null +++ b/system/lib/llvm-libc/include/llvm-libc-types/EFI_MEMORY_DESCRIPTOR.h @@ -0,0 +1,43 @@ +//===-- Definition of EFI_MEMORY_DESCRIPTOR type --------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_TYPES_EFI_MEMORY_DESCRIPTOR_H +#define LLVM_LIBC_TYPES_EFI_MEMORY_DESCRIPTOR_H + +#include "../llvm-libc-macros/stdint-macros.h" +#include "EFI_PHYSICAL_ADDRESS.h" +#include "EFI_VIRTUAL_ADDRESS.h" + +#define EFI_MEMORY_DESCRIPTOR_VERSION 1 + +#define EFI_MEMORY_UC 0x0000000000000001 +#define EFI_MEMORY_WC 0x0000000000000002 +#define EFI_MEMORY_WT 0x0000000000000004 +#define EFI_MEMORY_WB 0x0000000000000008 +#define EFI_MEMORY_UCE 0x0000000000000010 +#define EFI_MEMORY_WP 0x0000000000001000 +#define EFI_MEMORY_RP 0x0000000000002000 +#define EFI_MEMORY_XP 0x0000000000004000 +#define EFI_MEMORY_NV 0x0000000000008000 +#define EFI_MEMORY_MORE_RELIABLE 0x0000000000010000 +#define EFI_MEMORY_RO 0x0000000000020000 +#define EFI_MEMORY_SP 0x0000000000040000 +#define EFI_MEMORY_CPU_CRYPTO 0x0000000000080000 +#define EFI_MEMORY_RUNTIME 0x8000000000000000 +#define EFI_MEMORY_ISA_VALID 0x4000000000000000 +#define EFI_MEMORY_ISA_MASK 0x0FFFF00000000000 + +typedef struct { + uint32_t Type; + EFI_PHYSICAL_ADDRESS PhysicalStart; + EFI_VIRTUAL_ADDRESS VirtualStart; + uint64_t NumberOfPages; + uint64_t Attribute; +} EFI_MEMORY_DESCRIPTOR; + +#endif // LLVM_LIBC_TYPES_EFI_MEMORY_DESCRIPTOR_H diff --git a/system/lib/llvm-libc/include/llvm-libc-types/EFI_MEMORY_TYPE.h b/system/lib/llvm-libc/include/llvm-libc-types/EFI_MEMORY_TYPE.h new file mode 100644 index 0000000000000..c8921cda2c388 --- /dev/null +++ b/system/lib/llvm-libc/include/llvm-libc-types/EFI_MEMORY_TYPE.h @@ -0,0 +1,32 @@ +//===-- Definition of EFI_MEMORY_TYPE type --------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_TYPES_EFI_MEMORY_TYPE_H +#define LLVM_LIBC_TYPES_EFI_MEMORY_TYPE_H + +typedef enum { + EfiReservedMemoryType, + EfiLoaderCode, + EfiLoaderData, + EfiBootServicesCode, + EfiBootServicesData, + EfiRuntimeServicesCode, + EfiRuntimeServicesData, + EfiConventionalMemory, + EfiUnusableMemory, + EfiACPIReclaimMemory, + EfiACPIMemoryNVS, + EfiMemoryMappedIO, + EfiMemoryMappedIOPortSpace, + EfiPalCode, + EfiPersistentMemory, + EfiUnacceptedMemoryType, + EfiMaxMemoryType +} EFI_MEMORY_TYPE; + +#endif // LLVM_LIBC_TYPES_EFI_MEMORY_TYPE_H diff --git a/system/lib/llvm-libc/include/llvm-libc-types/EFI_OPEN_PROTOCOL_INFORMATION_ENTRY.h b/system/lib/llvm-libc/include/llvm-libc-types/EFI_OPEN_PROTOCOL_INFORMATION_ENTRY.h new file mode 100644 index 0000000000000..de0c59c139efb --- /dev/null +++ b/system/lib/llvm-libc/include/llvm-libc-types/EFI_OPEN_PROTOCOL_INFORMATION_ENTRY.h @@ -0,0 +1,22 @@ +//===-- Definition of EFI_OPEN_PROTOCOL_INFORMATION_ENTRY type ------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_TYPES_EFI_OPEN_PROTOCOL_INFORMATION_ENTRY_H +#define LLVM_LIBC_TYPES_EFI_OPEN_PROTOCOL_INFORMATION_ENTRY_H + +#include "../llvm-libc-macros/stdint-macros.h" +#include "EFI_HANDLE.h" + +typedef struct { + EFI_HANDLE AgentHandle; + EFI_HANDLE ControllerHandle; + uint32_t Attributes; + uint32_t OpenCount; +} EFI_OPEN_PROTOCOL_INFORMATION_ENTRY; + +#endif // LLVM_LIBC_TYPES_EFI_OPEN_PROTOCOL_INFORMATION_ENTRY_H diff --git a/system/lib/llvm-libc/include/llvm-libc-types/EFI_PHYSICAL_ADDRESS.h b/system/lib/llvm-libc/include/llvm-libc-types/EFI_PHYSICAL_ADDRESS.h new file mode 100644 index 0000000000000..8880ee66c0f8d --- /dev/null +++ b/system/lib/llvm-libc/include/llvm-libc-types/EFI_PHYSICAL_ADDRESS.h @@ -0,0 +1,16 @@ +//===-- Definition of EFI_PHYSICAL_ADDRESS type ---------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_TYPES_EFI_PHYSICAL_ADDRESS_H +#define LLVM_LIBC_TYPES_EFI_PHYSICAL_ADDRESS_H + +#include "../llvm-libc-macros/stdint-macros.h" + +typedef uint64_t EFI_PHYSICAL_ADDRESS; + +#endif // LLVM_LIBC_TYPES_EFI_PHYSICAL_ADDRESS_H diff --git a/system/lib/llvm-libc/include/llvm-libc-types/EFI_RUNTIME_SERVICES.h b/system/lib/llvm-libc/include/llvm-libc-types/EFI_RUNTIME_SERVICES.h new file mode 100644 index 0000000000000..8913118b0844c --- /dev/null +++ b/system/lib/llvm-libc/include/llvm-libc-types/EFI_RUNTIME_SERVICES.h @@ -0,0 +1,137 @@ +//===-- Definition of EFI_RUNTIME_SERVICES type ---------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_TYPES_EFI_RUNTIME_SERVICES_H +#define LLVM_LIBC_TYPES_EFI_RUNTIME_SERVICES_H + +#include "../llvm-libc-macros/EFIAPI-macros.h" +#include "../llvm-libc-macros/stdint-macros.h" +#include "EFI_CAPSULE.h" +#include "EFI_MEMORY_DESCRIPTOR.h" +#include "EFI_PHYSICAL_ADDRESS.h" +#include "EFI_STATUS.h" +#include "EFI_TABLE_HEADER.h" +#include "EFI_TIME.h" +#include "char16_t.h" +#include "size_t.h" + +#define EFI_RUNTIME_SERVICES_SIGNATURE 0x56524553544e5552 +#define EFI_RUNTIME_SERVICES_REVISION EFI_SPECIFICATION_VERSION + +#define EFI_VARIABLE_NON_VOLATILE 0x00000001 +#define EFI_VARIABLE_BOOTSERVICE_ACCESS 0x00000002 +#define EFI_VARIABLE_RUNTIME_ACCESS 0x00000004 +#define EFI_VARIABLE_HARDWARE_ERROR_RECORD 0x00000008 +// This attribute is identified by the mnemonic 'HR' elsewhere +// in this specification. +#define EFI_VARIABLE_AUTHENTICATED_WRITE_ACCESS 0x00000010 +// NOTE: EFI_VARIABLE_AUTHENTICATED_WRITE_ACCESS is deprecated +// and should be considered reserved. +#define EFI_VARIABLE_TIME_BASED_AUTHENTICATED_WRITE_ACCESS 0x00000020 +#define EFI_VARIABLE_APPEND_WRITE 0x00000040 +#define EFI_VARIABLE_ENHANCED_AUTHENTICATED_ACCESS 0x00000080 + +typedef enum { + EfiResetCold, + EfiResetWarm, + EfiResetShutdown, + EfiResetPlatformSpecific, +} EFI_RESET_TYPE; + +#define EFI_VARIABLE_AUTHENTICATION_3_CERT_ID_SHA256 1 + +typedef struct { + uint8_t Type; + uint32_t IdSize; + // Value is defined as: + // uint8_t Id[IdSize]; +} EFI_VARIABLE_AUTHENTICATION_3_CERT_ID; + +typedef EFI_STATUS(EFIAPI *EFI_GET_TIME)(EFI_TIME *Time, + EFI_TIME_CAPABILITIES *Capabilities); +typedef EFI_STATUS(EFIAPI *EFI_SET_TIME)(EFI_TIME *Time); +typedef EFI_STATUS(EFIAPI *EFI_GET_WAKEUP_TIME)(bool *Enabled, bool *Pending, + EFI_TIME *Time); +typedef EFI_STATUS(EFIAPI *EFI_SET_WAKEUP_TIME)(bool *Enabled, EFI_TIME *Time); + +typedef EFI_STATUS(EFIAPI *EFI_SET_VIRTUAL_ADDRESS_MAP)( + size_t MemoryMapSize, size_t DescriptorSize, uint32_t DescriptorVersion, + EFI_MEMORY_DESCRIPTOR *VirtualMap); +typedef EFI_STATUS(EFIAPI *EFI_CONVERT_POINTER)(size_t DebugDisposition, + void **Address); + +typedef EFI_STATUS(EFIAPI *EFI_GET_VARIABLE)(char16_t *VariableName, + EFI_GUID *VendorGuid, + uint32_t *Attributes, + size_t *DataSize, void *Data); +typedef EFI_STATUS(EFIAPI *EFI_GET_NEXT_VARIABLE_NAME)(size_t *VariableNameSize, + char16_t *VariableName, + EFI_GUID *VendorGuid); +typedef EFI_STATUS(EFIAPI *EFI_SET_VARIABLE)(char16_t *VariableName, + EFI_GUID *VendorGuid, + uint32_t Attributes, + size_t DataSize, void *Data); + +typedef EFI_STATUS(EFIAPI *EFI_GET_NEXT_HIGH_MONO_COUNT)(uint32_t *HighCount); +typedef void(EFIAPI *EFI_RESET_SYSTEM)(EFI_RESET_TYPE ResetType, + EFI_STATUS ResetStatus, size_t DataSize, + void *ResetData); + +typedef EFI_STATUS(EFIAPI *EFI_UPDATE_CAPSULE)( + EFI_CAPSULE_HEADER **CapsuleHeaderArray, size_t CapsuleCount, + EFI_PHYSICAL_ADDRESS ScatterGatherList); +typedef EFI_STATUS(EFIAPI *EFI_QUERY_CAPSULE_CAPABILITIES)( + EFI_CAPSULE_HEADER **CapsuleHeaderArray, size_t CapsuleCount, + uint64_t *MaximumCapsuleSize, EFI_RESET_TYPE ResetType); + +typedef EFI_STATUS(EFIAPI *EFI_QUERY_VARIABLE_INFO)( + uint32_t Attributes, uint64_t *MaximumVariableStorageSize, + uint64_t *RemainingVariableStorageSize, uint64_t *MaximumVariableSize); + +typedef struct { + EFI_TABLE_HEADER Hdr; + + /// + /// Time Services + EFI_GET_TIME GetTime; + EFI_SET_TIME SetTime; + EFI_GET_WAKEUP_TIME GetWakeupTime; + EFI_SET_WAKEUP_TIME SetWakeupTime; + + // + // Virtual Memory Services + // + EFI_SET_VIRTUAL_ADDRESS_MAP SetVirtualAddressMap; + EFI_CONVERT_POINTER ConvertPointer; + + // + // Variable Services + // + EFI_GET_VARIABLE GetVariable; + EFI_GET_NEXT_VARIABLE_NAME GetNextVariableName; + EFI_SET_VARIABLE SetVariable; + + // + // Miscellaneous Services + // + EFI_GET_NEXT_HIGH_MONO_COUNT GetNextHighMonotonicCount; + EFI_RESET_SYSTEM ResetSystem; + + // + // UEFI 2.0 Capsule Services + // + EFI_UPDATE_CAPSULE UpdateCapsule; + EFI_QUERY_CAPSULE_CAPABILITIES QueryCapsuleCapabilities; + + // + // Miscellaneous UEFI 2.0 Service + // + EFI_QUERY_VARIABLE_INFO QueryVariableInfo; +} EFI_RUNTIME_SERVICES; + +#endif // LLVM_LIBC_TYPES_EFI_RUNTIME_SERVICES_H diff --git a/system/lib/llvm-libc/include/llvm-libc-types/EFI_SIMPLE_TEXT_INPUT_PROTOCOL.h b/system/lib/llvm-libc/include/llvm-libc-types/EFI_SIMPLE_TEXT_INPUT_PROTOCOL.h new file mode 100644 index 0000000000000..a6dc0952b6310 --- /dev/null +++ b/system/lib/llvm-libc/include/llvm-libc-types/EFI_SIMPLE_TEXT_INPUT_PROTOCOL.h @@ -0,0 +1,39 @@ +//===-- Definition of EFI_SIMPLE_TEXT_INPUT_PROTOCOL type -----------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_TYPES_EFI_SIMPLE_TEXT_INPUT_PROTOCOL_H +#define LLVM_LIBC_TYPES_EFI_SIMPLE_TEXT_INPUT_PROTOCOL_H + +#include "../llvm-libc-macros/EFIAPI-macros.h" +#include "../llvm-libc-macros/stdint-macros.h" +#include "EFI_EVENT.h" +#include "EFI_STATUS.h" +#include "char16_t.h" + +#define EFI_SIMPLE_TEXT_INPUT_PROTOCOL_GUID \ + {0x387477c1, 0x69c7, 0x11d2, {0x8e, 0x39, 0x00, 0xa0, 0xc9, 0x69, 0x72, 0x3b}} + +typedef struct { + uint16_t ScanCode; + char16_t UnicodeChar; +} EFI_INPUT_KEY; + +struct _EFI_SIMPLE_TEXT_INPUT_PROTOCOL; + +typedef EFI_STATUS(EFIAPI *EFI_INPUT_RESET)( + struct _EFI_SIMPLE_TEXT_INPUT_PROTOCOL *This, bool ExtendedVerification); +typedef EFI_STATUS(EFIAPI *EFI_INPUT_READ_KEY)( + struct _EFI_SIMPLE_TEXT_INPUT_PROTOCOL *This, EFI_INPUT_KEY *Key); + +typedef struct _EFI_SIMPLE_TEXT_INPUT_PROTOCOL { + EFI_INPUT_RESET Reset; + EFI_INPUT_READ_KEY ReadKeyStroke; + EFI_EVENT WaitForKey; +} EFI_SIMPLE_TEXT_INPUT_PROTOCOL; + +#endif // LLVM_LIBC_TYPES_EFI_SIMPLE_TEXT_INPUT_PROTOCOL_H diff --git a/system/lib/llvm-libc/include/llvm-libc-types/EFI_SIMPLE_TEXT_OUTPUT_PROTOCOL.h b/system/lib/llvm-libc/include/llvm-libc-types/EFI_SIMPLE_TEXT_OUTPUT_PROTOCOL.h new file mode 100644 index 0000000000000..b5014c46a0722 --- /dev/null +++ b/system/lib/llvm-libc/include/llvm-libc-types/EFI_SIMPLE_TEXT_OUTPUT_PROTOCOL.h @@ -0,0 +1,64 @@ +//===-- Definition of EFI_SIMPLE_TEXT_OUTPUT_PROTOCOL type ----------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_TYPES_EFI_SIMPLE_TEXT_OUTPUT_PROTOCOL_H +#define LLVM_LIBC_TYPES_EFI_SIMPLE_TEXT_OUTPUT_PROTOCOL_H + +#include "../llvm-libc-macros/stdint-macros.h" +#include "EFI_STATUS.h" +#include "size_t.h" + +#define EFI_SIMPLE_TEXT_OUTPUT_PROTOCOL_GUID \ + {0x387477c2, 0x69c7, 0x11d2, {0x8e, 0x39, 0x00, 0xa0, 0xc9, 0x69, 0x72, 0x3b}} + +struct _EFI_SIMPLE_TEXT_OUTPUT_PROTOCOL; + +typedef EFI_STATUS(EFIAPI *EFI_TEXT_RESET)( + struct _EFI_SIMPLE_TEXT_OUTPUT_PROTOCOL *This, bool ExtendedVerification); +typedef EFI_STATUS(EFIAPI *EFI_TEXT_STRING)( + struct _EFI_SIMPLE_TEXT_OUTPUT_PROTOCOL *This, const char16_t *String); +typedef EFI_STATUS(EFIAPI *EFI_TEXT_TEST_STRING)( + struct _EFI_SIMPLE_TEXT_OUTPUT_PROTOCOL *This, const char16_t *String); +typedef EFI_STATUS(EFIAPI *EFI_TEXT_QUERY_MODE)( + struct _EFI_SIMPLE_TEXT_OUTPUT_PROTOCOL *This, size_t ModeNumber, + size_t *Columns, size_t *Rows); + +typedef EFI_STATUS(EFIAPI *EFI_TEXT_SET_MODE)( + struct _EFI_SIMPLE_TEXT_OUTPUT_PROTOCOL *This, size_t ModeNumber); +typedef EFI_STATUS(EFIAPI *EFI_TEXT_SET_ATTRIBUTE)( + struct _EFI_SIMPLE_TEXT_OUTPUT_PROTOCOL *This, size_t Attribute); +typedef EFI_STATUS(EFIAPI *EFI_TEXT_CLEAR_SCREEN)( + struct _EFI_SIMPLE_TEXT_OUTPUT_PROTOCOL *This); +typedef EFI_STATUS(EFIAPI *EFI_TEXT_SET_CURSOR_POSITION)( + struct _EFI_SIMPLE_TEXT_OUTPUT_PROTOCOL *This, size_t Column, size_t Row); +typedef EFI_STATUS(EFIAPI *EFI_TEXT_ENABLE_CURSOR)( + struct _EFI_SIMPLE_TEXT_OUTPUT_PROTOCOL *This, bool Visible); + +typedef struct { + int32_t MaxMode; + int32_t Mode; + int32_t Attribute; + int32_t CursorColumn; + int32_t CursorRow; + bool CursorVisible; +} SIMPLE_TEXT_OUTPUT_MODE; + +typedef struct _EFI_SIMPLE_TEXT_OUTPUT_PROTOCOL { + EFI_TEXT_RESET Reset; + EFI_TEXT_STRING OutputString; + EFI_TEXT_TEST_STRING TestString; + EFI_TEXT_QUERY_MODE QueryMode; + EFI_TEXT_SET_MODE SetMode; + EFI_TEXT_SET_ATTRIBUTE SetAttribute; + EFI_TEXT_CLEAR_SCREEN ClearScreen; + EFI_TEXT_SET_CURSOR_POSITION SetCursorPosition; + EFI_TEXT_ENABLE_CURSOR EnableCursor; + SIMPLE_TEXT_OUTPUT_MODE *Mode; +} EFI_SIMPLE_TEXT_OUTPUT_PROTOCOL; + +#endif // LLVM_LIBC_TYPES_EFI_SIMPLE_TEXT_OUTPUT_PROTOCOL_H diff --git a/system/lib/llvm-libc/include/llvm-libc-types/EFI_STATUS.h b/system/lib/llvm-libc/include/llvm-libc-types/EFI_STATUS.h new file mode 100644 index 0000000000000..bb9542bed8efc --- /dev/null +++ b/system/lib/llvm-libc/include/llvm-libc-types/EFI_STATUS.h @@ -0,0 +1,60 @@ +//===-- Definition of EFI_STATUS type ---------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_TYPES_EFI_STATUS_H +#define LLVM_LIBC_TYPES_EFI_STATUS_H + +#include "../llvm-libc-macros/stdint-macros.h" + +typedef uintptr_t EFI_STATUS; + +#define EFI_SUCCESS 0 + +#define EFI_LOAD_ERROR 1 +#define EFI_INVALID_PARAMETER 2 +#define EFI_UNSUPPORTED 3 +#define EFI_BAD_BUFFER_SIZE 4 +#define EFI_BUFFER_TOO_SMALL 5 +#define EFI_NOT_READY 6 +#define EFI_DEVICE_ERROR 7 +#define EFI_WRITE_PROTECTED 8 +#define EFI_OUT_OF_RESOURCES 9 +#define EFI_VOLUME_CORRUPTED 10 +#define EFI_VOLUME_FULL 11 +#define EFI_NO_MEDIA 12 +#define EFI_MEDIA_CHANGED 13 +#define EFI_NOT_FOUND 14 +#define EFI_ACCESS_DENIED 15 +#define EFI_NO_RESPONSE 16 +#define EFI_NO_MAPPING 17 +#define EFI_TIMEOUT 18 +#define EFI_NOT_STARTED 19 +#define EFI_ALREADY_STARTED 20 +#define EFI_ABORTED 21 +#define EFI_ICMP_ERROR 22 +#define EFI_TFTP_ERROR 23 +#define EFI_PROTOCOL_ERROR 24 +#define EFI_INCOMPATIBLE_VERSION 25 +#define EFI_SECURITY_VIOLATION 26 +#define EFI_CRC_ERROR 27 +#define EFI_END_OF_MEDIA 28 +#define EFI_END_OF_FILE 31 +#define EFI_INVALID_LANGUAGE 32 +#define EFI_COMPROMISED_DATA 33 +#define EFI_IP_ADDRESS_CONFLICT 34 +#define EFI_HTTP_ERROR 35 + +#define EFI_WARN_UNKNOWN_GLYPH 1 +#define EFI_WARN_DELETE_FAILURE 2 +#define EFI_WARN_WRITE_FAILURE 3 +#define EFI_WARN_BUFFER_TOO_SMALL 4 +#define EFI_WARN_STALE_DATA 5 +#define EFI_WARN_FILE_SYSTEM 6 +#define EFI_WARN_RESET_REQUIRED 7 + +#endif // LLVM_LIBC_TYPES_EFI_STATUS_H diff --git a/system/lib/llvm-libc/include/llvm-libc-types/EFI_SYSTEM_TABLE.h b/system/lib/llvm-libc/include/llvm-libc-types/EFI_SYSTEM_TABLE.h new file mode 100644 index 0000000000000..290067ad862e1 --- /dev/null +++ b/system/lib/llvm-libc/include/llvm-libc-types/EFI_SYSTEM_TABLE.h @@ -0,0 +1,65 @@ +//===-- Definition of EFI_SYSTEM_TABLE type -------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===---------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_TYPES_EFI_SYSTEM_TABLE_H +#define LLVM_LIBC_TYPES_EFI_SYSTEM_TABLE_H + +#include "../llvm-libc-macros/stdint-macros.h" +#include "EFI_BOOT_SERVICES.h" +#include "EFI_CONFIGURATION_TABLE.h" +#include "EFI_HANDLE.h" +#include "EFI_RUNTIME_SERVICES.h" +#include "EFI_SIMPLE_TEXT_INPUT_PROTOCOL.h" +#include "EFI_SIMPLE_TEXT_OUTPUT_PROTOCOL.h" +#include "EFI_STATUS.h" +#include "EFI_TABLE_HEADER.h" + +#include "char16_t.h" +#include "size_t.h" + +#define EFI_SYSTEM_TABLE_SIGNATURE 0x5453595320494249 +#define EFI_2_100_SYSTEM_TABLE_REVISION ((2 << 16) | (100)) +#define EFI_2_90_SYSTEM_TABLE_REVISION ((2 << 16) | (90)) +#define EFI_2_80_SYSTEM_TABLE_REVISION ((2 << 16) | (80)) +#define EFI_2_70_SYSTEM_TABLE_REVISION ((2 << 16) | (70)) +#define EFI_2_60_SYSTEM_TABLE_REVISION ((2 << 16) | (60)) +#define EFI_2_50_SYSTEM_TABLE_REVISION ((2 << 16) | (50)) +#define EFI_2_40_SYSTEM_TABLE_REVISION ((2 << 16) | (40)) +#define EFI_2_31_SYSTEM_TABLE_REVISION ((2 << 16) | (31)) +#define EFI_2_30_SYSTEM_TABLE_REVISION ((2 << 16) | (30)) +#define EFI_2_20_SYSTEM_TABLE_REVISION ((2 << 16) | (20)) +#define EFI_2_10_SYSTEM_TABLE_REVISION ((2 << 16) | (10)) +#define EFI_2_00_SYSTEM_TABLE_REVISION ((2 << 16) | (00)) +#define EFI_1_10_SYSTEM_TABLE_REVISION ((1 << 16) | (10)) +#define EFI_1_02_SYSTEM_TABLE_REVISION ((1 << 16) | (02)) +#define EFI_SPECIFICATION_VERSION EFI_SYSTEM_TABLE_REVISION +#define EFI_SYSTEM_TABLE_REVISION EFI_2_100_SYSTEM_TABLE_REVISION + +typedef struct { + EFI_TABLE_HEADER Hdr; + + char16_t *FirmwareVendor; + uint32_t FirmwareRevision; + + EFI_HANDLE ConsoleInHandle; + EFI_SIMPLE_TEXT_INPUT_PROTOCOL *ConIn; + + EFI_HANDLE ConsoleOutHandle; + EFI_SIMPLE_TEXT_OUTPUT_PROTOCOL *ConOut; + + EFI_HANDLE StandardErrorHandle; + EFI_SIMPLE_TEXT_OUTPUT_PROTOCOL *StdErr; + + EFI_RUNTIME_SERVICES *RuntimeServices; + EFI_BOOT_SERVICES *BootServices; + + size_t NumberOfTableEntries; + EFI_CONFIGURATION_TABLE *ConfigurationTable; +} EFI_SYSTEM_TABLE; + +#endif // LLVM_LIBC_TYPES_EFI_SYSTEM_TABLE_H diff --git a/system/lib/llvm-libc/include/llvm-libc-types/EFI_TABLE_HEADER.h b/system/lib/llvm-libc/include/llvm-libc-types/EFI_TABLE_HEADER.h new file mode 100644 index 0000000000000..293968ecc4d1b --- /dev/null +++ b/system/lib/llvm-libc/include/llvm-libc-types/EFI_TABLE_HEADER.h @@ -0,0 +1,22 @@ +//===-- Definition of EFI_TABLE_HEADER type -------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===---------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_TYPES_EFI_TABLE_HEADER_H +#define LLVM_LIBC_TYPES_EFI_TABLE_HEADER_H + +#include "../llvm-libc-macros/stdint-macros.h" + +typedef struct { + uint64_t Signature; + uint32_t Revision; + uint32_t HeaderSize; + uint32_t CRC32; + uint32_t Reserved; +} EFI_TABLE_HEADER; + +#endif // LLVM_LIBC_TYPES_EFI_TABLE_HEADER_H diff --git a/system/lib/llvm-libc/include/llvm-libc-types/EFI_TIME.h b/system/lib/llvm-libc/include/llvm-libc-types/EFI_TIME.h new file mode 100644 index 0000000000000..b0e38b987d44e --- /dev/null +++ b/system/lib/llvm-libc/include/llvm-libc-types/EFI_TIME.h @@ -0,0 +1,37 @@ +//===-- Definition of EFI_TIME type ---------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_TYPES_EFI_TIME_H +#define LLVM_LIBC_TYPES_EFI_TIME_H + +#include "../llvm-libc-macros/stdint-macros.h" + +typedef struct { + uint16_t Year; // 1900 - 9999 + uint8_t Month; // 1 - 12 + uint8_t Day; // 1 - 31 + uint8_t Hour; // 0 - 23 + uint8_t Minute; // 0 - 59 + uint8_t Second; // 0 - 59 + uint8_t Pad1; + uint32_t Nanosecond; // 0 - 999,999,999 + int16_t TimeZone; // --1440 to 1440 or 2047 +} EFI_TIME; + +#define EFI_TIME_ADJUST_DAYLIGHT 0x01 +#define EFI_TIME_IN_DAYLIGHT 0x02 + +#define EFI_UNSPECIFIED_TIMEZONE 0x07FF + +typedef struct { + uint32_t Resolution; + uint32_t Accuracy; + bool SetsToZero; +} EFI_TIME_CAPABILITIES; + +#endif // LLVM_LIBC_TYPES_EFI_TIME_H diff --git a/system/lib/llvm-libc/include/llvm-libc-types/EFI_TIMER_DELAY.h b/system/lib/llvm-libc/include/llvm-libc-types/EFI_TIMER_DELAY.h new file mode 100644 index 0000000000000..2a6872c69c8b3 --- /dev/null +++ b/system/lib/llvm-libc/include/llvm-libc-types/EFI_TIMER_DELAY.h @@ -0,0 +1,18 @@ +//===-- Definition of EFI_TIMER_DELAY type --------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_TYPES_EFI_TIMER_DELAY_H +#define LLVM_LIBC_TYPES_EFI_TIMER_DELAY_H + +typedef enum { + TimerCancel, + TimerPeriodic, + TimerRelative, +} EFI_TIMER_DELAY; + +#endif // LLVM_LIBC_TYPES_EFI_TIMER_DELAY_H diff --git a/system/lib/llvm-libc/include/llvm-libc-types/EFI_TPL.h b/system/lib/llvm-libc/include/llvm-libc-types/EFI_TPL.h new file mode 100644 index 0000000000000..8361ccfacd6f5 --- /dev/null +++ b/system/lib/llvm-libc/include/llvm-libc-types/EFI_TPL.h @@ -0,0 +1,21 @@ +//===-- Definition of EFI_TPL type ----------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_TYPES_EFI_TPL_H +#define LLVM_LIBC_TYPES_EFI_TPL_H + +#include "size_t.h" + +typedef size_t EFI_TPL; + +#define TPL_APPLICATION 4 +#define TPL_CALLBACK 8 +#define TPL_NOTIFY 16 +#define TPL_HIGH_LEVEL 31 + +#endif // LLVM_LIBC_TYPES_EFI_TPL_H diff --git a/system/lib/llvm-libc/include/llvm-libc-types/EFI_VIRTUAL_ADDRESS.h b/system/lib/llvm-libc/include/llvm-libc-types/EFI_VIRTUAL_ADDRESS.h new file mode 100644 index 0000000000000..46cbec734dadc --- /dev/null +++ b/system/lib/llvm-libc/include/llvm-libc-types/EFI_VIRTUAL_ADDRESS.h @@ -0,0 +1,16 @@ +//===-- Definition of EFI_VIRTUAL_ADDRESS type ----------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_TYPES_EFI_VIRTUAL_ADDRESS_H +#define LLVM_LIBC_TYPES_EFI_VIRTUAL_ADDRESS_H + +#include "../llvm-libc-macros/stdint-macros.h" + +typedef uint64_t EFI_VIRTUAL_ADDRESS; + +#endif // LLVM_LIBC_TYPES_EFI_VIRTUAL_ADDRESS_H diff --git a/system/lib/llvm-libc/include/llvm-libc-types/ENTRY.h b/system/lib/llvm-libc/include/llvm-libc-types/ENTRY.h index ccbd777e2475b..b007c8edba2f5 100644 --- a/system/lib/llvm-libc/include/llvm-libc-types/ENTRY.h +++ b/system/lib/llvm-libc/include/llvm-libc-types/ENTRY.h @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_TYPES_ENTRY_H #define LLVM_LIBC_TYPES_ENTRY_H -typedef struct { +typedef struct entry { char *key; void *data; } ENTRY; diff --git a/system/lib/llvm-libc/include/llvm-libc-types/VISIT.h b/system/lib/llvm-libc/include/llvm-libc-types/VISIT.h new file mode 100644 index 0000000000000..0a7ff6e50d469 --- /dev/null +++ b/system/lib/llvm-libc/include/llvm-libc-types/VISIT.h @@ -0,0 +1,14 @@ +//===-- Definition of VISIT type ------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_TYPES_VISIT_H +#define LLVM_LIBC_TYPES_VISIT_H + +typedef enum { preorder, postorder, endorder, leaf } VISIT; + +#endif // LLVM_LIBC_TYPES_VISIT_H diff --git a/system/lib/llvm-libc/include/llvm-libc-types/__dl_iterate_phdr_callback_t.h b/system/lib/llvm-libc/include/llvm-libc-types/__dl_iterate_phdr_callback_t.h new file mode 100644 index 0000000000000..52078daf7d3ce --- /dev/null +++ b/system/lib/llvm-libc/include/llvm-libc-types/__dl_iterate_phdr_callback_t.h @@ -0,0 +1,19 @@ +//===-- Definition of type __dl_iterate_phdr_callback_t ------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===---------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_TYPES___DL_ITERATE_PHDR_CALLBACK_T_H +#define LLVM_LIBC_TYPES___DL_ITERATE_PHDR_CALLBACK_T_H + +#include "size_t.h" + +struct dl_phdr_info; + +typedef int (*__dl_iterate_phdr_callback_t)(struct dl_phdr_info *, size_t, + void *); + +#endif // LLVM_LIBC_TYPES___DL_ITERATE_PHDR_CALLBACK_T_H diff --git a/system/lib/llvm-libc/include/llvm-libc-types/__search_compare_t.h b/system/lib/llvm-libc/include/llvm-libc-types/__search_compare_t.h new file mode 100644 index 0000000000000..7033fef49b49a --- /dev/null +++ b/system/lib/llvm-libc/include/llvm-libc-types/__search_compare_t.h @@ -0,0 +1,14 @@ +//===-- Definition of type __search_compare_t -----------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_TYPES___SEARCH_COMPARE_T_H +#define LLVM_LIBC_TYPES___SEARCH_COMPARE_T_H + +typedef int (*__search_compare_t)(const void *, const void *); + +#endif // LLVM_LIBC_TYPES___SEARCH_COMPARE_T_H diff --git a/system/lib/llvm-libc/include/llvm-libc-types/char8_t.h b/system/lib/llvm-libc/include/llvm-libc-types/char8_t.h index ddadab1afa219..a343be77d810b 100644 --- a/system/lib/llvm-libc/include/llvm-libc-types/char8_t.h +++ b/system/lib/llvm-libc/include/llvm-libc-types/char8_t.h @@ -9,8 +9,7 @@ #ifndef LLVM_LIBC_TYPES_CHAR8_T_H #define LLVM_LIBC_TYPES_CHAR8_T_H -#if !defined(__cplusplus) && defined(__STDC_VERSION__) && \ - __STDC_VERSION__ >= 202311L +#if !(defined(__cplusplus) && defined(__cpp_char8_t)) typedef unsigned char char8_t; #endif diff --git a/system/lib/llvm-libc/include/llvm-libc-types/cpu_set_t.h b/system/lib/llvm-libc/include/llvm-libc-types/cpu_set_t.h index e7f52597e147e..8c6859de5f1d4 100644 --- a/system/lib/llvm-libc/include/llvm-libc-types/cpu_set_t.h +++ b/system/lib/llvm-libc/include/llvm-libc-types/cpu_set_t.h @@ -9,6 +9,9 @@ #ifndef LLVM_LIBC_TYPES_CPU_SET_T_H #define LLVM_LIBC_TYPES_CPU_SET_T_H +#define __CPU_SETSIZE 1024 +#define __NCPUBITS (8 * sizeof(unsigned long)) + typedef struct { // If a processor with more than 1024 CPUs is to be supported in future, // we need to adjust the size of this array. diff --git a/system/lib/llvm-libc/include/llvm-libc-types/jmp_buf.h b/system/lib/llvm-libc/include/llvm-libc-types/jmp_buf.h index f246e6491cf55..2fec35dbc9163 100644 --- a/system/lib/llvm-libc/include/llvm-libc-types/jmp_buf.h +++ b/system/lib/llvm-libc/include/llvm-libc-types/jmp_buf.h @@ -9,6 +9,19 @@ #ifndef LLVM_LIBC_TYPES_JMP_BUF_H #define LLVM_LIBC_TYPES_JMP_BUF_H +// TODO: implement sigjmp_buf related functions for other architectures +// Issue: https://github.com/llvm/llvm-project/issues/136358 +#if defined(__linux__) +#if defined(__i386__) || defined(__x86_64__) || defined(__aarch64__) || \ + defined(__riscv) +#define __LIBC_HAS_SIGJMP_BUF +#endif +#endif + +#if defined(__LIBC_HAS_SIGJMP_BUF) +#include "sigset_t.h" +#endif + typedef struct { #ifdef __x86_64__ __UINT64_TYPE__ rbx; @@ -50,8 +63,22 @@ typedef struct { #else #error "__jmp_buf not available for your target architecture." #endif +#if defined(__LIBC_HAS_SIGJMP_BUF) + // return address + void *sig_retaddr; + // extra register buffer to avoid indefinite stack growth in sigsetjmp + void *sig_extra; + // signal masks + sigset_t sigmask; +#endif } __jmp_buf; typedef __jmp_buf jmp_buf[1]; +#if defined(__LIBC_HAS_SIGJMP_BUF) +typedef __jmp_buf sigjmp_buf[1]; +#endif + +#undef __LIBC_HAS_SIGJMP_BUF + #endif // LLVM_LIBC_TYPES_JMP_BUF_H diff --git a/system/lib/llvm-libc/include/llvm-libc-types/nfds_t.h b/system/lib/llvm-libc/include/llvm-libc-types/nfds_t.h new file mode 100644 index 0000000000000..c0abccee8a7da --- /dev/null +++ b/system/lib/llvm-libc/include/llvm-libc-types/nfds_t.h @@ -0,0 +1,14 @@ +//===-- Definition of type nfds_t -----------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_TYPES_NFDS_T_H +#define LLVM_LIBC_TYPES_NFDS_T_H + +typedef unsigned int nfds_t; + +#endif // LLVM_LIBC_TYPES_NFDS_T_H diff --git a/system/lib/llvm-libc/include/llvm-libc-types/sighandler_t.h b/system/lib/llvm-libc/include/llvm-libc-types/sighandler_t.h new file mode 100644 index 0000000000000..f39ab04685200 --- /dev/null +++ b/system/lib/llvm-libc/include/llvm-libc-types/sighandler_t.h @@ -0,0 +1,17 @@ +//===-- Definition of sighandler_t ----------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_TYPES_SIGHANDLER_T_H +#define LLVM_LIBC_TYPES_SIGHANDLER_T_H + +#ifdef __linux__ +// For compatibility with glibc. +typedef void (*sighandler_t)(int); +#endif + +#endif // LLVM_LIBC_TYPES_SIGHANDLER_T_H diff --git a/system/lib/llvm-libc/include/llvm-libc-types/size_t.h b/system/lib/llvm-libc/include/llvm-libc-types/size_t.h index 3b31b0820f237..26ae68abe0ee7 100644 --- a/system/lib/llvm-libc/include/llvm-libc-types/size_t.h +++ b/system/lib/llvm-libc/include/llvm-libc-types/size_t.h @@ -9,11 +9,6 @@ #ifndef LLVM_LIBC_TYPES_SIZE_T_H #define LLVM_LIBC_TYPES_SIZE_T_H -// Since __need_size_t is defined, we get the definition of size_t from the -// standalone C header stddef.h. Also, because __need_size_t is defined, -// including stddef.h will pull only the type size_t and nothing else. -#define __need_size_t -#include -#undef __need_size_t +typedef __SIZE_TYPE__ size_t; #endif // LLVM_LIBC_TYPES_SIZE_T_H diff --git a/system/lib/llvm-libc/include/llvm-libc-types/ssize_t.h b/system/lib/llvm-libc/include/llvm-libc-types/ssize_t.h index 41e4b6d2c500a..8f579e2749bac 100644 --- a/system/lib/llvm-libc/include/llvm-libc-types/ssize_t.h +++ b/system/lib/llvm-libc/include/llvm-libc-types/ssize_t.h @@ -9,6 +9,6 @@ #ifndef LLVM_LIBC_TYPES_SSIZE_T_H #define LLVM_LIBC_TYPES_SSIZE_T_H -typedef __INT64_TYPE__ ssize_t; +typedef __PTRDIFF_TYPE__ ssize_t; #endif // LLVM_LIBC_TYPES_SSIZE_T_H diff --git a/system/lib/llvm-libc/include/llvm-libc-types/struct_dl_phdr_info.h b/system/lib/llvm-libc/include/llvm-libc-types/struct_dl_phdr_info.h new file mode 100644 index 0000000000000..2b9a5d21b4a13 --- /dev/null +++ b/system/lib/llvm-libc/include/llvm-libc-types/struct_dl_phdr_info.h @@ -0,0 +1,30 @@ +//===-- Definition of type struct dl_phdr_info ---------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===---------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_TYPES_STRUCT_DL_PHDR_INFO_H +#define LLVM_LIBC_TYPES_STRUCT_DL_PHDR_INFO_H + +#include "../llvm-libc-macros/link-macros.h" +#include "size_t.h" +#include +#include + +struct dl_phdr_info { + ElfW(Addr) dlpi_addr; + const char *dlpi_name; + const ElfW(Phdr) * dlpi_phdr; + ElfW(Half) dlpi_phnum; + + uint64_t dlpi_adds; + uint64_t dlpi_subs; + + size_t dlpi_tls_modid; + void *dlpi_tls_data; +}; + +#endif // LLVM_LIBC_TYPES_STRUCT_DL_PHDR_INFO_H diff --git a/system/lib/llvm-libc/include/llvm-libc-types/struct_itimerval.h b/system/lib/llvm-libc/include/llvm-libc-types/struct_itimerval.h new file mode 100644 index 0000000000000..e23f1e6076dfe --- /dev/null +++ b/system/lib/llvm-libc/include/llvm-libc-types/struct_itimerval.h @@ -0,0 +1,19 @@ +//===-- Definition of struct itimerval ------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_TYPES_STRUCT_ITIMERVAL_H +#define LLVM_LIBC_TYPES_STRUCT_ITIMERVAL_H + +#include "struct_timeval.h" + +struct itimerval { + struct timeval it_interval; /* Interval for periodic timer */ + struct timeval it_value; /* Time until next expiration */ +}; + +#endif // LLVM_LIBC_TYPES_STRUCT_ITIMERVAL_H diff --git a/system/lib/llvm-libc/include/llvm-libc-types/struct_pollfd.h b/system/lib/llvm-libc/include/llvm-libc-types/struct_pollfd.h new file mode 100644 index 0000000000000..80abc8e76efc6 --- /dev/null +++ b/system/lib/llvm-libc/include/llvm-libc-types/struct_pollfd.h @@ -0,0 +1,18 @@ +//===-- Definition of type struct pollfd ----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_TYPES_STRUCT_POLLFD_H +#define LLVM_LIBC_TYPES_STRUCT_POLLFD_H + +struct pollfd { + int fd; + short events; + short revents; +}; + +#endif // LLVM_LIBC_TYPES_STRUCT_POLLFD_H diff --git a/system/lib/llvm-libc/include/llvm-libc-types/struct_sigaction.h b/system/lib/llvm-libc/include/llvm-libc-types/struct_sigaction.h index b4d0c965a4c63..907418b5e0f9a 100644 --- a/system/lib/llvm-libc/include/llvm-libc-types/struct_sigaction.h +++ b/system/lib/llvm-libc/include/llvm-libc-types/struct_sigaction.h @@ -25,6 +25,4 @@ struct sigaction { #endif }; -typedef void (*__sighandler_t)(int); - #endif // LLVM_LIBC_TYPES_STRUCT_SIGACTION_H diff --git a/system/lib/llvm-libc/include/llvm-libc-types/struct_tm.h b/system/lib/llvm-libc/include/llvm-libc-types/struct_tm.h index 9fef7c5718ea4..2ec74ecac0293 100644 --- a/system/lib/llvm-libc/include/llvm-libc-types/struct_tm.h +++ b/system/lib/llvm-libc/include/llvm-libc-types/struct_tm.h @@ -19,6 +19,7 @@ struct tm { int tm_wday; // days since Sunday int tm_yday; // days since January int tm_isdst; // Daylight Saving Time flag + // TODO: add tm_gmtoff and tm_zone? (posix extensions) }; #endif // LLVM_LIBC_TYPES_STRUCT_TM_H diff --git a/system/lib/llvm-libc/include/llvm-libc-types/suseconds_t.h b/system/lib/llvm-libc/include/llvm-libc-types/suseconds_t.h index 32ecc9f537d00..8e926e8401f5c 100644 --- a/system/lib/llvm-libc/include/llvm-libc-types/suseconds_t.h +++ b/system/lib/llvm-libc/include/llvm-libc-types/suseconds_t.h @@ -9,6 +9,11 @@ #ifndef LLVM_LIBC_TYPES_SUSECONDS_T_H #define LLVM_LIBC_TYPES_SUSECONDS_T_H -typedef __INT32_TYPE__ suseconds_t; +// Per posix: suseconds_t shall be a signed integer type capable of storing +// values at least in the range [-1, 1000000]. [...] the widths of [other +// types...] and suseconds_t are no greater than the width of type long. + +// The kernel expects 64 bit suseconds_t at least on x86_64. +typedef long suseconds_t; #endif // LLVM_LIBC_TYPES_SUSECONDS_T_H diff --git a/system/lib/llvm-libc/patches/0001-emscripten-related-changes-for-llvm-libc.patch b/system/lib/llvm-libc/patches/0001-emscripten-related-changes-for-llvm-libc.patch new file mode 100644 index 0000000000000..dc1ab1fcb0dd5 --- /dev/null +++ b/system/lib/llvm-libc/patches/0001-emscripten-related-changes-for-llvm-libc.patch @@ -0,0 +1,77 @@ +From 71c098f061e74c5214087728b028da4ff9db9679 Mon Sep 17 00:00:00 2001 +From: google-yfyang +Date: Wed, 18 Jun 2025 13:57:16 -0400 +Subject: [PATCH] emscripten related changes for llvm-libc + +--- + libc/src/__support/StringUtil/platform_errors.h | 2 +- + libc/src/__support/StringUtil/tables/posix_errors.h | 5 +++++ + libc/src/__support/StringUtil/tables/stdc_errors.h | 5 +++++ + libc/src/__support/macros/properties/architectures.h | 4 ++++ + 4 files changed, 15 insertions(+), 1 deletion(-) + +diff --git a/libc/src/__support/StringUtil/platform_errors.h b/libc/src/__support/StringUtil/platform_errors.h +index 32e8414b3e3d..5f83865482e7 100644 +--- a/libc/src/__support/StringUtil/platform_errors.h ++++ b/libc/src/__support/StringUtil/platform_errors.h +@@ -9,7 +9,7 @@ + #ifndef LLVM_LIBC_SRC___SUPPORT_STRINGUTIL_PLATFORM_ERRORS_H + #define LLVM_LIBC_SRC___SUPPORT_STRINGUTIL_PLATFORM_ERRORS_H + +-#if defined(__linux__) || defined(__Fuchsia__) ++#if defined(__linux__) || defined(__Fuchsia__) || defined(__EMSCRIPTEN__) + #include "tables/linux_platform_errors.h" + #else + #include "tables/minimal_platform_errors.h" +diff --git a/libc/src/__support/StringUtil/tables/posix_errors.h b/libc/src/__support/StringUtil/tables/posix_errors.h +index b21f28f0b132..031886dcc5cc 100644 +--- a/libc/src/__support/StringUtil/tables/posix_errors.h ++++ b/libc/src/__support/StringUtil/tables/posix_errors.h +@@ -63,7 +63,12 @@ LIBC_INLINE_VAR constexpr MsgTable<76> POSIX_ERRORS = { + MsgMapping(EPROTO, "Protocol error"), + MsgMapping(EMULTIHOP, "Multihop attempted"), + MsgMapping(EBADMSG, "Bad message"), ++#ifdef __EMSCRIPTEN__ ++ // For now, match the musl string ++ MsgMapping(EOVERFLOW, "Value too large for data type"), ++#else + MsgMapping(EOVERFLOW, "Value too large for defined data type"), ++#endif + MsgMapping(ENOTSOCK, "Socket operation on non-socket"), + MsgMapping(EDESTADDRREQ, "Destination address required"), + MsgMapping(EMSGSIZE, "Message too long"), +diff --git a/libc/src/__support/StringUtil/tables/stdc_errors.h b/libc/src/__support/StringUtil/tables/stdc_errors.h +index a326616f20ef..9a23d0718ea1 100644 +--- a/libc/src/__support/StringUtil/tables/stdc_errors.h ++++ b/libc/src/__support/StringUtil/tables/stdc_errors.h +@@ -16,7 +16,12 @@ + namespace LIBC_NAMESPACE_DECL { + + LIBC_INLINE_VAR constexpr const MsgTable<4> STDC_ERRORS = { ++#ifdef __EMSCRIPTEN__ ++ // For now, match the musl name for errno 0. ++ MsgMapping(0, "No error information"), ++#else + MsgMapping(0, "Success"), ++#endif + MsgMapping(EDOM, "Numerical argument out of domain"), + MsgMapping(ERANGE, "Numerical result out of range"), + MsgMapping(EILSEQ, "Invalid or incomplete multibyte or wide character"), +diff --git a/libc/src/__support/macros/properties/architectures.h b/libc/src/__support/macros/properties/architectures.h +index c88956ff4114..817ced4e95c2 100644 +--- a/libc/src/__support/macros/properties/architectures.h ++++ b/libc/src/__support/macros/properties/architectures.h +@@ -41,6 +41,10 @@ + #define LIBC_TARGET_ARCH_IS_ARM + #endif + ++#if defined(__wasm__) ++#define LIBC_TARGET_ARCH_IS_WASM ++#endif ++ + #if defined(__aarch64__) || defined(__arm64__) || defined(_M_ARM64) + #define LIBC_TARGET_ARCH_IS_AARCH64 + #endif +-- +2.50.0.rc2.701.gf1e915cc24-goog + diff --git a/system/lib/llvm-libc/readme.txt b/system/lib/llvm-libc/readme.txt index 7d257d1255dec..ba85171546b46 100644 --- a/system/lib/llvm-libc/readme.txt +++ b/system/lib/llvm-libc/readme.txt @@ -1,7 +1,7 @@ llvm's libc ----------- -These files are from llvm-project based on release 20.1.4. +These files are from llvm-project's HEAD. We maintain a local fork of llvm-project that contains any emscripten specific patches: @@ -10,18 +10,23 @@ specific patches: The current patch is based on the emscripten-libs-20 branch. -We do not use LLVM's libc directly, but libcxx uses a subset of headers from -libc. So we only import these directories that libcxx depend on: +Currently in producgtion, we do not use LLVM's libc directly yet, but libcxx uses a subset +of headers from libc. So libcxx directly depends on the following directories: - libc/hdr - libc/include/llvm-libc-macros - libc/include/llvm-libc-types - libc/shared - libc/src/__support +In addition, we are experimenting with using llvm-libc in overlay mode. You can enable this +by running with `EMCC_CFLAGS=-lllvmlibc`. See https://libc.llvm.org/overlay_mode.html for more +details. + Update Instructions ------------------- Run `system/lib/update_libcxx.py path/to/llvm-project` +Run `system/lib/update_llvm_libc.py path/to/llvm-project` Modifications ------------- diff --git a/system/lib/llvm-libc/shared/fp_bits.h b/system/lib/llvm-libc/shared/fp_bits.h index 2898c508b7772..e6bb1e17b80c9 100644 --- a/system/lib/llvm-libc/shared/fp_bits.h +++ b/system/lib/llvm-libc/shared/fp_bits.h @@ -9,6 +9,7 @@ #ifndef LLVM_LIBC_SHARED_FP_BITS_H #define LLVM_LIBC_SHARED_FP_BITS_H +#include "libc_common.h" #include "src/__support/FPUtil/FPBits.h" namespace LIBC_NAMESPACE_DECL { diff --git a/system/lib/llvm-libc/shared/libc_common.h b/system/lib/llvm-libc/shared/libc_common.h new file mode 100644 index 0000000000000..c4560bbb02763 --- /dev/null +++ b/system/lib/llvm-libc/shared/libc_common.h @@ -0,0 +1,26 @@ +//===-- Common defines for sharing LLVM libc with LLVM projects -*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SHARED_LIBC_COMMON_H +#define LLVM_LIBC_SHARED_LIBC_COMMON_H + +// Use system errno. +#ifdef LIBC_ERRNO_MODE +#if LIBC_ERRNO_MODE != LIBC_ERRNO_MODE_SYSTEM_INLINE +#error \ + "LIBC_ERRNO_MODE was set to something different from LIBC_ERRNO_MODE_SYSTEM_INLINE." +#endif // LIBC_ERRNO_MODE != LIBC_ERRNO_MODE_SYSTEM_INLINE +#else +#define LIBC_ERRNO_MODE LIBC_ERRNO_MODE_SYSTEM_INLINE +#endif // LIBC_ERRNO_MODE + +#ifndef LIBC_NAMESPACE +#define LIBC_NAMESPACE __llvm_libc +#endif // LIBC_NAMESPACE + +#endif // LLVM_LIBC_SHARED_LIBC_COMMON_H diff --git a/system/lib/llvm-libc/shared/math.h b/system/lib/llvm-libc/shared/math.h new file mode 100644 index 0000000000000..4ddc29c7ae834 --- /dev/null +++ b/system/lib/llvm-libc/shared/math.h @@ -0,0 +1,16 @@ +//===-- Floating point math functions ---------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SHARED_MATH_H +#define LLVM_LIBC_SHARED_MATH_H + +#include "libc_common.h" + +#include "math/expf.h" + +#endif // LLVM_LIBC_SHARED_MATH_H diff --git a/system/lib/llvm-libc/shared/math/expf.h b/system/lib/llvm-libc/shared/math/expf.h new file mode 100644 index 0000000000000..a4e8b0751bb42 --- /dev/null +++ b/system/lib/llvm-libc/shared/math/expf.h @@ -0,0 +1,23 @@ +//===-- Shared expf function ------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SHARED_MATH_EXPF_H +#define LLVM_LIBC_SHARED_MATH_EXPF_H + +#include "shared/libc_common.h" +#include "src/__support/math/expf.h" + +namespace LIBC_NAMESPACE_DECL { +namespace shared { + +using math::expf; + +} // namespace shared +} // namespace LIBC_NAMESPACE_DECL + +#endif // LLVM_LIBC_SHARED_MATH_EXPF_H diff --git a/system/lib/llvm-libc/shared/rpc.h b/system/lib/llvm-libc/shared/rpc.h index dd46d5dcb3dc0..7295efd4eee33 100644 --- a/system/lib/llvm-libc/shared/rpc.h +++ b/system/lib/llvm-libc/shared/rpc.h @@ -275,7 +275,7 @@ template struct Process { } }; -/// Invokes a function accross every active buffer across the total lane size. +/// Invokes a function across every active buffer across the total lane size. template RPC_ATTRS static void invoke_rpc(F &&fn, uint32_t lane_size, uint64_t lane_mask, Buffer *slot) { @@ -375,8 +375,8 @@ struct Server { uint32_t start = 0); RPC_ATTRS Port open(uint32_t lane_size); - RPC_ATTRS static uint64_t allocation_size(uint32_t lane_size, - uint32_t port_count) { + RPC_ATTRS static constexpr uint64_t allocation_size(uint32_t lane_size, + uint32_t port_count) { return Process::allocation_size(port_count, lane_size); } diff --git a/system/lib/llvm-libc/shared/rpc_opcodes.h b/system/lib/llvm-libc/shared/rpc_opcodes.h index 270c35dec28b8..6de41cd1899e7 100644 --- a/system/lib/llvm-libc/shared/rpc_opcodes.h +++ b/system/lib/llvm-libc/shared/rpc_opcodes.h @@ -50,10 +50,4 @@ typedef enum { #undef LLVM_LIBC_OPCODE -namespace rpc { -// The implementation of this function currently lives in the utility directory -// at 'utils/gpu/server/rpc_server.cpp'. -rpc::Status handle_libc_opcodes(rpc::Server::Port &port, uint32_t num_lanes); -} // namespace rpc - #endif // LLVM_LIBC_SHARED_RPC_OPCODES_H diff --git a/system/lib/llvm-libc/shared/rpc_server.h b/system/lib/llvm-libc/shared/rpc_server.h new file mode 100644 index 0000000000000..46e35f13f0eac --- /dev/null +++ b/system/lib/llvm-libc/shared/rpc_server.h @@ -0,0 +1,23 @@ +//===-- Shared RPC server interface -----------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SHARED_RPC_SERVER_H +#define LLVM_LIBC_SHARED_RPC_SERVER_H + +#include "libc_common.h" +#include "src/__support/RPC/rpc_server.h" + +namespace LIBC_NAMESPACE_DECL { +namespace shared { + +using LIBC_NAMESPACE::rpc::handle_libc_opcodes; + +} // namespace shared +} // namespace LIBC_NAMESPACE_DECL + +#endif // LLVM_LIBC_SHARED_RPC_SERVER_H diff --git a/system/lib/llvm-libc/shared/str_to_float.h b/system/lib/llvm-libc/shared/str_to_float.h index b133a28e26efc..dcc6027d6c77f 100644 --- a/system/lib/llvm-libc/shared/str_to_float.h +++ b/system/lib/llvm-libc/shared/str_to_float.h @@ -9,6 +9,7 @@ #ifndef LLVM_LIBC_SHARED_STR_TO_FLOAT_H #define LLVM_LIBC_SHARED_STR_TO_FLOAT_H +#include "libc_common.h" #include "src/__support/str_to_float.h" namespace LIBC_NAMESPACE_DECL { diff --git a/system/lib/llvm-libc/shared/str_to_integer.h b/system/lib/llvm-libc/shared/str_to_integer.h index 15bee698d5a6b..6ed38c932662e 100644 --- a/system/lib/llvm-libc/shared/str_to_integer.h +++ b/system/lib/llvm-libc/shared/str_to_integer.h @@ -9,6 +9,7 @@ #ifndef LLVM_LIBC_SHARED_STR_TO_INTEGER_H #define LLVM_LIBC_SHARED_STR_TO_INTEGER_H +#include "libc_common.h" #include "src/__support/str_to_integer.h" namespace LIBC_NAMESPACE_DECL { diff --git a/system/lib/llvm-libc/src/__support/CPP/algorithm.h b/system/lib/llvm-libc/src/__support/CPP/algorithm.h index f5dc9067409eb..7704b3fa81f0c 100644 --- a/system/lib/llvm-libc/src/__support/CPP/algorithm.h +++ b/system/lib/llvm-libc/src/__support/CPP/algorithm.h @@ -26,6 +26,8 @@ template LIBC_INLINE constexpr const T &min(const T &a, const T &b) { return (a < b) ? a : b; } +template LIBC_INLINE constexpr T abs(T a) { return a < 0 ? -a : a; } + template LIBC_INLINE constexpr InputIt find_if_not(InputIt first, InputIt last, UnaryPred q) { diff --git a/system/lib/llvm-libc/src/__support/CPP/atomic.h b/system/lib/llvm-libc/src/__support/CPP/atomic.h index 287dcac98fbb6..2f00b3ed32811 100644 --- a/system/lib/llvm-libc/src/__support/CPP/atomic.h +++ b/system/lib/llvm-libc/src/__support/CPP/atomic.h @@ -40,6 +40,28 @@ enum class MemoryScope : int { #endif }; +namespace impl { +LIBC_INLINE constexpr int order(MemoryOrder mem_ord) { + return static_cast(mem_ord); +} + +LIBC_INLINE constexpr int scope(MemoryScope mem_scope) { + return static_cast(mem_scope); +} + +template LIBC_INLINE T *addressof(T &ref) { + return __builtin_addressof(ref); +} + +LIBC_INLINE constexpr int infer_failure_order(MemoryOrder mem_ord) { + if (mem_ord == MemoryOrder::RELEASE) + return order(MemoryOrder::RELAXED); + if (mem_ord == MemoryOrder::ACQ_REL) + return order(MemoryOrder::ACQUIRE); + return order(mem_ord); +} +} // namespace impl + template struct Atomic { static_assert(is_trivially_copyable_v && is_copy_constructible_v && is_move_constructible_v && is_copy_assignable_v && @@ -54,15 +76,6 @@ template struct Atomic { private: // type conversion helper to avoid long c++ style casts - LIBC_INLINE static int order(MemoryOrder mem_ord) { - return static_cast(mem_ord); - } - - LIBC_INLINE static int scope(MemoryScope mem_scope) { - return static_cast(mem_scope); - } - - LIBC_INLINE static T *addressof(T &ref) { return __builtin_addressof(ref); } // Require types that are 1, 2, 4, 8, or 16 bytes in length to be aligned to // at least their size to be potentially used lock-free. @@ -84,7 +97,7 @@ template struct Atomic { LIBC_INLINE constexpr Atomic() = default; - // Intializes the value without using atomic operations. + // Initializes the value without using atomic operations. LIBC_INLINE constexpr Atomic(value_type v) : val(v) {} LIBC_INLINE Atomic(const Atomic &) = delete; @@ -98,10 +111,11 @@ template struct Atomic { [[maybe_unused]] MemoryScope mem_scope = MemoryScope::DEVICE) { T res; #if __has_builtin(__scoped_atomic_load) - __scoped_atomic_load(addressof(val), addressof(res), order(mem_ord), - scope(mem_scope)); + __scoped_atomic_load(impl::addressof(val), impl::addressof(res), + impl::order(mem_ord), impl::scope(mem_scope)); #else - __atomic_load(addressof(val), addressof(res), order(mem_ord)); + __atomic_load(impl::addressof(val), impl::addressof(res), + impl::order(mem_ord)); #endif return res; } @@ -116,10 +130,11 @@ template struct Atomic { store(T rhs, MemoryOrder mem_ord = MemoryOrder::SEQ_CST, [[maybe_unused]] MemoryScope mem_scope = MemoryScope::DEVICE) { #if __has_builtin(__scoped_atomic_store) - __scoped_atomic_store(addressof(val), addressof(rhs), order(mem_ord), - scope(mem_scope)); + __scoped_atomic_store(impl::addressof(val), impl::addressof(rhs), + impl::order(mem_ord), impl::scope(mem_scope)); #else - __atomic_store(addressof(val), addressof(rhs), order(mem_ord)); + __atomic_store(impl::addressof(val), impl::addressof(rhs), + impl::order(mem_ord)); #endif } @@ -127,9 +142,10 @@ template struct Atomic { LIBC_INLINE bool compare_exchange_strong( T &expected, T desired, MemoryOrder mem_ord = MemoryOrder::SEQ_CST, [[maybe_unused]] MemoryScope mem_scope = MemoryScope::DEVICE) { - return __atomic_compare_exchange(addressof(val), addressof(expected), - addressof(desired), false, order(mem_ord), - order(mem_ord)); + return __atomic_compare_exchange( + impl::addressof(val), impl::addressof(expected), + impl::addressof(desired), false, impl::order(mem_ord), + impl::infer_failure_order(mem_ord)); } // Atomic compare exchange (separate success and failure memory orders) @@ -138,17 +154,19 @@ template struct Atomic { MemoryOrder failure_order, [[maybe_unused]] MemoryScope mem_scope = MemoryScope::DEVICE) { return __atomic_compare_exchange( - addressof(val), addressof(expected), addressof(desired), false, - order(success_order), order(failure_order)); + impl::addressof(val), impl::addressof(expected), + impl::addressof(desired), false, impl::order(success_order), + impl::order(failure_order)); } // Atomic compare exchange (weak version) LIBC_INLINE bool compare_exchange_weak( T &expected, T desired, MemoryOrder mem_ord = MemoryOrder::SEQ_CST, [[maybe_unused]] MemoryScope mem_scope = MemoryScope::DEVICE) { - return __atomic_compare_exchange(addressof(val), addressof(expected), - addressof(desired), true, order(mem_ord), - order(mem_ord)); + return __atomic_compare_exchange( + impl::addressof(val), impl::addressof(expected), + impl::addressof(desired), true, impl::order(mem_ord), + impl::infer_failure_order(mem_ord)); } // Atomic compare exchange (weak version with separate success and failure @@ -158,8 +176,9 @@ template struct Atomic { MemoryOrder failure_order, [[maybe_unused]] MemoryScope mem_scope = MemoryScope::DEVICE) { return __atomic_compare_exchange( - addressof(val), addressof(expected), addressof(desired), true, - order(success_order), order(failure_order)); + impl::addressof(val), impl::addressof(expected), + impl::addressof(desired), true, impl::order(success_order), + impl::order(failure_order)); } LIBC_INLINE T @@ -167,11 +186,12 @@ template struct Atomic { [[maybe_unused]] MemoryScope mem_scope = MemoryScope::DEVICE) { T ret; #if __has_builtin(__scoped_atomic_exchange) - __scoped_atomic_exchange(addressof(val), addressof(desired), addressof(ret), - order(mem_ord), scope(mem_scope)); + __scoped_atomic_exchange(impl::addressof(val), impl::addressof(desired), + impl::addressof(ret), impl::order(mem_ord), + impl::scope(mem_scope)); #else - __atomic_exchange(addressof(val), addressof(desired), addressof(ret), - order(mem_ord)); + __atomic_exchange(impl::addressof(val), impl::addressof(desired), + impl::addressof(ret), impl::order(mem_ord)); #endif return ret; } @@ -181,10 +201,12 @@ template struct Atomic { [[maybe_unused]] MemoryScope mem_scope = MemoryScope::DEVICE) { static_assert(cpp::is_integral_v, "T must be an integral type."); #if __has_builtin(__scoped_atomic_fetch_add) - return __scoped_atomic_fetch_add(addressof(val), increment, order(mem_ord), - scope(mem_scope)); + return __scoped_atomic_fetch_add(impl::addressof(val), increment, + impl::order(mem_ord), + impl::scope(mem_scope)); #else - return __atomic_fetch_add(addressof(val), increment, order(mem_ord)); + return __atomic_fetch_add(impl::addressof(val), increment, + impl::order(mem_ord)); #endif } @@ -193,10 +215,11 @@ template struct Atomic { [[maybe_unused]] MemoryScope mem_scope = MemoryScope::DEVICE) { static_assert(cpp::is_integral_v, "T must be an integral type."); #if __has_builtin(__scoped_atomic_fetch_or) - return __scoped_atomic_fetch_or(addressof(val), mask, order(mem_ord), - scope(mem_scope)); + return __scoped_atomic_fetch_or(impl::addressof(val), mask, + impl::order(mem_ord), + impl::scope(mem_scope)); #else - return __atomic_fetch_or(addressof(val), mask, order(mem_ord)); + return __atomic_fetch_or(impl::addressof(val), mask, impl::order(mem_ord)); #endif } @@ -205,10 +228,11 @@ template struct Atomic { [[maybe_unused]] MemoryScope mem_scope = MemoryScope::DEVICE) { static_assert(cpp::is_integral_v, "T must be an integral type."); #if __has_builtin(__scoped_atomic_fetch_and) - return __scoped_atomic_fetch_and(addressof(val), mask, order(mem_ord), - scope(mem_scope)); + return __scoped_atomic_fetch_and(impl::addressof(val), mask, + impl::order(mem_ord), + impl::scope(mem_scope)); #else - return __atomic_fetch_and(addressof(val), mask, order(mem_ord)); + return __atomic_fetch_and(impl::addressof(val), mask, impl::order(mem_ord)); #endif } @@ -217,10 +241,12 @@ template struct Atomic { [[maybe_unused]] MemoryScope mem_scope = MemoryScope::DEVICE) { static_assert(cpp::is_integral_v, "T must be an integral type."); #if __has_builtin(__scoped_atomic_fetch_sub) - return __scoped_atomic_fetch_sub(addressof(val), decrement, order(mem_ord), - scope(mem_scope)); + return __scoped_atomic_fetch_sub(impl::addressof(val), decrement, + impl::order(mem_ord), + impl::scope(mem_scope)); #else - return __atomic_fetch_sub(addressof(val), decrement, order(mem_ord)); + return __atomic_fetch_sub(impl::addressof(val), decrement, + impl::order(mem_ord)); #endif } @@ -229,6 +255,149 @@ template struct Atomic { LIBC_INLINE void set(T rhs) { val = rhs; } }; +template struct AtomicRef { + static_assert(is_trivially_copyable_v && is_copy_constructible_v && + is_move_constructible_v && is_copy_assignable_v && + is_move_assignable_v, + "AtomicRef requires T to be trivially copyable, copy " + "constructible, move constructible, copy assignable, " + "and move assignable."); + + static_assert(cpp::has_unique_object_representations_v, + "AtomicRef only supports types with unique object " + "representations."); + +private: + T *ptr; + +public: + // Constructor from T reference + LIBC_INLINE explicit constexpr AtomicRef(T &obj) : ptr(&obj) {} + + // Non-standard Implicit conversion from T* + LIBC_INLINE constexpr AtomicRef(T *obj) : ptr(obj) {} + + LIBC_INLINE AtomicRef(const AtomicRef &) = default; + LIBC_INLINE AtomicRef &operator=(const AtomicRef &) = default; + + // Atomic load + LIBC_INLINE operator T() const { return load(); } + + LIBC_INLINE T + load(MemoryOrder mem_ord = MemoryOrder::SEQ_CST, + [[maybe_unused]] MemoryScope mem_scope = MemoryScope::DEVICE) const { + T res; +#if __has_builtin(__scoped_atomic_load) + __scoped_atomic_load(ptr, &res, impl::order(mem_ord), + impl::scope(mem_scope)); +#else + __atomic_load(ptr, &res, impl::order(mem_ord)); +#endif + return res; + } + + // Atomic store + LIBC_INLINE T operator=(T rhs) const { + store(rhs); + return rhs; + } + + LIBC_INLINE void + store(T rhs, MemoryOrder mem_ord = MemoryOrder::SEQ_CST, + [[maybe_unused]] MemoryScope mem_scope = MemoryScope::DEVICE) const { +#if __has_builtin(__scoped_atomic_store) + __scoped_atomic_store(ptr, &rhs, impl::order(mem_ord), + impl::scope(mem_scope)); +#else + __atomic_store(ptr, &rhs, impl::order(mem_ord)); +#endif + } + + // Atomic compare exchange (strong) + LIBC_INLINE bool compare_exchange_strong( + T &expected, T desired, MemoryOrder mem_ord = MemoryOrder::SEQ_CST, + [[maybe_unused]] MemoryScope mem_scope = MemoryScope::DEVICE) const { + return __atomic_compare_exchange(ptr, &expected, &desired, false, + impl::order(mem_ord), + impl::infer_failure_order(mem_ord)); + } + + // Atomic compare exchange (strong, separate success/failure memory orders) + LIBC_INLINE bool compare_exchange_strong( + T &expected, T desired, MemoryOrder success_order, + MemoryOrder failure_order, + [[maybe_unused]] MemoryScope mem_scope = MemoryScope::DEVICE) const { + return __atomic_compare_exchange(ptr, &expected, &desired, false, + impl::order(success_order), + impl::order(failure_order)); + } + + // Atomic exchange + LIBC_INLINE T + exchange(T desired, MemoryOrder mem_ord = MemoryOrder::SEQ_CST, + [[maybe_unused]] MemoryScope mem_scope = MemoryScope::DEVICE) const { + T ret; +#if __has_builtin(__scoped_atomic_exchange) + __scoped_atomic_exchange(ptr, &desired, &ret, impl::order(mem_ord), + impl::scope(mem_scope)); +#else + __atomic_exchange(ptr, &desired, &ret, impl::order(mem_ord)); +#endif + return ret; + } + + LIBC_INLINE T fetch_add( + T increment, MemoryOrder mem_ord = MemoryOrder::SEQ_CST, + [[maybe_unused]] MemoryScope mem_scope = MemoryScope::DEVICE) const { + static_assert(cpp::is_integral_v, "T must be an integral type."); +#if __has_builtin(__scoped_atomic_fetch_add) + return __scoped_atomic_fetch_add(ptr, increment, impl::order(mem_ord), + impl::scope(mem_scope)); +#else + return __atomic_fetch_add(ptr, increment, impl::order(mem_ord)); +#endif + } + + LIBC_INLINE T + fetch_or(T mask, MemoryOrder mem_ord = MemoryOrder::SEQ_CST, + [[maybe_unused]] MemoryScope mem_scope = MemoryScope::DEVICE) const { + static_assert(cpp::is_integral_v, "T must be an integral type."); +#if __has_builtin(__scoped_atomic_fetch_or) + return __scoped_atomic_fetch_or(ptr, mask, impl::order(mem_ord), + impl::scope(mem_scope)); +#else + return __atomic_fetch_or(ptr, mask, impl::order(mem_ord)); +#endif + } + + LIBC_INLINE T fetch_and( + T mask, MemoryOrder mem_ord = MemoryOrder::SEQ_CST, + [[maybe_unused]] MemoryScope mem_scope = MemoryScope::DEVICE) const { + static_assert(cpp::is_integral_v, "T must be an integral type."); +#if __has_builtin(__scoped_atomic_fetch_and) + return __scoped_atomic_fetch_and(ptr, mask, impl::order(mem_ord), + impl::scope(mem_scope)); +#else + return __atomic_fetch_and(ptr, mask, impl::order(mem_ord)); +#endif + } + + LIBC_INLINE T fetch_sub( + T decrement, MemoryOrder mem_ord = MemoryOrder::SEQ_CST, + [[maybe_unused]] MemoryScope mem_scope = MemoryScope::DEVICE) const { + static_assert(cpp::is_integral_v, "T must be an integral type."); +#if __has_builtin(__scoped_atomic_fetch_sub) + return __scoped_atomic_fetch_sub(ptr, decrement, impl::order(mem_ord), + impl::scope(mem_scope)); +#else + return __atomic_fetch_sub(ptr, decrement, impl::order(mem_ord)); +#endif + } +}; + +// Permit CTAD when generating an atomic reference. +template AtomicRef(T &) -> AtomicRef; + // Issue a thread fence with the given memory ordering. LIBC_INLINE void atomic_thread_fence( MemoryOrder mem_ord, @@ -254,7 +423,6 @@ LIBC_INLINE void atomic_signal_fence([[maybe_unused]] MemoryOrder mem_ord) { asm volatile("" ::: "memory"); #endif } - } // namespace cpp } // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/__support/CPP/bit.h b/system/lib/llvm-libc/src/__support/CPP/bit.h index adcd0472747d0..e491f3e032669 100644 --- a/system/lib/llvm-libc/src/__support/CPP/bit.h +++ b/system/lib/llvm-libc/src/__support/CPP/bit.h @@ -101,7 +101,7 @@ countr_zero(T value) { shift >>= 1; mask >>= shift; } - return zero_bits; + return static_cast(zero_bits); } #if __has_builtin(__builtin_ctzs) ADD_SPECIALIZATION(countr_zero, unsigned short, __builtin_ctzs) @@ -140,7 +140,7 @@ countl_zero(T value) { else zero_bits |= shift; } - return zero_bits; + return static_cast(zero_bits); } #if __has_builtin(__builtin_clzs) ADD_SPECIALIZATION(countl_zero, unsigned short, __builtin_clzs) @@ -162,7 +162,7 @@ ADD_SPECIALIZATION(countl_zero, unsigned long long, __builtin_clzll) template [[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, int> countl_one(T value) { - return cpp::countl_zero(~value); + return cpp::countl_zero(static_cast(~value)); } /// Count the number of ones from the least significant bit to the first @@ -175,7 +175,7 @@ countl_one(T value) { template [[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, int> countr_one(T value) { - return cpp::countr_zero(~value); + return cpp::countr_zero(static_cast(~value)); } /// Returns the number of bits needed to represent value if value is nonzero. @@ -226,25 +226,25 @@ rotr(T value, int rotate); template [[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, T> rotl(T value, int rotate) { - constexpr unsigned N = cpp::numeric_limits::digits; + constexpr int N = cpp::numeric_limits::digits; rotate = rotate % N; if (!rotate) return value; if (rotate < 0) return cpp::rotr(value, -rotate); - return (value << rotate) | (value >> (N - rotate)); + return static_cast((value << rotate) | (value >> (N - rotate))); } template [[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, T> rotr(T value, int rotate) { - constexpr unsigned N = cpp::numeric_limits::digits; + constexpr int N = cpp::numeric_limits::digits; rotate = rotate % N; if (!rotate) return value; if (rotate < 0) return cpp::rotl(value, -rotate); - return (value >> rotate) | (value << (N - rotate)); + return static_cast((value >> rotate) | (value << (N - rotate))); } // TODO: Do we need this function at all? How is it different from diff --git a/system/lib/llvm-libc/src/__support/CPP/span.h b/system/lib/llvm-libc/src/__support/CPP/span.h index e9e3dbf169ce0..9234a26d201cd 100644 --- a/system/lib/llvm-libc/src/__support/CPP/span.h +++ b/system/lib/llvm-libc/src/__support/CPP/span.h @@ -10,7 +10,8 @@ #include // For size_t -#include "array.h" // For array +#include "array.h" // For array +#include "limits.h" #include "src/__support/macros/config.h" #include "type_traits.h" // For remove_cv_t, enable_if_t, is_same_v, is_const_v @@ -48,15 +49,18 @@ template class span { using const_reference = const T &; using iterator = T *; - LIBC_INLINE_VAR static constexpr size_type dynamic_extent = -1; + LIBC_INLINE_VAR static constexpr size_type dynamic_extent = + cpp::numeric_limits::max(); LIBC_INLINE constexpr span() : span_data(nullptr), span_size(0) {} + LIBC_INLINE constexpr span(const span &) = default; + LIBC_INLINE constexpr span(pointer first, size_type count) : span_data(first), span_size(count) {} LIBC_INLINE constexpr span(pointer first, pointer end) - : span_data(first), span_size(end - first) {} + : span_data(first), span_size(static_cast(end - first)) {} template , bool> = true> diff --git a/system/lib/llvm-libc/src/__support/CPP/string.h b/system/lib/llvm-libc/src/__support/CPP/string.h index dbc0ae04e5e6f..1ac04c7f1f9dc 100644 --- a/system/lib/llvm-libc/src/__support/CPP/string.h +++ b/system/lib/llvm-libc/src/__support/CPP/string.h @@ -67,7 +67,8 @@ class string { : string(cstr, ::LIBC_NAMESPACE::internal::string_length(cstr)) {} LIBC_INLINE string(size_t size_, char value) { resize(size_); - inline_memset((void *)buffer_, value, size_); + static_assert(sizeof(char) == sizeof(uint8_t)); + inline_memset((void *)buffer_, static_cast(value), size_); } LIBC_INLINE string &operator=(const string &other) { diff --git a/system/lib/llvm-libc/src/__support/CPP/string_view.h b/system/lib/llvm-libc/src/__support/CPP/string_view.h index 745c62c35f0a0..aa15814b2e149 100644 --- a/system/lib/llvm-libc/src/__support/CPP/string_view.h +++ b/system/lib/llvm-libc/src/__support/CPP/string_view.h @@ -9,6 +9,7 @@ #ifndef LLVM_LIBC_SRC___SUPPORT_CPP_STRING_VIEW_H #define LLVM_LIBC_SRC___SUPPORT_CPP_STRING_VIEW_H +#include "limits.h" #include "src/__support/common.h" #include "src/__support/macros/config.h" @@ -40,7 +41,7 @@ class string_view { LIBC_INLINE static constexpr size_t length(const char *Str) { for (const char *End = Str;; ++End) if (*End == '\0') - return End - Str; + return static_cast(End - Str); } LIBC_INLINE bool equals(string_view Other) const { @@ -61,7 +62,8 @@ class string_view { // special value equal to the maximum value representable by the type // size_type. - LIBC_INLINE_VAR static constexpr size_t npos = -1; + LIBC_INLINE_VAR static constexpr size_t npos = + cpp::numeric_limits::max(); LIBC_INLINE constexpr string_view() : Data(nullptr), Len(0) {} diff --git a/system/lib/llvm-libc/src/__support/CPP/type_traits/is_signed.h b/system/lib/llvm-libc/src/__support/CPP/type_traits/is_signed.h index 3f56fb38aabb0..2ddb43ac4ee3e 100644 --- a/system/lib/llvm-libc/src/__support/CPP/type_traits/is_signed.h +++ b/system/lib/llvm-libc/src/__support/CPP/type_traits/is_signed.h @@ -8,20 +8,43 @@ #ifndef LLVM_LIBC_SRC___SUPPORT_CPP_TYPE_TRAITS_IS_SIGNED_H #define LLVM_LIBC_SRC___SUPPORT_CPP_TYPE_TRAITS_IS_SIGNED_H +#include "include/llvm-libc-macros/stdfix-macros.h" #include "src/__support/CPP/type_traits/bool_constant.h" #include "src/__support/CPP/type_traits/is_arithmetic.h" +#include "src/__support/CPP/type_traits/is_same.h" +#include "src/__support/CPP/type_traits/remove_cv.h" #include "src/__support/macros/attributes.h" #include "src/__support/macros/config.h" namespace LIBC_NAMESPACE_DECL { namespace cpp { -// is_signed +#ifndef LIBC_COMPILER_HAS_FIXED_POINT template struct is_signed : bool_constant<(is_arithmetic_v && (T(-1) < T(0)))> { LIBC_INLINE constexpr operator bool() const { return is_signed::value; } LIBC_INLINE constexpr bool operator()() const { return is_signed::value; } }; +#else +template struct is_signed { +private: + template + LIBC_INLINE static constexpr bool __is_unqualified_any_of() { + return (... || is_same_v, Args>); + } + +public: + LIBC_INLINE_VAR static constexpr bool value = + (is_arithmetic_v && (T(-1) < T(0))) || + __is_unqualified_any_of(); + LIBC_INLINE constexpr operator bool() const { return is_signed::value; } + LIBC_INLINE constexpr bool operator()() const { return is_signed::value; } +}; +#endif // LIBC_COMPILER_HAS_FIXED_POINT + template LIBC_INLINE_VAR constexpr bool is_signed_v = is_signed::value; diff --git a/system/lib/llvm-libc/src/__support/CPP/type_traits/is_unsigned.h b/system/lib/llvm-libc/src/__support/CPP/type_traits/is_unsigned.h index eed519b1c067e..3ae6337ceb50a 100644 --- a/system/lib/llvm-libc/src/__support/CPP/type_traits/is_unsigned.h +++ b/system/lib/llvm-libc/src/__support/CPP/type_traits/is_unsigned.h @@ -8,20 +8,45 @@ #ifndef LLVM_LIBC_SRC___SUPPORT_CPP_TYPE_TRAITS_IS_UNSIGNED_H #define LLVM_LIBC_SRC___SUPPORT_CPP_TYPE_TRAITS_IS_UNSIGNED_H +#include "include/llvm-libc-macros/stdfix-macros.h" #include "src/__support/CPP/type_traits/bool_constant.h" #include "src/__support/CPP/type_traits/is_arithmetic.h" +#include "src/__support/CPP/type_traits/is_same.h" +#include "src/__support/CPP/type_traits/remove_cv.h" #include "src/__support/macros/attributes.h" #include "src/__support/macros/config.h" namespace LIBC_NAMESPACE_DECL { namespace cpp { -// is_unsigned +#ifndef LIBC_COMPILER_HAS_FIXED_POINT template struct is_unsigned : bool_constant<(is_arithmetic_v && (T(-1) > T(0)))> { LIBC_INLINE constexpr operator bool() const { return is_unsigned::value; } LIBC_INLINE constexpr bool operator()() const { return is_unsigned::value; } }; +#else +template struct is_unsigned { +private: + template + LIBC_INLINE static constexpr bool __is_unqualified_any_of() { + return (... || is_same_v, Args>); + } + +public: + LIBC_INLINE_VAR static constexpr bool value = + (is_arithmetic_v && (T(-1) > T(0))) || + __is_unqualified_any_of(); + LIBC_INLINE constexpr operator bool() const { return is_unsigned::value; } + LIBC_INLINE constexpr bool operator()() const { return is_unsigned::value; } +}; +#endif // LIBC_COMPILER_HAS_FIXED_POINT + template LIBC_INLINE_VAR constexpr bool is_unsigned_v = is_unsigned::value; diff --git a/system/lib/llvm-libc/src/__support/FPUtil/FEnvImpl.h b/system/lib/llvm-libc/src/__support/FPUtil/FEnvImpl.h index 1c5a1108ff9e0..50a101f833c55 100644 --- a/system/lib/llvm-libc/src/__support/FPUtil/FEnvImpl.h +++ b/system/lib/llvm-libc/src/__support/FPUtil/FEnvImpl.h @@ -12,12 +12,12 @@ #include "hdr/fenv_macros.h" #include "hdr/math_macros.h" #include "hdr/types/fenv_t.h" +#include "src/__support/libc_errno.h" #include "src/__support/macros/attributes.h" // LIBC_INLINE #include "src/__support/macros/config.h" #include "src/__support/macros/properties/architectures.h" -#include "src/errno/libc_errno.h" -#if defined(LIBC_TARGET_ARCH_IS_AARCH64) +#if defined(LIBC_TARGET_ARCH_IS_AARCH64) && defined(__ARM_FP) #if defined(__APPLE__) #include "aarch64/fenv_darwin_impl.h" #else diff --git a/system/lib/llvm-libc/src/__support/FPUtil/FMA.h b/system/lib/llvm-libc/src/__support/FPUtil/FMA.h index 807d685239732..2cafb4c0974e3 100644 --- a/system/lib/llvm-libc/src/__support/FPUtil/FMA.h +++ b/system/lib/llvm-libc/src/__support/FPUtil/FMA.h @@ -24,13 +24,26 @@ LIBC_INLINE OutType fma(InType x, InType y, InType z) { } #ifdef LIBC_TARGET_CPU_HAS_FMA + +#ifdef LIBC_TARGET_CPU_HAS_FMA_FLOAT template <> LIBC_INLINE float fma(float x, float y, float z) { +#if __has_builtin(__builtin_elementwise_fma) + return __builtin_elementwise_fma(x, y, z); +#else return __builtin_fmaf(x, y, z); +#endif } +#endif // LIBC_TARGET_CPU_HAS_FMA_FLOAT +#ifdef LIBC_TARGET_CPU_HAS_FMA_DOUBLE template <> LIBC_INLINE double fma(double x, double y, double z) { +#if __has_builtin(__builtin_elementwise_fma) + return __builtin_elementwise_fma(x, y, z); +#else return __builtin_fma(x, y, z); +#endif } +#endif // LIBC_TARGET_CPU_HAS_FMA_DOUBLE #endif // LIBC_TARGET_CPU_HAS_FMA } // namespace fputil diff --git a/system/lib/llvm-libc/src/__support/FPUtil/FPBits.h b/system/lib/llvm-libc/src/__support/FPUtil/FPBits.h index 90b6e406e0f31..4fa3bc30ecd12 100644 --- a/system/lib/llvm-libc/src/__support/FPUtil/FPBits.h +++ b/system/lib/llvm-libc/src/__support/FPUtil/FPBits.h @@ -247,11 +247,11 @@ template struct FPStorage : public FPLayout { using UP::UP; LIBC_INLINE constexpr BiasedExponent(Exponent exp) - : UP(static_cast(exp) + EXP_BIAS) {} + : UP(static_cast(static_cast(exp) + EXP_BIAS)) {} // Cast operator to get convert from BiasedExponent to Exponent. LIBC_INLINE constexpr operator Exponent() const { - return Exponent(UP::value - EXP_BIAS); + return Exponent(static_cast(UP::value - EXP_BIAS)); } LIBC_INLINE constexpr BiasedExponent &operator++() { @@ -686,7 +686,7 @@ struct FPRepImpl : public FPRepSem { } LIBC_INLINE constexpr void set_biased_exponent(StorageType biased) { - UP::set_biased_exponent(BiasedExponent((int32_t)biased)); + UP::set_biased_exponent(BiasedExponent(static_cast(biased))); } LIBC_INLINE constexpr int get_exponent() const { @@ -757,7 +757,7 @@ struct FPRepImpl : public FPRepSem { result.set_significand(number); result.set_biased_exponent(static_cast(ep + 1)); } else { - result.set_significand(number >> -ep); + result.set_significand(number >> static_cast(-ep)); } return RetT(result.uintval()); } diff --git a/system/lib/llvm-libc/src/__support/FPUtil/Hypot.h b/system/lib/llvm-libc/src/__support/FPUtil/Hypot.h index 6aa808446d6d9..94da259cd42f0 100644 --- a/system/lib/llvm-libc/src/__support/FPUtil/Hypot.h +++ b/system/lib/llvm-libc/src/__support/FPUtil/Hypot.h @@ -30,7 +30,7 @@ LIBC_INLINE T find_leading_one(T mant, int &shift_length) { if (mant > 0) { shift_length = (sizeof(mant) * 8) - 1 - cpp::countl_zero(mant); } - return T(1) << shift_length; + return static_cast((T(1) << shift_length)); } } // namespace internal @@ -207,8 +207,10 @@ LIBC_INLINE T hypot(T x, T y) { for (StorageType current_bit = leading_one >> 1; current_bit; current_bit >>= 1) { - r = (r << 1) + ((tail_bits & current_bit) ? 1 : 0); - StorageType tmp = (y_new << 1) + current_bit; // 2*y_new(n - 1) + 2^(-n) + r = static_cast((r << 1)) + + ((tail_bits & current_bit) ? 1 : 0); + StorageType tmp = static_cast((y_new << 1)) + + current_bit; // 2*y_new(n - 1) + 2^(-n) if (r >= tmp) { r -= tmp; y_new += current_bit; diff --git a/system/lib/llvm-libc/src/__support/FPUtil/NormalFloat.h b/system/lib/llvm-libc/src/__support/FPUtil/NormalFloat.h index b4cbb5042a68b..a2f285fc6fb95 100644 --- a/system/lib/llvm-libc/src/__support/FPUtil/NormalFloat.h +++ b/system/lib/llvm-libc/src/__support/FPUtil/NormalFloat.h @@ -105,7 +105,7 @@ template struct NormalFloat { constexpr int SUBNORMAL_EXPONENT = -FPBits::EXP_BIAS + 1; if (exponent < SUBNORMAL_EXPONENT) { - unsigned shift = SUBNORMAL_EXPONENT - exponent; + unsigned shift = static_cast(SUBNORMAL_EXPONENT - exponent); // Since exponent > subnormalExponent, shift is strictly greater than // zero. if (shift <= FPBits::FRACTION_LEN + 1) { @@ -160,7 +160,7 @@ template struct NormalFloat { if (bits.is_subnormal()) { unsigned shift = evaluate_normalization_shift(bits.get_mantissa()); mantissa = static_cast(bits.get_mantissa() << shift); - exponent = 1 - FPBits::EXP_BIAS - shift; + exponent = 1 - FPBits::EXP_BIAS - static_cast(shift); } else { exponent = bits.get_biased_exponent() - FPBits::EXP_BIAS; mantissa = ONE | bits.get_mantissa(); diff --git a/system/lib/llvm-libc/src/__support/FPUtil/aarch64/FEnvImpl.h b/system/lib/llvm-libc/src/__support/FPUtil/aarch64/FEnvImpl.h index 18b0631324f8f..914155a01631d 100644 --- a/system/lib/llvm-libc/src/__support/FPUtil/aarch64/FEnvImpl.h +++ b/system/lib/llvm-libc/src/__support/FPUtil/aarch64/FEnvImpl.h @@ -110,7 +110,7 @@ LIBC_INLINE int enable_except(int excepts) { (controlWord >> FEnv::ExceptionControlFlagsBitPosition) & 0x1F; controlWord |= (newExcepts << FEnv::ExceptionControlFlagsBitPosition); FEnv::writeControlWord(controlWord); - return FEnv::exceptionStatusToMacro(oldExcepts); + return FEnv::exceptionStatusToMacro(static_cast(oldExcepts)); } LIBC_INLINE int disable_except(int excepts) { @@ -120,12 +120,12 @@ LIBC_INLINE int disable_except(int excepts) { (controlWord >> FEnv::ExceptionControlFlagsBitPosition) & 0x1F; controlWord &= ~(disabledExcepts << FEnv::ExceptionControlFlagsBitPosition); FEnv::writeControlWord(controlWord); - return FEnv::exceptionStatusToMacro(oldExcepts); + return FEnv::exceptionStatusToMacro(static_cast(oldExcepts)); } LIBC_INLINE int get_except() { uint32_t controlWord = FEnv::getControlWord(); - int enabledExcepts = + uint32_t enabledExcepts = (controlWord >> FEnv::ExceptionControlFlagsBitPosition) & 0x1F; return FEnv::exceptionStatusToMacro(enabledExcepts); } @@ -250,8 +250,10 @@ LIBC_INLINE int set_round(int mode) { } uint32_t controlWord = FEnv::getControlWord(); - controlWord &= ~(0x3 << FEnv::RoundingControlBitPosition); - controlWord |= (bitValue << FEnv::RoundingControlBitPosition); + controlWord &= + static_cast(~(0x3 << FEnv::RoundingControlBitPosition)); + controlWord |= + static_cast(bitValue << FEnv::RoundingControlBitPosition); FEnv::writeControlWord(controlWord); return 0; diff --git a/system/lib/llvm-libc/src/__support/FPUtil/aarch64/fenv_darwin_impl.h b/system/lib/llvm-libc/src/__support/FPUtil/aarch64/fenv_darwin_impl.h index 969e70796d1f1..dcce76b6116be 100644 --- a/system/lib/llvm-libc/src/__support/FPUtil/aarch64/fenv_darwin_impl.h +++ b/system/lib/llvm-libc/src/__support/FPUtil/aarch64/fenv_darwin_impl.h @@ -63,7 +63,7 @@ struct FEnv { // __fpcr_flush_to_zero bit in the FPCR register. This control bit is // located in a different place from FE_FLUSHTOZERO status bit relative to // the other exceptions. - LIBC_INLINE static uint32_t exception_value_from_status(int status) { + LIBC_INLINE static uint32_t exception_value_from_status(uint32_t status) { return ((status & FE_INVALID) ? EX_INVALID : 0) | ((status & FE_DIVBYZERO) ? EX_DIVBYZERO : 0) | ((status & FE_OVERFLOW) ? EX_OVERFLOW : 0) | @@ -72,7 +72,7 @@ struct FEnv { ((status & FE_FLUSHTOZERO) ? EX_FLUSHTOZERO : 0); } - LIBC_INLINE static uint32_t exception_value_from_control(int control) { + LIBC_INLINE static uint32_t exception_value_from_control(uint32_t control) { return ((control & __fpcr_trap_invalid) ? EX_INVALID : 0) | ((control & __fpcr_trap_divbyzero) ? EX_DIVBYZERO : 0) | ((control & __fpcr_trap_overflow) ? EX_OVERFLOW : 0) | @@ -81,7 +81,7 @@ struct FEnv { ((control & __fpcr_flush_to_zero) ? EX_FLUSHTOZERO : 0); } - LIBC_INLINE static int exception_value_to_status(uint32_t excepts) { + LIBC_INLINE static uint32_t exception_value_to_status(uint32_t excepts) { return ((excepts & EX_INVALID) ? FE_INVALID : 0) | ((excepts & EX_DIVBYZERO) ? FE_DIVBYZERO : 0) | ((excepts & EX_OVERFLOW) ? FE_OVERFLOW : 0) | @@ -90,7 +90,7 @@ struct FEnv { ((excepts & EX_FLUSHTOZERO) ? FE_FLUSHTOZERO : 0); } - LIBC_INLINE static int exception_value_to_control(uint32_t excepts) { + LIBC_INLINE static uint32_t exception_value_to_control(uint32_t excepts) { return ((excepts & EX_INVALID) ? __fpcr_trap_invalid : 0) | ((excepts & EX_DIVBYZERO) ? __fpcr_trap_divbyzero : 0) | ((excepts & EX_OVERFLOW) ? __fpcr_trap_overflow : 0) | @@ -113,34 +113,37 @@ struct FEnv { }; LIBC_INLINE int enable_except(int excepts) { - uint32_t new_excepts = FEnv::exception_value_from_status(excepts); + uint32_t new_excepts = + FEnv::exception_value_from_status(static_cast(excepts)); uint32_t control_word = FEnv::get_control_word(); uint32_t old_excepts = FEnv::exception_value_from_control(control_word); if (new_excepts != old_excepts) { control_word |= FEnv::exception_value_to_control(new_excepts); FEnv::set_control_word(control_word); } - return FEnv::exception_value_to_status(old_excepts); + return static_cast(FEnv::exception_value_to_status(old_excepts)); } LIBC_INLINE int disable_except(int excepts) { - uint32_t disabled_excepts = FEnv::exception_value_from_status(excepts); + uint32_t disabled_excepts = + FEnv::exception_value_from_status(static_cast(excepts)); uint32_t control_word = FEnv::get_control_word(); uint32_t old_excepts = FEnv::exception_value_from_control(control_word); control_word &= ~FEnv::exception_value_to_control(disabled_excepts); FEnv::set_control_word(control_word); - return FEnv::exception_value_to_status(old_excepts); + return static_cast(FEnv::exception_value_to_status(old_excepts)); } LIBC_INLINE int get_except() { uint32_t control_word = FEnv::get_control_word(); uint32_t enabled_excepts = FEnv::exception_value_from_control(control_word); - return FEnv::exception_value_to_status(enabled_excepts); + return static_cast(FEnv::exception_value_to_status(enabled_excepts)); } LIBC_INLINE int clear_except(int excepts) { uint32_t status_word = FEnv::get_status_word(); - uint32_t except_value = FEnv::exception_value_from_status(excepts); + uint32_t except_value = + FEnv::exception_value_from_status(static_cast(excepts)); status_word &= ~FEnv::exception_value_to_status(except_value); FEnv::set_status_word(status_word); return 0; @@ -148,13 +151,16 @@ LIBC_INLINE int clear_except(int excepts) { LIBC_INLINE int test_except(int excepts) { uint32_t statusWord = FEnv::get_status_word(); - uint32_t ex_value = FEnv::exception_value_from_status(excepts); - return statusWord & FEnv::exception_value_to_status(ex_value); + uint32_t ex_value = + FEnv::exception_value_from_status(static_cast(excepts)); + return static_cast(statusWord & + FEnv::exception_value_to_status(ex_value)); } LIBC_INLINE int set_except(int excepts) { uint32_t status_word = FEnv::get_status_word(); - uint32_t new_exceptions = FEnv::exception_value_from_status(excepts); + uint32_t new_exceptions = + FEnv::exception_value_from_status(static_cast(excepts)); status_word |= FEnv::exception_value_to_status(new_exceptions); FEnv::set_status_word(status_word); return 0; @@ -174,7 +180,8 @@ LIBC_INLINE int raise_except(int excepts) { : "s0", "s1" /* s0 and s1 are clobbered */); }; - uint32_t to_raise = FEnv::exception_value_from_status(excepts); + uint32_t to_raise = + FEnv::exception_value_from_status(static_cast(excepts)); int result = 0; if (to_raise & FEnv::EX_INVALID) { @@ -237,7 +244,7 @@ LIBC_INLINE int get_round() { } LIBC_INLINE int set_round(int mode) { - uint16_t bit_value; + uint32_t bit_value; switch (mode) { case FE_TONEAREST: bit_value = FEnv::TONEAREST; @@ -256,7 +263,7 @@ LIBC_INLINE int set_round(int mode) { } uint32_t control_word = FEnv::get_control_word(); - control_word &= ~(0x3 << FEnv::ROUNDING_CONTROL_BIT_POSITION); + control_word &= ~(0x3u << FEnv::ROUNDING_CONTROL_BIT_POSITION); control_word |= (bit_value << FEnv::ROUNDING_CONTROL_BIT_POSITION); FEnv::set_control_word(control_word); diff --git a/system/lib/llvm-libc/src/__support/FPUtil/aarch64/sqrt.h b/system/lib/llvm-libc/src/__support/FPUtil/aarch64/sqrt.h index b69267ff91f5c..4eb576bf2244c 100644 --- a/system/lib/llvm-libc/src/__support/FPUtil/aarch64/sqrt.h +++ b/system/lib/llvm-libc/src/__support/FPUtil/aarch64/sqrt.h @@ -12,6 +12,7 @@ #include "src/__support/common.h" #include "src/__support/macros/config.h" #include "src/__support/macros/properties/architectures.h" +#include "src/__support/macros/properties/cpu_features.h" #if !defined(LIBC_TARGET_ARCH_IS_AARCH64) #error "Invalid include" @@ -22,17 +23,21 @@ namespace LIBC_NAMESPACE_DECL { namespace fputil { +#ifdef LIBC_TARGET_CPU_HAS_FPU_FLOAT template <> LIBC_INLINE float sqrt(float x) { float y; - __asm__ __volatile__("fsqrt %s0, %s1\n\t" : "=w"(y) : "w"(x)); + asm("fsqrt %s0, %s1\n\t" : "=w"(y) : "w"(x)); return y; } +#endif // LIBC_TARGET_CPU_HAS_FPU_FLOAT +#ifdef LIBC_TARGET_CPU_HAS_FPU_DOUBLE template <> LIBC_INLINE double sqrt(double x) { double y; - __asm__ __volatile__("fsqrt %d0, %d1\n\t" : "=w"(y) : "w"(x)); + asm("fsqrt %d0, %d1\n\t" : "=w"(y) : "w"(x)); return y; } +#endif // LIBC_TARGET_CPU_HAS_FPU_DOUBLE } // namespace fputil } // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/__support/FPUtil/arm/sqrt.h b/system/lib/llvm-libc/src/__support/FPUtil/arm/sqrt.h new file mode 100644 index 0000000000000..e6cb58c0be817 --- /dev/null +++ b/system/lib/llvm-libc/src/__support/FPUtil/arm/sqrt.h @@ -0,0 +1,45 @@ +//===-- Square root of IEEE 754 floating point numbers ----------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC___SUPPORT_FPUTIL_ARM_SQRT_H +#define LLVM_LIBC_SRC___SUPPORT_FPUTIL_ARM_SQRT_H + +#include "src/__support/common.h" +#include "src/__support/macros/config.h" +#include "src/__support/macros/properties/architectures.h" +#include "src/__support/macros/properties/cpu_features.h" + +#if !defined(LIBC_TARGET_ARCH_IS_ARM) +#error "Invalid include" +#endif + +#include "src/__support/FPUtil/generic/sqrt.h" + +namespace LIBC_NAMESPACE_DECL { +namespace fputil { + +#ifdef LIBC_TARGET_CPU_HAS_FPU_FLOAT +template <> LIBC_INLINE float sqrt(float x) { + float y; + asm("vsqrt %0, %1\n\t" : "=w"(y) : "w"(x)); + return y; +} +#endif // LIBC_TARGET_CPU_HAS_FPU_FLOAT + +#ifdef LIBC_TARGET_CPU_HAS_FPU_DOUBLE +template <> LIBC_INLINE double sqrt(double x) { + double y; + asm("vsqrt %0, %1\n\t" : "=w"(y) : "w"(x)); + return y; +} +#endif // LIBC_TARGET_CPU_HAS_FPU_DOUBLE + +} // namespace fputil +} // namespace LIBC_NAMESPACE_DECL + +#endif // LLVM_LIBC_SRC___SUPPORT_FPUTIL_ARM_SQRT_H diff --git a/system/lib/llvm-libc/src/__support/FPUtil/cast.h b/system/lib/llvm-libc/src/__support/FPUtil/cast.h index 126f3852137b7..7578bb42b18f1 100644 --- a/system/lib/llvm-libc/src/__support/FPUtil/cast.h +++ b/system/lib/llvm-libc/src/__support/FPUtil/cast.h @@ -18,6 +18,9 @@ namespace LIBC_NAMESPACE::fputil { +// TODO: Add optimization for known good targets with fast +// float to float16 conversion: +// https://github.com/llvm/llvm-project/issues/133517 template LIBC_INLINE constexpr cpp::enable_if_t && cpp::is_floating_point_v, diff --git a/system/lib/llvm-libc/src/__support/FPUtil/double_double.h b/system/lib/llvm-libc/src/__support/FPUtil/double_double.h index db3c2c8a3d7a6..c27885aadc028 100644 --- a/system/lib/llvm-libc/src/__support/FPUtil/double_double.h +++ b/system/lib/llvm-libc/src/__support/FPUtil/double_double.h @@ -18,43 +18,52 @@ namespace LIBC_NAMESPACE_DECL { namespace fputil { -#define DEFAULT_DOUBLE_SPLIT 27 +template struct DefaultSplit; +template <> struct DefaultSplit { + static constexpr size_t VALUE = 12; +}; +template <> struct DefaultSplit { + static constexpr size_t VALUE = 27; +}; -using DoubleDouble = LIBC_NAMESPACE::NumberPair; +using DoubleDouble = NumberPair; +using FloatFloat = NumberPair; // The output of Dekker's FastTwoSum algorithm is correct, i.e.: // r.hi + r.lo = a + b exactly // and |r.lo| < eps(r.lo) // Assumption: |a| >= |b|, or a = 0. -template -LIBC_INLINE constexpr DoubleDouble exact_add(double a, double b) { - DoubleDouble r{0.0, 0.0}; +template +LIBC_INLINE constexpr NumberPair exact_add(T a, T b) { + NumberPair r{0.0, 0.0}; if constexpr (FAST2SUM) { r.hi = a + b; - double t = r.hi - a; + T t = r.hi - a; r.lo = b - t; } else { r.hi = a + b; - double t1 = r.hi - a; - double t2 = r.hi - t1; - double t3 = b - t1; - double t4 = a - t2; + T t1 = r.hi - a; + T t2 = r.hi - t1; + T t3 = b - t1; + T t4 = a - t2; r.lo = t3 + t4; } return r; } // Assumption: |a.hi| >= |b.hi| -LIBC_INLINE constexpr DoubleDouble add(const DoubleDouble &a, - const DoubleDouble &b) { - DoubleDouble r = exact_add(a.hi, b.hi); - double lo = a.lo + b.lo; +template +LIBC_INLINE constexpr NumberPair add(const NumberPair &a, + const NumberPair &b) { + NumberPair r = exact_add(a.hi, b.hi); + T lo = a.lo + b.lo; return exact_add(r.hi, r.lo + lo); } // Assumption: |a.hi| >= |b| -LIBC_INLINE constexpr DoubleDouble add(const DoubleDouble &a, double b) { - DoubleDouble r = exact_add(a.hi, b); +template +LIBC_INLINE constexpr NumberPair add(const NumberPair &a, T b) { + NumberPair r = exact_add(a.hi, b); return exact_add(r.hi, r.lo + a.lo); } @@ -63,12 +72,12 @@ LIBC_INLINE constexpr DoubleDouble add(const DoubleDouble &a, double b) { // Zimmermann, P., "Note on the Veltkamp/Dekker Algorithms with Directed // Roundings," https://inria.hal.science/hal-04480440. // Default splitting constant = 2^ceil(prec(double)/2) + 1 = 2^27 + 1. -template -LIBC_INLINE constexpr DoubleDouble split(double a) { - DoubleDouble r{0.0, 0.0}; +template ::VALUE> +LIBC_INLINE constexpr NumberPair split(T a) { + NumberPair r{0.0, 0.0}; // CN = 2^N. - constexpr double CN = static_cast(1 << N); - constexpr double C = CN + 1.0; + constexpr T CN = static_cast(1 << N); + constexpr T C = CN + 1.0; double t1 = C * a; double t2 = a - t1; r.hi = t1 + t2; @@ -77,21 +86,40 @@ LIBC_INLINE constexpr DoubleDouble split(double a) { } // Helper for non-fma exact mult where the first number is already split. -template -LIBC_INLINE DoubleDouble exact_mult(const DoubleDouble &as, double a, - double b) { - DoubleDouble bs = split(b); - DoubleDouble r{0.0, 0.0}; +template ::VALUE> +LIBC_INLINE NumberPair exact_mult(const NumberPair &as, T a, T b) { + NumberPair bs = split(b); + NumberPair r{0.0, 0.0}; r.hi = a * b; - double t1 = as.hi * bs.hi - r.hi; - double t2 = as.hi * bs.lo + t1; - double t3 = as.lo * bs.hi + t2; + T t1 = as.hi * bs.hi - r.hi; + T t2 = as.hi * bs.lo + t1; + T t3 = as.lo * bs.hi + t2; r.lo = as.lo * bs.lo + t3; return r; } +// The templated exact multiplication needs template version of +// LIBC_TARGET_CPU_HAS_FMA_* macro to correctly select the implementation. +// These can be moved to "src/__support/macros/properties/cpu_features.h" if +// other part of libc needed. +template struct TargetHasFmaInstruction { + static constexpr bool VALUE = false; +}; + +#ifdef LIBC_TARGET_CPU_HAS_FMA_FLOAT +template <> struct TargetHasFmaInstruction { + static constexpr bool VALUE = true; +}; +#endif // LIBC_TARGET_CPU_HAS_FMA_FLOAT + +#ifdef LIBC_TARGET_CPU_HAS_FMA_DOUBLE +template <> struct TargetHasFmaInstruction { + static constexpr bool VALUE = true; +}; +#endif // LIBC_TARGET_CPU_HAS_FMA_DOUBLE + // Note: When FMA instruction is not available, the `exact_mult` function is // only correct for round-to-nearest mode. See: // Zimmermann, P., "Note on the Veltkamp/Dekker Algorithms with Directed @@ -99,19 +127,19 @@ LIBC_INLINE DoubleDouble exact_mult(const DoubleDouble &as, double a, // Using Theorem 1 in the paper above, without FMA instruction, if we restrict // the generated constants to precision <= 51, and splitting it by 2^28 + 1, // then a * b = r.hi + r.lo is exact for all rounding modes. -template -LIBC_INLINE DoubleDouble exact_mult(double a, double b) { - DoubleDouble r{0.0, 0.0}; +template ::VALUE> +LIBC_INLINE NumberPair exact_mult(T a, T b) { + NumberPair r{0.0, 0.0}; -#ifdef LIBC_TARGET_CPU_HAS_FMA - r.hi = a * b; - r.lo = fputil::multiply_add(a, b, -r.hi); -#else - // Dekker's Product. - DoubleDouble as = split(a); + if constexpr (TargetHasFmaInstruction::VALUE) { + r.hi = a * b; + r.lo = fputil::multiply_add(a, b, -r.hi); + } else { + // Dekker's Product. + NumberPair as = split(a); - r = exact_mult(as, a, b); -#endif // LIBC_TARGET_CPU_HAS_FMA + r = exact_mult(as, a, b); + } return r; } @@ -125,7 +153,7 @@ LIBC_INLINE DoubleDouble quick_mult(double a, const DoubleDouble &b) { template LIBC_INLINE DoubleDouble quick_mult(const DoubleDouble &a, const DoubleDouble &b) { - DoubleDouble r = exact_mult(a.hi, b.hi); + DoubleDouble r = exact_mult(a.hi, b.hi); double t1 = multiply_add(a.hi, b.lo, r.lo); double t2 = multiply_add(a.lo, b.hi, t1); r.lo = t2; @@ -157,19 +185,20 @@ LIBC_INLINE DoubleDouble multiply_add(const DoubleDouble &a, // rl = q * (ah - bh * rh) + q * (al - bl * rh) // as accurate as possible, then the error is bounded by: // |(ah + al) / (bh + bl) - (rh + rl)| < O(bl/bh) * (2^-52 + al/ah + bl/bh) -LIBC_INLINE DoubleDouble div(const DoubleDouble &a, const DoubleDouble &b) { - DoubleDouble r; - double q = 1.0 / b.hi; +template +LIBC_INLINE NumberPair div(const NumberPair &a, const NumberPair &b) { + NumberPair r; + T q = T(1) / b.hi; r.hi = a.hi * q; #ifdef LIBC_TARGET_CPU_HAS_FMA - double e_hi = fputil::multiply_add(b.hi, -r.hi, a.hi); - double e_lo = fputil::multiply_add(b.lo, -r.hi, a.lo); + T e_hi = fputil::multiply_add(b.hi, -r.hi, a.hi); + T e_lo = fputil::multiply_add(b.lo, -r.hi, a.lo); #else - DoubleDouble b_hi_r_hi = fputil::exact_mult(b.hi, -r.hi); - DoubleDouble b_lo_r_hi = fputil::exact_mult(b.lo, -r.hi); - double e_hi = (a.hi + b_hi_r_hi.hi) + b_hi_r_hi.lo; - double e_lo = (a.lo + b_lo_r_hi.hi) + b_lo_r_hi.lo; + NumberPair b_hi_r_hi = fputil::exact_mult(b.hi, -r.hi); + NumberPair b_lo_r_hi = fputil::exact_mult(b.lo, -r.hi); + T e_hi = (a.hi + b_hi_r_hi.hi) + b_hi_r_hi.lo; + T e_lo = (a.lo + b_lo_r_hi.hi) + b_lo_r_hi.lo; #endif // LIBC_TARGET_CPU_HAS_FMA r.lo = q * (e_hi + e_lo); diff --git a/system/lib/llvm-libc/src/__support/FPUtil/dyadic_float.h b/system/lib/llvm-libc/src/__support/FPUtil/dyadic_float.h index 289fd01680547..6c3e1520e5aff 100644 --- a/system/lib/llvm-libc/src/__support/FPUtil/dyadic_float.h +++ b/system/lib/llvm-libc/src/__support/FPUtil/dyadic_float.h @@ -26,6 +26,54 @@ namespace LIBC_NAMESPACE_DECL { namespace fputil { +// Decide whether to round a UInt up, down or not at all at a given bit +// position, based on the current rounding mode. The assumption is that the +// caller is going to make the integer `value >> rshift`, and then might need +// to round it up by 1 depending on the value of the bits shifted off the +// bottom. +// +// `logical_sign` causes the behavior of FE_DOWNWARD and FE_UPWARD to +// be reversed, which is what you'd want if this is the mantissa of a +// negative floating-point number. +// +// Return value is +1 if the value should be rounded up; -1 if it should be +// rounded down; 0 if it's exact and needs no rounding. +template +LIBC_INLINE constexpr int +rounding_direction(const LIBC_NAMESPACE::UInt &value, size_t rshift, + Sign logical_sign) { + if (rshift == 0 || (rshift < Bits && (value << (Bits - rshift)) == 0) || + (rshift >= Bits && value == 0)) + return 0; // exact + + switch (quick_get_round()) { + case FE_TONEAREST: + if (rshift > 0 && rshift <= Bits && value.get_bit(rshift - 1)) { + // We round up, unless the value is an exact halfway case and + // the bit that will end up in the units place is 0, in which + // case tie-break-to-even says round down. + bool round_bit = rshift < Bits ? value.get_bit(rshift) : 0; + return round_bit != 0 || (value << (Bits - rshift + 1)) != 0 ? +1 : -1; + } else { + return -1; + } + case FE_TOWARDZERO: + return -1; + case FE_DOWNWARD: + return logical_sign.is_neg() && + (rshift < Bits && (value << (Bits - rshift)) != 0) + ? +1 + : -1; + case FE_UPWARD: + return logical_sign.is_pos() && + (rshift < Bits && (value << (Bits - rshift)) != 0) + ? +1 + : -1; + default: + __builtin_unreachable(); + } +} + // A generic class to perform computations of high precision floating points. // We store the value in dyadic format, including 3 fields: // sign : boolean value - false means positive, true means negative @@ -56,7 +104,7 @@ template struct DyadicFloat { normalize(); } - LIBC_INLINE constexpr DyadicFloat(Sign s, int e, MantissaType m) + LIBC_INLINE constexpr DyadicFloat(Sign s, int e, const MantissaType &m) : sign(s), exponent(e), mantissa(m) { normalize(); } @@ -101,12 +149,33 @@ template struct DyadicFloat { return exponent + (Bits - 1); } + // Produce a correctly rounded DyadicFloat from a too-large mantissa, + // by shifting it down and rounding if necessary. + template + LIBC_INLINE constexpr static DyadicFloat + round(Sign result_sign, int result_exponent, + const LIBC_NAMESPACE::UInt &input_mantissa, + size_t rshift) { + MantissaType result_mantissa(input_mantissa >> rshift); + if (rounding_direction(input_mantissa, rshift, result_sign) > 0) { + ++result_mantissa; + if (result_mantissa == 0) { + // Rounding up made the mantissa integer wrap round to 0, + // carrying a bit off the top. So we've rounded up to the next + // exponent. + result_mantissa.set_bit(Bits - 1); + ++result_exponent; + } + } + return DyadicFloat(result_sign, result_exponent, result_mantissa); + } + #ifdef LIBC_TYPES_HAS_FLOAT16 template LIBC_INLINE constexpr cpp::enable_if_t< cpp::is_floating_point_v && (FPBits::FRACTION_LEN < Bits), T> generic_as() const { - using FPBits = FPBits; + using FPBits = FPBits; using StorageType = typename FPBits::StorageType; constexpr int EXTRA_FRACTION_LEN = Bits - 1 - FPBits::FRACTION_LEN; @@ -266,7 +335,7 @@ template struct DyadicFloat { .get_val(); MantissaType round_mask = - shift > MantissaType::BITS ? 0 : MantissaType(1) << (shift - 1); + shift - 1 >= MantissaType::BITS ? 0 : MantissaType(1) << (shift - 1); MantissaType sticky_mask = round_mask - MantissaType(1); bool round_bit = !(mantissa & round_mask).is_zero(); @@ -365,7 +434,12 @@ template struct DyadicFloat { if (exponent > 0) { new_mant <<= exponent; } else { - new_mant >>= (-exponent); + // Cast the exponent to size_t before negating it, rather than after, + // to avoid undefined behavior negating INT_MIN as an integer (although + // exponents coming in to this function _shouldn't_ be that large). The + // result should always end up as a positive size_t. + size_t shift = -static_cast(exponent); + new_mant >>= shift; } if (sign.is_neg()) { @@ -374,6 +448,43 @@ template struct DyadicFloat { return new_mant; } + + LIBC_INLINE constexpr MantissaType + as_mantissa_type_rounded(int *round_dir_out = nullptr) const { + int round_dir = 0; + MantissaType new_mant; + if (mantissa.is_zero()) { + new_mant = 0; + } else { + new_mant = mantissa; + if (exponent > 0) { + new_mant <<= exponent; + } else if (exponent < 0) { + // Cast the exponent to size_t before negating it, rather than after, + // to avoid undefined behavior negating INT_MIN as an integer (although + // exponents coming in to this function _shouldn't_ be that large). The + // result should always end up as a positive size_t. + size_t shift = -static_cast(exponent); + new_mant >>= shift; + round_dir = rounding_direction(mantissa, shift, sign); + if (round_dir > 0) + ++new_mant; + } + + if (sign.is_neg()) { + new_mant = (~new_mant) + 1; + } + } + + if (round_dir_out) + *round_dir_out = round_dir; + + return new_mant; + } + + LIBC_INLINE constexpr DyadicFloat operator-() const { + return DyadicFloat(sign.negate(), exponent, mantissa); + } }; // Quick add - Add 2 dyadic floats with rounding toward 0 and then normalize the @@ -433,6 +544,12 @@ LIBC_INLINE constexpr DyadicFloat quick_add(DyadicFloat a, return result.normalize(); } +template +LIBC_INLINE constexpr DyadicFloat quick_sub(DyadicFloat a, + DyadicFloat b) { + return quick_add(a, -b); +} + // Quick Mul - Slightly less accurate but efficient multiplication of 2 dyadic // floats with rounding toward 0 and then normalize the output: // result.exponent = a.exponent + b.exponent + Bits, @@ -464,6 +581,95 @@ LIBC_INLINE constexpr DyadicFloat quick_mul(const DyadicFloat &a, return result; } +// Correctly rounded multiplication of 2 dyadic floats, assuming the +// exponent remains within range. +template +LIBC_INLINE constexpr DyadicFloat +rounded_mul(const DyadicFloat &a, const DyadicFloat &b) { + using DblMant = LIBC_NAMESPACE::UInt<(2 * Bits)>; + Sign result_sign = (a.sign != b.sign) ? Sign::NEG : Sign::POS; + int result_exponent = a.exponent + b.exponent + static_cast(Bits); + auto product = DblMant(a.mantissa) * DblMant(b.mantissa); + // As in quick_mul(), renormalize by 1 bit manually rather than countl_zero + if (product.get_bit(2 * Bits - 1) == 0) { + product <<= 1; + result_exponent -= 1; + } + + return DyadicFloat::round(result_sign, result_exponent, product, Bits); +} + +// Approximate reciprocal - given a nonzero a, make a good approximation to 1/a. +// The method is Newton-Raphson iteration, based on quick_mul. +template = 32)>> +LIBC_INLINE constexpr DyadicFloat +approx_reciprocal(const DyadicFloat &a) { + // Given an approximation x to 1/a, a better one is x' = x(2-ax). + // + // You can derive this by using the Newton-Raphson formula with the function + // f(x) = 1/x - a. But another way to see that it works is to say: suppose + // that ax = 1-e for some small error e. Then ax' = ax(2-ax) = (1-e)(1+e) = + // 1-e^2. So the error in x' is the square of the error in x, i.e. the number + // of correct bits in x' is double the number in x. + + // An initial approximation to the reciprocal + DyadicFloat x(Sign::POS, -32 - a.exponent - int(Bits), + uint64_t(0xFFFFFFFFFFFFFFFF) / + static_cast(a.mantissa >> (Bits - 32))); + + // The constant 2, which we'll need in every iteration + DyadicFloat two(Sign::POS, 1, 1); + + // We expect at least 31 correct bits from our 32-bit starting approximation + size_t ok_bits = 31; + + // The number of good bits doubles in each iteration, except that rounding + // errors introduce a little extra each time. Subtract a bit from our + // accuracy assessment to account for that. + while (ok_bits < Bits) { + x = quick_mul(x, quick_sub(two, quick_mul(a, x))); + ok_bits = 2 * ok_bits - 1; + } + + return x; +} + +// Correctly rounded division of 2 dyadic floats, assuming the +// exponent remains within range. +template +LIBC_INLINE constexpr DyadicFloat +rounded_div(const DyadicFloat &af, const DyadicFloat &bf) { + using DblMant = LIBC_NAMESPACE::UInt<(Bits * 2 + 64)>; + + // Make an approximation to the quotient as a * (1/b). Both the + // multiplication and the reciprocal are a bit sloppy, which doesn't + // matter, because we're going to correct for that below. + auto qf = fputil::quick_mul(af, fputil::approx_reciprocal(bf)); + + // Switch to BigInt and stop using quick_add and quick_mul: now + // we're working in exact integers so as to get the true remainder. + DblMant a = af.mantissa, b = bf.mantissa, q = qf.mantissa; + q <<= 2; // leave room for a round bit, even if exponent decreases + a <<= af.exponent - bf.exponent - qf.exponent + 2; + DblMant qb = q * b; + if (qb < a) { + DblMant too_small = a - b; + while (qb <= too_small) { + qb += b; + ++q; + } + } else { + while (qb > a) { + qb -= b; + --q; + } + } + + DyadicFloat<(Bits * 2)> qbig(qf.sign, qf.exponent - 2, q); + return DyadicFloat::round(qbig.sign, qbig.exponent + Bits, + qbig.mantissa, Bits); +} + // Simple polynomial approximation. template LIBC_INLINE constexpr DyadicFloat diff --git a/system/lib/llvm-libc/src/__support/FPUtil/generic/add_sub.h b/system/lib/llvm-libc/src/__support/FPUtil/generic/add_sub.h index 6bc9dcd23bafa..fda702931ef6f 100644 --- a/system/lib/llvm-libc/src/__support/FPUtil/generic/add_sub.h +++ b/system/lib/llvm-libc/src/__support/FPUtil/generic/add_sub.h @@ -9,7 +9,6 @@ #ifndef LLVM_LIBC_SRC___SUPPORT_FPUTIL_GENERIC_ADD_SUB_H #define LLVM_LIBC_SRC___SUPPORT_FPUTIL_GENERIC_ADD_SUB_H -#include "hdr/errno_macros.h" #include "hdr/fenv_macros.h" #include "src/__support/CPP/algorithm.h" #include "src/__support/CPP/bit.h" @@ -110,12 +109,8 @@ add_or_sub(InType x, InType y) { return cast(tmp); } - if (y_bits.is_zero()) { - volatile InType tmp = y; - if constexpr (IsSub) - tmp = -tmp; - return cast(tmp); - } + if (y_bits.is_zero()) + return cast(x); } InType x_abs = x_bits.abs().get_val(); @@ -160,20 +155,22 @@ add_or_sub(InType x, InType y) { } else { InStorageType max_mant = max_bits.get_explicit_mantissa() << GUARD_BITS_LEN; InStorageType min_mant = min_bits.get_explicit_mantissa() << GUARD_BITS_LEN; - int alignment = - max_bits.get_biased_exponent() - min_bits.get_biased_exponent(); + + int alignment = (max_bits.get_biased_exponent() - max_bits.is_normal()) - + (min_bits.get_biased_exponent() - min_bits.is_normal()); InStorageType aligned_min_mant = min_mant >> cpp::min(alignment, RESULT_MANTISSA_LEN); bool aligned_min_mant_sticky; - if (alignment <= 3) + if (alignment <= GUARD_BITS_LEN) aligned_min_mant_sticky = false; - else if (alignment <= InFPBits::FRACTION_LEN + 3) - aligned_min_mant_sticky = - (min_mant << (InFPBits::STORAGE_LEN - alignment)) != 0; - else + else if (alignment > InFPBits::FRACTION_LEN + GUARD_BITS_LEN) aligned_min_mant_sticky = true; + else + aligned_min_mant_sticky = + (static_cast( + min_mant << (InFPBits::STORAGE_LEN - alignment))) != 0; InStorageType min_mant_sticky(static_cast(aligned_min_mant_sticky)); @@ -183,7 +180,7 @@ add_or_sub(InType x, InType y) { result_mant = max_mant - (aligned_min_mant | min_mant_sticky); } - int result_exp = max_bits.get_exponent() - RESULT_FRACTION_LEN; + int result_exp = max_bits.get_explicit_exponent() - RESULT_FRACTION_LEN; DyadicFloat result(result_sign, result_exp, result_mant); return result.template as(); } diff --git a/system/lib/llvm-libc/src/__support/FPUtil/generic/sqrt_80_bit_long_double.h b/system/lib/llvm-libc/src/__support/FPUtil/generic/sqrt_80_bit_long_double.h index 9492d52da0455..0ba836d17a085 100644 --- a/system/lib/llvm-libc/src/__support/FPUtil/generic/sqrt_80_bit_long_double.h +++ b/system/lib/llvm-libc/src/__support/FPUtil/generic/sqrt_80_bit_long_double.h @@ -24,7 +24,7 @@ namespace x86 { LIBC_INLINE void normalize(int &exponent, FPBits::StorageType &mantissa) { const unsigned int shift = static_cast( - cpp::countl_zero(static_cast(mantissa)) - + static_cast(cpp::countl_zero(static_cast(mantissa))) - (8 * sizeof(uint64_t) - 1 - FPBits::FRACTION_LEN)); exponent -= shift; mantissa <<= shift; diff --git a/system/lib/llvm-libc/src/__support/FPUtil/multiply_add.h b/system/lib/llvm-libc/src/__support/FPUtil/multiply_add.h index a86067c401873..8260702e2c9f4 100644 --- a/system/lib/llvm-libc/src/__support/FPUtil/multiply_add.h +++ b/system/lib/llvm-libc/src/__support/FPUtil/multiply_add.h @@ -46,13 +46,25 @@ multiply_add(T x, T y, T z) { namespace LIBC_NAMESPACE_DECL { namespace fputil { +#ifdef LIBC_TARGET_CPU_HAS_FMA_FLOAT LIBC_INLINE float multiply_add(float x, float y, float z) { +#if __has_builtin(__builtin_elementwise_fma) + return __builtin_elementwise_fma(x, y, z); +#else return __builtin_fmaf(x, y, z); +#endif } +#endif // LIBC_TARGET_CPU_HAS_FMA_FLOAT +#ifdef LIBC_TARGET_CPU_HAS_FMA_DOUBLE LIBC_INLINE double multiply_add(double x, double y, double z) { +#if __has_builtin(__builtin_elementwise_fma) + return __builtin_elementwise_fma(x, y, z); +#else return __builtin_fma(x, y, z); +#endif } +#endif // LIBC_TARGET_CPU_HAS_FMA_DOUBLE } // namespace fputil } // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/__support/FPUtil/nearest_integer.h b/system/lib/llvm-libc/src/__support/FPUtil/nearest_integer.h index 5d0deddd751f5..768f13414bd95 100644 --- a/system/lib/llvm-libc/src/__support/FPUtil/nearest_integer.h +++ b/system/lib/llvm-libc/src/__support/FPUtil/nearest_integer.h @@ -16,7 +16,7 @@ #if (defined(LIBC_TARGET_ARCH_IS_X86_64) && defined(LIBC_TARGET_CPU_HAS_SSE4_2)) #include "x86_64/nearest_integer.h" -#elif defined(LIBC_TARGET_ARCH_IS_AARCH64) +#elif (defined(LIBC_TARGET_ARCH_IS_AARCH64) && defined(__ARM_FP)) #include "aarch64/nearest_integer.h" #elif defined(LIBC_TARGET_ARCH_IS_GPU) diff --git a/system/lib/llvm-libc/src/__support/FPUtil/riscv/sqrt.h b/system/lib/llvm-libc/src/__support/FPUtil/riscv/sqrt.h index 0363822a4e8af..8cff03a9fc73e 100644 --- a/system/lib/llvm-libc/src/__support/FPUtil/riscv/sqrt.h +++ b/system/lib/llvm-libc/src/__support/FPUtil/riscv/sqrt.h @@ -12,6 +12,7 @@ #include "src/__support/common.h" #include "src/__support/macros/config.h" #include "src/__support/macros/properties/architectures.h" +#include "src/__support/macros/properties/cpu_features.h" #if !defined(LIBC_TARGET_ARCH_IS_ANY_RISCV) #error "Invalid include" @@ -22,21 +23,21 @@ namespace LIBC_NAMESPACE_DECL { namespace fputil { -#ifdef __riscv_flen +#ifdef LIBC_TARGET_CPU_HAS_FPU_FLOAT template <> LIBC_INLINE float sqrt(float x) { float result; - __asm__ __volatile__("fsqrt.s %0, %1\n\t" : "=f"(result) : "f"(x)); + asm("fsqrt.s %0, %1\n\t" : "=f"(result) : "f"(x)); return result; } +#endif // LIBC_TARGET_CPU_HAS_FPU_FLOAT -#if __riscv_flen >= 64 +#ifdef LIBC_TARGET_CPU_HAS_FPU_DOUBLE template <> LIBC_INLINE double sqrt(double x) { double result; - __asm__ __volatile__("fsqrt.d %0, %1\n\t" : "=f"(result) : "f"(x)); + asm("fsqrt.d %0, %1\n\t" : "=f"(result) : "f"(x)); return result; } -#endif // __riscv_flen >= 64 -#endif // __riscv_flen +#endif // LIBC_TARGET_CPU_HAS_FPU_FLOAT } // namespace fputil } // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/__support/FPUtil/sqrt.h b/system/lib/llvm-libc/src/__support/FPUtil/sqrt.h index eb86ddfa89d8e..d4ed744b4010d 100644 --- a/system/lib/llvm-libc/src/__support/FPUtil/sqrt.h +++ b/system/lib/llvm-libc/src/__support/FPUtil/sqrt.h @@ -12,14 +12,54 @@ #include "src/__support/macros/properties/architectures.h" #include "src/__support/macros/properties/cpu_features.h" -#if defined(LIBC_TARGET_ARCH_IS_X86_64) && defined(LIBC_TARGET_CPU_HAS_SSE2) +#include "src/__support/FPUtil/generic/sqrt.h" + +// Generic instruction specializations with __builtin_elementwise_sqrt. +#if defined(LIBC_TARGET_CPU_HAS_FPU_FLOAT) || \ + defined(LIBC_TARGET_CPU_HAS_FPU_DOUBLE) + +#if __has_builtin(__builtin_elementwise_sqrt) + +namespace LIBC_NAMESPACE_DECL { +namespace fputil { + +#ifdef LIBC_TARGET_CPU_HAS_FPU_FLOAT +template <> LIBC_INLINE float sqrt(float x) { + return __builtin_elementwise_sqrt(x); +} +#endif // LIBC_TARGET_CPU_HAS_FPU_FLOAT + +#ifdef LIBC_TARGET_CPU_HAS_FPU_DOUBLE +template <> LIBC_INLINE double sqrt(double x) { + return __builtin_elementwise_sqrt(x); +} +#endif // LIBC_TARGET_CPU_HAS_FPU_DOUBLE + +// Use 80-bit long double instruction on x86. +// https://godbolt.org/z/oWEaj6hxK +#ifdef LIBC_TYPES_LONG_DOUBLE_IS_X86_FLOAT80 +template <> LIBC_INLINE long double sqrt(long double x) { + return __builtin_elementwise_sqrt(x); +} +#endif // LIBC_TYPES_LONG_DOUBLE_IS_X86_FLOAT80 + +} // namespace fputil +} // namespace LIBC_NAMESPACE_DECL + +#else // __builtin_elementwise_sqrt +// Use inline assembly when __builtin_elementwise_sqrt is not available. +#if defined(LIBC_TARGET_CPU_HAS_SSE2) #include "x86_64/sqrt.h" -#elif defined(LIBC_TARGET_ARCH_IS_AARCH64) +#elif defined(LIBC_TARGET_ARCH_IS_AARCH64) && defined(__ARM_FP) #include "aarch64/sqrt.h" +#elif defined(LIBC_TARGET_ARCH_IS_ARM) +#include "arm/sqrt.h" #elif defined(LIBC_TARGET_ARCH_IS_ANY_RISCV) #include "riscv/sqrt.h" -#else -#include "generic/sqrt.h" +#endif // Target specific header of inline asm. + +#endif // __builtin_elementwise_sqrt + +#endif // LIBC_TARGET_CPU_HAS_FPU_FLOAT or DOUBLE -#endif #endif // LLVM_LIBC_SRC___SUPPORT_FPUTIL_SQRT_H diff --git a/system/lib/llvm-libc/src/__support/File/dir.cpp b/system/lib/llvm-libc/src/__support/File/dir.cpp index 21b0106f70106..aea8862c15f7f 100644 --- a/system/lib/llvm-libc/src/__support/File/dir.cpp +++ b/system/lib/llvm-libc/src/__support/File/dir.cpp @@ -11,8 +11,8 @@ #include "src/__support/CPP/mutex.h" // lock_guard #include "src/__support/CPP/new.h" #include "src/__support/error_or.h" +#include "src/__support/libc_errno.h" // For error macros #include "src/__support/macros/config.h" -#include "src/errno/libc_errno.h" // For error macros namespace LIBC_NAMESPACE_DECL { diff --git a/system/lib/llvm-libc/src/__support/File/file.cpp b/system/lib/llvm-libc/src/__support/File/file.cpp index 528542cccf324..303852dbbb717 100644 --- a/system/lib/llvm-libc/src/__support/File/file.cpp +++ b/system/lib/llvm-libc/src/__support/File/file.cpp @@ -13,8 +13,8 @@ #include "hdr/types/off_t.h" #include "src/__support/CPP/new.h" #include "src/__support/CPP/span.h" +#include "src/__support/libc_errno.h" // For error macros #include "src/__support/macros/config.h" -#include "src/errno/libc_errno.h" // For error macros namespace LIBC_NAMESPACE_DECL { diff --git a/system/lib/llvm-libc/src/__support/File/linux/file.cpp b/system/lib/llvm-libc/src/__support/File/linux/file.cpp index 824c1f200e8c5..4594dadf1ccdf 100644 --- a/system/lib/llvm-libc/src/__support/File/linux/file.cpp +++ b/system/lib/llvm-libc/src/__support/File/linux/file.cpp @@ -15,12 +15,12 @@ #include "src/__support/File/linux/lseekImpl.h" #include "src/__support/OSUtil/fcntl.h" #include "src/__support/OSUtil/syscall.h" // For internal syscall function. +#include "src/__support/libc_errno.h" // For error macros #include "src/__support/macros/config.h" -#include "src/errno/libc_errno.h" // For error macros #include "hdr/fcntl_macros.h" // For mode_t and other flags to the open syscall -#include // For S_IS*, S_IF*, and S_IR* flags. -#include // For syscall numbers +#include // For S_IS*, S_IF*, and S_IR* flags. +#include // For syscall numbers namespace LIBC_NAMESPACE_DECL { @@ -128,10 +128,11 @@ ErrorOr create_file_from_fd(int fd, const char *mode) { return Error(EINVAL); } - int fd_flags = internal::fcntl(fd, F_GETFL); - if (fd_flags == -1) { + auto result = internal::fcntl(fd, F_GETFL); + if (!result.has_value()) { return Error(EBADF); } + int fd_flags = result.value(); using OpenMode = File::OpenMode; if (((fd_flags & O_ACCMODE) == O_RDONLY && @@ -145,8 +146,9 @@ ErrorOr create_file_from_fd(int fd, const char *mode) { if ((modeflags & static_cast(OpenMode::APPEND)) && !(fd_flags & O_APPEND)) { do_seek = true; - if (internal::fcntl(fd, F_SETFL, - reinterpret_cast(fd_flags | O_APPEND)) == -1) { + if (!internal::fcntl(fd, F_SETFL, + reinterpret_cast(fd_flags | O_APPEND)) + .has_value()) { return Error(EBADF); } } diff --git a/system/lib/llvm-libc/src/__support/File/linux/lseekImpl.h b/system/lib/llvm-libc/src/__support/File/linux/lseekImpl.h index a034913d9f6ec..300e5c5dd55bf 100644 --- a/system/lib/llvm-libc/src/__support/File/linux/lseekImpl.h +++ b/system/lib/llvm-libc/src/__support/File/linux/lseekImpl.h @@ -13,8 +13,8 @@ #include "src/__support/OSUtil/syscall.h" // For internal syscall function. #include "src/__support/common.h" #include "src/__support/error_or.h" +#include "src/__support/libc_errno.h" #include "src/__support/macros/config.h" -#include "src/errno/libc_errno.h" #include // For uint64_t. #include // For syscall numbers. diff --git a/system/lib/llvm-libc/src/__support/GPU/allocator.cpp b/system/lib/llvm-libc/src/__support/GPU/allocator.cpp index ac335a1b9aab0..66ab155e5c299 100644 --- a/system/lib/llvm-libc/src/__support/GPU/allocator.cpp +++ b/system/lib/llvm-libc/src/__support/GPU/allocator.cpp @@ -5,17 +5,49 @@ // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// +// +// This file implements a parallel allocator intended for use on a GPU device. +// The core algorithm is slab allocator using a random walk over a bitfield for +// maximum parallel progress. Slab handling is done by a wait-free reference +// counted guard. The first use of a slab will create it from system memory for +// re-use. The last use will invalidate it and free the memory. +// +//===----------------------------------------------------------------------===// #include "allocator.h" +#include "src/__support/CPP/atomic.h" +#include "src/__support/CPP/bit.h" +#include "src/__support/CPP/new.h" #include "src/__support/GPU/utils.h" #include "src/__support/RPC/rpc_client.h" -#include "src/__support/macros/config.h" +#include "src/__support/threads/sleep.h" namespace LIBC_NAMESPACE_DECL { -namespace { -void *rpc_allocate(uint64_t size) { +constexpr static uint64_t MAX_SIZE = /* 64 GiB */ 64ull * 1024 * 1024 * 1024; +constexpr static uint64_t SLAB_SIZE = /* 2 MiB */ 2ull * 1024 * 1024; +constexpr static uint64_t ARRAY_SIZE = MAX_SIZE / SLAB_SIZE; +constexpr static uint64_t SLAB_ALIGNMENT = SLAB_SIZE - 1; +constexpr static uint32_t BITS_IN_WORD = sizeof(uint32_t) * 8; +constexpr static uint32_t MIN_SIZE = 16; +constexpr static uint32_t MIN_ALIGNMENT = MIN_SIZE - 1; + +// A sentinel used to indicate an invalid but non-null pointer value. +constexpr static uint64_t SENTINEL = cpp::numeric_limits::max(); + +// The number of times we will try starting on a single index before skipping +// past it. +constexpr static uint32_t MAX_TRIES = 512; + +static_assert(!(ARRAY_SIZE & (ARRAY_SIZE - 1)), "Must be a power of two"); + +namespace impl { +// Allocates more memory from the system through the RPC interface. All +// allocations from the system MUST be aligned on a 2MiB barrier. The default +// HSA allocator has this behavior for any allocation >= 2MiB and the CUDA +// driver provides an alignment field for virtual memory allocations. +static void *rpc_allocate(uint64_t size) { void *ptr = nullptr; rpc::Client::Port port = rpc::client.open(); port.send_and_recv( @@ -27,7 +59,8 @@ void *rpc_allocate(uint64_t size) { return ptr; } -void rpc_free(void *ptr) { +// Deallocates the associated system memory. +static void rpc_free(void *ptr) { rpc::Client::Port port = rpc::client.open(); port.send([=](rpc::Buffer *buffer, uint32_t) { buffer->data[0] = reinterpret_cast(ptr); @@ -35,13 +68,487 @@ void rpc_free(void *ptr) { port.close(); } -} // namespace +// Convert a potentially disjoint bitmask into an increasing integer per-lane +// for use with indexing between gpu lanes. +static inline uint32_t lane_count(uint64_t lane_mask) { + return cpp::popcount(lane_mask & ((uint64_t(1) << gpu::get_lane_id()) - 1)); +} + +// Obtain an initial value to seed a random number generator. We use the rounded +// multiples of the golden ratio from xorshift* as additional spreading. +static inline uint32_t entropy() { + return (static_cast(gpu::processor_clock()) ^ + (gpu::get_thread_id_x() * 0x632be59b) ^ + (gpu::get_block_id_x() * 0x85157af5)) * + 0x9e3779bb; +} + +// Generate a random number and update the state using the xorshift32* PRNG. +static inline uint32_t xorshift32(uint32_t &state) { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + return state * 0x9e3779bb; +} + +// Final stage of murmurhash used to get a unique index for the global array +static inline uint32_t hash(uint32_t x) { + x ^= x >> 16; + x *= 0x85ebca6b; + x ^= x >> 13; + x *= 0xc2b2ae35; + x ^= x >> 16; + return x; +} + +// Rounds the input value to the closest permitted chunk size. Here we accept +// the sum of the closest three powers of two. For a 2MiB slab size this is 48 +// different chunk sizes. This gives us average internal fragmentation of 87.5%. +static inline uint32_t get_chunk_size(uint32_t x) { + uint32_t y = x < MIN_SIZE ? MIN_SIZE : x; + uint32_t pow2 = BITS_IN_WORD - cpp::countl_zero(y - 1); + + uint32_t s0 = 0b0100 << (pow2 - 3); + uint32_t s1 = 0b0110 << (pow2 - 3); + uint32_t s2 = 0b0111 << (pow2 - 3); + uint32_t s3 = 0b1000 << (pow2 - 3); + + if (s0 > y) + return (s0 + MIN_ALIGNMENT) & ~MIN_ALIGNMENT; + if (s1 > y) + return (s1 + MIN_ALIGNMENT) & ~MIN_ALIGNMENT; + if (s2 > y) + return (s2 + MIN_ALIGNMENT) & ~MIN_ALIGNMENT; + return (s3 + MIN_ALIGNMENT) & ~MIN_ALIGNMENT; +} + +// Rounds to the nearest power of two. +template +static inline constexpr T round_up(const T x) { + static_assert(((N - 1) & N) == 0, "N must be a power of two"); + return (x + N) & ~(N - 1); +} + +// Perform a lane parallel memset on a uint32_t pointer. +void uniform_memset(uint32_t *s, uint32_t c, uint32_t n, uint64_t uniform) { + uint64_t mask = gpu::get_lane_mask(); + uint32_t workers = cpp::popcount(uniform); + for (uint32_t i = impl::lane_count(mask & uniform); i < n; i += workers) + s[i] = c; +} + +} // namespace impl + +/// A slab allocator used to hand out identically sized slabs of memory. +/// Allocation is done through random walks of a bitfield until a free bit is +/// encountered. This reduces contention and is highly parallel on a GPU. +/// +/// 0 4 8 16 ... 2 MiB +/// ┌────────┬──────────┬────────┬──────────────────┬──────────────────────────┐ +/// │ chunk │ index │ pad │ bitfield[] │ memory[] │ +/// └────────┴──────────┴────────┴──────────────────┴──────────────────────────┘ +/// +/// The size of the bitfield is the slab size divided by the chunk size divided +/// by the number of bits per word. We pad the interface to ensure 16 byte +/// alignment and to indicate that if the pointer is not aligned by 2MiB it +/// belongs to a slab rather than the global allocator. +struct Slab { + // Header metadata for the slab, aligned to the minimum alignment. + struct alignas(MIN_SIZE) Header { + uint32_t chunk_size; + uint32_t global_index; + }; + + // Initialize the slab with its chunk size and index in the global table for + // use when freeing. + Slab(uint32_t chunk_size, uint32_t global_index) { + Header *header = reinterpret_cast
(memory); + header->chunk_size = chunk_size; + header->global_index = global_index; + } + + // Set the necessary bitfield bytes to zero in parallel using many lanes. This + // must be called before the bitfield can be accessed safely, memory is not + // guaranteed to be zero initialized in the current implementation. + void initialize(uint64_t uniform) { + uint32_t size = (bitfield_bytes(get_chunk_size()) + sizeof(uint32_t) - 1) / + sizeof(uint32_t); + impl::uniform_memset(get_bitfield(), 0, size, uniform); + } + + // Get the number of chunks that can theoretically fit inside this slab. + constexpr static uint32_t num_chunks(uint32_t chunk_size) { + return SLAB_SIZE / chunk_size; + } + + // Get the number of bytes needed to contain the bitfield bits. + constexpr static uint32_t bitfield_bytes(uint32_t chunk_size) { + return ((num_chunks(chunk_size) + BITS_IN_WORD - 1) / BITS_IN_WORD) * 8; + } + + // The actual amount of memory available excluding the bitfield and metadata. + constexpr static uint32_t available_bytes(uint32_t chunk_size) { + return SLAB_SIZE - bitfield_bytes(chunk_size) - sizeof(Header); + } + + // The number of chunks that can be stored in this slab. + constexpr static uint32_t available_chunks(uint32_t chunk_size) { + return available_bytes(chunk_size) / chunk_size; + } + + // The length in bits of the bitfield. + constexpr static uint32_t usable_bits(uint32_t chunk_size) { + return available_bytes(chunk_size) / chunk_size; + } + + // Get the location in the memory where we will store the chunk size. + uint32_t get_chunk_size() const { + return reinterpret_cast(memory)->chunk_size; + } + + // Get the location in the memory where we will store the global index. + uint32_t get_global_index() const { + return reinterpret_cast(memory)->global_index; + } + + // Get a pointer to where the bitfield is located in the memory. + uint32_t *get_bitfield() { + return reinterpret_cast(memory + sizeof(Header)); + } + + // Get a pointer to where the actual memory to be allocated lives. + uint8_t *get_memory(uint32_t chunk_size) { + return reinterpret_cast(get_bitfield()) + + bitfield_bytes(chunk_size); + } + + // Get a pointer to the actual memory given an index into the bitfield. + void *ptr_from_index(uint32_t index, uint32_t chunk_size) { + return get_memory(chunk_size) + index * chunk_size; + } + + // Convert a pointer back into its bitfield index using its offset. + uint32_t index_from_ptr(void *ptr, uint32_t chunk_size) { + return static_cast(reinterpret_cast(ptr) - + get_memory(chunk_size)) / + chunk_size; + } + + // Randomly walks the bitfield until it finds a free bit. Allocations attempt + // to put lanes right next to each other for better caching and convergence. + void *allocate(uint64_t lane_mask, uint64_t uniform) { + uint32_t chunk_size = get_chunk_size(); + uint32_t state = impl::entropy(); + + // The uniform mask represents which lanes contain a uniform target pointer. + // We attempt to place these next to each other. + void *result = nullptr; + for (uint64_t mask = lane_mask; mask; + mask = gpu::ballot(lane_mask, !result)) { + if (result) + continue; + + uint32_t start = gpu::broadcast_value(lane_mask, impl::xorshift32(state)); + + uint32_t id = impl::lane_count(uniform & mask); + uint32_t index = (start + id) % usable_bits(chunk_size); + uint32_t slot = index / BITS_IN_WORD; + uint32_t bit = index % BITS_IN_WORD; + + // Get the mask of bits destined for the same slot and coalesce it. + uint64_t match = uniform & gpu::match_any(mask, slot); + uint32_t length = cpp::popcount(match); + uint32_t bitmask = static_cast((uint64_t(1) << length) - 1) + << bit; + + uint32_t before = 0; + if (gpu::get_lane_id() == static_cast(cpp::countr_zero(match))) + before = cpp::AtomicRef(get_bitfield()[slot]) + .fetch_or(bitmask, cpp::MemoryOrder::RELAXED); + before = gpu::shuffle(mask, cpp::countr_zero(match), before); + if (~before & (1 << bit)) + result = ptr_from_index(index, chunk_size); + else + sleep_briefly(); + } + + cpp::atomic_thread_fence(cpp::MemoryOrder::ACQUIRE); + return result; + } + + // Deallocates memory by resetting its corresponding bit in the bitfield. + void deallocate(void *ptr) { + uint32_t chunk_size = get_chunk_size(); + uint32_t index = index_from_ptr(ptr, chunk_size); + uint32_t slot = index / BITS_IN_WORD; + uint32_t bit = index % BITS_IN_WORD; + + cpp::atomic_thread_fence(cpp::MemoryOrder::RELEASE); + cpp::AtomicRef(get_bitfield()[slot]) + .fetch_and(~(1u << bit), cpp::MemoryOrder::RELAXED); + } + + // The actual memory the slab will manage. All offsets are calculated at + // runtime with the chunk size to keep the interface convergent when a warp or + // wavefront is handling multiple sizes at once. + uint8_t memory[SLAB_SIZE]; +}; + +/// A wait-free guard around a pointer resource to be created dynamically if +/// space is available and freed once there are no more users. +struct GuardPtr { +private: + struct RefCounter { + // Indicates that the object is in its deallocation phase and thus invalid. + static constexpr uint64_t INVALID = uint64_t(1) << 63; + + // If a read preempts an unlock call we indicate this so the following + // unlock call can swap out the helped bit and maintain exclusive ownership. + static constexpr uint64_t HELPED = uint64_t(1) << 62; + + // Resets the reference counter, cannot be reset to zero safely. + void reset(uint32_t n, uint64_t &count) { + counter.store(n, cpp::MemoryOrder::RELAXED); + count = n; + } + + // Acquire a slot in the reference counter if it is not invalid. + bool acquire(uint32_t n, uint64_t &count) { + count = counter.fetch_add(n, cpp::MemoryOrder::RELAXED) + n; + return (count & INVALID) == 0; + } + + // Release a slot in the reference counter. This function should only be + // called following a valid acquire call. + bool release(uint32_t n) { + // If this thread caused the counter to reach zero we try to invalidate it + // and obtain exclusive rights to deconstruct it. If the CAS failed either + // another thread resurrected the counter and we quit, or a parallel read + // helped us invalidating it. For the latter, claim that flag and return. + if (counter.fetch_sub(n, cpp::MemoryOrder::RELAXED) == n) { + uint64_t expected = 0; + if (counter.compare_exchange_strong(expected, INVALID, + cpp::MemoryOrder::RELAXED, + cpp::MemoryOrder::RELAXED)) + return true; + else if ((expected & HELPED) && + (counter.exchange(INVALID, cpp::MemoryOrder::RELAXED) & + HELPED)) + return true; + } + return false; + } + + // Returns the current reference count, potentially helping a releasing + // thread. + uint64_t read() { + auto val = counter.load(cpp::MemoryOrder::RELAXED); + if (val == 0 && counter.compare_exchange_strong( + val, INVALID | HELPED, cpp::MemoryOrder::RELAXED)) + return 0; + return (val & INVALID) ? 0 : val; + } + + cpp::Atomic counter{0}; + }; + + cpp::Atomic ptr{nullptr}; + RefCounter ref{}; + + // Should be called be a single lane for each different pointer. + template + Slab *try_lock_impl(uint32_t n, uint64_t &count, Args &&...args) { + Slab *expected = ptr.load(cpp::MemoryOrder::RELAXED); + if (!expected && + ptr.compare_exchange_strong( + expected, reinterpret_cast(SENTINEL), + cpp::MemoryOrder::RELAXED, cpp::MemoryOrder::RELAXED)) { + count = cpp::numeric_limits::max(); + void *raw = impl::rpc_allocate(sizeof(Slab)); + if (!raw) + return nullptr; + return new (raw) Slab(cpp::forward(args)...); + } + + if (!expected || expected == reinterpret_cast(SENTINEL)) + return nullptr; + + if (!ref.acquire(n, count)) + return nullptr; + + cpp::atomic_thread_fence(cpp::MemoryOrder::ACQUIRE); + return ptr.load(cpp::MemoryOrder::RELAXED); + } + + // Finalize the associated memory and signal that it is ready to use by + // resetting the counter. + void finalize(Slab *mem, uint32_t n, uint64_t &count) { + cpp::atomic_thread_fence(cpp::MemoryOrder::RELEASE); + ptr.store(mem, cpp::MemoryOrder::RELAXED); + cpp::atomic_thread_fence(cpp::MemoryOrder::ACQUIRE); + if (!ref.acquire(n, count)) + ref.reset(n, count); + } + +public: + // Attempt to lock access to the pointer, potentially creating it if empty. + // The uniform mask represents which lanes share the same pointer. For each + // uniform value we elect a leader to handle it on behalf of the other lanes. + template + Slab *try_lock(uint64_t lane_mask, uint64_t uniform, uint64_t &count, + Args &&...args) { + count = 0; + Slab *result = nullptr; + if (gpu::get_lane_id() == uint32_t(cpp::countr_zero(uniform))) + result = try_lock_impl(cpp::popcount(uniform), count, + cpp::forward(args)...); + result = gpu::shuffle(lane_mask, cpp::countr_zero(uniform), result); + count = gpu::shuffle(lane_mask, cpp::countr_zero(uniform), count); + + if (!result) + return nullptr; + + // We defer storing the newly allocated slab until now so that we can use + // multiple lanes to initialize it and release it for use. + if (count == cpp::numeric_limits::max()) { + result->initialize(uniform); + if (gpu::get_lane_id() == uint32_t(cpp::countr_zero(uniform))) + finalize(result, cpp::popcount(uniform), count); + } + + if (count != cpp::numeric_limits::max()) + count = count - cpp::popcount(uniform) + impl::lane_count(uniform) + 1; + + return result; + } + + // Release the associated lock on the pointer, potentially destroying it. + void unlock(uint64_t lane_mask, uint64_t mask) { + cpp::atomic_thread_fence(cpp::MemoryOrder::RELEASE); + if (gpu::get_lane_id() == uint32_t(cpp::countr_zero(mask)) && + ref.release(cpp::popcount(mask))) { + Slab *p = ptr.load(cpp::MemoryOrder::RELAXED); + p->~Slab(); + impl::rpc_free(p); + cpp::atomic_thread_fence(cpp::MemoryOrder::RELEASE); + ptr.store(nullptr, cpp::MemoryOrder::RELAXED); + } + gpu::sync_lane(lane_mask); + } + + // Get the current value of the reference counter. + uint64_t use_count() { return ref.read(); } +}; + +// The global array used to search for a valid slab to allocate from. +static GuardPtr slots[ARRAY_SIZE] = {}; + +// Tries to find a slab in the table that can support the given chunk size. +static Slab *find_slab(uint32_t chunk_size) { + // We start at a hashed value to spread out different chunk sizes. + uint32_t start = impl::hash(chunk_size); + uint64_t lane_mask = gpu::get_lane_mask(); + uint64_t uniform = gpu::match_any(lane_mask, chunk_size); + + Slab *result = nullptr; + uint32_t nudge = 0; + for (uint64_t mask = lane_mask; mask; + mask = gpu::ballot(lane_mask, !result), ++nudge) { + uint32_t index = cpp::numeric_limits::max(); + for (uint32_t offset = nudge / MAX_TRIES; + gpu::ballot(lane_mask, index == cpp::numeric_limits::max()); + offset += cpp::popcount(uniform & lane_mask)) { + uint32_t candidate = + (start + offset + impl::lane_count(uniform & lane_mask)) % ARRAY_SIZE; + uint64_t available = + gpu::ballot(lane_mask, slots[candidate].use_count() < + Slab::available_chunks(chunk_size)); + uint32_t new_index = gpu::shuffle( + lane_mask, cpp::countr_zero(available & uniform), candidate); + + // Each uniform group will use the first empty slot they find. + if ((index == cpp::numeric_limits::max() && + (available & uniform))) + index = new_index; + + // Guaruntees that this loop will eventuall exit if there is no space. + if (offset >= ARRAY_SIZE) { + result = reinterpret_cast(SENTINEL); + index = 0; + } + } + + // Try to claim a slot for the found slot. + if (!result) { + uint64_t reserved = 0; + Slab *slab = slots[index].try_lock(lane_mask & mask, uniform & mask, + reserved, chunk_size, index); + // If we find a slab with a matching chunk size then we store the result. + // Otherwise, we need to free the claimed lock and continue. In the case + // of out-of-memory we return a sentinel value. + if (slab && reserved <= Slab::available_chunks(chunk_size) && + slab->get_chunk_size() == chunk_size) { + result = slab; + } else if (slab && (reserved > Slab::available_chunks(chunk_size) || + slab->get_chunk_size() != chunk_size)) { + if (slab->get_chunk_size() != chunk_size) + start = index + 1; + slots[index].unlock(gpu::get_lane_mask(), + gpu::get_lane_mask() & uniform); + } else if (!slab && reserved == cpp::numeric_limits::max()) { + result = reinterpret_cast(SENTINEL); + } else { + sleep_briefly(); + } + } + } + return result; +} + +// Release the lock associated with a given slab. +static void release_slab(Slab *slab) { + uint32_t index = slab->get_global_index(); + uint64_t lane_mask = gpu::get_lane_mask(); + uint64_t uniform = gpu::match_any(lane_mask, index); + slots[index].unlock(lane_mask, uniform); +} namespace gpu { -void *allocate(uint64_t size) { return rpc_allocate(size); } +void *allocate(uint64_t size) { + if (!size) + return nullptr; + + // Allocations requiring a full slab or more go directly to memory. + if (size >= SLAB_SIZE / 2) + return impl::rpc_allocate(impl::round_up(size)); + + // Try to find a slab for the rounded up chunk size and allocate from it. + uint32_t chunk_size = impl::get_chunk_size(static_cast(size)); + Slab *slab = find_slab(chunk_size); + if (!slab || slab == reinterpret_cast(SENTINEL)) + return nullptr; + + uint64_t lane_mask = gpu::get_lane_mask(); + uint64_t uniform = gpu::match_any(lane_mask, slab->get_global_index()); + void *ptr = slab->allocate(lane_mask, uniform); + return ptr; +} + +void deallocate(void *ptr) { + if (!ptr) + return; -void deallocate(void *ptr) { rpc_free(ptr); } + // All non-slab allocations will be aligned on a 2MiB boundary. + if ((reinterpret_cast(ptr) & SLAB_ALIGNMENT) == 0) + return impl::rpc_free(ptr); + + // The original slab pointer is the 2MiB boundary using the given pointer. + Slab *slab = reinterpret_cast( + (reinterpret_cast(ptr) & ~SLAB_ALIGNMENT)); + slab->deallocate(ptr); + release_slab(slab); +} } // namespace gpu } // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/__support/GPU/utils.h b/system/lib/llvm-libc/src/__support/GPU/utils.h index 323c003f1ff07..1b3e6edfc4e0d 100644 --- a/system/lib/llvm-libc/src/__support/GPU/utils.h +++ b/system/lib/llvm-libc/src/__support/GPU/utils.h @@ -92,6 +92,26 @@ LIBC_INLINE uint32_t shuffle(uint64_t lane_mask, uint32_t idx, uint32_t x, return __gpu_shuffle_idx_u32(lane_mask, idx, x, width); } +LIBC_INLINE uint64_t shuffle(uint64_t lane_mask, uint32_t idx, uint64_t x, + uint32_t width = __gpu_num_lanes()) { + return __gpu_shuffle_idx_u64(lane_mask, idx, x, width); +} + +template +LIBC_INLINE T *shuffle(uint64_t lane_mask, uint32_t idx, T *x, + uint32_t width = __gpu_num_lanes()) { + return reinterpret_cast(__gpu_shuffle_idx_u64( + lane_mask, idx, reinterpret_cast(x), width)); +} + +LIBC_INLINE uint64_t match_any(uint64_t lane_mask, uint32_t x) { + return __gpu_match_any_u32(lane_mask, x); +} + +LIBC_INLINE uint64_t match_all(uint64_t lane_mask, uint32_t x) { + return __gpu_match_all_u32(lane_mask, x); +} + [[noreturn]] LIBC_INLINE void end_program() { __gpu_exit(); } LIBC_INLINE bool is_first_lane(uint64_t lane_mask) { diff --git a/system/lib/llvm-libc/src/__support/HashTable/randomness.h b/system/lib/llvm-libc/src/__support/HashTable/randomness.h index 244dd41be3eec..6b58a4125f785 100644 --- a/system/lib/llvm-libc/src/__support/HashTable/randomness.h +++ b/system/lib/llvm-libc/src/__support/HashTable/randomness.h @@ -14,7 +14,7 @@ #include "src/__support/macros/attributes.h" #include "src/__support/macros/config.h" #if defined(LIBC_HASHTABLE_USE_GETRANDOM) -#include "src/errno/libc_errno.h" +#include "src/__support/libc_errno.h" #include "src/sys/random/getrandom.h" #endif diff --git a/system/lib/llvm-libc/src/__support/HashTable/table.h b/system/lib/llvm-libc/src/__support/HashTable/table.h index d50a9482ec4ce..10dd9711afbf6 100644 --- a/system/lib/llvm-libc/src/__support/HashTable/table.h +++ b/system/lib/llvm-libc/src/__support/HashTable/table.h @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_SRC___SUPPORT_HASHTABLE_TABLE_H #define LLVM_LIBC_SRC___SUPPORT_HASHTABLE_TABLE_H -#include "include/llvm-libc-types/ENTRY.h" +#include "hdr/types/ENTRY.h" #include "src/__support/CPP/bit.h" // bit_ceil #include "src/__support/CPP/new.h" #include "src/__support/HashTable/bitmask.h" @@ -18,9 +18,8 @@ #include "src/__support/macros/config.h" #include "src/__support/macros/optimization.h" #include "src/__support/memory_size.h" -#include "src/string/memset.h" -#include "src/string/strcmp.h" -#include "src/string/strlen.h" +#include "src/string/memory_utils/inline_strcmp.h" +#include "src/string/string_utils.h" #include #include @@ -158,7 +157,9 @@ struct HashTable { for (size_t i : masks) { size_t index = (pos + i) & entries_mask; ENTRY &entry = this->entry(index); - if (LIBC_LIKELY(entry.key != nullptr && strcmp(entry.key, key) == 0)) + auto comp = [](char l, char r) -> int { return l - r; }; + if (LIBC_LIKELY(entry.key != nullptr && + inline_strcmp(entry.key, key, comp) == 0)) return index; } BitMask available = ctrls.mask_available(); @@ -176,7 +177,7 @@ struct HashTable { LIBC_INLINE uint64_t oneshot_hash(const char *key) const { LIBC_NAMESPACE::internal::HashState hasher = state; - hasher.update(key, strlen(key)); + hasher.update(key, internal::string_length(key)); return hasher.finish(); } @@ -282,8 +283,8 @@ struct HashTable { table->entries_mask = entries - 1u; table->available_slots = entries / 8 * 7; table->state = HashState{randomness}; - memset(&table->control(0), 0x80, ctrl_sizes); - memset(mem, 0, table->offset_from_entries()); + __builtin_memset(&table->control(0), 0x80, ctrl_sizes); + __builtin_memset(mem, 0, table->offset_from_entries()); } return table; } diff --git a/system/lib/llvm-libc/src/__support/OSUtil/darwin/arm/syscall.h b/system/lib/llvm-libc/src/__support/OSUtil/darwin/aarch64/syscall.h similarity index 100% rename from system/lib/llvm-libc/src/__support/OSUtil/darwin/arm/syscall.h rename to system/lib/llvm-libc/src/__support/OSUtil/darwin/aarch64/syscall.h diff --git a/system/lib/llvm-libc/src/__support/OSUtil/darwin/io.h b/system/lib/llvm-libc/src/__support/OSUtil/darwin/io.h index a5f7ecbd70362..69df99da522fb 100644 --- a/system/lib/llvm-libc/src/__support/OSUtil/darwin/io.h +++ b/system/lib/llvm-libc/src/__support/OSUtil/darwin/io.h @@ -17,7 +17,8 @@ namespace LIBC_NAMESPACE_DECL { LIBC_INLINE void write_to_stderr(cpp::string_view msg) { LIBC_NAMESPACE::syscall_impl(4 /*SYS_write*/, 2 /* stderr */, - reinterpret_cast(msg.data()), msg.size()); + reinterpret_cast(msg.data()), + static_cast(msg.size())); } } // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/__support/OSUtil/darwin/syscall.h b/system/lib/llvm-libc/src/__support/OSUtil/darwin/syscall.h index eab96366a21a3..463407dbe19ef 100644 --- a/system/lib/llvm-libc/src/__support/OSUtil/darwin/syscall.h +++ b/system/lib/llvm-libc/src/__support/OSUtil/darwin/syscall.h @@ -15,7 +15,7 @@ #include "src/__support/macros/properties/architectures.h" #ifdef LIBC_TARGET_ARCH_IS_ANY_ARM -#include "arm/syscall.h" +#include "aarch64/syscall.h" #else #error "Unsupported architecture" #endif diff --git a/system/lib/llvm-libc/src/__support/OSUtil/fcntl.h b/system/lib/llvm-libc/src/__support/OSUtil/fcntl.h index 46f7d28132396..3983d78f7f89c 100644 --- a/system/lib/llvm-libc/src/__support/OSUtil/fcntl.h +++ b/system/lib/llvm-libc/src/__support/OSUtil/fcntl.h @@ -8,12 +8,18 @@ #ifndef LLVM_LIBC_SRC___SUPPORT_OSUTIL_FCNTL_H #define LLVM_LIBC_SRC___SUPPORT_OSUTIL_FCNTL_H +#include "hdr/types/mode_t.h" +#include "src/__support/error_or.h" #include "src/__support/macros/config.h" namespace LIBC_NAMESPACE_DECL { namespace internal { -int fcntl(int fd, int cmd, void *arg = nullptr); +ErrorOr fcntl(int fd, int cmd, void *arg = nullptr); + +ErrorOr open(const char *path, int flags, mode_t mode_flags = 0); + +ErrorOr close(int fd); } // namespace internal } // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/__support/OSUtil/io.h b/system/lib/llvm-libc/src/__support/OSUtil/io.h index 80119da77fc02..66af31f3cc8c6 100644 --- a/system/lib/llvm-libc/src/__support/OSUtil/io.h +++ b/system/lib/llvm-libc/src/__support/OSUtil/io.h @@ -24,6 +24,8 @@ #elif defined(__ELF__) // TODO: Ideally we would have LIBC_TARGET_OS_IS_BAREMETAL. #include "baremetal/io.h" +#elif defined(__UEFI__) +#include "uefi/io.h" #endif #endif // LLVM_LIBC_SRC___SUPPORT_OSUTIL_IO_H diff --git a/system/lib/llvm-libc/src/__support/OSUtil/linux/fcntl.cpp b/system/lib/llvm-libc/src/__support/OSUtil/linux/fcntl.cpp index 4742b2a00220b..bb76eee90efd2 100644 --- a/system/lib/llvm-libc/src/__support/OSUtil/linux/fcntl.cpp +++ b/system/lib/llvm-libc/src/__support/OSUtil/linux/fcntl.cpp @@ -8,23 +8,24 @@ #include "src/__support/OSUtil/fcntl.h" +#include "hdr/errno_macros.h" #include "hdr/fcntl_macros.h" +#include "hdr/types/mode_t.h" #include "hdr/types/off_t.h" #include "hdr/types/struct_f_owner_ex.h" #include "hdr/types/struct_flock.h" #include "hdr/types/struct_flock64.h" #include "src/__support/OSUtil/syscall.h" // For internal syscall function. #include "src/__support/common.h" +#include "src/__support/error_or.h" #include "src/__support/macros/config.h" -#include "src/errno/libc_errno.h" -#include #include // For syscall numbers. namespace LIBC_NAMESPACE_DECL { namespace internal { -int fcntl(int fd, int cmd, void *arg) { +ErrorOr fcntl(int fd, int cmd, void *arg) { #if SYS_fcntl constexpr auto FCNTL_SYSCALL_ID = SYS_fcntl; #elif defined(SYS_fcntl64) @@ -33,8 +34,7 @@ int fcntl(int fd, int cmd, void *arg) { #error "fcntl and fcntl64 syscalls not available." #endif - int new_cmd = cmd; - switch (new_cmd) { + switch (cmd) { case F_OFD_SETLKW: { struct flock *flk = reinterpret_cast(arg); // convert the struct to a flock64 @@ -45,8 +45,11 @@ int fcntl(int fd, int cmd, void *arg) { flk64.l_len = flk->l_len; flk64.l_pid = flk->l_pid; // create a syscall - return LIBC_NAMESPACE::syscall_impl(FCNTL_SYSCALL_ID, fd, new_cmd, - &flk64); + int ret = + LIBC_NAMESPACE::syscall_impl(FCNTL_SYSCALL_ID, fd, cmd, &flk64); + if (ret < 0) + return Error(-ret); + return ret; } case F_OFD_GETLK: case F_OFD_SETLK: { @@ -59,60 +62,80 @@ int fcntl(int fd, int cmd, void *arg) { flk64.l_len = flk->l_len; flk64.l_pid = flk->l_pid; // create a syscall - int retVal = LIBC_NAMESPACE::syscall_impl(FCNTL_SYSCALL_ID, fd, - new_cmd, &flk64); + int ret = + LIBC_NAMESPACE::syscall_impl(FCNTL_SYSCALL_ID, fd, cmd, &flk64); // On failure, return - if (retVal == -1) - return -1; + if (ret < 0) + return Error(-1); // Check for overflow, i.e. the offsets are not the same when cast // to off_t from off64_t. if (static_cast(flk64.l_len) != flk64.l_len || - static_cast(flk64.l_start) != flk64.l_start) { - libc_errno = EOVERFLOW; - return -1; - } + static_cast(flk64.l_start) != flk64.l_start) + return Error(EOVERFLOW); + // Now copy back into flk, in case flk64 got modified flk->l_type = flk64.l_type; flk->l_whence = flk64.l_whence; flk->l_start = static_castl_start)>(flk64.l_start); flk->l_len = static_castl_len)>(flk64.l_len); flk->l_pid = flk64.l_pid; - return retVal; + return ret; } case F_GETOWN: { struct f_owner_ex fex; int ret = LIBC_NAMESPACE::syscall_impl(FCNTL_SYSCALL_ID, fd, F_GETOWN_EX, &fex); - if (ret >= 0) - return fex.type == F_OWNER_PGRP ? -fex.pid : fex.pid; - libc_errno = -ret; - return -1; + if (ret < 0) + return Error(-ret); + return fex.type == F_OWNER_PGRP ? -fex.pid : fex.pid; } #ifdef SYS_fcntl64 case F_GETLK: { if constexpr (FCNTL_SYSCALL_ID == SYS_fcntl64) - new_cmd = F_GETLK64; + cmd = F_GETLK64; break; } case F_SETLK: { if constexpr (FCNTL_SYSCALL_ID == SYS_fcntl64) - new_cmd = F_SETLK64; + cmd = F_SETLK64; break; } case F_SETLKW: { if constexpr (FCNTL_SYSCALL_ID == SYS_fcntl64) - new_cmd = F_SETLKW64; + cmd = F_SETLKW64; break; } #endif } - int retVal = LIBC_NAMESPACE::syscall_impl(FCNTL_SYSCALL_ID, fd, new_cmd, - reinterpret_cast(arg)); - if (retVal >= 0) { - return retVal; - } - libc_errno = -retVal; - return -1; + + // default, but may use rewritten cmd from above. + int ret = LIBC_NAMESPACE::syscall_impl(FCNTL_SYSCALL_ID, fd, cmd, + reinterpret_cast(arg)); + if (ret < 0) + return Error(-ret); + return ret; +} + +ErrorOr open(const char *path, int flags, mode_t mode_flags) { +#ifdef SYS_open + int fd = LIBC_NAMESPACE::syscall_impl(SYS_open, path, flags, mode_flags); +#else + int fd = LIBC_NAMESPACE::syscall_impl(SYS_openat, AT_FDCWD, path, flags, + mode_flags); +#endif + if (fd < 0) + return Error(-fd); + + return fd; +} + +ErrorOr close(int fd) { + int ret = LIBC_NAMESPACE::syscall_impl(SYS_close, fd); + + if (ret < 0) + return Error(-ret); + + return ret; } } // namespace internal diff --git a/system/lib/llvm-libc/src/__support/OSUtil/linux/vdso.cpp b/system/lib/llvm-libc/src/__support/OSUtil/linux/vdso.cpp index 8c9bd3e1bcc72..e4e53c3c2a0f2 100644 --- a/system/lib/llvm-libc/src/__support/OSUtil/linux/vdso.cpp +++ b/system/lib/llvm-libc/src/__support/OSUtil/linux/vdso.cpp @@ -11,9 +11,9 @@ #include "src/__support/CPP/array.h" #include "src/__support/CPP/optional.h" #include "src/__support/CPP/string_view.h" +#include "src/__support/libc_errno.h" #include "src/__support/threads/callonce.h" #include "src/__support/threads/linux/futex_word.h" -#include "src/errno/libc_errno.h" #include "src/sys/auxv/getauxval.h" #include diff --git a/system/lib/llvm-libc/src/__support/OSUtil/uefi/error.h b/system/lib/llvm-libc/src/__support/OSUtil/uefi/error.h new file mode 100644 index 0000000000000..9fdc569bab574 --- /dev/null +++ b/system/lib/llvm-libc/src/__support/OSUtil/uefi/error.h @@ -0,0 +1,104 @@ +//===----------- UEFI implementation of error utils --------------*- C++-*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC___SUPPORT_OSUTIL_UEFI_ERROR_H +#define LLVM_LIBC_SRC___SUPPORT_OSUTIL_UEFI_ERROR_H + +#include "hdr/errno_macros.h" +#include "include/llvm-libc-types/EFI_STATUS.h" +#include "src/__support/CPP/array.h" +#include "src/__support/CPP/limits.h" +#include "src/__support/macros/attributes.h" +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +static constexpr int EFI_ERROR_MAX_BIT = cpp::numeric_limits::max(); + +static constexpr int EFI_ENCODE_ERROR(int value) { + return EFI_ERROR_MAX_BIT | (EFI_ERROR_MAX_BIT >> 2) | (value); +} + +static constexpr int EFI_ENCODE_WARNING(int value) { + return (EFI_ERROR_MAX_BIT >> 2) | (value); +} + +struct UefiStatusErrnoEntry { + EFI_STATUS status; + int errno_value; +}; + +static constexpr cpp::array UEFI_STATUS_ERRNO_MAP = {{ + {EFI_SUCCESS, 0}, + {EFI_ENCODE_ERROR(EFI_LOAD_ERROR), EINVAL}, + {EFI_ENCODE_ERROR(EFI_INVALID_PARAMETER), EINVAL}, + {EFI_ENCODE_ERROR(EFI_BAD_BUFFER_SIZE), EINVAL}, + {EFI_ENCODE_ERROR(EFI_NOT_READY), EBUSY}, + {EFI_ENCODE_ERROR(EFI_DEVICE_ERROR), EIO}, + {EFI_ENCODE_ERROR(EFI_WRITE_PROTECTED), EPERM}, + {EFI_ENCODE_ERROR(EFI_OUT_OF_RESOURCES), ENOMEM}, + {EFI_ENCODE_ERROR(EFI_VOLUME_CORRUPTED), EROFS}, + {EFI_ENCODE_ERROR(EFI_VOLUME_FULL), ENOSPC}, + {EFI_ENCODE_ERROR(EFI_NO_MEDIA), ENODEV}, + {EFI_ENCODE_ERROR(EFI_MEDIA_CHANGED), ENXIO}, + {EFI_ENCODE_ERROR(EFI_NOT_FOUND), ENOENT}, + {EFI_ENCODE_ERROR(EFI_ACCESS_DENIED), EACCES}, + {EFI_ENCODE_ERROR(EFI_NO_RESPONSE), EBUSY}, + {EFI_ENCODE_ERROR(EFI_NO_MAPPING), ENODEV}, + {EFI_ENCODE_ERROR(EFI_TIMEOUT), EBUSY}, + {EFI_ENCODE_ERROR(EFI_NOT_STARTED), EAGAIN}, + {EFI_ENCODE_ERROR(EFI_ALREADY_STARTED), EINVAL}, + {EFI_ENCODE_ERROR(EFI_ABORTED), EFAULT}, + {EFI_ENCODE_ERROR(EFI_ICMP_ERROR), EIO}, + {EFI_ENCODE_ERROR(EFI_TFTP_ERROR), EIO}, + {EFI_ENCODE_ERROR(EFI_PROTOCOL_ERROR), EINVAL}, + {EFI_ENCODE_ERROR(EFI_INCOMPATIBLE_VERSION), EINVAL}, + {EFI_ENCODE_ERROR(EFI_SECURITY_VIOLATION), EPERM}, + {EFI_ENCODE_ERROR(EFI_CRC_ERROR), EINVAL}, + {EFI_ENCODE_ERROR(EFI_END_OF_MEDIA), EPIPE}, + {EFI_ENCODE_ERROR(EFI_END_OF_FILE), EPIPE}, + {EFI_ENCODE_ERROR(EFI_INVALID_LANGUAGE), EINVAL}, + {EFI_ENCODE_ERROR(EFI_COMPROMISED_DATA), EINVAL}, + {EFI_ENCODE_ERROR(EFI_IP_ADDRESS_CONFLICT), EINVAL}, + {EFI_ENCODE_ERROR(EFI_HTTP_ERROR), EIO}, + {EFI_ENCODE_WARNING(EFI_WARN_UNKNOWN_GLYPH), EINVAL}, + {EFI_ENCODE_WARNING(EFI_WARN_DELETE_FAILURE), EROFS}, + {EFI_ENCODE_WARNING(EFI_WARN_WRITE_FAILURE), EROFS}, + {EFI_ENCODE_WARNING(EFI_WARN_BUFFER_TOO_SMALL), E2BIG}, + {EFI_ENCODE_WARNING(EFI_WARN_STALE_DATA), EINVAL}, + {EFI_ENCODE_WARNING(EFI_WARN_FILE_SYSTEM), EROFS}, + {EFI_ENCODE_WARNING(EFI_WARN_RESET_REQUIRED), EINTR}, +}}; + +LIBC_INLINE int uefi_status_to_errno(EFI_STATUS status) { + for (auto it = UEFI_STATUS_ERRNO_MAP.begin(); + it != UEFI_STATUS_ERRNO_MAP.end(); it++) { + const struct UefiStatusErrnoEntry entry = *it; + if (entry.status == status) + return entry.errno_value; + } + + // Unknown type + return EINVAL; +} + +LIBC_INLINE EFI_STATUS errno_to_uefi_status(int errno_value) { + for (auto it = UEFI_STATUS_ERRNO_MAP.begin(); + it != UEFI_STATUS_ERRNO_MAP.end(); it++) { + const struct UefiStatusErrnoEntry entry = *it; + if (entry.errno_value == errno_value) + return entry.status; + } + + // Unknown type + return EFI_INVALID_PARAMETER; +} + +} // namespace LIBC_NAMESPACE_DECL + +#endif // LLVM_LIBC_SRC___SUPPORT_OSUTIL_UEFI_ERROR_H diff --git a/system/lib/llvm-libc/src/__support/OSUtil/uefi/exit.cpp b/system/lib/llvm-libc/src/__support/OSUtil/uefi/exit.cpp new file mode 100644 index 0000000000000..e734983cd125b --- /dev/null +++ b/system/lib/llvm-libc/src/__support/OSUtil/uefi/exit.cpp @@ -0,0 +1,24 @@ +//===-------- UEFI implementation of an exit function ------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===-----------------------------------------------------------------===// + +#include "src/__support/OSUtil/exit.h" +#include "config/uefi.h" +#include "include/llvm-libc-types/EFI_SYSTEM_TABLE.h" +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { +namespace internal { + +[[noreturn]] void exit(int status) { + app.system_table->BootServices->Exit(__llvm_libc_efi_image_handle, status, 0, + nullptr); + __builtin_unreachable(); +} + +} // namespace internal +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/__support/OSUtil/uefi/io.cpp b/system/lib/llvm-libc/src/__support/OSUtil/uefi/io.cpp new file mode 100644 index 0000000000000..e1e50fbad3931 --- /dev/null +++ b/system/lib/llvm-libc/src/__support/OSUtil/uefi/io.cpp @@ -0,0 +1,41 @@ +//===---------- UEFI implementation of IO utils ------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===-----------------------------------------------------------------===// + +#include "io.h" + +#include "Uefi.h" +#include "config/app.h" +#include "src/__support/CPP/string_view.h" +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +ssize_t read_from_stdin([[gnu::unused]] char *buf, + [[gnu::unused]] size_t size) { + return 0; +} + +void write_to_stdout(cpp::string_view msg) { + // TODO: use mbstowcs once implemented + for (size_t i = 0; i < msg.size(); i++) { + char16_t e[2] = {msg[i], 0}; + app.system_table->ConOut->OutputString( + app.system_table->ConOut, reinterpret_cast(&e)); + } +} + +void write_to_stderr(cpp::string_view msg) { + // TODO: use mbstowcs once implemented + for (size_t i = 0; i < msg.size(); i++) { + char16_t e[2] = {msg[i], 0}; + app.system_table->StdErr->OutputString( + app.system_table->StdErr, reinterpret_cast(&e)); + } +} + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/__support/OSUtil/uefi/io.h b/system/lib/llvm-libc/src/__support/OSUtil/uefi/io.h new file mode 100644 index 0000000000000..088ae09b8c602 --- /dev/null +++ b/system/lib/llvm-libc/src/__support/OSUtil/uefi/io.h @@ -0,0 +1,25 @@ +//===---------- UEFI implementation of IO utils ------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===-----------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC___SUPPORT_OSUTIL_UEFI_IO_H +#define LLVM_LIBC_SRC___SUPPORT_OSUTIL_UEFI_IO_H + +#include "include/llvm-libc-types/size_t.h" +#include "include/llvm-libc-types/ssize_t.h" +#include "src/__support/CPP/string_view.h" +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +ssize_t read_from_stdin(char *buf, size_t size); +void write_to_stderr(cpp::string_view msg); +void write_to_stdout(cpp::string_view msg); + +} // namespace LIBC_NAMESPACE_DECL + +#endif // LLVM_LIBC_SRC___SUPPORT_OSUTIL_UEFI_IO_H diff --git a/system/lib/llvm-libc/src/__support/RPC/rpc_server.h b/system/lib/llvm-libc/src/__support/RPC/rpc_server.h new file mode 100644 index 0000000000000..dc3d8030caa47 --- /dev/null +++ b/system/lib/llvm-libc/src/__support/RPC/rpc_server.h @@ -0,0 +1,551 @@ +//===-- Shared memory RPC server instantiation ------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is intended to be used externally as part of the `shared/` +// interface. For that purpose, we manually define a few options normally +// handled by the libc build system. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC___SUPPORT_RPC_RPC_SERVER_H +#define LLVM_LIBC_SRC___SUPPORT_RPC_RPC_SERVER_H + +// Workaround for missing __has_builtin in < GCC 10. +#ifndef __has_builtin +#define __has_builtin(x) 0 +#endif + +// Workaround for missing __builtin_is_constant_evaluated in < GCC 10. +#ifndef __builtin_is_constant_evaluated +#define __builtin_is_constant_evaluated(x) 0 +#endif + +// Configs for using the LLVM libc writer interface. +#define LIBC_COPT_USE_C_ASSERT +#define LIBC_COPT_MEMCPY_USE_EMBEDDED_TINY +#define LIBC_COPT_ARRAY_ARG_LIST +#define LIBC_COPT_PRINTF_DISABLE_WRITE_INT +#define LIBC_COPT_PRINTF_DISABLE_INDEX_MODE +#define LIBC_COPT_PRINTF_DISABLE_STRERROR + +// The 'long double' type is 8 bytes. +#define LIBC_TYPES_LONG_DOUBLE_IS_FLOAT64 + +#include "shared/rpc.h" +#include "shared/rpc_opcodes.h" + +#include "src/__support/arg_list.h" +#include "src/stdio/printf_core/converter.h" +#include "src/stdio/printf_core/parser.h" +#include "src/stdio/printf_core/writer.h" + +#include "hdr/stdio_overlay.h" +#include "hdr/stdlib_overlay.h" + +namespace LIBC_NAMESPACE_DECL { +namespace internal { + +// Minimal replacement for 'std::vector' that works for trivial types. +template class TempVector { + static_assert(cpp::is_trivially_constructible::value && + cpp::is_trivially_destructible::value, + "Not a trivial type."); + T *data; + size_t current; + size_t capacity; + +public: + LIBC_INLINE TempVector() : data(nullptr), current(0), capacity(0) {} + + LIBC_INLINE ~TempVector() { free(data); } + + LIBC_INLINE void push_back(const T &value) { + if (current == capacity) + grow(); + data[current] = T(value); + ++current; + } + + LIBC_INLINE void push_back(T &&value) { + if (current == capacity) + grow(); + data[current] = T(static_cast(value)); + ++current; + } + + LIBC_INLINE void pop_back() { --current; } + + LIBC_INLINE bool empty() { return current == 0; } + + LIBC_INLINE size_t size() { return current; } + + LIBC_INLINE T &operator[](size_t index) { return data[index]; } + + LIBC_INLINE T &back() { return data[current - 1]; } + +private: + LIBC_INLINE void grow() { + size_t new_capacity = capacity ? capacity * 2 : 1; + void *new_data = realloc(data, new_capacity * sizeof(T)); + data = static_cast(new_data); + capacity = new_capacity; + } +}; + +struct TempStorage { + LIBC_INLINE char *alloc(size_t size) { + storage.push_back(reinterpret_cast(malloc(size))); + return storage.back(); + } + + LIBC_INLINE ~TempStorage() { + for (size_t i = 0; i < storage.size(); ++i) + free(storage[i]); + } + + TempVector storage; +}; + +// Get the associated stream out of an encoded number. +LIBC_INLINE static ::FILE *to_stream(uintptr_t f) { + enum Stream { + File = 0, + Stdin = 1, + Stdout = 2, + Stderr = 3, + }; + + ::FILE *stream = reinterpret_cast(f & ~0x3ull); + Stream type = static_cast(f & 0x3ull); + if (type == Stdin) + return stdin; + if (type == Stdout) + return stdout; + if (type == Stderr) + return stderr; + return stream; +} + +template +LIBC_INLINE static void handle_printf(rpc::Server::Port &port, + TempStorage &temp_storage) { + FILE *files[num_lanes] = {nullptr}; + // Get the appropriate output stream to use. + if (port.get_opcode() == LIBC_PRINTF_TO_STREAM || + port.get_opcode() == LIBC_PRINTF_TO_STREAM_PACKED) { + port.recv([&](rpc::Buffer *buffer, uint32_t id) { + files[id] = reinterpret_cast(buffer->data[0]); + }); + } else if (port.get_opcode() == LIBC_PRINTF_TO_STDOUT || + port.get_opcode() == LIBC_PRINTF_TO_STDOUT_PACKED) { + for (uint32_t i = 0; i < num_lanes; ++i) + files[i] = stdout; + } else { + for (uint32_t i = 0; i < num_lanes; ++i) + files[i] = stderr; + } + + uint64_t format_sizes[num_lanes] = {0}; + void *format[num_lanes] = {nullptr}; + + uint64_t args_sizes[num_lanes] = {0}; + void *args[num_lanes] = {nullptr}; + + // Recieve the format string and arguments from the client. + port.recv_n(format, format_sizes, + [&](uint64_t size) { return temp_storage.alloc(size); }); + + // Parse the format string to get the expected size of the buffer. + for (uint32_t lane = 0; lane < num_lanes; ++lane) { + if (!format[lane]) + continue; + + printf_core::WriteBuffer< + printf_core::WriteMode::FILL_BUFF_AND_DROP_OVERFLOW> + wb(nullptr, 0); + printf_core::Writer writer(wb); + + internal::DummyArgList printf_args; + printf_core::Parser &> parser( + reinterpret_cast(format[lane]), printf_args); + + for (printf_core::FormatSection cur_section = parser.get_next_section(); + !cur_section.raw_string.empty(); + cur_section = parser.get_next_section()) + ; + args_sizes[lane] = printf_args.read_count(); + } + port.send([&](rpc::Buffer *buffer, uint32_t id) { + buffer->data[0] = args_sizes[id]; + }); + port.recv_n(args, args_sizes, + [&](uint64_t size) { return temp_storage.alloc(size); }); + + // Identify any arguments that are actually pointers to strings on the client. + // Additionally we want to determine how much buffer space we need to print. + TempVector strs_to_copy[num_lanes]; + int buffer_size[num_lanes] = {0}; + for (uint32_t lane = 0; lane < num_lanes; ++lane) { + if (!format[lane]) + continue; + + printf_core::WriteBuffer< + printf_core::WriteMode::FILL_BUFF_AND_DROP_OVERFLOW> + wb(nullptr, 0); + printf_core::Writer writer(wb); + + internal::StructArgList printf_args(args[lane], args_sizes[lane]); + printf_core::Parser> parser( + reinterpret_cast(format[lane]), printf_args); + + for (printf_core::FormatSection cur_section = parser.get_next_section(); + !cur_section.raw_string.empty(); + cur_section = parser.get_next_section()) { + if (cur_section.has_conv && cur_section.conv_name == 's' && + cur_section.conv_val_ptr) { + strs_to_copy[lane].push_back(cur_section.conv_val_ptr); + // Get the minimum size of the string in the case of padding. + char c = '\0'; + cur_section.conv_val_ptr = &c; + convert(&writer, cur_section); + } else if (cur_section.has_conv) { + // Ignore conversion errors for the first pass. + convert(&writer, cur_section); + } else { + writer.write(cur_section.raw_string); + } + } + buffer_size[lane] = writer.get_chars_written(); + } + + // Recieve any strings from the client and push them into a buffer. + TempVector copied_strs[num_lanes]; + auto HasPendingCopies = [](TempVector v[num_lanes]) { + for (uint32_t i = 0; i < num_lanes; ++i) + if (!v[i].empty() && v[i].back()) + return true; + return false; + }; + while (HasPendingCopies(strs_to_copy)) { + port.send([&](rpc::Buffer *buffer, uint32_t id) { + void *ptr = !strs_to_copy[id].empty() ? strs_to_copy[id].back() : nullptr; + buffer->data[1] = reinterpret_cast(ptr); + if (!strs_to_copy[id].empty()) + strs_to_copy[id].pop_back(); + }); + uint64_t str_sizes[num_lanes] = {0}; + void *strs[num_lanes] = {nullptr}; + port.recv_n(strs, str_sizes, + [&](uint64_t size) { return temp_storage.alloc(size); }); + for (uint32_t lane = 0; lane < num_lanes; ++lane) { + if (!strs[lane]) + continue; + + copied_strs[lane].push_back(strs[lane]); + buffer_size[lane] += str_sizes[lane]; + } + } + + // Perform the final formatting and printing using the LLVM C library printf. + int results[num_lanes] = {0}; + for (uint32_t lane = 0; lane < num_lanes; ++lane) { + if (!format[lane]) + continue; + + char *buffer = temp_storage.alloc(buffer_size[lane]); + printf_core::WriteBuffer< + printf_core::WriteMode::FILL_BUFF_AND_DROP_OVERFLOW> + wb(buffer, buffer_size[lane]); + printf_core::Writer writer(wb); + + internal::StructArgList printf_args(args[lane], args_sizes[lane]); + printf_core::Parser> parser( + reinterpret_cast(format[lane]), printf_args); + + // Parse and print the format string using the arguments we copied from + // the client. + int ret = 0; + for (printf_core::FormatSection cur_section = parser.get_next_section(); + !cur_section.raw_string.empty(); + cur_section = parser.get_next_section()) { + // If this argument was a string we use the memory buffer we copied from + // the client by replacing the raw pointer with the copied one. + if (cur_section.has_conv && cur_section.conv_name == 's') { + if (!copied_strs[lane].empty()) { + cur_section.conv_val_ptr = copied_strs[lane].back(); + copied_strs[lane].pop_back(); + } else { + cur_section.conv_val_ptr = nullptr; + } + } + if (cur_section.has_conv) { + ret = convert(&writer, cur_section); + if (ret == -1) + break; + } else { + writer.write(cur_section.raw_string); + } + } + + results[lane] = static_cast( + fwrite(buffer, 1, writer.get_chars_written(), files[lane])); + if (results[lane] != writer.get_chars_written() || ret == -1) + results[lane] = -1; + } + + // Send the final return value and signal completion by setting the string + // argument to null. + port.send([&](rpc::Buffer *buffer, uint32_t id) { + buffer->data[0] = static_cast(results[id]); + buffer->data[1] = reinterpret_cast(nullptr); + }); +} + +template +LIBC_INLINE static rpc::Status handle_port_impl(rpc::Server::Port &port) { + TempStorage temp_storage; + + switch (port.get_opcode()) { + case LIBC_WRITE_TO_STREAM: + case LIBC_WRITE_TO_STDERR: + case LIBC_WRITE_TO_STDOUT: + case LIBC_WRITE_TO_STDOUT_NEWLINE: { + uint64_t sizes[num_lanes] = {0}; + void *strs[num_lanes] = {nullptr}; + FILE *files[num_lanes] = {nullptr}; + if (port.get_opcode() == LIBC_WRITE_TO_STREAM) { + port.recv([&](rpc::Buffer *buffer, uint32_t id) { + files[id] = reinterpret_cast(buffer->data[0]); + }); + } else { + for (uint32_t i = 0; i < num_lanes; ++i) + files[i] = port.get_opcode() == LIBC_WRITE_TO_STDERR ? stderr : stdout; + } + + port.recv_n(strs, sizes, + [&](uint64_t size) { return temp_storage.alloc(size); }); + port.send([&](rpc::Buffer *buffer, uint32_t id) { + flockfile(files[id]); + buffer->data[0] = fwrite_unlocked(strs[id], 1, sizes[id], files[id]); + if (port.get_opcode() == LIBC_WRITE_TO_STDOUT_NEWLINE && + buffer->data[0] == sizes[id]) + buffer->data[0] += fwrite_unlocked("\n", 1, 1, files[id]); + funlockfile(files[id]); + }); + break; + } + case LIBC_READ_FROM_STREAM: { + uint64_t sizes[num_lanes] = {0}; + void *data[num_lanes] = {nullptr}; + port.recv([&](rpc::Buffer *buffer, uint32_t id) { + data[id] = temp_storage.alloc(buffer->data[0]); + sizes[id] = + fread(data[id], 1, buffer->data[0], to_stream(buffer->data[1])); + }); + port.send_n(data, sizes); + port.send([&](rpc::Buffer *buffer, uint32_t id) { + __builtin_memcpy(buffer->data, &sizes[id], sizeof(uint64_t)); + }); + break; + } + case LIBC_READ_FGETS: { + uint64_t sizes[num_lanes] = {0}; + void *data[num_lanes] = {nullptr}; + port.recv([&](rpc::Buffer *buffer, uint32_t id) { + data[id] = temp_storage.alloc(buffer->data[0]); + const char *str = ::fgets(reinterpret_cast(data[id]), + static_cast(buffer->data[0]), + to_stream(buffer->data[1])); + sizes[id] = !str ? 0 : __builtin_strlen(str) + 1; + }); + port.send_n(data, sizes); + break; + } + case LIBC_OPEN_FILE: { + uint64_t sizes[num_lanes] = {0}; + void *paths[num_lanes] = {nullptr}; + port.recv_n(paths, sizes, + [&](uint64_t size) { return temp_storage.alloc(size); }); + port.recv_and_send([&](rpc::Buffer *buffer, uint32_t id) { + FILE *file = fopen(reinterpret_cast(paths[id]), + reinterpret_cast(buffer->data)); + buffer->data[0] = reinterpret_cast(file); + }); + break; + } + case LIBC_CLOSE_FILE: { + port.recv_and_send([&](rpc::Buffer *buffer, uint32_t) { + FILE *file = reinterpret_cast(buffer->data[0]); + buffer->data[0] = ::fclose(file); + }); + break; + } + case LIBC_EXIT: { + // Send a response to the client to signal that we are ready to exit. + port.recv_and_send([](rpc::Buffer *, uint32_t) {}); + port.recv([](rpc::Buffer *buffer, uint32_t) { + int status = 0; + __builtin_memcpy(&status, buffer->data, sizeof(int)); + exit(status); + }); + break; + } + case LIBC_ABORT: { + // Send a response to the client to signal that we are ready to abort. + port.recv_and_send([](rpc::Buffer *, uint32_t) {}); + port.recv([](rpc::Buffer *, uint32_t) {}); + abort(); + break; + } + case LIBC_HOST_CALL: { + uint64_t sizes[num_lanes] = {0}; + unsigned long long results[num_lanes] = {0}; + void *args[num_lanes] = {nullptr}; + port.recv_n(args, sizes, + [&](uint64_t size) { return temp_storage.alloc(size); }); + port.recv([&](rpc::Buffer *buffer, uint32_t id) { + using func_ptr_t = unsigned long long (*)(void *); + auto func = reinterpret_cast(buffer->data[0]); + results[id] = func(args[id]); + }); + port.send([&](rpc::Buffer *buffer, uint32_t id) { + buffer->data[0] = static_cast(results[id]); + }); + break; + } + case LIBC_FEOF: { + port.recv_and_send([](rpc::Buffer *buffer, uint32_t) { + buffer->data[0] = feof(to_stream(buffer->data[0])); + }); + break; + } + case LIBC_FERROR: { + port.recv_and_send([](rpc::Buffer *buffer, uint32_t) { + buffer->data[0] = ferror(to_stream(buffer->data[0])); + }); + break; + } + case LIBC_CLEARERR: { + port.recv_and_send([](rpc::Buffer *buffer, uint32_t) { + clearerr(to_stream(buffer->data[0])); + }); + break; + } + case LIBC_FSEEK: { + port.recv_and_send([](rpc::Buffer *buffer, uint32_t) { + buffer->data[0] = + fseek(to_stream(buffer->data[0]), static_cast(buffer->data[1]), + static_cast(buffer->data[2])); + }); + break; + } + case LIBC_FTELL: { + port.recv_and_send([](rpc::Buffer *buffer, uint32_t) { + buffer->data[0] = ftell(to_stream(buffer->data[0])); + }); + break; + } + case LIBC_FFLUSH: { + port.recv_and_send([](rpc::Buffer *buffer, uint32_t) { + buffer->data[0] = fflush(to_stream(buffer->data[0])); + }); + break; + } + case LIBC_UNGETC: { + port.recv_and_send([](rpc::Buffer *buffer, uint32_t) { + buffer->data[0] = + ungetc(static_cast(buffer->data[0]), to_stream(buffer->data[1])); + }); + break; + } + case LIBC_PRINTF_TO_STREAM_PACKED: + case LIBC_PRINTF_TO_STDOUT_PACKED: + case LIBC_PRINTF_TO_STDERR_PACKED: { + handle_printf(port, temp_storage); + break; + } + case LIBC_PRINTF_TO_STREAM: + case LIBC_PRINTF_TO_STDOUT: + case LIBC_PRINTF_TO_STDERR: { + handle_printf(port, temp_storage); + break; + } + case LIBC_REMOVE: { + uint64_t sizes[num_lanes] = {0}; + void *args[num_lanes] = {nullptr}; + port.recv_n(args, sizes, + [&](uint64_t size) { return temp_storage.alloc(size); }); + port.send([&](rpc::Buffer *buffer, uint32_t id) { + buffer->data[0] = static_cast( + remove(reinterpret_cast(args[id]))); + }); + break; + } + case LIBC_RENAME: { + uint64_t oldsizes[num_lanes] = {0}; + uint64_t newsizes[num_lanes] = {0}; + void *oldpath[num_lanes] = {nullptr}; + void *newpath[num_lanes] = {nullptr}; + port.recv_n(oldpath, oldsizes, + [&](uint64_t size) { return temp_storage.alloc(size); }); + port.recv_n(newpath, newsizes, + [&](uint64_t size) { return temp_storage.alloc(size); }); + port.send([&](rpc::Buffer *buffer, uint32_t id) { + buffer->data[0] = static_cast( + rename(reinterpret_cast(oldpath[id]), + reinterpret_cast(newpath[id]))); + }); + break; + } + case LIBC_SYSTEM: { + uint64_t sizes[num_lanes] = {0}; + void *args[num_lanes] = {nullptr}; + port.recv_n(args, sizes, + [&](uint64_t size) { return temp_storage.alloc(size); }); + port.send([&](rpc::Buffer *buffer, uint32_t id) { + buffer->data[0] = static_cast( + system(reinterpret_cast(args[id]))); + }); + break; + } + case LIBC_NOOP: { + port.recv([](rpc::Buffer *, uint32_t) {}); + break; + } + default: + return rpc::RPC_UNHANDLED_OPCODE; + } + + return rpc::RPC_SUCCESS; +} + +} // namespace internal +} // namespace LIBC_NAMESPACE_DECL + +namespace LIBC_NAMESPACE_DECL { +namespace rpc { + +// Handles any opcode generated from the 'libc' client code. +LIBC_INLINE ::rpc::Status handle_libc_opcodes(::rpc::Server::Port &port, + uint32_t num_lanes) { + switch (num_lanes) { + case 1: + return internal::handle_port_impl<1>(port); + case 32: + return internal::handle_port_impl<32>(port); + case 64: + return internal::handle_port_impl<64>(port); + default: + return ::rpc::RPC_ERROR; + } +} + +} // namespace rpc +} // namespace LIBC_NAMESPACE_DECL + +#endif // LLVM_LIBC_SRC___SUPPORT_RPC_RPC_SERVER_H diff --git a/system/lib/llvm-libc/src/__support/StringUtil/tables/linux_extension_errors.h b/system/lib/llvm-libc/src/__support/StringUtil/tables/linux_extension_errors.h index 425590f6e91c9..de637d60bea97 100644 --- a/system/lib/llvm-libc/src/__support/StringUtil/tables/linux_extension_errors.h +++ b/system/lib/llvm-libc/src/__support/StringUtil/tables/linux_extension_errors.h @@ -10,8 +10,8 @@ #define LLVM_LIBC_SRC___SUPPORT_STRINGUTIL_TABLES_LINUX_EXTENSION_ERRORS_H #include "src/__support/StringUtil/message_mapper.h" +#include "src/__support/libc_errno.h" #include "src/__support/macros/config.h" -#include "src/errno/libc_errno.h" namespace LIBC_NAMESPACE_DECL { diff --git a/system/lib/llvm-libc/src/__support/big_int.h b/system/lib/llvm-libc/src/__support/big_int.h index a95ab4ff8e1ab..85db31d01399a 100644 --- a/system/lib/llvm-libc/src/__support/big_int.h +++ b/system/lib/llvm-libc/src/__support/big_int.h @@ -241,7 +241,7 @@ LIBC_INLINE constexpr void quick_mul_hi(cpp::array &dst, } template -LIBC_INLINE constexpr bool is_negative(cpp::array &array) { +LIBC_INLINE constexpr bool is_negative(const cpp::array &array) { using signed_word = cpp::make_signed_t; return cpp::bit_cast(array.back()) < 0; } @@ -284,8 +284,8 @@ LIBC_INLINE constexpr cpp::array shift(cpp::array array, if (i < 0) return 0; if (i >= int(N)) - return is_neg ? -1 : 0; - return array[i]; + return is_neg ? cpp::numeric_limits::max() : 0; + return array[static_cast(i)]; }; const size_t index_offset = offset / WORD_BITS; const size_t bit_offset = offset % WORD_BITS; @@ -296,7 +296,7 @@ LIBC_INLINE constexpr cpp::array shift(cpp::array array, for (size_t index = 0; index < N; ++index) { const word part1 = safe_get_at(index + index_offset); const word part2 = safe_get_at(index + index_offset + 1); - word &dst = out[at(index)]; + word &dst = out[static_cast(at(index))]; if (bit_offset == 0) dst = part1; // no crosstalk between parts. else if constexpr (direction == LEFT) @@ -465,8 +465,7 @@ struct BigInt { } // Initialize the first word to |v| and the rest to 0. - template && - !cpp::is_same_v>> + template >> LIBC_INLINE constexpr BigInt(T v) { constexpr size_t T_SIZE = sizeof(T) * CHAR_BIT; const bool is_neg = v < 0; @@ -697,7 +696,8 @@ struct BigInt { } BigInt quotient; WordType x_word = static_cast(x); - constexpr size_t LOG2_WORD_SIZE = cpp::bit_width(WORD_SIZE) - 1; + constexpr size_t LOG2_WORD_SIZE = + static_cast(cpp::bit_width(WORD_SIZE) - 1); constexpr size_t HALF_WORD_SIZE = WORD_SIZE >> 1; constexpr WordType HALF_MASK = ((WordType(1) << HALF_WORD_SIZE) - 1); // lower = smallest multiple of WORD_SIZE that is >= e. @@ -866,7 +866,7 @@ struct BigInt { LIBC_INLINE constexpr BigInt operator~() const { BigInt result; for (size_t i = 0; i < WORD_COUNT; ++i) - result[i] = ~val[i]; + result[i] = static_cast(~val[i]); return result; } @@ -936,6 +936,18 @@ struct BigInt { // Return the i-th word of the number. LIBC_INLINE constexpr WordType &operator[](size_t i) { return val[i]; } + // Return the i-th bit of the number. + LIBC_INLINE constexpr bool get_bit(size_t i) const { + const size_t word_index = i / WORD_SIZE; + return 1 & (val[word_index] >> (i % WORD_SIZE)); + } + + // Set the i-th bit of the number. + LIBC_INLINE constexpr void set_bit(size_t i) { + const size_t word_index = i / WORD_SIZE; + val[word_index] |= WordType(1) << (i % WORD_SIZE); + } + private: LIBC_INLINE friend constexpr int cmp(const BigInt &lhs, const BigInt &rhs) { constexpr auto compare = [](WordType a, WordType b) { @@ -955,7 +967,7 @@ struct BigInt { LIBC_INLINE constexpr void bitwise_not() { for (auto &part : val) - part = ~part; + part = static_cast(~part); } LIBC_INLINE constexpr void negate() { @@ -968,7 +980,7 @@ struct BigInt { } LIBC_INLINE constexpr void decrement() { - multiword::add_with_carry(val, cpp::array{1}); + multiword::sub_with_borrow(val, cpp::array{1}); } LIBC_INLINE constexpr void extend(size_t index, bool is_neg) { @@ -989,12 +1001,6 @@ struct BigInt { LIBC_INLINE constexpr void clear_msb() { val.back() &= mask_trailing_ones(); } - - LIBC_INLINE constexpr void set_bit(size_t i) { - const size_t word_index = i / WORD_SIZE; - val[word_index] |= WordType(1) << (i % WORD_SIZE); - } - LIBC_INLINE constexpr static Division divide_unsigned(const BigInt ÷nd, const BigInt ÷r) { BigInt remainder = dividend; @@ -1003,12 +1009,12 @@ struct BigInt { BigInt subtractor = divider; int cur_bit = multiword::countl_zero(subtractor.val) - multiword::countl_zero(remainder.val); - subtractor <<= cur_bit; + subtractor <<= static_cast(cur_bit); for (; cur_bit >= 0 && remainder > 0; --cur_bit, subtractor >>= 1) { if (remainder < subtractor) continue; remainder -= subtractor; - quotient.set_bit(cur_bit); + quotient.set_bit(static_cast(cur_bit)); } } return Division{quotient, remainder}; @@ -1270,26 +1276,28 @@ rotr(T value, int rotate); template [[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, T> rotl(T value, int rotate) { - constexpr unsigned N = cpp::numeric_limits::digits; + constexpr int N = cpp::numeric_limits::digits; rotate = rotate % N; if (!rotate) return value; if (rotate < 0) return cpp::rotr(value, -rotate); - return (value << rotate) | (value >> (N - rotate)); + return (value << static_cast(rotate)) | + (value >> (N - static_cast(rotate))); } // Specialization of cpp::rotr ('bit.h') for BigInt. template [[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, T> rotr(T value, int rotate) { - constexpr unsigned N = cpp::numeric_limits::digits; + constexpr int N = cpp::numeric_limits::digits; rotate = rotate % N; if (!rotate) return value; if (rotate < 0) return cpp::rotl(value, -rotate); - return (value >> rotate) | (value << (N - rotate)); + return (value >> static_cast(rotate)) | + (value << (N - static_cast(rotate))); } } // namespace cpp @@ -1306,7 +1314,7 @@ mask_trailing_ones() { T out; // zero initialized for (size_t i = 0; i <= QUOTIENT; ++i) out[i] = i < QUOTIENT - ? -1 + ? cpp::numeric_limits::max() : mask_trailing_ones(); return out; } @@ -1322,7 +1330,7 @@ LIBC_INLINE constexpr cpp::enable_if_t, T> mask_leading_ones() { T out; // zero initialized for (size_t i = QUOTIENT; i < T::WORD_COUNT; ++i) out[i] = i > QUOTIENT - ? -1 + ? cpp::numeric_limits::max() : mask_leading_ones(); return out; } @@ -1375,8 +1383,7 @@ first_trailing_zero(T value) { template [[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, int> first_trailing_one(T value) { - return value == cpp::numeric_limits::max() ? 0 - : cpp::countr_zero(value) + 1; + return value == 0 ? 0 : cpp::countr_zero(value) + 1; } } // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/__support/common.h b/system/lib/llvm-libc/src/__support/common.h index 42e8a79187fac..15209b76978af 100644 --- a/system/lib/llvm-libc/src/__support/common.h +++ b/system/lib/llvm-libc/src/__support/common.h @@ -37,17 +37,27 @@ #define LLVM_LIBC_ATTR(name) EXPAND_THEN_SECOND(LLVM_LIBC_FUNCTION_ATTR_##name) -// MacOS needs to be excluded because it does not support aliasing. -#if defined(LIBC_COPT_PUBLIC_PACKAGING) && (!defined(__APPLE__)) +// At the moment, [[gnu::alias()]] is not supported on MacOS, and it is needed +// to cleanly export and alias the C++ symbol `LIBC_NAMESPACE::func` with the C +// symbol `func`. So for public packaging on MacOS, we will only export the C +// symbol. Moreover, a C symbol `func` in macOS is mangled as `_func`. +#if defined(LIBC_COPT_PUBLIC_PACKAGING) +#ifndef __APPLE__ #define LLVM_LIBC_FUNCTION_IMPL(type, name, arglist) \ LLVM_LIBC_ATTR(name) \ LLVM_LIBC_FUNCTION_ATTR decltype(LIBC_NAMESPACE::name) \ __##name##_impl__ __asm__(#name); \ decltype(LIBC_NAMESPACE::name) name [[gnu::alias(#name)]]; \ type __##name##_impl__ arglist -#else +#else // __APPLE__ +#define LLVM_LIBC_FUNCTION_IMPL(type, name, arglist) \ + LLVM_LIBC_ATTR(name) \ + LLVM_LIBC_FUNCTION_ATTR decltype(LIBC_NAMESPACE::name) name asm("_" #name); \ + type name arglist +#endif // __APPLE__ +#else // LIBC_COPT_PUBLIC_PACKAGING #define LLVM_LIBC_FUNCTION_IMPL(type, name, arglist) type name arglist -#endif +#endif // LIBC_COPT_PUBLIC_PACKAGING // This extra layer of macro allows `name` to be a macro to rename a function. #define LLVM_LIBC_FUNCTION(type, name, arglist) \ diff --git a/system/lib/llvm-libc/src/__support/fixed_point/fx_bits.h b/system/lib/llvm-libc/src/__support/fixed_point/fx_bits.h index 225ea417760a0..00c6119b4f353 100644 --- a/system/lib/llvm-libc/src/__support/fixed_point/fx_bits.h +++ b/system/lib/llvm-libc/src/__support/fixed_point/fx_bits.h @@ -11,9 +11,11 @@ #include "include/llvm-libc-macros/stdfix-macros.h" #include "src/__support/CPP/bit.h" +#include "src/__support/CPP/limits.h" // numeric_limits #include "src/__support/CPP/type_traits.h" -#include "src/__support/macros/attributes.h" // LIBC_INLINE -#include "src/__support/macros/config.h" +#include "src/__support/macros/attributes.h" // LIBC_INLINE +#include "src/__support/macros/config.h" // LIBC_NAMESPACE_DECL +#include "src/__support/macros/null_check.h" // LIBC_CRASH_ON_VALUE #include "src/__support/macros/optimization.h" // LIBC_UNLIKELY #include "src/__support/math_extras.h" @@ -50,6 +52,12 @@ template struct FXBits { static constexpr StorageType SIGN_MASK = (fx_rep::SIGN_LEN == 0 ? 0 : StorageType(1) << SIGN_OFFSET); + // mask for + static constexpr StorageType VALUE_MASK = INTEGRAL_MASK | FRACTION_MASK; + + // mask for + static constexpr StorageType TOTAL_MASK = SIGN_MASK | VALUE_MASK; + public: LIBC_INLINE constexpr FXBits() = default; @@ -74,6 +82,12 @@ template struct FXBits { return (value & INTEGRAL_MASK) >> INTEGRAL_OFFSET; } + // returns complete bitstring representation the fixed point number + // the bitstring is of the form: padding | sign | integral | fraction + LIBC_INLINE constexpr StorageType get_bits() { + return (value & TOTAL_MASK) >> FRACTION_OFFSET; + } + // TODO: replace bool with Sign LIBC_INLINE constexpr bool get_sign() { return static_cast((value & SIGN_MASK) >> SIGN_OFFSET); @@ -163,6 +177,53 @@ template LIBC_INLINE constexpr T round(T x, int n) { return bit_and((x + round_bit), rounding_mask); } +// count leading sign bits +// TODO: support fixed_point_padding +template +LIBC_INLINE constexpr cpp::enable_if_t, int> +countls(T f) { + using FXRep = FXRep; + using BitType = typename FXRep::StorageType; + using FXBits = FXBits; + + if constexpr (FXRep::SIGN_LEN > 0) { + if (f < 0) + f = bit_not(f); + } + + BitType value_bits = FXBits(f).get_bits(); + return cpp::countl_zero(value_bits) - FXRep::SIGN_LEN; +} + +// fixed-point to integer conversion +template +LIBC_INLINE constexpr cpp::enable_if_t, XType> +bitsfx(T f) { + return cpp::bit_cast(f); +} + +// divide the two fixed-point types and return an integer result +template +LIBC_INLINE constexpr cpp::enable_if_t, XType> +idiv(T x, T y) { + using FXBits = FXBits; + using FXRep = FXRep; + using CompType = typename FXRep::CompType; + + // If the value of the second operand of the / operator is zero, the + // behavior is undefined. Ref: ISO/IEC TR 18037:2008(E) p.g. 16 + LIBC_CRASH_ON_VALUE(y, FXRep::ZERO()); + + CompType x_comp = static_cast(FXBits(x).get_bits()); + CompType y_comp = static_cast(FXBits(y).get_bits()); + + // If an integer result of one of these functions overflows, the behavior is + // undefined. Ref: ISO/IEC TR 18037:2008(E) p.g. 16 + CompType result = x_comp / y_comp; + + return static_cast(result); +} + } // namespace fixed_point } // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/__support/fixed_point/fx_rep.h b/system/lib/llvm-libc/src/__support/fixed_point/fx_rep.h index 186938947694e..7227fffa683a8 100644 --- a/system/lib/llvm-libc/src/__support/fixed_point/fx_rep.h +++ b/system/lib/llvm-libc/src/__support/fixed_point/fx_rep.h @@ -43,8 +43,8 @@ template <> struct FXRep { LIBC_INLINE_VAR static constexpr int SIGN_LEN = 1; LIBC_INLINE_VAR static constexpr int INTEGRAL_LEN = 0; LIBC_INLINE_VAR static constexpr int FRACTION_LEN = SFRACT_FBIT; - LIBC_INLINE_VAR static constexpr int TOTAL_LEN = - SIGN_LEN + INTEGRAL_LEN + FRACTION_LEN; + LIBC_INLINE_VAR static constexpr int VALUE_LEN = INTEGRAL_LEN + FRACTION_LEN; + LIBC_INLINE_VAR static constexpr int TOTAL_LEN = SIGN_LEN + VALUE_LEN; LIBC_INLINE static constexpr Type MIN() { return SFRACT_MIN; } LIBC_INLINE static constexpr Type MAX() { return SFRACT_MAX; } @@ -63,8 +63,8 @@ template <> struct FXRep { LIBC_INLINE_VAR static constexpr int SIGN_LEN = 0; LIBC_INLINE_VAR static constexpr int INTEGRAL_LEN = 0; LIBC_INLINE_VAR static constexpr int FRACTION_LEN = USFRACT_FBIT; - LIBC_INLINE_VAR static constexpr int TOTAL_LEN = - SIGN_LEN + INTEGRAL_LEN + FRACTION_LEN; + LIBC_INLINE_VAR static constexpr int VALUE_LEN = INTEGRAL_LEN + FRACTION_LEN; + LIBC_INLINE_VAR static constexpr int TOTAL_LEN = SIGN_LEN + VALUE_LEN; LIBC_INLINE static constexpr Type MIN() { return USFRACT_MIN; } LIBC_INLINE static constexpr Type MAX() { return USFRACT_MAX; } @@ -83,8 +83,8 @@ template <> struct FXRep { LIBC_INLINE_VAR static constexpr int SIGN_LEN = 1; LIBC_INLINE_VAR static constexpr int INTEGRAL_LEN = 0; LIBC_INLINE_VAR static constexpr int FRACTION_LEN = FRACT_FBIT; - LIBC_INLINE_VAR static constexpr int TOTAL_LEN = - SIGN_LEN + INTEGRAL_LEN + FRACTION_LEN; + LIBC_INLINE_VAR static constexpr int VALUE_LEN = INTEGRAL_LEN + FRACTION_LEN; + LIBC_INLINE_VAR static constexpr int TOTAL_LEN = SIGN_LEN + VALUE_LEN; LIBC_INLINE static constexpr Type MIN() { return FRACT_MIN; } LIBC_INLINE static constexpr Type MAX() { return FRACT_MAX; } @@ -103,8 +103,8 @@ template <> struct FXRep { LIBC_INLINE_VAR static constexpr int SIGN_LEN = 0; LIBC_INLINE_VAR static constexpr int INTEGRAL_LEN = 0; LIBC_INLINE_VAR static constexpr int FRACTION_LEN = UFRACT_FBIT; - LIBC_INLINE_VAR static constexpr int TOTAL_LEN = - SIGN_LEN + INTEGRAL_LEN + FRACTION_LEN; + LIBC_INLINE_VAR static constexpr int VALUE_LEN = INTEGRAL_LEN + FRACTION_LEN; + LIBC_INLINE_VAR static constexpr int TOTAL_LEN = SIGN_LEN + VALUE_LEN; LIBC_INLINE static constexpr Type MIN() { return UFRACT_MIN; } LIBC_INLINE static constexpr Type MAX() { return UFRACT_MAX; } @@ -123,8 +123,8 @@ template <> struct FXRep { LIBC_INLINE_VAR static constexpr int SIGN_LEN = 1; LIBC_INLINE_VAR static constexpr int INTEGRAL_LEN = 0; LIBC_INLINE_VAR static constexpr int FRACTION_LEN = LFRACT_FBIT; - LIBC_INLINE_VAR static constexpr int TOTAL_LEN = - SIGN_LEN + INTEGRAL_LEN + FRACTION_LEN; + LIBC_INLINE_VAR static constexpr int VALUE_LEN = INTEGRAL_LEN + FRACTION_LEN; + LIBC_INLINE_VAR static constexpr int TOTAL_LEN = SIGN_LEN + VALUE_LEN; LIBC_INLINE static constexpr Type MIN() { return LFRACT_MIN; } LIBC_INLINE static constexpr Type MAX() { return LFRACT_MAX; } @@ -143,8 +143,8 @@ template <> struct FXRep { LIBC_INLINE_VAR static constexpr int SIGN_LEN = 0; LIBC_INLINE_VAR static constexpr int INTEGRAL_LEN = 0; LIBC_INLINE_VAR static constexpr int FRACTION_LEN = ULFRACT_FBIT; - LIBC_INLINE_VAR static constexpr int TOTAL_LEN = - SIGN_LEN + INTEGRAL_LEN + FRACTION_LEN; + LIBC_INLINE_VAR static constexpr int VALUE_LEN = INTEGRAL_LEN + FRACTION_LEN; + LIBC_INLINE_VAR static constexpr int TOTAL_LEN = SIGN_LEN + VALUE_LEN; LIBC_INLINE static constexpr Type MIN() { return ULFRACT_MIN; } LIBC_INLINE static constexpr Type MAX() { return ULFRACT_MAX; } @@ -163,8 +163,8 @@ template <> struct FXRep { LIBC_INLINE_VAR static constexpr int SIGN_LEN = 1; LIBC_INLINE_VAR static constexpr int INTEGRAL_LEN = SACCUM_IBIT; LIBC_INLINE_VAR static constexpr int FRACTION_LEN = SACCUM_FBIT; - LIBC_INLINE_VAR static constexpr int TOTAL_LEN = - SIGN_LEN + INTEGRAL_LEN + FRACTION_LEN; + LIBC_INLINE_VAR static constexpr int VALUE_LEN = INTEGRAL_LEN + FRACTION_LEN; + LIBC_INLINE_VAR static constexpr int TOTAL_LEN = SIGN_LEN + VALUE_LEN; LIBC_INLINE static constexpr Type MIN() { return SACCUM_MIN; } LIBC_INLINE static constexpr Type MAX() { return SACCUM_MAX; } @@ -183,8 +183,8 @@ template <> struct FXRep { LIBC_INLINE_VAR static constexpr int SIGN_LEN = 0; LIBC_INLINE_VAR static constexpr int INTEGRAL_LEN = USACCUM_IBIT; LIBC_INLINE_VAR static constexpr int FRACTION_LEN = USACCUM_FBIT; - LIBC_INLINE_VAR static constexpr int TOTAL_LEN = - SIGN_LEN + INTEGRAL_LEN + FRACTION_LEN; + LIBC_INLINE_VAR static constexpr int VALUE_LEN = INTEGRAL_LEN + FRACTION_LEN; + LIBC_INLINE_VAR static constexpr int TOTAL_LEN = SIGN_LEN + VALUE_LEN; LIBC_INLINE static constexpr Type MIN() { return USACCUM_MIN; } LIBC_INLINE static constexpr Type MAX() { return USACCUM_MAX; } @@ -203,8 +203,8 @@ template <> struct FXRep { LIBC_INLINE_VAR static constexpr int SIGN_LEN = 1; LIBC_INLINE_VAR static constexpr int INTEGRAL_LEN = ACCUM_IBIT; LIBC_INLINE_VAR static constexpr int FRACTION_LEN = ACCUM_FBIT; - LIBC_INLINE_VAR static constexpr int TOTAL_LEN = - SIGN_LEN + INTEGRAL_LEN + FRACTION_LEN; + LIBC_INLINE_VAR static constexpr int VALUE_LEN = INTEGRAL_LEN + FRACTION_LEN; + LIBC_INLINE_VAR static constexpr int TOTAL_LEN = SIGN_LEN + VALUE_LEN; LIBC_INLINE static constexpr Type MIN() { return ACCUM_MIN; } LIBC_INLINE static constexpr Type MAX() { return ACCUM_MAX; } @@ -223,8 +223,8 @@ template <> struct FXRep { LIBC_INLINE_VAR static constexpr int SIGN_LEN = 0; LIBC_INLINE_VAR static constexpr int INTEGRAL_LEN = UACCUM_IBIT; LIBC_INLINE_VAR static constexpr int FRACTION_LEN = UACCUM_FBIT; - LIBC_INLINE_VAR static constexpr int TOTAL_LEN = - SIGN_LEN + INTEGRAL_LEN + FRACTION_LEN; + LIBC_INLINE_VAR static constexpr int VALUE_LEN = INTEGRAL_LEN + FRACTION_LEN; + LIBC_INLINE_VAR static constexpr int TOTAL_LEN = SIGN_LEN + VALUE_LEN; LIBC_INLINE static constexpr Type MIN() { return UACCUM_MIN; } LIBC_INLINE static constexpr Type MAX() { return UACCUM_MAX; } @@ -243,8 +243,8 @@ template <> struct FXRep { LIBC_INLINE_VAR static constexpr int SIGN_LEN = 1; LIBC_INLINE_VAR static constexpr int INTEGRAL_LEN = LACCUM_IBIT; LIBC_INLINE_VAR static constexpr int FRACTION_LEN = LACCUM_FBIT; - LIBC_INLINE_VAR static constexpr int TOTAL_LEN = - SIGN_LEN + INTEGRAL_LEN + FRACTION_LEN; + LIBC_INLINE_VAR static constexpr int VALUE_LEN = INTEGRAL_LEN + FRACTION_LEN; + LIBC_INLINE_VAR static constexpr int TOTAL_LEN = SIGN_LEN + VALUE_LEN; LIBC_INLINE static constexpr Type MIN() { return LACCUM_MIN; } LIBC_INLINE static constexpr Type MAX() { return LACCUM_MAX; } @@ -263,8 +263,8 @@ template <> struct FXRep { LIBC_INLINE_VAR static constexpr int SIGN_LEN = 0; LIBC_INLINE_VAR static constexpr int INTEGRAL_LEN = ULACCUM_IBIT; LIBC_INLINE_VAR static constexpr int FRACTION_LEN = ULACCUM_FBIT; - LIBC_INLINE_VAR static constexpr int TOTAL_LEN = - SIGN_LEN + INTEGRAL_LEN + FRACTION_LEN; + LIBC_INLINE_VAR static constexpr int VALUE_LEN = INTEGRAL_LEN + FRACTION_LEN; + LIBC_INLINE_VAR static constexpr int TOTAL_LEN = SIGN_LEN + VALUE_LEN; LIBC_INLINE static constexpr Type MIN() { return ULACCUM_MIN; } LIBC_INLINE static constexpr Type MAX() { return ULACCUM_MAX; } diff --git a/system/lib/llvm-libc/src/__support/float_to_string.h b/system/lib/llvm-libc/src/__support/float_to_string.h index d5de6f38cb655..d88bf844b15af 100644 --- a/system/lib/llvm-libc/src/__support/float_to_string.h +++ b/system/lib/llvm-libc/src/__support/float_to_string.h @@ -287,7 +287,7 @@ LIBC_INLINE UInt get_table_negative(int exponent, size_t i) { size_t ten_blocks = i; size_t five_blocks = 0; if (shift_amount < 0) { - int block_shifts = (-shift_amount) / BLOCK_SIZE; + int block_shifts = (-shift_amount) / static_cast(BLOCK_SIZE); if (block_shifts < static_cast(ten_blocks)) { ten_blocks = ten_blocks - block_shifts; five_blocks = block_shifts; @@ -491,7 +491,7 @@ class FloatToString { LIBC_INLINE constexpr BlockInt get_negative_block(int block_index) { if (exponent < 0) { - const int32_t idx = -exponent / IDX_SIZE; + const int32_t idx = -exponent / static_cast(IDX_SIZE); UInt val; @@ -579,7 +579,7 @@ class FloatToString { return num_requested_digits > -exponent; #else - const int32_t idx = -exponent / IDX_SIZE; + const int32_t idx = -exponent / static_cast(IDX_SIZE); const size_t p = POW10_OFFSET_2[idx] + negative_block_index - MIN_BLOCK_2[idx]; // If the remaining digits are all 0, then this is the lowest block. @@ -601,7 +601,7 @@ class FloatToString { } return 0; #else - return MIN_BLOCK_2[-exponent / IDX_SIZE]; + return MIN_BLOCK_2[-exponent / static_cast(IDX_SIZE)]; #endif } }; diff --git a/system/lib/llvm-libc/src/__support/high_precision_decimal.h b/system/lib/llvm-libc/src/__support/high_precision_decimal.h index 922dce484aa6b..cb4b50c315447 100644 --- a/system/lib/llvm-libc/src/__support/high_precision_decimal.h +++ b/system/lib/llvm-libc/src/__support/high_precision_decimal.h @@ -264,7 +264,7 @@ class HighPrecisionDecimal { LIBC_INLINE void left_shift(uint32_t shift_amount) { uint32_t new_digits = this->get_num_new_digits(shift_amount); - int32_t read_index = this->num_digits - 1; + int32_t read_index = static_cast(this->num_digits - 1); uint32_t write_index = this->num_digits + new_digits; uint64_t accumulator = 0; @@ -329,7 +329,7 @@ class HighPrecisionDecimal { if (saw_dot) { break; } - this->decimal_point = total_digits; + this->decimal_point = static_cast(total_digits); saw_dot = true; } else { if (num_string[num_cur] == '0' && this->num_digits == 0) { @@ -350,7 +350,7 @@ class HighPrecisionDecimal { } if (!saw_dot) - this->decimal_point = total_digits; + this->decimal_point = static_cast(total_digits); if (num_cur < num_len && (num_string[num_cur] == 'e' || num_string[num_cur] == 'E')) { @@ -393,7 +393,7 @@ class HighPrecisionDecimal { this->left_shift(MAX_SHIFT_AMOUNT); shift_amount -= MAX_SHIFT_AMOUNT; } - this->left_shift(shift_amount); + this->left_shift(static_cast(shift_amount)); } // Right else { @@ -401,7 +401,7 @@ class HighPrecisionDecimal { this->right_shift(MAX_SHIFT_AMOUNT); shift_amount += MAX_SHIFT_AMOUNT; } - this->right_shift(-shift_amount); + this->right_shift(static_cast(-shift_amount)); } } @@ -424,8 +424,8 @@ class HighPrecisionDecimal { result *= 10; ++cur_digit; } - return result + static_cast( - this->should_round_up(this->decimal_point, round)); + return result + + static_cast(this->should_round_up(this->decimal_point, round)); } // Extra functions for testing. diff --git a/system/lib/llvm-libc/src/__support/integer_literals.h b/system/lib/llvm-libc/src/__support/integer_literals.h index 0298ec7d088d6..f68b7ef12c879 100644 --- a/system/lib/llvm-libc/src/__support/integer_literals.h +++ b/system/lib/llvm-libc/src/__support/integer_literals.h @@ -47,7 +47,7 @@ LIBC_INLINE constexpr T accumulate(int base, const uint8_t *digits, size_t size) { T value{}; for (; size; ++digits, --size) { - value *= base; + value *= static_cast(base); value += *digits; } return value; diff --git a/system/lib/llvm-libc/src/__support/integer_to_string.h b/system/lib/llvm-libc/src/__support/integer_to_string.h index ea620087584cb..65bdcf16b3867 100644 --- a/system/lib/llvm-libc/src/__support/integer_to_string.h +++ b/system/lib/llvm-libc/src/__support/integer_to_string.h @@ -164,6 +164,170 @@ template using Custom = details::Fmt; } // namespace radix +// Extract the low-order decimal digit from a value of integer type T. The +// returned value is the digit itself, from 0 to 9. The input value is passed +// by reference, and modified by dividing by 10, so that iterating this +// function extracts all the digits of the original number one at a time from +// low to high. +template +LIBC_INLINE cpp::enable_if_t, uint8_t> +extract_decimal_digit(T &value) { + const uint8_t digit(static_cast(value % 10)); + // For built-in integer types, we assume that an adequately fast division is + // available. If hardware division isn't implemented, then with a divisor + // known at compile time the compiler might be able to generate an optimized + // sequence instead. + value /= 10; + return digit; +} + +// A specialization of extract_decimal_digit for the BigInt type in big_int.h, +// avoiding the use of general-purpose BigInt division which is very slow. +template +LIBC_INLINE cpp::enable_if_t, uint8_t> +extract_decimal_digit(T &value) { + // There are two essential ways you can turn n into (n/10,n%10). One is + // ordinary integer division. The other is a modular-arithmetic approach in + // which you first compute n%10 by bit twiddling, then subtract it off to get + // a value that is definitely a multiple of 10. Then you divide that by 10 in + // two steps: shift right to divide off a factor of 2, and then divide off a + // factor of 5 by multiplying by the modular inverse of 5 mod 2^BITS. (That + // last step only works if you know there's no remainder, which is why you + // had to subtract off the output digit first.) + // + // Either approach can be made to work in linear time. This code uses the + // modular-arithmetic technique, because the other approach either does a lot + // of integer divisions (requiring a fast hardware divider), or else uses a + // "multiply by an approximation to the reciprocal" technique which depends + // on careful error analysis which might go wrong in an untested edge case. + + using Word = typename T::word_type; + + // Find the remainder (value % 10). We do this by breaking up the input + // integer into chunks of size WORD_SIZE/2, so that the sum of them doesn't + // overflow a Word. Then we sum all the half-words times 6, except the bottom + // one, which is added to that sum without scaling. + // + // Why 6? Because you can imagine that the original number had the form + // + // halfwords[0] + K*halfwords[1] + K^2*halfwords[2] + ... + // + // where K = 2^(WORD_SIZE/2). Since WORD_SIZE is expected to be a multiple of + // 8, that makes WORD_SIZE/2 a multiple of 4, so that K is a power of 16. And + // all powers of 16 (larger than 1) are congruent to 6 mod 10, by induction: + // 16 itself is, and 6^2=36 is also congruent to 6. + Word acc_remainder = 0; + constexpr Word HALFWORD_BITS = T::WORD_SIZE / 2; + constexpr Word HALFWORD_MASK = ((Word(1) << HALFWORD_BITS) - 1); + // Sum both halves of all words except the low one. + for (size_t i = 1; i < T::WORD_COUNT; i++) { + acc_remainder += value.val[i] >> HALFWORD_BITS; + acc_remainder += value.val[i] & HALFWORD_MASK; + } + // Add the high half of the low word. Then we have everything that needs to + // be multiplied by 6, so do that. + acc_remainder += value.val[0] >> HALFWORD_BITS; + acc_remainder *= 6; + // Having multiplied it by 6, add the lowest half-word, and then reduce mod + // 10 by normal integer division to finish. + acc_remainder += value.val[0] & HALFWORD_MASK; + uint8_t digit = static_cast(acc_remainder % 10); + + // Now we have the output digit. Subtract it from the input value, and shift + // right to divide by 2. + value -= digit; + value >>= 1; + + // Now all that's left is to multiply by the inverse of 5 mod 2^BITS. No + // matter what the value of BITS, the inverse of 5 has the very convenient + // form 0xCCCC...CCCD, with as many C hex digits in the middle as necessary. + // + // We could construct a second BigInt with all words 0xCCCCCCCCCCCCCCCC, + // increment the bottom word, and call a general-purpose multiply function. + // But we can do better, by taking advantage of the regularity: we can do + // this particular operation in linear time, whereas a general multiplier + // would take superlinear time (quadratic in small cases). + // + // To begin with, instead of computing n*0xCCCC...CCCD, we'll compute + // n*0xCCCC...CCCC and then add it to the original n. Then all the words of + // the multiplier have the same value 0xCCCCCCCCCCCCCCCC, which I'll just + // denote as C. If we also write t = 2^WORD_SIZE, and imagine (as an example) + // that the input number has three words x,y,z with x being the low word, + // then we're computing + // + // (x + y t + z t^2) * (C + C t + C t^2) + // + // = x C + y C t + z C t^2 + // + x C t + y C t^2 + z C t^3 + // + x C t^2 + y C t^3 + z C t^4 + // + // but we're working mod t^3, so the high-order terms vanish and this becomes + // + // x C + y C t + z C t^2 + // + x C t + y C t^2 + // + x C t^2 + // + // = x C + (x+y) C t + (x+y+z) C t^2 + // + // So all you have to do is to work from the low word of the integer upwards, + // accumulating C times the sum of all the words you've seen so far to get + // x*C, (x+y)*C, (x+y+z)*C and so on. In each step you add another product to + // the accumulator, and add the accumulator to the corresponding word of the + // original number (so that we end up with value*CCCD, not just value*CCCC). + // + // If you do that literally, then your accumulator has to be three words + // wide, because the sum of words can overflow into a second word, and + // multiplying by C adds another word. But we can do slightly better by + // breaking each product word*C up into a bottom half and a top half. If we + // write x*C = xl + xh*t, and similarly for y and z, then our sum becomes + // + // (xl + xh t) + (yl + yh t) t + (zl + zh t) t^2 + // + (xl + xh t) t + (yl + yh t) t^2 + // + (xl + xh t) t^2 + // + // and if you expand out again, collect terms, and discard t^3 terms, you get + // + // (xl) + // + (xl + xh + yl) t + // + (xl + xh + yl + yh + zl) t^2 + // + // in which each coefficient is the sum of all the low words of the products + // up to _and including_ the current word, plus all the high words up to but + // _not_ including the current word. So now you only have to retain two words + // of sum instead of three. + // + // We do this entire procedure in a single in-place pass over the input + // number, reading each word to make its product with C and then adding the + // low word of the accumulator to it. + constexpr Word C = Word(-1) / 5 * 4; // calculate 0xCCCC as 4/5 of 0xFFFF + Word acc_lo = 0, acc_hi = 0; // accumulator of all the half-products so far + Word carry_bit, carry_word = 0; + + for (size_t i = 0; i < T::WORD_COUNT; i++) { + // Make the two-word product of C with the current input word. + multiword::DoubleWide product = multiword::mul2(C, value.val[i]); + + // Add the low half of the product to our accumulator, but not yet the high + // half. + acc_lo = add_with_carry(acc_lo, product[0], 0, carry_bit); + acc_hi += carry_bit; + + // Now the accumulator contains exactly the value we need to add to the + // current input word. Add it, plus any carries from lower words, and make + // a new word of carry data to propagate into the next iteration. + value.val[i] = add_with_carry(value.val[i], carry_word, 0, carry_bit); + carry_word = acc_hi + carry_bit; + value.val[i] = add_with_carry(value.val[i], acc_lo, 0, carry_bit); + carry_word += carry_bit; + + // Now add the high half of the current product to our accumulator. + acc_lo = add_with_carry(acc_lo, product[1], 0, carry_bit); + acc_hi += carry_bit; + } + + return digit; +} + // See file header for documentation. template class IntegerToString { static_assert(cpp::is_integral_v || is_big_int_v); @@ -229,10 +393,19 @@ template class IntegerToString { } } + LIBC_INLINE static void + write_unsigned_number_dec(UNSIGNED_T value, + details::BackwardStringBufferWriter &sink) { + while (sink.ok() && value != 0) { + const uint8_t digit = extract_decimal_digit(value); + sink.push(digit_char(digit)); + } + } + // Returns the absolute value of 'value' as 'UNSIGNED_T'. LIBC_INLINE static UNSIGNED_T abs(T value) { if (cpp::is_unsigned_v || value >= 0) - return value; // already of the right sign. + return static_cast(value); // already of the right sign. // Signed integers are asymmetric (e.g., int8_t ∈ [-128, 127]). // Thus negating the type's minimum value would overflow. @@ -249,14 +422,15 @@ template class IntegerToString { if (value == cpp::numeric_limits::min()) { return cpp::bit_cast(value); } else { - return -value; // legal and representable both as T and UNSIGNED_T.` + return static_cast( + -value); // legal and representable both as T and UNSIGNED_T.` } } LIBC_INLINE static void write(T value, details::BackwardStringBufferWriter &sink) { if constexpr (Fmt::BASE == 10) { - write_unsigned_number(abs(value), sink); + write_unsigned_number_dec(abs(value), sink); } else { write_unsigned_number(static_cast(value), sink); } diff --git a/system/lib/llvm-libc/src/__support/libc_errno.h b/system/lib/llvm-libc/src/__support/libc_errno.h new file mode 100644 index 0000000000000..ab5f6a9c4b9d9 --- /dev/null +++ b/system/lib/llvm-libc/src/__support/libc_errno.h @@ -0,0 +1,108 @@ +//===-- Implementation header for libc_errno --------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC___SUPPORT_LIBC_ERRNO_H +#define LLVM_LIBC_SRC___SUPPORT_LIBC_ERRNO_H + +// This header is to be consumed by internal implementations, in which all of +// them should refer to `libc_errno` instead of using `errno` directly from +// header. + +// Unit and hermetic tests should: +// - #include "src/__support/libc_errno.h" +// - NOT #include +// - Only use `libc_errno` in the code +// - Depend on libc.src.errno.errno + +// Integration tests should: +// - NOT #include "src/__support/libc_errno.h" +// - #include +// - Use regular `errno` in the code +// - Still depend on libc.src.errno.errno + +// libc uses a fallback default value, either system or thread local. +#define LIBC_ERRNO_MODE_DEFAULT 0 +// libc never stores a value; `errno` macro uses get link-time failure. +#define LIBC_ERRNO_MODE_UNDEFINED 1 +// libc maintains per-thread state (requires C++ `thread_local` support). +#define LIBC_ERRNO_MODE_THREAD_LOCAL 2 +// libc maintains shared state used by all threads, contrary to standard C +// semantics unless always single-threaded; nothing prevents data races. +#define LIBC_ERRNO_MODE_SHARED 3 +// libc doesn't maintain any internal state, instead the embedder must define +// `int *__llvm_libc_errno(void);` C function. +#define LIBC_ERRNO_MODE_EXTERNAL 4 +// libc uses system `` `errno` macro directly in the overlay mode; in +// fullbuild mode, effectively the same as `LIBC_ERRNO_MODE_EXTERNAL`. +// In this mode, the public C++ symbol `LIBC_NAMESPACE::libc_errno ` is still +// exported and get redirected to the system `errno` inside its implementation. + +// TODO: Investigate deprecating LIBC_ERRNO_MODE_SYSTEM in favor of +// LIBC_ERRNO_MODE_SYSTEM_INLINE. +// https://github.com/llvm/llvm-project/issues/143454 +#define LIBC_ERRNO_MODE_SYSTEM 5 +// In this mode, the libc_errno is simply a macro resolved to `errno` from the +// system header . There is no need to link against the +// `libc.src.errno.errno` object. +#define LIBC_ERRNO_MODE_SYSTEM_INLINE 6 + +#if !defined(LIBC_ERRNO_MODE) || LIBC_ERRNO_MODE == LIBC_ERRNO_MODE_DEFAULT +#undef LIBC_ERRNO_MODE +#if defined(LIBC_FULL_BUILD) || !defined(LIBC_COPT_PUBLIC_PACKAGING) +#define LIBC_ERRNO_MODE LIBC_ERRNO_MODE_THREAD_LOCAL +#else +#define LIBC_ERRNO_MODE LIBC_ERRNO_MODE_SYSTEM +#endif +#endif // LIBC_ERRNO_MODE + +#if LIBC_ERRNO_MODE != LIBC_ERRNO_MODE_DEFAULT && \ + LIBC_ERRNO_MODE != LIBC_ERRNO_MODE_UNDEFINED && \ + LIBC_ERRNO_MODE != LIBC_ERRNO_MODE_THREAD_LOCAL && \ + LIBC_ERRNO_MODE != LIBC_ERRNO_MODE_SHARED && \ + LIBC_ERRNO_MODE != LIBC_ERRNO_MODE_EXTERNAL && \ + LIBC_ERRNO_MODE != LIBC_ERRNO_MODE_SYSTEM && \ + LIBC_ERRNO_MODE != LIBC_ERRNO_MODE_SYSTEM_INLINE +#error LIBC_ERRNO_MODE must be one of the following values: \ +LIBC_ERRNO_MODE_DEFAULT, \ +LIBC_ERRNO_MODE_UNDEFINED, \ +LIBC_ERRNO_MODE_THREAD_LOCAL, \ +LIBC_ERRNO_MODE_SHARED, \ +LIBC_ERRNO_MODE_EXTERNAL, \ +LIBC_ERRNO_MODE_SYSTEM, \ +LIBC_ERRNO_MODE_SYSTEM_INLINE. +#endif + +#if LIBC_ERRNO_MODE == LIBC_ERRNO_MODE_SYSTEM_INLINE + +#include + +#define libc_errno errno + +#else // !LIBC_ERRNO_MODE_SYSTEM_INLINE + +#include "hdr/errno_macros.h" +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +extern "C" int *__llvm_libc_errno() noexcept; + +struct Errno { + void operator=(int); + operator int(); +}; + +extern Errno libc_errno; + +} // namespace LIBC_NAMESPACE_DECL + +using LIBC_NAMESPACE::libc_errno; + +#endif // LIBC_ERRNO_MODE_SYSTEM_INLINE + +#endif // LLVM_LIBC_SRC___SUPPORT_LIBC_ERRNO_H diff --git a/system/lib/llvm-libc/src/__support/macros/null_check.h b/system/lib/llvm-libc/src/__support/macros/null_check.h index eda19f889235e..abf65c56c404b 100644 --- a/system/lib/llvm-libc/src/__support/macros/null_check.h +++ b/system/lib/llvm-libc/src/__support/macros/null_check.h @@ -19,10 +19,19 @@ if (LIBC_UNLIKELY((ptr) == nullptr)) \ __builtin_trap(); \ } while (0) +#define LIBC_CRASH_ON_VALUE(var, value) \ + do { \ + if (LIBC_UNLIKELY((var) == (value))) \ + __builtin_trap(); \ + } while (0) + #else #define LIBC_CRASH_ON_NULLPTR(ptr) \ do { \ } while (0) +#define LIBC_CRASH_ON_VALUE(var, value) \ + do { \ + } while (0) #endif #endif // LLVM_LIBC_SRC___SUPPORT_MACROS_NULL_CHECK_H diff --git a/system/lib/llvm-libc/src/__support/macros/optimization.h b/system/lib/llvm-libc/src/__support/macros/optimization.h index a2634950d431b..253843e5e37aa 100644 --- a/system/lib/llvm-libc/src/__support/macros/optimization.h +++ b/system/lib/llvm-libc/src/__support/macros/optimization.h @@ -45,6 +45,7 @@ LIBC_INLINE constexpr bool expects_bool_condition(T value, T expected) { #define LIBC_MATH_FAST \ (LIBC_MATH_SKIP_ACCURATE_PASS | LIBC_MATH_SMALL_TABLES | \ LIBC_MATH_NO_ERRNO | LIBC_MATH_NO_EXCEPT) +#define LIBC_MATH_INTERMEDIATE_COMP_IN_FLOAT 0x10 #ifndef LIBC_MATH #define LIBC_MATH 0 @@ -58,4 +59,8 @@ LIBC_INLINE constexpr bool expects_bool_condition(T value, T expected) { #define LIBC_MATH_HAS_SMALL_TABLES #endif +#if (LIBC_MATH & LIBC_MATH_INTERMEDIATE_COMP_IN_FLOAT) +#define LIBC_MATH_HAS_INTERMEDIATE_COMP_IN_FLOAT +#endif + #endif // LLVM_LIBC_SRC___SUPPORT_MACROS_OPTIMIZATION_H diff --git a/system/lib/llvm-libc/src/__support/macros/properties/cpu_features.h b/system/lib/llvm-libc/src/__support/macros/properties/cpu_features.h index d2cea367516db..3677e1fc3275c 100644 --- a/system/lib/llvm-libc/src/__support/macros/properties/cpu_features.h +++ b/system/lib/llvm-libc/src/__support/macros/properties/cpu_features.h @@ -20,6 +20,8 @@ #if defined(__SSE2__) #define LIBC_TARGET_CPU_HAS_SSE2 +#define LIBC_TARGET_CPU_HAS_FPU_FLOAT +#define LIBC_TARGET_CPU_HAS_FPU_DOUBLE #endif #if defined(__SSE4_2__) @@ -42,9 +44,55 @@ #define LIBC_TARGET_CPU_HAS_AVX512BW #endif +#if defined(__ARM_FP) +#if (__ARM_FP & 0x2) +#define LIBC_TARGET_CPU_HAS_ARM_FPU_HALF +#define LIBC_TARGET_CPU_HAS_FPU_HALF +#endif // LIBC_TARGET_CPU_HAS_ARM_FPU_HALF +#if (__ARM_FP & 0x4) +#define LIBC_TARGET_CPU_HAS_ARM_FPU_FLOAT +#define LIBC_TARGET_CPU_HAS_FPU_FLOAT +#endif // LIBC_TARGET_CPU_HAS_ARM_FPU_FLOAT +#if (__ARM_FP & 0x8) +#define LIBC_TARGET_CPU_HAS_ARM_FPU_DOUBLE +#define LIBC_TARGET_CPU_HAS_FPU_DOUBLE +#endif // LIBC_TARGET_CPU_HAS_ARM_FPU_DOUBLE +#endif // __ARM_FP + +#if defined(__riscv_flen) +// https://github.com/riscv-non-isa/riscv-c-api-doc/blob/main/src/c-api.adoc +#if (__riscv_flen & 0x10) +#define LIBC_TARGET_CPU_HAS_RISCV_FPU_HALF +#define LIBC_TARGET_CPU_HAS_FPU_HALF +#endif // LIBC_TARGET_CPU_HAS_RISCV_FPU_HALF +#if (__riscv_flen & 0x20) +#define LIBC_TARGET_CPU_HAS_RISCV_FPU_FLOAT +#define LIBC_TARGET_CPU_HAS_FPU_FLOAT +#endif // LIBC_TARGET_CPU_HAS_RISCV_FPU_FLOAT +#if (__riscv_flen & 0x40) +#define LIBC_TARGET_CPU_HAS_RISCV_FPU_DOUBLE +#define LIBC_TARGET_CPU_HAS_FPU_DOUBLE +#endif // LIBC_TARGET_CPU_HAS_RISCV_FPU_DOUBLE +#endif // __riscv_flen + +#if defined(__NVPTX__) || defined(__AMDGPU__) +#define LIBC_TARGET_CPU_HAS_FPU_FLOAT +#define LIBC_TARGET_CPU_HAS_FPU_DOUBLE +#endif + #if defined(__ARM_FEATURE_FMA) || (defined(__AVX2__) && defined(__FMA__)) || \ defined(__NVPTX__) || defined(__AMDGPU__) || defined(__LIBC_RISCV_USE_FMA) #define LIBC_TARGET_CPU_HAS_FMA +// Provide a more fine-grained control of FMA instruction for ARM targets. +#if defined(LIBC_TARGET_CPU_HAS_FPU_HALF) +#define LIBC_TARGET_CPU_HAS_FMA_HALF +#endif // LIBC_TARGET_CPU_HAS_FMA_HALF +#if defined(LIBC_TARGET_CPU_HAS_FPU_FLOAT) +#define LIBC_TARGET_CPU_HAS_FMA_FLOAT +#endif // LIBC_TARGET_CPU_HAS_FMA_FLOAT +#if defined(LIBC_TARGET_CPU_HAS_FPU_DOUBLE) +#define LIBC_TARGET_CPU_HAS_FMA_DOUBLE +#endif // LIBC_TARGET_CPU_HAS_FMA_DOUBLE #endif #if defined(LIBC_TARGET_ARCH_IS_AARCH64) || \ diff --git a/system/lib/llvm-libc/src/__support/math/exp_float_constants.h b/system/lib/llvm-libc/src/__support/math/exp_float_constants.h new file mode 100644 index 0000000000000..cabb227a034b5 --- /dev/null +++ b/system/lib/llvm-libc/src/__support/math/exp_float_constants.h @@ -0,0 +1,145 @@ +//===-- Look-up tables for exp*f functions ----------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC___SUPPORT_MATH_EXP_FLOAT_CONSTANTS_H +#define LLVM_LIBC_SRC___SUPPORT_MATH_EXP_FLOAT_CONSTANTS_H + +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +namespace math { +// Lookup table for exp(m) with m = -104, ..., 89. +// -104 = floor(log(single precision's min denormal)) +// 89 = ceil(log(single precision's max normal)) +// Table is generated with Sollya as follow: +// > display = hexadecimal; +// > for i from -104 to 89 do { D(exp(i)); }; +static constexpr double EXP_M1[195] = { + 0x1.f1e6b68529e33p-151, 0x1.525be4e4e601dp-149, 0x1.cbe0a45f75eb1p-148, + 0x1.3884e838aea68p-146, 0x1.a8c1f14e2af5dp-145, 0x1.20a717e64a9bdp-143, + 0x1.8851d84118908p-142, 0x1.0a9bdfb02d240p-140, 0x1.6a5bea046b42ep-139, + 0x1.ec7f3b269efa8p-138, 0x1.4eafb87eab0f2p-136, 0x1.c6e2d05bbc000p-135, + 0x1.35208867c2683p-133, 0x1.a425b317eeacdp-132, 0x1.1d8508fa8246ap-130, + 0x1.840fbc08fdc8ap-129, 0x1.07b7112bc1ffep-127, 0x1.666d0dad2961dp-126, + 0x1.e726c3f64d0fep-125, 0x1.4b0dc07cabf98p-123, 0x1.c1f2daf3b6a46p-122, + 0x1.31c5957a47de2p-120, 0x1.9f96445648b9fp-119, 0x1.1a6baeadb4fd1p-117, + 0x1.7fd974d372e45p-116, 0x1.04da4d1452919p-114, 0x1.62891f06b3450p-113, + 0x1.e1dd273aa8a4ap-112, 0x1.4775e0840bfddp-110, 0x1.bd109d9d94bdap-109, + 0x1.2e73f53fba844p-107, 0x1.9b138170d6bfep-106, 0x1.175af0cf60ec5p-104, + 0x1.7baee1bffa80bp-103, 0x1.02057d1245cebp-101, 0x1.5eafffb34ba31p-100, + 0x1.dca23bae16424p-99, 0x1.43e7fc88b8056p-97, 0x1.b83bf23a9a9ebp-96, + 0x1.2b2b8dd05b318p-94, 0x1.969d47321e4ccp-93, 0x1.1452b7723aed2p-91, + 0x1.778fe2497184cp-90, 0x1.fe7116182e9ccp-89, 0x1.5ae191a99585ap-87, + 0x1.d775d87da854dp-86, 0x1.4063f8cc8bb98p-84, 0x1.b374b315f87c1p-83, + 0x1.27ec458c65e3cp-81, 0x1.923372c67a074p-80, 0x1.1152eaeb73c08p-78, + 0x1.737c5645114b5p-77, 0x1.f8e6c24b5592ep-76, 0x1.571db733a9d61p-74, + 0x1.d257d547e083fp-73, 0x1.3ce9b9de78f85p-71, 0x1.aebabae3a41b5p-70, + 0x1.24b6031b49bdap-68, 0x1.8dd5e1bb09d7ep-67, 0x1.0e5b73d1ff53dp-65, + 0x1.6f741de1748ecp-64, 0x1.f36bd37f42f3ep-63, 0x1.536452ee2f75cp-61, + 0x1.cd480a1b74820p-60, 0x1.39792499b1a24p-58, 0x1.aa0de4bf35b38p-57, + 0x1.2188ad6ae3303p-55, 0x1.898471fca6055p-54, 0x1.0b6c3afdde064p-52, + 0x1.6b7719a59f0e0p-51, 0x1.ee001eed62aa0p-50, 0x1.4fb547c775da8p-48, + 0x1.c8464f7616468p-47, 0x1.36121e24d3bbap-45, 0x1.a56e0c2ac7f75p-44, + 0x1.1e642baeb84a0p-42, 0x1.853f01d6d53bap-41, 0x1.0885298767e9ap-39, + 0x1.67852a7007e42p-38, 0x1.e8a37a45fc32ep-37, 0x1.4c1078fe9228ap-35, + 0x1.c3527e433fab1p-34, 0x1.32b48bf117da2p-32, 0x1.a0db0d0ddb3ecp-31, + 0x1.1b48655f37267p-29, 0x1.81056ff2c5772p-28, 0x1.05a628c699fa1p-26, + 0x1.639e3175a689dp-25, 0x1.e355bbaee85cbp-24, 0x1.4875ca227ec38p-22, + 0x1.be6c6fdb01612p-21, 0x1.2f6053b981d98p-19, 0x1.9c54c3b43bc8bp-18, + 0x1.18354238f6764p-16, 0x1.7cd79b5647c9bp-15, 0x1.02cf22526545ap-13, + 0x1.5fc21041027adp-12, 0x1.de16b9c24a98fp-11, 0x1.44e51f113d4d6p-9, + 0x1.b993fe00d5376p-8, 0x1.2c155b8213cf4p-6, 0x1.97db0ccceb0afp-5, + 0x1.152aaa3bf81ccp-3, 0x1.78b56362cef38p-2, 0x1.0000000000000p+0, + 0x1.5bf0a8b145769p+1, 0x1.d8e64b8d4ddaep+2, 0x1.415e5bf6fb106p+4, + 0x1.b4c902e273a58p+5, 0x1.28d389970338fp+7, 0x1.936dc5690c08fp+8, + 0x1.122885aaeddaap+10, 0x1.749ea7d470c6ep+11, 0x1.fa7157c470f82p+12, + 0x1.5829dcf950560p+14, 0x1.d3c4488ee4f7fp+15, 0x1.3de1654d37c9ap+17, + 0x1.b00b5916ac955p+18, 0x1.259ac48bf05d7p+20, 0x1.8f0ccafad2a87p+21, + 0x1.0f2ebd0a80020p+23, 0x1.709348c0ea4f9p+24, 0x1.f4f22091940bdp+25, + 0x1.546d8f9ed26e1p+27, 0x1.ceb088b68e804p+28, 0x1.3a6e1fd9eecfdp+30, + 0x1.ab5adb9c43600p+31, 0x1.226af33b1fdc1p+33, 0x1.8ab7fb5475fb7p+34, + 0x1.0c3d3920962c9p+36, 0x1.6c932696a6b5dp+37, 0x1.ef822f7f6731dp+38, + 0x1.50bba3796379ap+40, 0x1.c9aae4631c056p+41, 0x1.370470aec28edp+43, + 0x1.a6b765d8cdf6dp+44, 0x1.1f43fcc4b662cp+46, 0x1.866f34a725782p+47, + 0x1.0953e2f3a1ef7p+49, 0x1.689e221bc8d5bp+50, 0x1.ea215a1d20d76p+51, + 0x1.4d13fbb1a001ap+53, 0x1.c4b334617cc67p+54, 0x1.33a43d282a519p+56, + 0x1.a220d397972ebp+57, 0x1.1c25c88df6862p+59, 0x1.8232558201159p+60, + 0x1.0672a3c9eb871p+62, 0x1.64b41c6d37832p+63, 0x1.e4cf766fe49bep+64, + 0x1.49767bc0483e3p+66, 0x1.bfc951eb8bb76p+67, 0x1.304d6aeca254bp+69, + 0x1.9d97010884251p+70, 0x1.19103e4080b45p+72, 0x1.7e013cd114461p+73, + 0x1.03996528e074cp+75, 0x1.60d4f6fdac731p+76, 0x1.df8c5af17ba3bp+77, + 0x1.45e3076d61699p+79, 0x1.baed16a6e0da7p+80, 0x1.2cffdfebde1a1p+82, + 0x1.9919cabefcb69p+83, 0x1.160345c9953e3p+85, 0x1.79dbc9dc53c66p+86, + 0x1.00c810d464097p+88, 0x1.5d009394c5c27p+89, 0x1.da57de8f107a8p+90, + 0x1.425982cf597cdp+92, 0x1.b61e5ca3a5e31p+93, 0x1.29bb825dfcf87p+95, + 0x1.94a90db0d6fe2p+96, 0x1.12fec759586fdp+98, 0x1.75c1dc469e3afp+99, + 0x1.fbfd219c43b04p+100, 0x1.5936d44e1a146p+102, 0x1.d531d8a7ee79cp+103, + 0x1.3ed9d24a2d51bp+105, 0x1.b15cfe5b6e17bp+106, 0x1.268038c2c0e00p+108, + 0x1.9044a73545d48p+109, 0x1.1002ab6218b38p+111, 0x1.71b3540cbf921p+112, + 0x1.f6799ea9c414ap+113, 0x1.55779b984f3ebp+115, 0x1.d01a210c44aa4p+116, + 0x1.3b63da8e91210p+118, 0x1.aca8d6b0116b8p+119, 0x1.234de9e0c74e9p+121, + 0x1.8bec7503ca477p+122, 0x1.0d0eda9796b90p+124, 0x1.6db0118477245p+125, + 0x1.f1056dc7bf22dp+126, 0x1.51c2cc3433801p+128, 0x1.cb108ffbec164p+129, +}; + +// Lookup table for exp(m * 2^(-7)) with m = 0, ..., 127. +// Table is generated with Sollya as follow: +// > display = hexadecimal; +// > for i from 0 to 127 do { D(exp(i / 128)); }; +static constexpr double EXP_M2[128] = { + 0x1.0000000000000p0, 0x1.0202015600446p0, 0x1.04080ab55de39p0, + 0x1.06122436410ddp0, 0x1.08205601127edp0, 0x1.0a32a84e9c1f6p0, + 0x1.0c49236829e8cp0, 0x1.0e63cfa7ab09dp0, 0x1.1082b577d34edp0, + 0x1.12a5dd543ccc5p0, 0x1.14cd4fc989cd6p0, 0x1.16f9157587069p0, + 0x1.192937074e0cdp0, 0x1.1b5dbd3f68122p0, 0x1.1d96b0eff0e79p0, + 0x1.1fd41afcba45ep0, 0x1.2216045b6f5cdp0, 0x1.245c7613b8a9bp0, + 0x1.26a7793f60164p0, 0x1.28f7170a755fdp0, 0x1.2b4b58b372c79p0, + 0x1.2da4478b620c7p0, 0x1.3001ecf601af7p0, 0x1.32645269ea829p0, + 0x1.34cb8170b5835p0, 0x1.373783a722012p0, 0x1.39a862bd3c106p0, + 0x1.3c1e2876834aap0, 0x1.3e98deaa11dccp0, 0x1.41188f42c3e32p0, + 0x1.439d443f5f159p0, 0x1.462707b2bac21p0, 0x1.48b5e3c3e8186p0, + 0x1.4b49e2ae5ac67p0, 0x1.4de30ec211e60p0, 0x1.50817263c13cdp0, + 0x1.5325180cfacf7p0, 0x1.55ce0a4c58c7cp0, 0x1.587c53c5a7af0p0, + 0x1.5b2fff3210fd9p0, 0x1.5de9176045ff5p0, 0x1.60a7a734ab0e8p0, + 0x1.636bb9a983258p0, 0x1.663559cf1bc7cp0, 0x1.690492cbf9433p0, + 0x1.6bd96fdd034a2p0, 0x1.6eb3fc55b1e76p0, 0x1.719443a03acb9p0, + 0x1.747a513dbef6ap0, 0x1.776630c678bc1p0, 0x1.7a57ede9ea23ep0, + 0x1.7d4f946f0ba8dp0, 0x1.804d30347b546p0, 0x1.8350cd30ac390p0, + 0x1.865a7772164c5p0, 0x1.896a3b1f66a0ep0, 0x1.8c802477b0010p0, + 0x1.8f9c3fd29beafp0, 0x1.92be99a09bf00p0, 0x1.95e73e6b1b75ep0, + 0x1.99163ad4b1dccp0, 0x1.9c4b9b995509bp0, 0x1.9f876d8e8c566p0, + 0x1.a2c9bda3a3e78p0, 0x1.a61298e1e069cp0, 0x1.a9620c6cb3374p0, + 0x1.acb82581eee54p0, 0x1.b014f179fc3b8p0, 0x1.b3787dc80f95fp0, + 0x1.b6e2d7fa5eb18p0, 0x1.ba540dba56e56p0, 0x1.bdcc2cccd3c85p0, + 0x1.c14b431256446p0, 0x1.c4d15e873c193p0, 0x1.c85e8d43f7cd0p0, + 0x1.cbf2dd7d490f2p0, 0x1.cf8e5d84758a9p0, 0x1.d3311bc7822b4p0, + 0x1.d6db26d16cd67p0, 0x1.da8c8d4a66969p0, 0x1.de455df80e3c0p0, + 0x1.e205a7bdab73ep0, 0x1.e5cd799c6a54ep0, 0x1.e99ce2b397649p0, + 0x1.ed73f240dc142p0, 0x1.f152b7a07bb76p0, 0x1.f539424d90f5ep0, + 0x1.f927a1e24bb76p0, 0x1.fd1de6182f8c9p0, 0x1.008e0f64294abp1, + 0x1.02912df5ce72ap1, 0x1.049856cd84339p1, 0x1.06a39207f0a09p1, + 0x1.08b2e7d2035cfp1, 0x1.0ac6606916501p1, 0x1.0cde041b0e9aep1, + 0x1.0ef9db467dcf8p1, 0x1.1119ee5ac36b6p1, 0x1.133e45d82e952p1, + 0x1.1566ea50201d7p1, 0x1.1793e4652cc50p1, 0x1.19c53ccb3fc6bp1, + 0x1.1bfafc47bda73p1, 0x1.1e352bb1a74adp1, 0x1.2073d3f1bd518p1, + 0x1.22b6fe02a3b9cp1, 0x1.24feb2f105cb8p1, 0x1.274afbdbba4a6p1, + 0x1.299be1f3e7f1cp1, 0x1.2bf16e7d2a38cp1, 0x1.2e4baacdb6614p1, + 0x1.30aaa04e80d05p1, 0x1.330e587b62b28p1, 0x1.3576dce33feadp1, + 0x1.37e437282d4eep1, 0x1.3a5670ff972edp1, 0x1.3ccd9432682b4p1, + 0x1.3f49aa9d30590p1, 0x1.41cabe304cb34p1, 0x1.4450d8f00edd4p1, + 0x1.46dc04f4e5338p1, 0x1.496c4c6b832dap1, 0x1.4c01b9950a111p1, + 0x1.4e9c56c731f5dp1, 0x1.513c2e6c731d7p1, 0x1.53e14b042f9cap1, + 0x1.568bb722dd593p1, 0x1.593b7d72305bbp1, +}; + +} // namespace math + +} // namespace LIBC_NAMESPACE_DECL + +#endif // LLVM_LIBC_SRC___SUPPORT_MATH_EXP_FLOAT_CONSTANTS_H diff --git a/system/lib/llvm-libc/src/__support/math/expf.h b/system/lib/llvm-libc/src/__support/math/expf.h new file mode 100644 index 0000000000000..88c151492a041 --- /dev/null +++ b/system/lib/llvm-libc/src/__support/math/expf.h @@ -0,0 +1,116 @@ +//===-- Implementation header for expf --------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC___SUPPORT_MATH_EXPF_H +#define LLVM_LIBC_SRC___SUPPORT_MATH_EXPF_H + +#include "exp_float_constants.h" // Lookup tables EXP_M1 and EXP_M2. +#include "src/__support/FPUtil/FEnvImpl.h" +#include "src/__support/FPUtil/FPBits.h" +#include "src/__support/FPUtil/PolyEval.h" +#include "src/__support/FPUtil/multiply_add.h" +#include "src/__support/FPUtil/nearest_integer.h" +#include "src/__support/FPUtil/rounding_mode.h" +#include "src/__support/common.h" +#include "src/__support/macros/config.h" +#include "src/__support/macros/optimization.h" // LIBC_UNLIKELY + +namespace LIBC_NAMESPACE_DECL { + +namespace math { + +static constexpr float expf(float x) { + using FPBits = typename fputil::FPBits; + FPBits xbits(x); + + uint32_t x_u = xbits.uintval(); + uint32_t x_abs = x_u & 0x7fff'ffffU; + +#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS + // Exceptional values + if (LIBC_UNLIKELY(x_u == 0xc236'bd8cU)) { // x = -0x1.6d7b18p+5f + return 0x1.108a58p-66f - x * 0x1.0p-95f; + } +#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS + + // When |x| >= 89, |x| < 2^-25, or x is nan + if (LIBC_UNLIKELY(x_abs >= 0x42b2'0000U || x_abs <= 0x3280'0000U)) { + // |x| < 2^-25 + if (xbits.get_biased_exponent() <= 101) { + return 1.0f + x; + } + + // When x < log(2^-150) or nan + if (xbits.uintval() >= 0xc2cf'f1b5U) { + // exp(-Inf) = 0 + if (xbits.is_inf()) + return 0.0f; + // exp(nan) = nan + if (xbits.is_nan()) + return x; + if (fputil::fenv_is_round_up()) + return FPBits::min_subnormal().get_val(); + fputil::set_errno_if_required(ERANGE); + fputil::raise_except_if_required(FE_UNDERFLOW); + return 0.0f; + } + // x >= 89 or nan + if (xbits.is_pos() && (xbits.uintval() >= 0x42b2'0000)) { + // x is finite + if (xbits.uintval() < 0x7f80'0000U) { + int rounding = fputil::quick_get_round(); + if (rounding == FE_DOWNWARD || rounding == FE_TOWARDZERO) + return FPBits::max_normal().get_val(); + + fputil::set_errno_if_required(ERANGE); + fputil::raise_except_if_required(FE_OVERFLOW); + } + // x is +inf or nan + return x + FPBits::inf().get_val(); + } + } + // For -104 < x < 89, to compute exp(x), we perform the following range + // reduction: find hi, mid, lo such that: + // x = hi + mid + lo, in which + // hi is an integer, + // mid * 2^7 is an integer + // -2^(-8) <= lo < 2^-8. + // In particular, + // hi + mid = round(x * 2^7) * 2^(-7). + // Then, + // exp(x) = exp(hi + mid + lo) = exp(hi) * exp(mid) * exp(lo). + // We store exp(hi) and exp(mid) in the lookup tables EXP_M1 and EXP_M2 + // respectively. exp(lo) is computed using a degree-4 minimax polynomial + // generated by Sollya. + + // x_hi = (hi + mid) * 2^7 = round(x * 2^7). + float kf = fputil::nearest_integer(x * 0x1.0p7f); + // Subtract (hi + mid) from x to get lo. + double xd = static_cast(fputil::multiply_add(kf, -0x1.0p-7f, x)); + int x_hi = static_cast(kf); + x_hi += 104 << 7; + // hi = x_hi >> 7 + double exp_hi = EXP_M1[x_hi >> 7]; + // mid * 2^7 = x_hi & 0x0000'007fU; + double exp_mid = EXP_M2[x_hi & 0x7f]; + // Degree-4 minimax polynomial generated by Sollya with the following + // commands: + // > display = hexadecimal; + // > Q = fpminimax(expm1(x)/x, 3, [|D...|], [-2^-8, 2^-8]); + // > Q; + double exp_lo = + fputil::polyeval(xd, 0x1p0, 0x1.ffffffffff777p-1, 0x1.000000000071cp-1, + 0x1.555566668e5e7p-3, 0x1.55555555ef243p-5); + return static_cast(exp_hi * exp_mid * exp_lo); +} + +} // namespace math + +} // namespace LIBC_NAMESPACE_DECL + +#endif // LLVM_LIBC_SRC___SUPPORT_MATH_EXPF_H diff --git a/system/lib/llvm-libc/src/__support/math_extras.h b/system/lib/llvm-libc/src/__support/math_extras.h index 6f4a006aad270..47df2a43250c8 100644 --- a/system/lib/llvm-libc/src/__support/math_extras.h +++ b/system/lib/llvm-libc/src/__support/math_extras.h @@ -146,8 +146,7 @@ first_trailing_zero(T value) { template [[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t, int> first_trailing_one(T value) { - return value == cpp::numeric_limits::max() ? 0 - : cpp::countr_zero(value) + 1; + return value == 0 ? 0 : cpp::countr_zero(value) + 1; } template diff --git a/system/lib/llvm-libc/src/__support/memory_size.h b/system/lib/llvm-libc/src/__support/memory_size.h index cdd6a10222de1..3d40b113bcb68 100644 --- a/system/lib/llvm-libc/src/__support/memory_size.h +++ b/system/lib/llvm-libc/src/__support/memory_size.h @@ -77,7 +77,8 @@ class SafeMemSize { if (!cpp::has_single_bit(alignment) || alignment > MAX_MEM_SIZE || !valid()) return SafeMemSize{type{-1}}; - type offset = offset_to(value, alignment); + type offset = + static_cast(offset_to(static_cast(value), alignment)); if (LIBC_UNLIKELY(offset > static_cast(MAX_MEM_SIZE) - value)) return SafeMemSize{type{-1}}; diff --git a/system/lib/llvm-libc/src/__support/sign.h b/system/lib/llvm-libc/src/__support/sign.h index 4a629e4488195..e0de0e0798acb 100644 --- a/system/lib/llvm-libc/src/__support/sign.h +++ b/system/lib/llvm-libc/src/__support/sign.h @@ -29,6 +29,8 @@ struct Sign { static const Sign POS; static const Sign NEG; + LIBC_INLINE constexpr Sign negate() const { return Sign(!is_negative); } + private: LIBC_INLINE constexpr explicit Sign(bool is_negative) : is_negative(is_negative) {} diff --git a/system/lib/llvm-libc/src/__support/str_to_float.h b/system/lib/llvm-libc/src/__support/str_to_float.h index dcbc0fe97fea7..0748e1cb8a8b4 100644 --- a/system/lib/llvm-libc/src/__support/str_to_float.h +++ b/system/lib/llvm-libc/src/__support/str_to_float.h @@ -108,11 +108,11 @@ eisel_lemire(ExpandedFloat init_num, } // Normalization - uint32_t clz = cpp::countl_zero(mantissa); + uint32_t clz = static_cast(cpp::countl_zero(mantissa)); mantissa <<= clz; - int32_t exp2 = - exp10_to_exp2(exp10) + FPBits::STORAGE_LEN + FPBits::EXP_BIAS - clz; + int32_t exp2 = exp10_to_exp2(exp10) + FPBits::STORAGE_LEN + FPBits::EXP_BIAS - + static_cast(clz); // Multiplication const uint64_t *power_of_ten = @@ -225,8 +225,8 @@ eisel_lemire(ExpandedFloat init_num, } // Normalization - uint32_t clz = cpp::countl_zero(mantissa) - - ((sizeof(UInt128) - sizeof(StorageType)) * CHAR_BIT); + int32_t clz = static_cast(cpp::countl_zero(mantissa)) - + ((sizeof(UInt128) - sizeof(StorageType)) * CHAR_BIT); mantissa <<= clz; int32_t exp2 = @@ -802,7 +802,7 @@ LIBC_INLINE FloatConvertReturn binary_exp_to_float(ExpandedFloat init_num, // Handle subnormals. if (biased_exponent <= 0) { - amount_to_shift_right += 1 - biased_exponent; + amount_to_shift_right += static_cast(1 - biased_exponent); biased_exponent = 0; if (amount_to_shift_right > FPBits::STORAGE_LEN) { @@ -909,7 +909,7 @@ decimal_string_to_float(const char *__restrict src, const char DECIMAL_POINT, cpp::numeric_limits::max() / BASE; while (true) { if (isdigit(src[index])) { - uint32_t digit = b36_char_to_int(src[index]); + uint32_t digit = static_cast(b36_char_to_int(src[index])); seen_digit = true; if (mantissa < bitstype_max_div_by_base) { @@ -956,7 +956,7 @@ decimal_string_to_float(const char *__restrict src, const char DECIMAL_POINT, if (result.has_error()) output.error = result.error; int32_t add_to_exponent = result.value; - index += result.parsed_len; + index += static_cast(result.parsed_len); // Here we do this operation as int64 to avoid overflow. int64_t temp_exponent = static_cast(exponent) + @@ -1020,7 +1020,7 @@ hexadecimal_string_to_float(const char *__restrict src, cpp::numeric_limits::max() / BASE; while (true) { if (isalnum(src[index])) { - uint32_t digit = b36_char_to_int(src[index]); + uint32_t digit = static_cast(b36_char_to_int(src[index])); if (digit < BASE) seen_digit = true; else @@ -1070,7 +1070,7 @@ hexadecimal_string_to_float(const char *__restrict src, output.error = result.error; int32_t add_to_exponent = result.value; - index += result.parsed_len; + index += static_cast(result.parsed_len); // Here we do this operation as int64 to avoid overflow. int64_t temp_exponent = static_cast(exponent) + @@ -1135,7 +1135,7 @@ LIBC_INLINE StrToNumResult strtofloatingpoint(const char *__restrict src) { int error = 0; - ptrdiff_t index = first_non_whitespace(src) - src; + size_t index = static_cast(first_non_whitespace(src) - src); if (src[index] == '+' || src[index] == '-') { sign = src[index]; @@ -1245,7 +1245,7 @@ LIBC_INLINE StrToNumResult strtofloatingpoint(const char *__restrict src) { // special 80 bit long doubles. Otherwise it should be inlined out. set_implicit_bit(result); - return {result.get_val(), index, error}; + return {result.get_val(), static_cast(index), error}; } template LIBC_INLINE StrToNumResult strtonan(const char *arg) { diff --git a/system/lib/llvm-libc/src/__support/str_to_integer.h b/system/lib/llvm-libc/src/__support/str_to_integer.h index dca1d402a5da5..76a99a8948941 100644 --- a/system/lib/llvm-libc/src/__support/str_to_integer.h +++ b/system/lib/llvm-libc/src/__support/str_to_integer.h @@ -96,7 +96,7 @@ strtointeger(const char *__restrict src, int base, if (base < 0 || base == 1 || base > 36) return {0, 0, EINVAL}; - src_cur = first_non_whitespace(src, src_len) - src; + src_cur = static_cast(first_non_whitespace(src, src_len) - src); char result_sign = '+'; if (src[src_cur] == '+' || src[src_cur] == '-') { @@ -119,7 +119,7 @@ strtointeger(const char *__restrict src, int base, ResultType const abs_max = (is_positive ? cpp::numeric_limits::max() : NEGATIVE_MAX); ResultType const abs_max_div_by_base = - static_cast(abs_max / base); + abs_max / static_cast(base); while (src_cur < src_len && isalnum(src[src_cur])) { int cur_digit = b36_char_to_int(src[src_cur]); @@ -141,17 +141,17 @@ strtointeger(const char *__restrict src, int base, result = abs_max; error_val = ERANGE; } else { - result = static_cast(result * base); + result = result * static_cast(base); } - if (result > abs_max - cur_digit) { + if (result > abs_max - static_cast(cur_digit)) { result = abs_max; error_val = ERANGE; } else { - result = static_cast(result + cur_digit); + result = result + static_cast(cur_digit); } } - ptrdiff_t str_len = is_number ? (src_cur) : 0; + ptrdiff_t str_len = is_number ? static_cast(src_cur) : 0; if (error_val == ERANGE) { if (is_positive || IS_UNSIGNED) diff --git a/system/lib/llvm-libc/src/__support/threads/linux/thread.cpp b/system/lib/llvm-libc/src/__support/threads/linux/thread.cpp index c531d74c53355..baad26aed6851 100644 --- a/system/lib/llvm-libc/src/__support/threads/linux/thread.cpp +++ b/system/lib/llvm-libc/src/__support/threads/linux/thread.cpp @@ -14,9 +14,9 @@ #include "src/__support/OSUtil/syscall.h" // For syscall functions. #include "src/__support/common.h" #include "src/__support/error_or.h" +#include "src/__support/libc_errno.h" // For error macros #include "src/__support/macros/config.h" #include "src/__support/threads/linux/futex_utils.h" // For FutexWordType -#include "src/errno/libc_errno.h" // For error macros #ifdef LIBC_TARGET_ARCH_IS_AARCH64 #include diff --git a/system/lib/llvm-libc/src/__support/wchar/character_converter.cpp b/system/lib/llvm-libc/src/__support/wchar/character_converter.cpp new file mode 100644 index 0000000000000..3b9046dfb9a76 --- /dev/null +++ b/system/lib/llvm-libc/src/__support/wchar/character_converter.cpp @@ -0,0 +1,150 @@ +//===-- Implementation of a class for conversion --------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "hdr/types/char32_t.h" +#include "hdr/types/char8_t.h" +#include "src/__support/CPP/bit.h" +#include "src/__support/common.h" +#include "src/__support/error_or.h" +#include "src/__support/math_extras.h" +#include "src/__support/wchar/mbstate.h" + +#include "character_converter.h" + +namespace LIBC_NAMESPACE_DECL { +namespace internal { + +CharacterConverter::CharacterConverter(mbstate *mbstate) { state = mbstate; } + +void CharacterConverter::clear() { + state->partial = 0; + state->bytes_processed = 0; + state->total_bytes = 0; +} + +bool CharacterConverter::isComplete() { + return state->bytes_processed == state->total_bytes; +} + +int CharacterConverter::push(char8_t utf8_byte) { + uint8_t num_ones = static_cast(cpp::countl_one(utf8_byte)); + // Checking the first byte if first push + if (state->bytes_processed == 0) { + // UTF-8 char has 1 byte total + if (num_ones == 0) { + state->total_bytes = 1; + } + // UTF-8 char has 2 through 4 bytes total + else if (num_ones >= 2 && num_ones <= 4) { + /* Since the format is 110xxxxx, 1110xxxx, and 11110xxx for 2, 3, and 4, + we will make the base mask with 7 ones and right shift it as necessary. */ + constexpr size_t SIGNIFICANT_BITS = 7; + char8_t base_mask = + static_cast(mask_trailing_ones()); + state->total_bytes = num_ones; + utf8_byte &= (base_mask >> num_ones); + } + // Invalid first byte + else { + // bytes_processed and total_bytes will always be 0 here + state->partial = static_cast(0); + return -1; + } + state->partial = static_cast(utf8_byte); + state->bytes_processed++; + return 0; + } + // Any subsequent push + // Adding 6 more bits so need to left shift + constexpr size_t ENCODED_BITS_PER_UTF8 = 6; + if (num_ones == 1 && !isComplete()) { + char32_t byte = + utf8_byte & mask_trailing_ones(); + state->partial = state->partial << ENCODED_BITS_PER_UTF8; + state->partial |= byte; + state->bytes_processed++; + return 0; + } + // Invalid byte -> reset the state + clear(); + return -1; +} + +int CharacterConverter::push(char32_t utf32) { + // we can't be partially through a conversion when pushing a utf32 value + if (!isComplete()) + return -1; + + state->partial = utf32; + state->bytes_processed = 0; + + // determine number of utf-8 bytes needed to represent this utf32 value + constexpr char32_t MAX_VALUE_PER_UTF8_LEN[] = {0x7f, 0x7ff, 0xffff, 0x10ffff}; + constexpr int NUM_RANGES = 4; + for (uint8_t i = 0; i < NUM_RANGES; i++) { + if (state->partial <= MAX_VALUE_PER_UTF8_LEN[i]) { + state->total_bytes = i + 1; + return 0; + } + } + + // `utf32` contains a value that is too large to actually represent a valid + // unicode character + clear(); + return -1; +} + +ErrorOr CharacterConverter::pop_utf32() { + // If pop is called too early, do not reset the state, use error to determine + // whether enough bytes have been pushed + if (!isComplete() || state->bytes_processed == 0) + return Error(-1); + char32_t utf32 = state->partial; + // reset if successful pop + clear(); + return utf32; +} + +ErrorOr CharacterConverter::pop_utf8() { + if (isComplete()) + return Error(-1); + + constexpr char8_t FIRST_BYTE_HEADERS[] = {0, 0xC0, 0xE0, 0xF0}; + constexpr char8_t CONTINUING_BYTE_HEADER = 0x80; + + // the number of bits per utf-8 byte that actually encode character + // information not metadata (# of bits excluding the byte headers) + constexpr size_t ENCODED_BITS_PER_UTF8 = 6; + constexpr int MASK_ENCODED_BITS = + mask_trailing_ones(); + + char32_t output; + + // Shift to get the next 6 bits from the utf32 encoding + const size_t shift_amount = + (state->total_bytes - state->bytes_processed - 1) * ENCODED_BITS_PER_UTF8; + if (state->bytes_processed == 0) { + /* + Choose the correct set of most significant bits to encode the length + of the utf8 sequence. The remaining bits contain the most significant + bits of the unicode value of the character. + */ + output = FIRST_BYTE_HEADERS[state->total_bytes - 1] | + (state->partial >> shift_amount); + } else { + // Get the next 6 bits and format it like so: 10xxxxxx + output = CONTINUING_BYTE_HEADER | + ((state->partial >> shift_amount) & MASK_ENCODED_BITS); + } + + state->bytes_processed++; + return static_cast(output); +} + +} // namespace internal +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/__support/wchar/character_converter.h b/system/lib/llvm-libc/src/__support/wchar/character_converter.h new file mode 100644 index 0000000000000..c4ba7cf6b689f --- /dev/null +++ b/system/lib/llvm-libc/src/__support/wchar/character_converter.h @@ -0,0 +1,41 @@ +//===-- Definition of a class for mbstate_t and conversion -----*-- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC___SUPPORT_CHARACTER_CONVERTER_H +#define LLVM_LIBC_SRC___SUPPORT_CHARACTER_CONVERTER_H + +#include "hdr/types/char32_t.h" +#include "hdr/types/char8_t.h" +#include "src/__support/common.h" +#include "src/__support/error_or.h" +#include "src/__support/wchar/mbstate.h" + +namespace LIBC_NAMESPACE_DECL { +namespace internal { + +class CharacterConverter { +private: + mbstate *state; + +public: + CharacterConverter(mbstate *mbstate); + + void clear(); + bool isComplete(); + + int push(char8_t utf8_byte); + int push(char32_t utf32); + + ErrorOr pop_utf8(); + ErrorOr pop_utf32(); +}; + +} // namespace internal +} // namespace LIBC_NAMESPACE_DECL + +#endif // LLVM_LIBC_SRC___SUPPORT_CHARACTER_CONVERTER_H diff --git a/system/lib/llvm-libc/src/__support/wchar/mbstate.h b/system/lib/llvm-libc/src/__support/wchar/mbstate.h new file mode 100644 index 0000000000000..fb08fb4eaa188 --- /dev/null +++ b/system/lib/llvm-libc/src/__support/wchar/mbstate.h @@ -0,0 +1,37 @@ +//===-- Definition of mbstate-----------------------------------*-- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC___SUPPORT_MBSTATE_H +#define LLVM_LIBC_SRC___SUPPORT_MBSTATE_H + +#include "hdr/types/char32_t.h" +#include "src/__support/common.h" +#include + +namespace LIBC_NAMESPACE_DECL { +namespace internal { + +struct mbstate { + // store a partial codepoint (in UTF-32) + char32_t partial; + + /* + Progress towards a conversion + For utf8 -> utf32, increases with each CharacterConverter::push(utf8_byte) + For utf32 -> utf8, increases with each CharacterConverter::pop_utf8() + */ + uint8_t bytes_processed; + + // Total number of bytes that will be needed to represent this character + uint8_t total_bytes; +}; + +} // namespace internal +} // namespace LIBC_NAMESPACE_DECL + +#endif // LLVM_LIBC_SRC___SUPPORT_MBSTATE_H diff --git a/system/lib/llvm-libc/src/errno/libc_errno.cpp b/system/lib/llvm-libc/src/errno/libc_errno.cpp index d1600d1b050e3..8ff1eec1b1035 100644 --- a/system/lib/llvm-libc/src/errno/libc_errno.cpp +++ b/system/lib/llvm-libc/src/errno/libc_errno.cpp @@ -6,51 +6,14 @@ // //===----------------------------------------------------------------------===// -#include "libc_errno.h" +#include "src/__support/libc_errno.h" +#include "src/__support/macros/attributes.h" #include "src/__support/macros/config.h" -// libc uses a fallback default value, either system or thread local. -#define LIBC_ERRNO_MODE_DEFAULT 0 -// libc never stores a value; `errno` macro uses get link-time failure. -#define LIBC_ERRNO_MODE_UNDEFINED 1 -// libc maintains per-thread state (requires C++ `thread_local` support). -#define LIBC_ERRNO_MODE_THREAD_LOCAL 2 -// libc maintains shared state used by all threads, contrary to standard C -// semantics unless always single-threaded; nothing prevents data races. -#define LIBC_ERRNO_MODE_SHARED 3 -// libc doesn't maintain any internal state, instead the embedder must define -// `int *__llvm_libc_errno(void);` C function. -#define LIBC_ERRNO_MODE_EXTERNAL 4 -// libc uses system `` `errno` macro directly in the overlay mode; in -// fullbuild mode, effectively the same as `LIBC_ERRNO_MODE_EXTERNAL`. -#define LIBC_ERRNO_MODE_SYSTEM 5 - -#if !defined(LIBC_ERRNO_MODE) || LIBC_ERRNO_MODE == LIBC_ERRNO_MODE_DEFAULT -#undef LIBC_ERRNO_MODE -#if defined(LIBC_FULL_BUILD) || !defined(LIBC_COPT_PUBLIC_PACKAGING) -#define LIBC_ERRNO_MODE LIBC_ERRNO_MODE_THREAD_LOCAL -#else -#define LIBC_ERRNO_MODE LIBC_ERRNO_MODE_SYSTEM -#endif -#endif // LIBC_ERRNO_MODE - -#if LIBC_ERRNO_MODE != LIBC_ERRNO_MODE_DEFAULT && \ - LIBC_ERRNO_MODE != LIBC_ERRNO_MODE_UNDEFINED && \ - LIBC_ERRNO_MODE != LIBC_ERRNO_MODE_THREAD_LOCAL && \ - LIBC_ERRNO_MODE != LIBC_ERRNO_MODE_SHARED && \ - LIBC_ERRNO_MODE != LIBC_ERRNO_MODE_EXTERNAL && \ - LIBC_ERRNO_MODE != LIBC_ERRNO_MODE_SYSTEM -#error LIBC_ERRNO_MODE must be one of the following values: \ -LIBC_ERRNO_MODE_DEFAULT, \ -LIBC_ERRNO_MODE_UNDEFINED, \ -LIBC_ERRNO_MODE_THREAD_LOCAL, \ -LIBC_ERRNO_MODE_SHARED, \ -LIBC_ERRNO_MODE_EXTERNAL, \ -LIBC_ERRNO_MODE_SYSTEM -#endif - namespace LIBC_NAMESPACE_DECL { +#if LIBC_ERRNO_MODE != LIBC_ERRNO_MODE_SYSTEM_INLINE + #if LIBC_ERRNO_MODE == LIBC_ERRNO_MODE_UNDEFINED void Errno::operator=(int) {} @@ -93,4 +56,6 @@ Errno::operator int() { return errno; } // Define the global `libc_errno` instance. Errno libc_errno; +#endif // LIBC_ERRNO_MODE != LIBC_ERRNO_MODE_SYSTEM_INLINE + } // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/errno/libc_errno.h b/system/lib/llvm-libc/src/errno/libc_errno.h deleted file mode 100644 index 44ee2714843ba..0000000000000 --- a/system/lib/llvm-libc/src/errno/libc_errno.h +++ /dev/null @@ -1,47 +0,0 @@ -//===-- Implementation header for libc_errno --------------------*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_LIBC_SRC_ERRNO_LIBC_ERRNO_H -#define LLVM_LIBC_SRC_ERRNO_LIBC_ERRNO_H - -#include "src/__support/macros/attributes.h" -#include "src/__support/macros/config.h" -#include "src/__support/macros/properties/architectures.h" - -#include "hdr/errno_macros.h" - -// This header is to be consumed by internal implementations, in which all of -// them should refer to `libc_errno` instead of using `errno` directly from -// header. - -// Unit and hermetic tests should: -// - #include "src/errno/libc_errno.h" -// - NOT #include -// - Only use `libc_errno` in the code -// - Depend on libc.src.errno.errno - -// Integration tests should: -// - NOT #include "src/errno/libc_errno.h" -// - #include -// - Use regular `errno` in the code -// - Still depend on libc.src.errno.errno - -namespace LIBC_NAMESPACE_DECL { - -extern "C" int *__llvm_libc_errno() noexcept; - -struct Errno { - void operator=(int); - operator int(); -}; - -extern Errno libc_errno; - -} // namespace LIBC_NAMESPACE_DECL - -#endif // LLVM_LIBC_SRC_ERRNO_LIBC_ERRNO_H diff --git a/system/lib/llvm-libc/src/inttypes/strtoimax.cpp b/system/lib/llvm-libc/src/inttypes/strtoimax.cpp index 85f197c75d90c..6e55a4b56aac7 100644 --- a/system/lib/llvm-libc/src/inttypes/strtoimax.cpp +++ b/system/lib/llvm-libc/src/inttypes/strtoimax.cpp @@ -8,9 +8,9 @@ #include "src/inttypes/strtoimax.h" #include "src/__support/common.h" +#include "src/__support/libc_errno.h" #include "src/__support/macros/config.h" #include "src/__support/str_to_integer.h" -#include "src/errno/libc_errno.h" namespace LIBC_NAMESPACE_DECL { diff --git a/system/lib/llvm-libc/src/inttypes/strtoumax.cpp b/system/lib/llvm-libc/src/inttypes/strtoumax.cpp index 2e9cbc9acba75..ce5a0a782d979 100644 --- a/system/lib/llvm-libc/src/inttypes/strtoumax.cpp +++ b/system/lib/llvm-libc/src/inttypes/strtoumax.cpp @@ -8,9 +8,9 @@ #include "src/inttypes/strtoumax.h" #include "src/__support/common.h" +#include "src/__support/libc_errno.h" #include "src/__support/macros/config.h" #include "src/__support/str_to_integer.h" -#include "src/errno/libc_errno.h" namespace LIBC_NAMESPACE_DECL { diff --git a/system/lib/llvm-libc/src/math/acosf16.h b/system/lib/llvm-libc/src/math/acosf16.h new file mode 100644 index 0000000000000..df30be2c537c9 --- /dev/null +++ b/system/lib/llvm-libc/src/math/acosf16.h @@ -0,0 +1,21 @@ +//===-- Implementation header for acosf16 -----------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_MATH_ACOSF16_H +#define LLVM_LIBC_SRC_MATH_ACOSF16_H + +#include "src/__support/macros/config.h" +#include "src/__support/macros/properties/types.h" + +namespace LIBC_NAMESPACE_DECL { + +float16 acosf16(float16 x); + +} // namespace LIBC_NAMESPACE_DECL + +#endif // LLVM_LIBC_SRC_MATH_ACOSF16_H diff --git a/system/lib/llvm-libc/src/math/acoshf16.h b/system/lib/llvm-libc/src/math/acoshf16.h new file mode 100644 index 0000000000000..f471ecfeb6154 --- /dev/null +++ b/system/lib/llvm-libc/src/math/acoshf16.h @@ -0,0 +1,21 @@ +//===-- Implementation header for acoshf16 ----------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_MATH_ACOSHF16_H +#define LLVM_LIBC_SRC_MATH_ACOSHF16_H + +#include "src/__support/macros/config.h" +#include "src/__support/macros/properties/types.h" + +namespace LIBC_NAMESPACE_DECL { + +float16 acoshf16(float16 x); + +} // namespace LIBC_NAMESPACE_DECL + +#endif // LLVM_LIBC_SRC_MATH_ACOSHF16_H diff --git a/system/lib/llvm-libc/src/math/acospif16.h b/system/lib/llvm-libc/src/math/acospif16.h new file mode 100644 index 0000000000000..e94a35bdf6fb7 --- /dev/null +++ b/system/lib/llvm-libc/src/math/acospif16.h @@ -0,0 +1,21 @@ +//===-- Implementation header for acospif16 ---------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_MATH_ACOSPIF16_H +#define LLVM_LIBC_SRC_MATH_ACOSPIF16_H + +#include "src/__support/macros/config.h" +#include "src/__support/macros/properties/types.h" + +namespace LIBC_NAMESPACE_DECL { + +float16 acospif16(float16 x); + +} // namespace LIBC_NAMESPACE_DECL + +#endif // LLVM_LIBC_SRC_MATH_ACOSPIF16_H diff --git a/system/lib/llvm-libc/src/math/amdgpu/ceil.cpp b/system/lib/llvm-libc/src/math/amdgpu/ceil.cpp new file mode 100644 index 0000000000000..8834c7b560a1b --- /dev/null +++ b/system/lib/llvm-libc/src/math/amdgpu/ceil.cpp @@ -0,0 +1,17 @@ +//===-- Implementation of the ceil function for GPU -----------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/ceil.h" +#include "src/__support/common.h" +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(double, ceil, (double x)) { return __builtin_ceil(x); } + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/amdgpu/ceilf.cpp b/system/lib/llvm-libc/src/math/amdgpu/ceilf.cpp new file mode 100644 index 0000000000000..5d26a30c849cd --- /dev/null +++ b/system/lib/llvm-libc/src/math/amdgpu/ceilf.cpp @@ -0,0 +1,17 @@ +//===-- Implementation of the ceilf function for GPU ----------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/ceilf.h" +#include "src/__support/common.h" +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(float, ceilf, (float x)) { return __builtin_ceilf(x); } + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/amdgpu/copysign.cpp b/system/lib/llvm-libc/src/math/amdgpu/copysign.cpp new file mode 100644 index 0000000000000..06ef36fb3595f --- /dev/null +++ b/system/lib/llvm-libc/src/math/amdgpu/copysign.cpp @@ -0,0 +1,19 @@ +//===-- Implementation of the copysign function for GPU -------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/copysign.h" +#include "src/__support/common.h" +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(double, copysign, (double x, double y)) { + return __builtin_copysign(x, y); +} + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/amdgpu/copysignf.cpp b/system/lib/llvm-libc/src/math/amdgpu/copysignf.cpp new file mode 100644 index 0000000000000..aea94f3577d8f --- /dev/null +++ b/system/lib/llvm-libc/src/math/amdgpu/copysignf.cpp @@ -0,0 +1,19 @@ +//===-- Implementation of the copysignf function for GPU ------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/copysignf.h" +#include "src/__support/common.h" +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(float, copysignf, (float x, float y)) { + return __builtin_copysignf(x, y); +} + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/amdgpu/fabs.cpp b/system/lib/llvm-libc/src/math/amdgpu/fabs.cpp new file mode 100644 index 0000000000000..bb37596b9d563 --- /dev/null +++ b/system/lib/llvm-libc/src/math/amdgpu/fabs.cpp @@ -0,0 +1,17 @@ +//===-- Implementation of the fabs function for GPU -----------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/fabs.h" +#include "src/__support/common.h" +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(double, fabs, (double x)) { return __builtin_fabs(x); } + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/amdgpu/fabsf.cpp b/system/lib/llvm-libc/src/math/amdgpu/fabsf.cpp new file mode 100644 index 0000000000000..2698618f3f1e1 --- /dev/null +++ b/system/lib/llvm-libc/src/math/amdgpu/fabsf.cpp @@ -0,0 +1,17 @@ +//===-- Implementation of the fabsf function for GPU ----------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/fabsf.h" +#include "src/__support/common.h" +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(float, fabsf, (float x)) { return __builtin_fabsf(x); } + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/amdgpu/floor.cpp b/system/lib/llvm-libc/src/math/amdgpu/floor.cpp new file mode 100644 index 0000000000000..564efa9a7da38 --- /dev/null +++ b/system/lib/llvm-libc/src/math/amdgpu/floor.cpp @@ -0,0 +1,17 @@ +//===-- Implementation of the floor function for GPU ----------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/floor.h" +#include "src/__support/common.h" +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(double, floor, (double x)) { return __builtin_floor(x); } + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/amdgpu/floorf.cpp b/system/lib/llvm-libc/src/math/amdgpu/floorf.cpp new file mode 100644 index 0000000000000..6717c8f60c992 --- /dev/null +++ b/system/lib/llvm-libc/src/math/amdgpu/floorf.cpp @@ -0,0 +1,17 @@ +//===-- Implementation of the floorf function for GPU ---------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/floorf.h" +#include "src/__support/common.h" +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(float, floorf, (float x)) { return __builtin_floorf(x); } + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/amdgpu/fma.cpp b/system/lib/llvm-libc/src/math/amdgpu/fma.cpp new file mode 100644 index 0000000000000..c4a117e42a3aa --- /dev/null +++ b/system/lib/llvm-libc/src/math/amdgpu/fma.cpp @@ -0,0 +1,19 @@ +//===-- Implementation of the fma function for GPU ------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/fma.h" +#include "src/__support/common.h" +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(double, fma, (double x, double y, double z)) { + return __builtin_fma(x, y, z); +} + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/amdgpu/fmaf.cpp b/system/lib/llvm-libc/src/math/amdgpu/fmaf.cpp new file mode 100644 index 0000000000000..c088bd5b30fea --- /dev/null +++ b/system/lib/llvm-libc/src/math/amdgpu/fmaf.cpp @@ -0,0 +1,19 @@ +//===-- Implementation of the fmaf function for GPU -----------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/fmaf.h" +#include "src/__support/common.h" +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(float, fmaf, (float x, float y, float z)) { + return __builtin_fmaf(x, y, z); +} + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/amdgpu/fmax.cpp b/system/lib/llvm-libc/src/math/amdgpu/fmax.cpp new file mode 100644 index 0000000000000..474019733598f --- /dev/null +++ b/system/lib/llvm-libc/src/math/amdgpu/fmax.cpp @@ -0,0 +1,22 @@ +//===-- Implementation of the fmax function for GPU -----------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/fmax.h" + +#include "src/__support/CPP/bit.h" +#include "src/__support/common.h" +#include "src/__support/macros/config.h" +#include "src/__support/macros/optimization.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(double, fmax, (double x, double y)) { + return __builtin_fmax(x, y); +} + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/amdgpu/fmaxf.cpp b/system/lib/llvm-libc/src/math/amdgpu/fmaxf.cpp new file mode 100644 index 0000000000000..59ee8259c025a --- /dev/null +++ b/system/lib/llvm-libc/src/math/amdgpu/fmaxf.cpp @@ -0,0 +1,20 @@ +//===-- Implementation of the fmaxf function for GPU ----------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/fmaxf.h" + +#include "src/__support/common.h" +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(float, fmaxf, (float x, float y)) { + return __builtin_fmaxf(x, y); +} + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/amdgpu/fmin.cpp b/system/lib/llvm-libc/src/math/amdgpu/fmin.cpp new file mode 100644 index 0000000000000..694eb664b599b --- /dev/null +++ b/system/lib/llvm-libc/src/math/amdgpu/fmin.cpp @@ -0,0 +1,20 @@ +//===-- Implementation of the fmin function for GPU -----------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/fmin.h" + +#include "src/__support/common.h" +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(double, fmin, (double x, double y)) { + return __builtin_fmin(x, y); +} + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/amdgpu/fminf.cpp b/system/lib/llvm-libc/src/math/amdgpu/fminf.cpp new file mode 100644 index 0000000000000..2060b71b0841f --- /dev/null +++ b/system/lib/llvm-libc/src/math/amdgpu/fminf.cpp @@ -0,0 +1,20 @@ +//===-- Implementation of the fminf function for GPU ----------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/fminf.h" + +#include "src/__support/common.h" +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(float, fminf, (float x, float y)) { + return __builtin_fminf(x, y); +} + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/amdgpu/fmod.cpp b/system/lib/llvm-libc/src/math/amdgpu/fmod.cpp new file mode 100644 index 0000000000000..49d19c4decb96 --- /dev/null +++ b/system/lib/llvm-libc/src/math/amdgpu/fmod.cpp @@ -0,0 +1,19 @@ +//===-- Implementation of the fmod function for GPU -----------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/fmod.h" +#include "src/__support/common.h" +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(double, fmod, (double x, double y)) { + return __builtin_fmod(x, y); +} + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/amdgpu/fmodf.cpp b/system/lib/llvm-libc/src/math/amdgpu/fmodf.cpp new file mode 100644 index 0000000000000..8fbcb0cc2ad9e --- /dev/null +++ b/system/lib/llvm-libc/src/math/amdgpu/fmodf.cpp @@ -0,0 +1,19 @@ +//===-- Implementation of the fmodf function for GPU ----------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/fmodf.h" +#include "src/__support/common.h" +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(float, fmodf, (float x, float y)) { + return __builtin_fmodf(x, y); +} + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/amdgpu/frexp.cpp b/system/lib/llvm-libc/src/math/amdgpu/frexp.cpp new file mode 100644 index 0000000000000..4ae2b00198ec3 --- /dev/null +++ b/system/lib/llvm-libc/src/math/amdgpu/frexp.cpp @@ -0,0 +1,20 @@ +//===-- Implementation of the frexp function for GPU ----------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/frexp.h" +#include "src/__support/common.h" + +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(double, frexp, (double x, int *p)) { + return __builtin_frexp(x, p); +} + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/amdgpu/frexpf.cpp b/system/lib/llvm-libc/src/math/amdgpu/frexpf.cpp new file mode 100644 index 0000000000000..fd53f65701f59 --- /dev/null +++ b/system/lib/llvm-libc/src/math/amdgpu/frexpf.cpp @@ -0,0 +1,20 @@ +//===-- Implementation of the frexpf function for GPU ---------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/frexpf.h" +#include "src/__support/common.h" + +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(float, frexpf, (float x, int *p)) { + return __builtin_frexpf(x, p); +} + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/amdgpu/ldexp.cpp b/system/lib/llvm-libc/src/math/amdgpu/ldexp.cpp new file mode 100644 index 0000000000000..5b786a9b21e70 --- /dev/null +++ b/system/lib/llvm-libc/src/math/amdgpu/ldexp.cpp @@ -0,0 +1,20 @@ +//===-- Implementation of the ldexp function for GPU ----------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/ldexp.h" +#include "src/__support/common.h" + +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(double, ldexp, (double x, int y)) { + return __builtin_ldexp(x, y); +} + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/amdgpu/ldexpf.cpp b/system/lib/llvm-libc/src/math/amdgpu/ldexpf.cpp new file mode 100644 index 0000000000000..d3aa77fe592cb --- /dev/null +++ b/system/lib/llvm-libc/src/math/amdgpu/ldexpf.cpp @@ -0,0 +1,20 @@ +//===-- Implementation of the ldexpf function for GPU ---------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/ldexpf.h" +#include "src/__support/common.h" + +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(float, ldexpf, (float x, int y)) { + return __builtin_ldexpf(x, y); +} + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/amdgpu/lgamma.cpp b/system/lib/llvm-libc/src/math/amdgpu/lgamma.cpp new file mode 100644 index 0000000000000..9666927e60c47 --- /dev/null +++ b/system/lib/llvm-libc/src/math/amdgpu/lgamma.cpp @@ -0,0 +1,19 @@ +//===-- Implementation of the lgamma function for GPU ---------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/lgamma.h" +#include "src/__support/common.h" + +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +// TODO: Implement this. +LLVM_LIBC_FUNCTION(double, lgamma, (double)) { return 0.0; } + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/amdgpu/lgamma_r.cpp b/system/lib/llvm-libc/src/math/amdgpu/lgamma_r.cpp new file mode 100644 index 0000000000000..8b69ca349456c --- /dev/null +++ b/system/lib/llvm-libc/src/math/amdgpu/lgamma_r.cpp @@ -0,0 +1,21 @@ +//===-- Implementation of the lgamma_r function for GPU -------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/lgamma_r.h" +#include "src/__support/common.h" + +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(double, lgamma_r, (double, int *signp)) { + *signp = 0; + return 0; +} + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/amdgpu/llrint.cpp b/system/lib/llvm-libc/src/math/amdgpu/llrint.cpp new file mode 100644 index 0000000000000..6e0f57a5a0387 --- /dev/null +++ b/system/lib/llvm-libc/src/math/amdgpu/llrint.cpp @@ -0,0 +1,20 @@ +//===-- Implementation of the llrint function for GPU ---------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/llrint.h" +#include "src/__support/common.h" + +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(long long, llrint, (double x)) { + return static_cast(__builtin_rint(x)); +} + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/amdgpu/llrintf.cpp b/system/lib/llvm-libc/src/math/amdgpu/llrintf.cpp new file mode 100644 index 0000000000000..d8de23fac3c8b --- /dev/null +++ b/system/lib/llvm-libc/src/math/amdgpu/llrintf.cpp @@ -0,0 +1,20 @@ +//===-- Implementation of the llrintf function for GPU --------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/llrintf.h" +#include "src/__support/common.h" + +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(long long, llrintf, (float x)) { + return static_cast(__builtin_rintf(x)); +} + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/amdgpu/lrint.cpp b/system/lib/llvm-libc/src/math/amdgpu/lrint.cpp new file mode 100644 index 0000000000000..5ba70ec890b2c --- /dev/null +++ b/system/lib/llvm-libc/src/math/amdgpu/lrint.cpp @@ -0,0 +1,20 @@ +//===-- Implementation of the lrint function for GPU ----------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/lrint.h" +#include "src/__support/common.h" + +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(long, lrint, (double x)) { + return static_cast(__builtin_rint(x)); +} + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/amdgpu/lrintf.cpp b/system/lib/llvm-libc/src/math/amdgpu/lrintf.cpp new file mode 100644 index 0000000000000..1c985b01c2dc1 --- /dev/null +++ b/system/lib/llvm-libc/src/math/amdgpu/lrintf.cpp @@ -0,0 +1,20 @@ +//===-- Implementation of the lrintf function for GPU ---------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/lrintf.h" +#include "src/__support/common.h" + +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(long, lrintf, (float x)) { + return static_cast(__builtin_rintf(x)); +} + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/amdgpu/nearbyint.cpp b/system/lib/llvm-libc/src/math/amdgpu/nearbyint.cpp new file mode 100644 index 0000000000000..7d78c7241d023 --- /dev/null +++ b/system/lib/llvm-libc/src/math/amdgpu/nearbyint.cpp @@ -0,0 +1,19 @@ +//===-- Implementation of the GPU nearbyint function ----------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/nearbyint.h" +#include "src/__support/common.h" +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(double, nearbyint, (double x)) { + return __builtin_nearbyint(x); +} + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/amdgpu/nearbyintf.cpp b/system/lib/llvm-libc/src/math/amdgpu/nearbyintf.cpp new file mode 100644 index 0000000000000..4bd20dc58fb89 --- /dev/null +++ b/system/lib/llvm-libc/src/math/amdgpu/nearbyintf.cpp @@ -0,0 +1,19 @@ +//===-- Implementation of the GPU nearbyintf function ---------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/nearbyintf.h" +#include "src/__support/common.h" +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(float, nearbyintf, (float x)) { + return __builtin_nearbyintf(x); +} + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/amdgpu/remainder.cpp b/system/lib/llvm-libc/src/math/amdgpu/remainder.cpp new file mode 100644 index 0000000000000..9027204312e00 --- /dev/null +++ b/system/lib/llvm-libc/src/math/amdgpu/remainder.cpp @@ -0,0 +1,19 @@ +//===-- Implementation of the GPU remainder function ----------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/remainder.h" +#include "src/__support/common.h" +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(double, remainder, (double x, double y)) { + return __builtin_remainder(x, y); +} + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/amdgpu/remainderf.cpp b/system/lib/llvm-libc/src/math/amdgpu/remainderf.cpp new file mode 100644 index 0000000000000..50df3b2ce25c1 --- /dev/null +++ b/system/lib/llvm-libc/src/math/amdgpu/remainderf.cpp @@ -0,0 +1,19 @@ +//===-- Implementation of the GPU remainderf function ---------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/remainderf.h" +#include "src/__support/common.h" +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(float, remainderf, (float x, float y)) { + return __builtin_remainderf(x, y); +} + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/amdgpu/rint.cpp b/system/lib/llvm-libc/src/math/amdgpu/rint.cpp new file mode 100644 index 0000000000000..ac6837a4abc37 --- /dev/null +++ b/system/lib/llvm-libc/src/math/amdgpu/rint.cpp @@ -0,0 +1,17 @@ +//===-- Implementation of the GPU rint function ---------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/rint.h" +#include "src/__support/common.h" +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(double, rint, (double x)) { return __builtin_rint(x); } + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/amdgpu/rintf.cpp b/system/lib/llvm-libc/src/math/amdgpu/rintf.cpp new file mode 100644 index 0000000000000..94093471a8d92 --- /dev/null +++ b/system/lib/llvm-libc/src/math/amdgpu/rintf.cpp @@ -0,0 +1,17 @@ +//===-- Implementation of the GPU rintf function --------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/rintf.h" +#include "src/__support/common.h" +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(float, rintf, (float x)) { return __builtin_rintf(x); } + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/amdgpu/round.cpp b/system/lib/llvm-libc/src/math/amdgpu/round.cpp new file mode 100644 index 0000000000000..0d2765f2e959f --- /dev/null +++ b/system/lib/llvm-libc/src/math/amdgpu/round.cpp @@ -0,0 +1,17 @@ +//===-- Implementation of the GPU round function --------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/round.h" +#include "src/__support/common.h" +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(double, round, (double x)) { return __builtin_round(x); } + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/amdgpu/roundf.cpp b/system/lib/llvm-libc/src/math/amdgpu/roundf.cpp new file mode 100644 index 0000000000000..86e8ba3ac83d6 --- /dev/null +++ b/system/lib/llvm-libc/src/math/amdgpu/roundf.cpp @@ -0,0 +1,17 @@ +//===-- Implementation of the GPU roundf function -------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/roundf.h" +#include "src/__support/common.h" +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(float, roundf, (float x)) { return __builtin_roundf(x); } + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/amdgpu/scalbn.cpp b/system/lib/llvm-libc/src/math/amdgpu/scalbn.cpp new file mode 100644 index 0000000000000..3c3c19ea0d9d4 --- /dev/null +++ b/system/lib/llvm-libc/src/math/amdgpu/scalbn.cpp @@ -0,0 +1,20 @@ +//===-- Implementation of the GPU scalbn function -------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/scalbn.h" +#include "src/__support/common.h" + +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(double, scalbn, (double x, int y)) { + return __builtin_amdgcn_ldexp(x, y); +} + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/amdgpu/scalbnf.cpp b/system/lib/llvm-libc/src/math/amdgpu/scalbnf.cpp new file mode 100644 index 0000000000000..c348aa7c3be22 --- /dev/null +++ b/system/lib/llvm-libc/src/math/amdgpu/scalbnf.cpp @@ -0,0 +1,20 @@ +//===-- Implementation of the GPU scalbnf function ------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/scalbnf.h" +#include "src/__support/common.h" + +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(float, scalbnf, (float x, int y)) { + return __builtin_amdgcn_ldexpf(x, y); +} + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/amdgpu/sqrt.cpp b/system/lib/llvm-libc/src/math/amdgpu/sqrt.cpp new file mode 100644 index 0000000000000..ed83b6e5c6cae --- /dev/null +++ b/system/lib/llvm-libc/src/math/amdgpu/sqrt.cpp @@ -0,0 +1,17 @@ +//===-- Implementation of the GPU sqrt function ---------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/sqrt.h" +#include "src/__support/common.h" +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(double, sqrt, (double x)) { return __builtin_sqrt(x); } + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/amdgpu/sqrtf.cpp b/system/lib/llvm-libc/src/math/amdgpu/sqrtf.cpp new file mode 100644 index 0000000000000..851922c316452 --- /dev/null +++ b/system/lib/llvm-libc/src/math/amdgpu/sqrtf.cpp @@ -0,0 +1,17 @@ +//===-- Implementation of the GPU sqrtf function --------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/sqrtf.h" +#include "src/__support/common.h" +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(float, sqrtf, (float x)) { return __builtin_sqrtf(x); } + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/amdgpu/tgamma.cpp b/system/lib/llvm-libc/src/math/amdgpu/tgamma.cpp new file mode 100644 index 0000000000000..0dfb0fd9ee13c --- /dev/null +++ b/system/lib/llvm-libc/src/math/amdgpu/tgamma.cpp @@ -0,0 +1,18 @@ +//===-- Implementation of the GPU tgamma function -------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/tgamma.h" +#include "src/__support/common.h" + +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(double, tgamma, (double)) { return 0.0; } + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/amdgpu/tgammaf.cpp b/system/lib/llvm-libc/src/math/amdgpu/tgammaf.cpp new file mode 100644 index 0000000000000..9dd17ab8de917 --- /dev/null +++ b/system/lib/llvm-libc/src/math/amdgpu/tgammaf.cpp @@ -0,0 +1,18 @@ +//===-- Implementation of the GPU tgammaf function ------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/tgammaf.h" +#include "src/__support/common.h" + +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(float, tgammaf, (float)) { return 0.0f; } + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/amdgpu/trunc.cpp b/system/lib/llvm-libc/src/math/amdgpu/trunc.cpp new file mode 100644 index 0000000000000..f60caa2a71d78 --- /dev/null +++ b/system/lib/llvm-libc/src/math/amdgpu/trunc.cpp @@ -0,0 +1,17 @@ +//===-- Implementation of the GPU trunc function --------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/trunc.h" +#include "src/__support/common.h" +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(double, trunc, (double x)) { return __builtin_trunc(x); } + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/amdgpu/truncf.cpp b/system/lib/llvm-libc/src/math/amdgpu/truncf.cpp new file mode 100644 index 0000000000000..a6c9b8f188f02 --- /dev/null +++ b/system/lib/llvm-libc/src/math/amdgpu/truncf.cpp @@ -0,0 +1,17 @@ +//===-- Implementation of the GPU truncf function -------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/truncf.h" +#include "src/__support/common.h" +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(float, truncf, (float x)) { return __builtin_truncf(x); } + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/asinf16.h b/system/lib/llvm-libc/src/math/asinf16.h new file mode 100644 index 0000000000000..f16647ec2a6f9 --- /dev/null +++ b/system/lib/llvm-libc/src/math/asinf16.h @@ -0,0 +1,21 @@ +//===-- Implementation header for asinf16 -----------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_MATH_ASINF16_H +#define LLVM_LIBC_SRC_MATH_ASINF16_H + +#include "src/__support/macros/config.h" +#include "src/__support/macros/properties/types.h" + +namespace LIBC_NAMESPACE_DECL { + +float16 asinf16(float16 x); + +} // namespace LIBC_NAMESPACE_DECL + +#endif // LLVM_LIBC_SRC_MATH_ASINF16_H diff --git a/system/lib/llvm-libc/src/math/asinhf16.h b/system/lib/llvm-libc/src/math/asinhf16.h new file mode 100644 index 0000000000000..bb40e208d7ee3 --- /dev/null +++ b/system/lib/llvm-libc/src/math/asinhf16.h @@ -0,0 +1,21 @@ +//===-- Implementation header for asinhf16 ----------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_MATH_ASINHF16_H +#define LLVM_LIBC_SRC_MATH_ASINHF16_H + +#include "src/__support/macros/config.h" +#include "src/__support/macros/properties/types.h" + +namespace LIBC_NAMESPACE_DECL { + +float16 asinhf16(float16 x); + +} // namespace LIBC_NAMESPACE_DECL + +#endif // LLVM_LIBC_SRC_MATH_ASINHF16_H diff --git a/system/lib/llvm-libc/src/math/atan2f128.h b/system/lib/llvm-libc/src/math/atan2f128.h new file mode 100644 index 0000000000000..26f7ec624940c --- /dev/null +++ b/system/lib/llvm-libc/src/math/atan2f128.h @@ -0,0 +1,21 @@ +//===-- Implementation header for atan2f128 ---------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_MATH_ATAN2F128_H +#define LLVM_LIBC_SRC_MATH_ATAN2F128_H + +#include "src/__support/macros/config.h" +#include "src/__support/macros/properties/types.h" + +namespace LIBC_NAMESPACE_DECL { + +float128 atan2f128(float128 x, float128 y); + +} // namespace LIBC_NAMESPACE_DECL + +#endif // LLVM_LIBC_SRC_MATH_ATAN2F128_H diff --git a/system/lib/llvm-libc/src/math/atanf16.h b/system/lib/llvm-libc/src/math/atanf16.h new file mode 100644 index 0000000000000..96ae35c2ba333 --- /dev/null +++ b/system/lib/llvm-libc/src/math/atanf16.h @@ -0,0 +1,21 @@ +//===-- Implementation header for atanf16 -----------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_MATH_ATANF16_H +#define LLVM_LIBC_SRC_MATH_ATANF16_H + +#include "src/__support/macros/config.h" +#include "src/__support/macros/properties/types.h" + +namespace LIBC_NAMESPACE_DECL { + +float16 atanf16(float16 x); + +} // namespace LIBC_NAMESPACE_DECL + +#endif // LLVM_LIBC_SRC_MATH_ATANF16_H diff --git a/system/lib/llvm-libc/src/math/atanhf16.h b/system/lib/llvm-libc/src/math/atanhf16.h new file mode 100644 index 0000000000000..9fbb262c16514 --- /dev/null +++ b/system/lib/llvm-libc/src/math/atanhf16.h @@ -0,0 +1,21 @@ +//===-- Implementation header for atanhf16 ----------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_MATH_ATANHF16_H +#define LLVM_LIBC_SRC_MATH_ATANHF16_H + +#include "src/__support/macros/config.h" +#include "src/__support/macros/properties/types.h" + +namespace LIBC_NAMESPACE_DECL { + +float16 atanhf16(float16 x); + +} // namespace LIBC_NAMESPACE_DECL + +#endif // LLVM_LIBC_SRC_MATH_ATANHF16_H diff --git a/system/lib/llvm-libc/src/math/fmaf16.h b/system/lib/llvm-libc/src/math/fmaf16.h new file mode 100644 index 0000000000000..1c4d468e67a7a --- /dev/null +++ b/system/lib/llvm-libc/src/math/fmaf16.h @@ -0,0 +1,21 @@ +//===-- Implementation header for fmaf16 ------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_MATH_FMAF16_H +#define LLVM_LIBC_SRC_MATH_FMAF16_H + +#include "src/__support/macros/config.h" +#include "src/__support/macros/properties/types.h" + +namespace LIBC_NAMESPACE_DECL { + +float16 fmaf16(float16 x, float16 y, float16 z); + +} // namespace LIBC_NAMESPACE_DECL + +#endif // LLVM_LIBC_SRC_MATH_FMAF16_H diff --git a/system/lib/llvm-libc/src/math/generic/acos.cpp b/system/lib/llvm-libc/src/math/generic/acos.cpp new file mode 100644 index 0000000000000..c14721faef3ce --- /dev/null +++ b/system/lib/llvm-libc/src/math/generic/acos.cpp @@ -0,0 +1,278 @@ +//===-- Double-precision acos function ------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/acos.h" +#include "asin_utils.h" +#include "src/__support/FPUtil/FEnvImpl.h" +#include "src/__support/FPUtil/FPBits.h" +#include "src/__support/FPUtil/PolyEval.h" +#include "src/__support/FPUtil/double_double.h" +#include "src/__support/FPUtil/dyadic_float.h" +#include "src/__support/FPUtil/multiply_add.h" +#include "src/__support/FPUtil/sqrt.h" +#include "src/__support/macros/config.h" +#include "src/__support/macros/optimization.h" // LIBC_UNLIKELY +#include "src/__support/macros/properties/cpu_features.h" // LIBC_TARGET_CPU_HAS_FMA + +namespace LIBC_NAMESPACE_DECL { + +using DoubleDouble = fputil::DoubleDouble; +using Float128 = fputil::DyadicFloat<128>; + +LLVM_LIBC_FUNCTION(double, acos, (double x)) { + using FPBits = fputil::FPBits; + + FPBits xbits(x); + int x_exp = xbits.get_biased_exponent(); + + // |x| < 0.5. + if (x_exp < FPBits::EXP_BIAS - 1) { + // |x| < 2^-55. + if (LIBC_UNLIKELY(x_exp < FPBits::EXP_BIAS - 55)) { + // When |x| < 2^-55, acos(x) = pi/2 +#if defined(LIBC_MATH_HAS_SKIP_ACCURATE_PASS) + return PI_OVER_TWO.hi; +#else + // Force the evaluation and prevent constant propagation so that it + // is rounded correctly for FE_UPWARD rounding mode. + return (xbits.abs().get_val() + 0x1.0p-160) + PI_OVER_TWO.hi; +#endif // LIBC_MATH_HAS_SKIP_ACCURATE_PASS + } + +#ifdef LIBC_MATH_HAS_SKIP_ACCURATE_PASS + // acos(x) = pi/2 - asin(x) + // = pi/2 - x * P(x^2) + double p = asin_eval(x * x); + return PI_OVER_TWO.hi + fputil::multiply_add(-x, p, PI_OVER_TWO.lo); +#else + unsigned idx; + DoubleDouble x_sq = fputil::exact_mult(x, x); + double err = xbits.abs().get_val() * 0x1.0p-51; + // Polynomial approximation: + // p ~ asin(x)/x + DoubleDouble p = asin_eval(x_sq, idx, err); + // asin(x) ~ x * p + DoubleDouble r0 = fputil::exact_mult(x, p.hi); + // acos(x) = pi/2 - asin(x) + // ~ pi/2 - x * p + // = pi/2 - x * (p.hi + p.lo) + double r_hi = fputil::multiply_add(-x, p.hi, PI_OVER_TWO.hi); + // Use Dekker's 2SUM algorithm to compute the lower part. + double r_lo = ((PI_OVER_TWO.hi - r_hi) - r0.hi) - r0.lo; + r_lo = fputil::multiply_add(-x, p.lo, r_lo + PI_OVER_TWO.lo); + + // Ziv's accuracy test. + + double r_upper = r_hi + (r_lo + err); + double r_lower = r_hi + (r_lo - err); + + if (LIBC_LIKELY(r_upper == r_lower)) + return r_upper; + + // Ziv's accuracy test failed, perform 128-bit calculation. + + // Recalculate mod 1/64. + idx = static_cast(fputil::nearest_integer(x_sq.hi * 0x1.0p6)); + + // Get x^2 - idx/64 exactly. When FMA is available, double-double + // multiplication will be correct for all rounding modes. Otherwise we use + // Float128 directly. + Float128 x_f128(x); + +#ifdef LIBC_TARGET_CPU_HAS_FMA_DOUBLE + // u = x^2 - idx/64 + Float128 u_hi( + fputil::multiply_add(static_cast(idx), -0x1.0p-6, x_sq.hi)); + Float128 u = fputil::quick_add(u_hi, Float128(x_sq.lo)); +#else + Float128 x_sq_f128 = fputil::quick_mul(x_f128, x_f128); + Float128 u = fputil::quick_add( + x_sq_f128, Float128(static_cast(idx) * (-0x1.0p-6))); +#endif // LIBC_TARGET_CPU_HAS_FMA_DOUBLE + + Float128 p_f128 = asin_eval(u, idx); + // Flip the sign of x_f128 to perform subtraction. + x_f128.sign = x_f128.sign.negate(); + Float128 r = + fputil::quick_add(PI_OVER_TWO_F128, fputil::quick_mul(x_f128, p_f128)); + + return static_cast(r); +#endif // LIBC_MATH_HAS_SKIP_ACCURATE_PASS + } + // |x| >= 0.5 + + double x_abs = xbits.abs().get_val(); + + // Maintaining the sign: + constexpr double SIGN[2] = {1.0, -1.0}; + double x_sign = SIGN[xbits.is_neg()]; + // |x| >= 1 + if (LIBC_UNLIKELY(x_exp >= FPBits::EXP_BIAS)) { + // x = +-1, asin(x) = +- pi/2 + if (x_abs == 1.0) { + // x = 1, acos(x) = 0, + // x = -1, acos(x) = pi + return x == 1.0 ? 0.0 : fputil::multiply_add(-x_sign, PI.hi, PI.lo); + } + // |x| > 1, return NaN. + if (xbits.is_quiet_nan()) + return x; + + // Set domain error for non-NaN input. + if (!xbits.is_nan()) + fputil::set_errno_if_required(EDOM); + + fputil::raise_except_if_required(FE_INVALID); + return FPBits::quiet_nan().get_val(); + } + + // When |x| >= 0.5, we perform range reduction as follow: + // + // When 0.5 <= x < 1, let: + // y = acos(x) + // We will use the double angle formula: + // cos(2y) = 1 - 2 sin^2(y) + // and the complement angle identity: + // x = cos(y) = 1 - 2 sin^2 (y/2) + // So: + // sin(y/2) = sqrt( (1 - x)/2 ) + // And hence: + // y/2 = asin( sqrt( (1 - x)/2 ) ) + // Equivalently: + // acos(x) = y = 2 * asin( sqrt( (1 - x)/2 ) ) + // Let u = (1 - x)/2, then: + // acos(x) = 2 * asin( sqrt(u) ) + // Moreover, since 0.5 <= x < 1: + // 0 < u <= 1/4, and 0 < sqrt(u) <= 0.5, + // And hence we can reuse the same polynomial approximation of asin(x) when + // |x| <= 0.5: + // acos(x) ~ 2 * sqrt(u) * P(u). + // + // When -1 < x <= -0.5, we reduce to the previous case using the formula: + // acos(x) = pi - acos(-x) + // = pi - 2 * asin ( sqrt( (1 + x)/2 ) ) + // ~ pi - 2 * sqrt(u) * P(u), + // where u = (1 - |x|)/2. + + // u = (1 - |x|)/2 + double u = fputil::multiply_add(x_abs, -0.5, 0.5); + // v_hi + v_lo ~ sqrt(u). + // Let: + // h = u - v_hi^2 = (sqrt(u) - v_hi) * (sqrt(u) + v_hi) + // Then: + // sqrt(u) = v_hi + h / (sqrt(u) + v_hi) + // ~ v_hi + h / (2 * v_hi) + // So we can use: + // v_lo = h / (2 * v_hi). + double v_hi = fputil::sqrt(u); + +#ifdef LIBC_MATH_HAS_SKIP_ACCURATE_PASS + constexpr DoubleDouble CONST_TERM[2] = {{0.0, 0.0}, PI}; + DoubleDouble const_term = CONST_TERM[xbits.is_neg()]; + + double p = asin_eval(u); + double scale = x_sign * 2.0 * v_hi; + double r = const_term.hi + fputil::multiply_add(scale, p, const_term.lo); + return r; +#else + +#ifdef LIBC_TARGET_CPU_HAS_FMA_DOUBLE + double h = fputil::multiply_add(v_hi, -v_hi, u); +#else + DoubleDouble v_hi_sq = fputil::exact_mult(v_hi, v_hi); + double h = (u - v_hi_sq.hi) - v_hi_sq.lo; +#endif // LIBC_TARGET_CPU_HAS_FMA_DOUBLE + + // Scale v_lo and v_hi by 2 from the formula: + // vh = v_hi * 2 + // vl = 2*v_lo = h / v_hi. + double vh = v_hi * 2.0; + double vl = h / v_hi; + + // Polynomial approximation: + // p ~ asin(sqrt(u))/sqrt(u) + unsigned idx; + double err = vh * 0x1.0p-51; + + DoubleDouble p = asin_eval(DoubleDouble{0.0, u}, idx, err); + + // Perform computations in double-double arithmetic: + // asin(x) = pi/2 - (v_hi + v_lo) * (ASIN_COEFFS[idx][0] + p) + DoubleDouble r0 = fputil::quick_mult(DoubleDouble{vl, vh}, p); + + double r_hi, r_lo; + if (xbits.is_pos()) { + r_hi = r0.hi; + r_lo = r0.lo; + } else { + DoubleDouble r = fputil::exact_add(PI.hi, -r0.hi); + r_hi = r.hi; + r_lo = (PI.lo - r0.lo) + r.lo; + } + + // Ziv's accuracy test. + + double r_upper = r_hi + (r_lo + err); + double r_lower = r_hi + (r_lo - err); + + if (LIBC_LIKELY(r_upper == r_lower)) + return r_upper; + + // Ziv's accuracy test failed, we redo the computations in Float128. + // Recalculate mod 1/64. + idx = static_cast(fputil::nearest_integer(u * 0x1.0p6)); + + // After the first step of Newton-Raphson approximating v = sqrt(u), we have + // that: + // sqrt(u) = v_hi + h / (sqrt(u) + v_hi) + // v_lo = h / (2 * v_hi) + // With error: + // sqrt(u) - (v_hi + v_lo) = h * ( 1/(sqrt(u) + v_hi) - 1/(2*v_hi) ) + // = -h^2 / (2*v * (sqrt(u) + v)^2). + // Since: + // (sqrt(u) + v_hi)^2 ~ (2sqrt(u))^2 = 4u, + // we can add another correction term to (v_hi + v_lo) that is: + // v_ll = -h^2 / (2*v_hi * 4u) + // = -v_lo * (h / 4u) + // = -vl * (h / 8u), + // making the errors: + // sqrt(u) - (v_hi + v_lo + v_ll) = O(h^3) + // well beyond 128-bit precision needed. + + // Get the rounding error of vl = 2 * v_lo ~ h / vh + // Get full product of vh * vl +#ifdef LIBC_TARGET_CPU_HAS_FMA_DOUBLE + double vl_lo = fputil::multiply_add(-v_hi, vl, h) / v_hi; +#else + DoubleDouble vh_vl = fputil::exact_mult(v_hi, vl); + double vl_lo = ((h - vh_vl.hi) - vh_vl.lo) / v_hi; +#endif // LIBC_TARGET_CPU_HAS_FMA_DOUBLE + // vll = 2*v_ll = -vl * (h / (4u)). + double t = h * (-0.25) / u; + double vll = fputil::multiply_add(vl, t, vl_lo); + // m_v = -(v_hi + v_lo + v_ll). + Float128 m_v = fputil::quick_add( + Float128(vh), fputil::quick_add(Float128(vl), Float128(vll))); + m_v.sign = xbits.sign(); + + // Perform computations in Float128: + // acos(x) = (v_hi + v_lo + vll) * P(u) , when 0.5 <= x < 1, + // = pi - (v_hi + v_lo + vll) * P(u) , when -1 < x <= -0.5. + Float128 y_f128(fputil::multiply_add(static_cast(idx), -0x1.0p-6, u)); + + Float128 p_f128 = asin_eval(y_f128, idx); + Float128 r_f128 = fputil::quick_mul(m_v, p_f128); + + if (xbits.is_neg()) + r_f128 = fputil::quick_add(PI_F128, r_f128); + + return static_cast(r_f128); +#endif // LIBC_MATH_HAS_SKIP_ACCURATE_PASS +} + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/generic/acosf.cpp b/system/lib/llvm-libc/src/math/generic/acosf.cpp index 3c097a7871380..8dd6de2ce7474 100644 --- a/system/lib/llvm-libc/src/math/generic/acosf.cpp +++ b/system/lib/llvm-libc/src/math/generic/acosf.cpp @@ -20,6 +20,7 @@ namespace LIBC_NAMESPACE_DECL { +#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS static constexpr size_t N_EXCEPTS = 4; // Exceptional values when |x| <= 0.5 @@ -34,6 +35,7 @@ static constexpr fputil::ExceptValues ACOSF_EXCEPTS = {{ // x = -0x1.04c444p-12, acosf(x) = 0x1.923p0 (RZ) {0xb9826222, 0x3fc91800, 1, 0, 1}, }}; +#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS LLVM_LIBC_FUNCTION(float, acosf, (float x)) { using FPBits = typename fputil::FPBits; @@ -51,9 +53,11 @@ LLVM_LIBC_FUNCTION(float, acosf, (float x)) { // acos(x) = pi/2 - asin(x) // ~ pi/2 - x - x^3 / 6 +#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS // Check for exceptional values if (auto r = ACOSF_EXCEPTS.lookup(x_uint); LIBC_UNLIKELY(r.has_value())) return r.value(); +#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS double xd = static_cast(x); return static_cast(fputil::multiply_add( @@ -80,10 +84,17 @@ LLVM_LIBC_FUNCTION(float, acosf, (float x)) { 0x1.921fb6p+1f) : /* x == 1.0f */ 0.0f; + if (xbits.is_signaling_nan()) { + fputil::raise_except_if_required(FE_INVALID); + return FPBits::quiet_nan().get_val(); + } + + // |x| <= +/-inf if (x_abs <= 0x7f80'0000U) { fputil::set_errno_if_required(EDOM); fputil::raise_except_if_required(FE_INVALID); } + return x + FPBits::quiet_nan().get_val(); } diff --git a/system/lib/llvm-libc/src/math/generic/acoshf.cpp b/system/lib/llvm-libc/src/math/generic/acoshf.cpp index 6158063d30521..c4927fa27a84b 100644 --- a/system/lib/llvm-libc/src/math/generic/acoshf.cpp +++ b/system/lib/llvm-libc/src/math/generic/acoshf.cpp @@ -22,7 +22,6 @@ namespace LIBC_NAMESPACE_DECL { LLVM_LIBC_FUNCTION(float, acoshf, (float x)) { using FPBits_t = typename fputil::FPBits; FPBits_t xbits(x); - uint32_t x_u = xbits.uintval(); if (LIBC_UNLIKELY(x <= 1.0f)) { if (x == 1.0f) @@ -33,6 +32,8 @@ LLVM_LIBC_FUNCTION(float, acoshf, (float x)) { return FPBits_t::quiet_nan().get_val(); } +#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS + uint32_t x_u = xbits.uintval(); if (LIBC_UNLIKELY(x_u >= 0x4f8ffb03)) { if (LIBC_UNLIKELY(xbits.is_inf_or_nan())) return x; @@ -64,6 +65,10 @@ LLVM_LIBC_FUNCTION(float, acoshf, (float x)) { return round_result_slightly_up(0x1.451436p6f); } } +#else + if (LIBC_UNLIKELY(xbits.is_inf_or_nan())) + return x; +#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS double x_d = static_cast(x); // acosh(x) = log(x + sqrt(x^2 - 1)) diff --git a/system/lib/llvm-libc/src/math/generic/asin.cpp b/system/lib/llvm-libc/src/math/generic/asin.cpp new file mode 100644 index 0000000000000..ad77683d1f880 --- /dev/null +++ b/system/lib/llvm-libc/src/math/generic/asin.cpp @@ -0,0 +1,288 @@ +//===-- Double-precision asin function ------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/asin.h" +#include "asin_utils.h" +#include "src/__support/FPUtil/FEnvImpl.h" +#include "src/__support/FPUtil/FPBits.h" +#include "src/__support/FPUtil/PolyEval.h" +#include "src/__support/FPUtil/double_double.h" +#include "src/__support/FPUtil/dyadic_float.h" +#include "src/__support/FPUtil/multiply_add.h" +#include "src/__support/FPUtil/sqrt.h" +#include "src/__support/macros/config.h" +#include "src/__support/macros/optimization.h" // LIBC_UNLIKELY +#include "src/__support/macros/properties/cpu_features.h" // LIBC_TARGET_CPU_HAS_FMA + +namespace LIBC_NAMESPACE_DECL { + +using DoubleDouble = fputil::DoubleDouble; +using Float128 = fputil::DyadicFloat<128>; + +LLVM_LIBC_FUNCTION(double, asin, (double x)) { + using FPBits = fputil::FPBits; + + FPBits xbits(x); + int x_exp = xbits.get_biased_exponent(); + + // |x| < 0.5. + if (x_exp < FPBits::EXP_BIAS - 1) { + // |x| < 2^-26. + if (LIBC_UNLIKELY(x_exp < FPBits::EXP_BIAS - 26)) { + // When |x| < 2^-26, the relative error of the approximation asin(x) ~ x + // is: + // |asin(x) - x| / |asin(x)| < |x^3| / (6|x|) + // = x^2 / 6 + // < 2^-54 + // < epsilon(1)/2. + // So the correctly rounded values of asin(x) are: + // = x + sign(x)*eps(x) if rounding mode = FE_TOWARDZERO, + // or (rounding mode = FE_UPWARD and x is + // negative), + // = x otherwise. + // To simplify the rounding decision and make it more efficient, we use + // fma(x, 2^-54, x) instead. + // Note: to use the formula x + 2^-54*x to decide the correct rounding, we + // do need fma(x, 2^-54, x) to prevent underflow caused by 2^-54*x when + // |x| < 2^-1022. For targets without FMA instructions, when x is close to + // denormal range, we normalize x, +#if defined(LIBC_MATH_HAS_SKIP_ACCURATE_PASS) + return x; +#elif defined(LIBC_TARGET_CPU_HAS_FMA_DOUBLE) + return fputil::multiply_add(x, 0x1.0p-54, x); +#else + if (xbits.abs().uintval() == 0) + return x; + // Get sign(x) * min_normal. + FPBits eps_bits = FPBits::min_normal(); + eps_bits.set_sign(xbits.sign()); + double eps = eps_bits.get_val(); + double normalize_const = (x_exp == 0) ? eps : 0.0; + double scaled_normal = + fputil::multiply_add(x + normalize_const, 0x1.0p54, eps); + return fputil::multiply_add(scaled_normal, 0x1.0p-54, -normalize_const); +#endif // LIBC_MATH_HAS_SKIP_ACCURATE_PASS + } + +#ifdef LIBC_MATH_HAS_SKIP_ACCURATE_PASS + return x * asin_eval(x * x); +#else + unsigned idx; + DoubleDouble x_sq = fputil::exact_mult(x, x); + double err = xbits.abs().get_val() * 0x1.0p-51; + // Polynomial approximation: + // p ~ asin(x)/x + + DoubleDouble p = asin_eval(x_sq, idx, err); + // asin(x) ~ x * (ASIN_COEFFS[idx][0] + p) + DoubleDouble r0 = fputil::exact_mult(x, p.hi); + double r_lo = fputil::multiply_add(x, p.lo, r0.lo); + + // Ziv's accuracy test. + + double r_upper = r0.hi + (r_lo + err); + double r_lower = r0.hi + (r_lo - err); + + if (LIBC_LIKELY(r_upper == r_lower)) + return r_upper; + + // Ziv's accuracy test failed, perform 128-bit calculation. + + // Recalculate mod 1/64. + idx = static_cast(fputil::nearest_integer(x_sq.hi * 0x1.0p6)); + + // Get x^2 - idx/64 exactly. When FMA is available, double-double + // multiplication will be correct for all rounding modes. Otherwise we use + // Float128 directly. + Float128 x_f128(x); + +#ifdef LIBC_TARGET_CPU_HAS_FMA_DOUBLE + // u = x^2 - idx/64 + Float128 u_hi( + fputil::multiply_add(static_cast(idx), -0x1.0p-6, x_sq.hi)); + Float128 u = fputil::quick_add(u_hi, Float128(x_sq.lo)); +#else + Float128 x_sq_f128 = fputil::quick_mul(x_f128, x_f128); + Float128 u = fputil::quick_add( + x_sq_f128, Float128(static_cast(idx) * (-0x1.0p-6))); +#endif // LIBC_TARGET_CPU_HAS_FMA_DOUBLE + + Float128 p_f128 = asin_eval(u, idx); + Float128 r = fputil::quick_mul(x_f128, p_f128); + + return static_cast(r); +#endif // LIBC_MATH_HAS_SKIP_ACCURATE_PASS + } + // |x| >= 0.5 + + double x_abs = xbits.abs().get_val(); + + // Maintaining the sign: + constexpr double SIGN[2] = {1.0, -1.0}; + double x_sign = SIGN[xbits.is_neg()]; + + // |x| >= 1 + if (LIBC_UNLIKELY(x_exp >= FPBits::EXP_BIAS)) { + // x = +-1, asin(x) = +- pi/2 + if (x_abs == 1.0) { + // return +- pi/2 + return fputil::multiply_add(x_sign, PI_OVER_TWO.hi, + x_sign * PI_OVER_TWO.lo); + } + // |x| > 1, return NaN. + if (xbits.is_quiet_nan()) + return x; + + // Set domain error for non-NaN input. + if (!xbits.is_nan()) + fputil::set_errno_if_required(EDOM); + + fputil::raise_except_if_required(FE_INVALID); + return FPBits::quiet_nan().get_val(); + } + + // When |x| >= 0.5, we perform range reduction as follow: + // + // Assume further that 0.5 <= x < 1, and let: + // y = asin(x) + // We will use the double angle formula: + // cos(2y) = 1 - 2 sin^2(y) + // and the complement angle identity: + // x = sin(y) = cos(pi/2 - y) + // = 1 - 2 sin^2 (pi/4 - y/2) + // So: + // sin(pi/4 - y/2) = sqrt( (1 - x)/2 ) + // And hence: + // pi/4 - y/2 = asin( sqrt( (1 - x)/2 ) ) + // Equivalently: + // asin(x) = y = pi/2 - 2 * asin( sqrt( (1 - x)/2 ) ) + // Let u = (1 - x)/2, then: + // asin(x) = pi/2 - 2 * asin( sqrt(u) ) + // Moreover, since 0.5 <= x < 1: + // 0 < u <= 1/4, and 0 < sqrt(u) <= 0.5, + // And hence we can reuse the same polynomial approximation of asin(x) when + // |x| <= 0.5: + // asin(x) ~ pi/2 - 2 * sqrt(u) * P(u), + + // u = (1 - |x|)/2 + double u = fputil::multiply_add(x_abs, -0.5, 0.5); + // v_hi + v_lo ~ sqrt(u). + // Let: + // h = u - v_hi^2 = (sqrt(u) - v_hi) * (sqrt(u) + v_hi) + // Then: + // sqrt(u) = v_hi + h / (sqrt(u) + v_hi) + // ~ v_hi + h / (2 * v_hi) + // So we can use: + // v_lo = h / (2 * v_hi). + // Then, + // asin(x) ~ pi/2 - 2*(v_hi + v_lo) * P(u) + double v_hi = fputil::sqrt(u); + +#ifdef LIBC_MATH_HAS_SKIP_ACCURATE_PASS + double p = asin_eval(u); + double r = x_sign * fputil::multiply_add(-2.0 * v_hi, p, PI_OVER_TWO.hi); + return r; +#else + +#ifdef LIBC_TARGET_CPU_HAS_FMA_DOUBLE + double h = fputil::multiply_add(v_hi, -v_hi, u); +#else + DoubleDouble v_hi_sq = fputil::exact_mult(v_hi, v_hi); + double h = (u - v_hi_sq.hi) - v_hi_sq.lo; +#endif // LIBC_TARGET_CPU_HAS_FMA_DOUBLE + + // Scale v_lo and v_hi by 2 from the formula: + // vh = v_hi * 2 + // vl = 2*v_lo = h / v_hi. + double vh = v_hi * 2.0; + double vl = h / v_hi; + + // Polynomial approximation: + // p ~ asin(sqrt(u))/sqrt(u) + unsigned idx; + double err = vh * 0x1.0p-51; + + DoubleDouble p = asin_eval(DoubleDouble{0.0, u}, idx, err); + + // Perform computations in double-double arithmetic: + // asin(x) = pi/2 - (v_hi + v_lo) * (ASIN_COEFFS[idx][0] + p) + DoubleDouble r0 = fputil::quick_mult(DoubleDouble{vl, vh}, p); + DoubleDouble r = fputil::exact_add(PI_OVER_TWO.hi, -r0.hi); + + double r_lo = PI_OVER_TWO.lo - r0.lo + r.lo; + + // Ziv's accuracy test. + +#ifdef LIBC_TARGET_CPU_HAS_FMA_DOUBLE + double r_upper = fputil::multiply_add( + r.hi, x_sign, fputil::multiply_add(r_lo, x_sign, err)); + double r_lower = fputil::multiply_add( + r.hi, x_sign, fputil::multiply_add(r_lo, x_sign, -err)); +#else + r_lo *= x_sign; + r.hi *= x_sign; + double r_upper = r.hi + (r_lo + err); + double r_lower = r.hi + (r_lo - err); +#endif // LIBC_TARGET_CPU_HAS_FMA_DOUBLE + + if (LIBC_LIKELY(r_upper == r_lower)) + return r_upper; + + // Ziv's accuracy test failed, we redo the computations in Float128. + // Recalculate mod 1/64. + idx = static_cast(fputil::nearest_integer(u * 0x1.0p6)); + + // After the first step of Newton-Raphson approximating v = sqrt(u), we have + // that: + // sqrt(u) = v_hi + h / (sqrt(u) + v_hi) + // v_lo = h / (2 * v_hi) + // With error: + // sqrt(u) - (v_hi + v_lo) = h * ( 1/(sqrt(u) + v_hi) - 1/(2*v_hi) ) + // = -h^2 / (2*v * (sqrt(u) + v)^2). + // Since: + // (sqrt(u) + v_hi)^2 ~ (2sqrt(u))^2 = 4u, + // we can add another correction term to (v_hi + v_lo) that is: + // v_ll = -h^2 / (2*v_hi * 4u) + // = -v_lo * (h / 4u) + // = -vl * (h / 8u), + // making the errors: + // sqrt(u) - (v_hi + v_lo + v_ll) = O(h^3) + // well beyond 128-bit precision needed. + + // Get the rounding error of vl = 2 * v_lo ~ h / vh + // Get full product of vh * vl +#ifdef LIBC_TARGET_CPU_HAS_FMA_DOUBLE + double vl_lo = fputil::multiply_add(-v_hi, vl, h) / v_hi; +#else + DoubleDouble vh_vl = fputil::exact_mult(v_hi, vl); + double vl_lo = ((h - vh_vl.hi) - vh_vl.lo) / v_hi; +#endif // LIBC_TARGET_CPU_HAS_FMA_DOUBLE + // vll = 2*v_ll = -vl * (h / (4u)). + double t = h * (-0.25) / u; + double vll = fputil::multiply_add(vl, t, vl_lo); + // m_v = -(v_hi + v_lo + v_ll). + Float128 m_v = fputil::quick_add( + Float128(vh), fputil::quick_add(Float128(vl), Float128(vll))); + m_v.sign = Sign::NEG; + + // Perform computations in Float128: + // asin(x) = pi/2 - (v_hi + v_lo + vll) * P(u). + Float128 y_f128(fputil::multiply_add(static_cast(idx), -0x1.0p-6, u)); + + Float128 p_f128 = asin_eval(y_f128, idx); + Float128 r0_f128 = fputil::quick_mul(m_v, p_f128); + Float128 r_f128 = fputil::quick_add(PI_OVER_TWO_F128, r0_f128); + + if (xbits.is_neg()) + r_f128.sign = Sign::NEG; + + return static_cast(r_f128); +#endif // LIBC_MATH_HAS_SKIP_ACCURATE_PASS +} + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/generic/asin_utils.h b/system/lib/llvm-libc/src/math/generic/asin_utils.h new file mode 100644 index 0000000000000..44913d573de2c --- /dev/null +++ b/system/lib/llvm-libc/src/math/generic/asin_utils.h @@ -0,0 +1,574 @@ +//===-- Collection of utils for asin/acos -----------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_MATH_GENERIC_ASIN_UTILS_H +#define LLVM_LIBC_SRC_MATH_GENERIC_ASIN_UTILS_H + +#include "src/__support/FPUtil/PolyEval.h" +#include "src/__support/FPUtil/double_double.h" +#include "src/__support/FPUtil/dyadic_float.h" +#include "src/__support/FPUtil/multiply_add.h" +#include "src/__support/FPUtil/nearest_integer.h" +#include "src/__support/integer_literals.h" +#include "src/__support/macros/config.h" +#include "src/__support/macros/optimization.h" + +namespace LIBC_NAMESPACE_DECL { + +namespace { + +using DoubleDouble = fputil::DoubleDouble; +using Float128 = fputil::DyadicFloat<128>; + +constexpr DoubleDouble PI = {0x1.1a62633145c07p-53, 0x1.921fb54442d18p1}; + +constexpr DoubleDouble PI_OVER_TWO = {0x1.1a62633145c07p-54, + 0x1.921fb54442d18p0}; + +#ifdef LIBC_MATH_HAS_SKIP_ACCURATE_PASS + +// When correct rounding is not needed, we use a degree-22 minimax polynomial to +// approximate asin(x)/x on [0, 0.5] using Sollya with: +// > P = fpminimax(asin(x)/x, [|0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22|], +// [|1, D...|], [0, 0.5]); +// > dirtyinfnorm(asin(x)/x - P, [0, 0.5]); +// 0x1.1a71ef0a0f26a9fb7ed7e41dee788b13d1770db3dp-52 + +constexpr double ASIN_COEFFS[12] = { + 0x1.0000000000000p0, 0x1.5555555556dcfp-3, 0x1.3333333082e11p-4, + 0x1.6db6dd14099edp-5, 0x1.f1c69b35bf81fp-6, 0x1.6e97194225a67p-6, + 0x1.1babddb82ce12p-6, 0x1.d55bd078600d6p-7, 0x1.33328959e63d6p-7, + 0x1.2b5993bda1d9bp-6, -0x1.806aff270bf25p-7, 0x1.02614e5ed3936p-5, +}; + +LIBC_INLINE double asin_eval(double u) { + double u2 = u * u; + double c0 = fputil::multiply_add(u, ASIN_COEFFS[1], ASIN_COEFFS[0]); + double c1 = fputil::multiply_add(u, ASIN_COEFFS[3], ASIN_COEFFS[2]); + double c2 = fputil::multiply_add(u, ASIN_COEFFS[5], ASIN_COEFFS[4]); + double c3 = fputil::multiply_add(u, ASIN_COEFFS[7], ASIN_COEFFS[6]); + double c4 = fputil::multiply_add(u, ASIN_COEFFS[9], ASIN_COEFFS[8]); + double c5 = fputil::multiply_add(u, ASIN_COEFFS[11], ASIN_COEFFS[10]); + + double u4 = u2 * u2; + double d0 = fputil::multiply_add(u2, c1, c0); + double d1 = fputil::multiply_add(u2, c3, c2); + double d2 = fputil::multiply_add(u2, c5, c4); + + return fputil::polyeval(u4, d0, d1, d2); +} + +#else + +// The Taylor expansion of asin(x) around 0 is: +// asin(x) = x + x^3/6 + 3x^5/40 + ... +// ~ x * P(x^2). +// Let u = x^2, then P(x^2) = P(u), and |x| = sqrt(u). Note that when +// |x| <= 0.5, we have |u| <= 0.25. +// We approximate P(u) by breaking it down by performing range reduction mod +// 2^-5 = 1/32. +// So for: +// k = round(u * 32), +// y = u - k/32, +// we have that: +// x = sqrt(u) = sqrt(k/32 + y), +// |y| <= 2^-5 = 1/32, +// and: +// P(u) = P(k/32 + y) = Q_k(y). +// Hence : +// asin(x) = sqrt(k/32 + y) * Q_k(y), +// Or equivalently: +// Q_k(y) = asin(sqrt(k/32 + y)) / sqrt(k/32 + y). +// We generate the coefficients of Q_k by Sollya as following: +// > procedure ASIN_APPROX(N, Deg) { +// abs_error = 0; +// rel_error = 0; +// deg = [||]; +// for i from 2 to Deg do deg = deg :. i; +// for i from 1 to N/4 do { +// F = asin(sqrt(i/N + x))/sqrt(i/N + x); +// T = taylor(F, 1, 0); +// T_DD = roundcoefficients(T, [|DD...|]); +// I = [-1/(2*N), 1/(2*N)]; +// Q = fpminimax(F, deg, [|D...|], I, T_DD); +// abs_err = dirtyinfnorm(F - Q, I); +// rel_err = dirtyinfnorm((F - Q)/x^2, I); +// if (abs_err > abs_error) then abs_error = abs_err; +// if (rel_err > rel_error) then rel_error = rel_err; +// d0 = D(coeff(Q, 0)); +// d1 = coeff(Q, 0) - d0; +// write("{", d0, ", ", d1); +// d0 = D(coeff(Q, 1)); d1 = coeff(Q, 1) - d0; write(", ", d0, ", ", d1); +// for j from 2 to Deg do { +// write(", ", coeff(Q, j)); +// }; +// print("},"); +// }; +// print("Absolute Errors:", D(abs_error)); +// print("Relative Errors:", D(rel_error)); +// }; +// > ASIN_APPROX(32, 9); +// Absolute Errors: 0x1.69837b5183654p-72 +// Relative Errors: 0x1.4d7f82835bf64p-55 + +// For k = 0, we use the degree-18 Taylor polynomial of asin(x)/x: +// +// > P = 1 + x^2 * DD(1/6) + x^4 * D(3/40) + x^6 * D(5/112) + x^8 * D(35/1152) + +// x^10 * D(63/2816) + x^12 * D(231/13312) + x^14 * D(143/10240) + +// x^16 * D(6435/557056) + x^18 * D(12155/1245184); +// > dirtyinfnorm(asin(x)/x - P, [-1/64, 1/64]); +// 0x1.999075402cafp-83 + +constexpr double ASIN_COEFFS[9][12] = { + {1.0, 0.0, 0x1.5555555555555p-3, 0x1.5555555555555p-57, + 0x1.3333333333333p-4, 0x1.6db6db6db6db7p-5, 0x1.f1c71c71c71c7p-6, + 0x1.6e8ba2e8ba2e9p-6, 0x1.1c4ec4ec4ec4fp-6, 0x1.c99999999999ap-7, + 0x1.7a87878787878p-7, 0x1.3fde50d79435ep-7}, + {0x1.015a397cf0f1cp0, -0x1.eebd6ccfe3ee3p-55, 0x1.5f3581be7b08bp-3, + -0x1.5df80d0e7237dp-57, 0x1.4519ddf1ae53p-4, 0x1.8eb4b6eeb1696p-5, + 0x1.17bc85420fec8p-5, 0x1.a8e39b5dcad81p-6, 0x1.53f8df127539bp-6, + 0x1.1a485a0b0130ap-6, 0x1.e20e6e493002p-7, 0x1.a466a7030f4c9p-7}, + {0x1.02be9ce0b87cdp0, 0x1.e5d09da2e0f04p-56, 0x1.69ab5325bc359p-3, + -0x1.92f480cfede2dp-57, 0x1.58a4c3097aab1p-4, 0x1.b3db36068dd8p-5, + 0x1.3b9482184625p-5, 0x1.eedc823765d21p-6, 0x1.98e35d756be6bp-6, + 0x1.5ea4f1b32731ap-6, 0x1.355115764148ep-6, 0x1.16a5853847c91p-6}, + {0x1.042dc6a65ffbfp0, -0x1.c7ea28dce95d1p-55, 0x1.74c4bd7412f9dp-3, + 0x1.447024c0a3c87p-58, 0x1.6e09c6d2b72b9p-4, 0x1.ddd9dcdae5315p-5, + 0x1.656f1f64058b8p-5, 0x1.21a42e4437101p-5, 0x1.eed0350b7edb2p-6, + 0x1.b6bc877e58c52p-6, 0x1.903a0872eb2a4p-6, 0x1.74da839ddd6d8p-6}, + {0x1.05a8621feb16bp0, -0x1.e5b33b1407c5fp-56, 0x1.809186c2e57ddp-3, + -0x1.3dcb4d6069407p-60, 0x1.8587d99442dc5p-4, 0x1.06c23d1e75be3p-4, + 0x1.969024051c67dp-5, 0x1.54e4f934aacfdp-5, 0x1.2d60a732dbc9cp-5, + 0x1.149f0c046eac7p-5, 0x1.053a56dba1fbap-5, 0x1.f7face3343992p-6}, + {0x1.072f2b6f1e601p0, -0x1.2dcbb0541997p-54, 0x1.8d2397127aebap-3, + 0x1.ead0c497955fbp-57, 0x1.9f68df88da518p-4, 0x1.21ee26a5900d7p-4, + 0x1.d08e7081b53a9p-5, 0x1.938dd661713f7p-5, 0x1.71b9f299b72e6p-5, + 0x1.5fbc7d2450527p-5, 0x1.58573247ec325p-5, 0x1.585a174a6a4cep-5}, + {0x1.08c2f1d638e4cp0, 0x1.b47c159534a3dp-56, 0x1.9a8f592078624p-3, + -0x1.ea339145b65cdp-57, 0x1.bc04165b57aabp-4, 0x1.410df5f58441dp-4, + 0x1.0ab6bdf5f8f7p-4, 0x1.e0b92eea1fce1p-5, 0x1.c9094e443a971p-5, + 0x1.c34651d64bc74p-5, 0x1.caa008d1af08p-5, 0x1.dc165bc0c4fc5p-5}, + {0x1.0a649a73e61f2p0, 0x1.74ac0d817e9c7p-55, 0x1.a8ec30dc9389p-3, + -0x1.8ab1c0eef300cp-59, 0x1.dbc11ea95061bp-4, 0x1.64e371d661328p-4, + 0x1.33e0023b3d895p-4, 0x1.2042269c243cep-4, 0x1.1cce74bda223p-4, + 0x1.244d425572ce9p-4, 0x1.34d475c7f1e3ep-4, 0x1.4d4e653082ad3p-4}, + {0x1.0c152382d7366p0, -0x1.ee6913347c2a6p-54, 0x1.b8550d62bfb6dp-3, + -0x1.d10aec3f116d5p-57, 0x1.ff1bde0fa3cap-4, 0x1.8e5f3ab69f6a4p-4, + 0x1.656be8b6527cep-4, 0x1.5c39755dc041ap-4, 0x1.661e6ebd40599p-4, + 0x1.7ea3dddee2a4fp-4, 0x1.a4f439abb4869p-4, 0x1.d9181c0fda658p-4}, +}; + +// We calculate the lower part of the approximation P(u). +LIBC_INLINE DoubleDouble asin_eval(const DoubleDouble &u, unsigned &idx, + double &err) { + using fputil::multiply_add; + // k = round(u * 32). + double k = fputil::nearest_integer(u.hi * 0x1.0p5); + idx = static_cast(k); + // y = u - k/32. + double y_hi = multiply_add(k, -0x1.0p-5, u.hi); // Exact + DoubleDouble y = fputil::exact_add(y_hi, u.lo); + double y2 = y.hi * y.hi; + // Add double-double errors in addition to the relative errors from y2. + err = fputil::multiply_add(err, y2, 0x1.0p-102); + DoubleDouble c0 = fputil::quick_mult( + y, DoubleDouble{ASIN_COEFFS[idx][3], ASIN_COEFFS[idx][2]}); + double c1 = multiply_add(y.hi, ASIN_COEFFS[idx][5], ASIN_COEFFS[idx][4]); + double c2 = multiply_add(y.hi, ASIN_COEFFS[idx][7], ASIN_COEFFS[idx][6]); + double c3 = multiply_add(y.hi, ASIN_COEFFS[idx][9], ASIN_COEFFS[idx][8]); + double c4 = multiply_add(y.hi, ASIN_COEFFS[idx][11], ASIN_COEFFS[idx][10]); + + double y4 = y2 * y2; + double d0 = multiply_add(y2, c2, c1); + double d1 = multiply_add(y2, c4, c3); + + DoubleDouble r = fputil::exact_add(ASIN_COEFFS[idx][0], c0.hi); + + double e1 = multiply_add(y4, d1, d0); + + r.lo = multiply_add(y2, e1, ASIN_COEFFS[idx][1] + c0.lo + r.lo); + + return r; +} + +// Follow the discussion above, we generate the coefficients of Q_k by Sollya as +// following: +// > procedure PRINTF128(a) { +// write("{"); +// if (a < 0) +// then write("Sign::NEG, ") else write("Sign::POS, "); +// a_exp = floor(log2(a)) + 1; +// write((a + 2 ^ a_exp) * 2 ^ -128); +// print("},"); +// }; +// > verbosity = 0; +// > procedure ASIN_APPROX(N, Deg) { +// abs_error = 0; +// rel_error = 0; +// for i from 1 to N / 4 do { +// Q = fpminimax(asin(sqrt(i / N + x)) / sqrt(i / N + x), Deg, +// [| 128... | ], [ -1 / (2 * N), 1 / (2 * N) ]); +// abs_err = dirtyinfnorm(asin(sqrt(i / N + x)) - sqrt(i / N + x) * Q, +// [ -1 / (2 * N), 1 / (2 * N) ]); +// rel_err = dirtyinfnorm(asin(sqrt(i / N + x)) / sqrt(i / N + x) - Q, +// [ -1 / (2 * N), 1 / (2 * N) ]); +// if (abs_err > abs_error) then abs_error = abs_err; +// if (rel_err > rel_error) then rel_error = rel_err; +// write("{"); +// for j from 0 to Deg do PRINTF128(coeff(Q, j)); +// print("},"); +// }; +// print("Absolute Errors:", abs_error); +// print("Relative Errors:", rel_error); +// }; +// > ASIN_APPROX(64, 15); +// ... +// Absolute Errors: 0x1.0b3...p-129 +// Relative Errors: 0x1.1db...p-128 +// +// For k = 0, we use Taylor polynomial of asin(x)/x around x = 0. +// asin(x)/x ~ 1 + x^2/6 + (3 x^4)/40 + (5 x^6)/112 + (35 x^8)/1152 + +// + (63 x^10)/2816 + (231 x^12)/13312 + (143 x^14)/10240 + +// + (6435 x^16)/557056 + (12155 x^18)/1245184 + +// + (46189 x^20)/5505024 + (88179 x^22)/12058624 + +// + (676039 x^24)/104857600 + (1300075 x^26)/226492416 + +// + (5014575 x^28)/973078528 + (9694845 x^30)/2080374784. + +constexpr Float128 ASIN_COEFFS_F128[17][16] = { + { + {Sign::POS, -127, 0x80000000'00000000'00000000'00000000_u128}, + {Sign::POS, -130, 0xaaaaaaaa'aaaaaaaa'aaaaaaaa'aaaaaaab_u128}, + {Sign::POS, -131, 0x99999999'99999999'99999999'9999999a_u128}, + {Sign::POS, -132, 0xb6db6db6'db6db6db'6db6db6d'b6db6db7_u128}, + {Sign::POS, -133, 0xf8e38e38'e38e38e3'8e38e38e'38e38e39_u128}, + {Sign::POS, -133, 0xb745d174'5d1745d1'745d1745'd1745d17_u128}, + {Sign::POS, -133, 0x8e276276'27627627'62762762'76276276_u128}, + {Sign::POS, -134, 0xe4cccccc'cccccccc'cccccccc'cccccccd_u128}, + {Sign::POS, -134, 0xbd43c3c3'c3c3c3c3'c3c3c3c3'c3c3c3c4_u128}, + {Sign::POS, -134, 0x9fef286b'ca1af286'bca1af28'6bca1af3_u128}, + {Sign::POS, -134, 0x89779e79'e79e79e7'9e79e79e'79e79e7a_u128}, + {Sign::POS, -135, 0xef9de9bd'37a6f4de'9bd37a6f'4de9bd38_u128}, + {Sign::POS, -135, 0xd3431eb8'51eb851e'b851eb85'1eb851ec_u128}, + {Sign::POS, -135, 0xbc16ed09'7b425ed0'97b425ed'097b425f_u128}, + {Sign::POS, -135, 0xa8dd1846'9ee58469'ee58469e'e58469ee_u128}, + {Sign::POS, -135, 0x98b41def'7bdef7bd'ef7bdef7'bdef7bdf_u128}, + }, + { + {Sign::POS, -127, 0x8055f060'94f0f05f'3ac3b927'50a701d9_u128}, + {Sign::POS, -130, 0xad19c2ea'e3dd2429'8d04f71d'b965ee1b_u128}, + {Sign::POS, -131, 0x9dfa882b'7b31af17'f9f19d33'0c45d24b_u128}, + {Sign::POS, -132, 0xbedd3b58'c9e605ef'1404e1f0'4ba57940_u128}, + {Sign::POS, -132, 0x83df2581'cb4fea82'b406f201'2fde6d5c_u128}, + {Sign::POS, -133, 0xc534fe61'9b82dd16'ed5d8a43'f7710526_u128}, + {Sign::POS, -133, 0x9b56fa62'88295ddf'ce8425fe'a04d733e_u128}, + {Sign::POS, -134, 0xfdeddb19'4a030da7'27158080'd24caf46_u128}, + {Sign::POS, -134, 0xd55827db'ff416ea8'042c4d8c'07cddeeb_u128}, + {Sign::POS, -134, 0xb71d73a9'f2ba0688'5eaeeae9'413a0f5f_u128}, + {Sign::POS, -134, 0x9fde87e2'ace91274'38f82666'd619c1ba_u128}, + {Sign::POS, -134, 0x8d876557'5e4626a1'1b621336'93587847_u128}, + {Sign::POS, -135, 0xfd801840'c8710595'6880fe13'a9657f8f_u128}, + {Sign::POS, -135, 0xe54245a9'4c8c2ebb'30488494'64b0e34d_u128}, + {Sign::POS, -135, 0xd11eb46f'4095a661'8890d123'15c96482_u128}, + {Sign::POS, -135, 0xc01a4201'467fbc0b'960618d5'ec2adaa8_u128}, + }, + { + {Sign::POS, -127, 0x80ad1cbe'7878de11'4293301c'11ce9d49_u128}, + {Sign::POS, -130, 0xaf9ac0df'3d845544'0fe5e31b'9051d03e_u128}, + {Sign::POS, -131, 0xa28ceef8'd7297e05'f94773ad'f4a695c6_u128}, + {Sign::POS, -132, 0xc75a5b77'58b4b11d'396c68ad'6733022b_u128}, + {Sign::POS, -132, 0x8bde42a1'084a6674'50c5bceb'005d4b62_u128}, + {Sign::POS, -133, 0xd471cdae'e2f35a96'bd4bc513'e0ccdf2c_u128}, + {Sign::POS, -133, 0xa9fc6fd5'd204a4e3'e609940c'6b991b67_u128}, + {Sign::POS, -133, 0x8d242d97'ba12b492'e25c7e7c'0c3fcf60_u128}, + {Sign::POS, -134, 0xf0f1ba74'b149afc3'2f0bbab5'a20c6199_u128}, + {Sign::POS, -134, 0xd21b42fb'd8e9098d'19612692'9a043332_u128}, + {Sign::POS, -134, 0xba5e5492'7896a3e7'193a74d5'78631587_u128}, + {Sign::POS, -134, 0xa7a17ae7'fc707f45'910e7a5d'c95251f4_u128}, + {Sign::POS, -134, 0x98889a6a'b0370464'50c950d3'61d79ed7_u128}, + {Sign::POS, -134, 0x8c29330e'4318fd29'25c5b528'84e39e7c_u128}, + {Sign::POS, -134, 0x81e7bf48'b25bc7c0'b9204a4f'd4f5fa8b_u128}, + {Sign::POS, -135, 0xf2801b09'11bf0768'773996dd'5224d852_u128}, + }, + { + {Sign::POS, -127, 0x81058e3e'f82ba622'ab81cd63'e1a91d57_u128}, + {Sign::POS, -130, 0xb22e7055'c80dd354'8a2f2e8e'860d3f33_u128}, + {Sign::POS, -131, 0xa753ce1a'7e3d1f57'247b37e6'03f93624_u128}, + {Sign::POS, -132, 0xd05c5604'8eca8d18'dcdd76b7'f4b1f185_u128}, + {Sign::POS, -132, 0x947cdd5e'f1d64df0'84f78df1'e2ecb854_u128}, + {Sign::POS, -133, 0xe5218370'2ebbf6e8'3727a755'57843b93_u128}, + {Sign::POS, -133, 0xba482553'383b92eb'186f78f1'8c35d6af_u128}, + {Sign::POS, -133, 0x9d2b034a'7266c6a1'54b78a98'1a547429_u128}, + {Sign::POS, -133, 0x8852f723'feea6046'e125f5a9'64e168e6_u128}, + {Sign::POS, -134, 0xf19c9891'6c896c99'732052fe'5c54e992_u128}, + {Sign::POS, -134, 0xd9cc81a5'c5ddf0f0'd651011e'a8ecd936_u128}, + {Sign::POS, -134, 0xc7173169'dcb6095f'a6160847'b595aaff_u128}, + {Sign::POS, -134, 0xb81cd3f6'4a422ebe'07aeb734'e4dcf3a1_u128}, + {Sign::POS, -134, 0xabf01b1c'd15932aa'698d4382'512318a9_u128}, + {Sign::POS, -134, 0xa1f1cf1b'd889a1ac'7120ca2f'bbbc1745_u128}, + {Sign::POS, -134, 0x99a1b838'e38fbf11'429a4350'76b7d191_u128}, + }, + { + {Sign::POS, -127, 0x815f4e70'5c3e68f2'e84ed170'78211dfd_u128}, + {Sign::POS, -130, 0xb4d5a992'de1ac4da'16fe6024'3a6cc371_u128}, + {Sign::POS, -131, 0xac526184'bd558c65'66642dce'edc4b04a_u128}, + {Sign::POS, -132, 0xd9ed9b03'46ec0bab'429ea221'4774bbc1_u128}, + {Sign::POS, -132, 0x9dca410c'1efaeb74'87956685'dd5fe848_u128}, + {Sign::POS, -133, 0xf76e411b'a926fc02'7f942265'9c39a882_u128}, + {Sign::POS, -133, 0xcc71b004'eeb60c0f'1d387f76'44b46bf8_u128}, + {Sign::POS, -133, 0xaf527a40'6f1084fb'5019904e'd12d384d_u128}, + {Sign::POS, -133, 0x9a9304b0'd8a9de19'e1803691'269be22c_u128}, + {Sign::POS, -133, 0x8b3d37c0'dbde09ef'342ddf4f'e80dd3fb_u128}, + {Sign::POS, -134, 0xff2e9111'3a961c78'92297bab'cc257804_u128}, + {Sign::POS, -134, 0xed1fb643'f2ca31c1'b0a1553a'e077285a_u128}, + {Sign::POS, -134, 0xdeeb0f5e'81ad5e30'78d79ae3'83be1c18_u128}, + {Sign::POS, -134, 0xd3a13ba6'8ce9abfc'a66eb1fd'c0c760fd_u128}, + {Sign::POS, -134, 0xcaa8c381'd44bb44f'0ab25126'9a5fae10_u128}, + {Sign::POS, -134, 0xc36fb2c4'244401cf'10dd8a39'78ccbf7f_u128}, + }, + { + {Sign::POS, -127, 0x81ba6750'6064f4dd'08015b7c'713688f0_u128}, + {Sign::POS, -130, 0xb791524b'd975fdd1'584037b7'103b42ca_u128}, + {Sign::POS, -131, 0xb18c26c5'3ced9856'db5bc672'cc95a64f_u128}, + {Sign::POS, -132, 0xe4199ce5'd25be89b'4a0ad208'da77022d_u128}, + {Sign::POS, -132, 0xa7d77999'0f80e3e9'7e97e9d1'0e337550_u128}, + {Sign::POS, -132, 0x85c3e039'8959c95b'e6e1e87f'7e6636b1_u128}, + {Sign::POS, -133, 0xe0b90ecd'95f7e6eb'a675bae0'628bd214_u128}, + {Sign::POS, -133, 0xc3edb6b4'ed0a684c'c7a3ee4d'f1dcd3f9_u128}, + {Sign::POS, -133, 0xafa274d2'e66e1f61'9e8ab3c7'7221214e_u128}, + {Sign::POS, -133, 0xa0dd903d'e110b71a'8a1fc9df'cc080308_u128}, + {Sign::POS, -133, 0x95e2f38c'60441961'72b90625'e3a37573_u128}, + {Sign::POS, -133, 0x8d9fe38f'2c705139'029f857c'9f628b2b_u128}, + {Sign::POS, -133, 0x8762410a'4967a974'6b609e83'7c025a39_u128}, + {Sign::POS, -133, 0x82b220be'd9ec0e5a'9ce9af7c'c65c94b9_u128}, + {Sign::POS, -134, 0xfe866073'2312c056'4265d82a'3afea10c_u128}, + {Sign::POS, -134, 0xf99b667c'5f8ef6a6'11fafa4d'5c76ebb3_u128}, + }, + { + {Sign::POS, -127, 0x8216e353'2ffdf638'15d72316'a2f327f2_u128}, + {Sign::POS, -130, 0xba625eba'097ce944'7024c0a3'c873729b_u128}, + {Sign::POS, -131, 0xb704e369'5b95ce44'cde30106'90e92cc3_u128}, + {Sign::POS, -132, 0xeeecee6d'7298b8a3'075da5d7'456bdcde_u128}, + {Sign::POS, -132, 0xb2b78fb1'fcfdc273'1d1ac11c'e29c16f1_u128}, + {Sign::POS, -132, 0x90d21722'148fdaf5'0d566a01'0bb8784b_u128}, + {Sign::POS, -133, 0xf7681c54'9771ebb6'17686858'eb5e1caf_u128}, + {Sign::POS, -133, 0xdb5e45c0'52ec0c1c'ff28765e'd4c44bfb_u128}, + {Sign::POS, -133, 0xc7ff0dd7'a34ee29b'7cb689af'fe887bf5_u128}, + {Sign::POS, -133, 0xba4e6f37'a98a3e3f'f1175427'20f45c82_u128}, + {Sign::POS, -133, 0xb08f6e11'688e4174'b3d48abe'c0a6d5cd_u128}, + {Sign::POS, -133, 0xa9af6a33'14aabe45'26da1218'05bbb52e_u128}, + {Sign::POS, -133, 0xa4fd22fa'1b4f0d7f'1456af96'cbd0cde6_u128}, + {Sign::POS, -133, 0xa20229b4'7e9c2e39'22c49987'66a05c5a_u128}, + {Sign::POS, -133, 0xa0775ca8'4409c735'351d01f1'34467927_u128}, + {Sign::POS, -133, 0xa010d2d9'08428a53'53603f20'66c8b8ba_u128}, + }, + { + {Sign::POS, -127, 0x8274cd6a'f25e642d'0b1a02fb'03f53f3e_u128}, + {Sign::POS, -130, 0xbd49d2c8'b9005b2a'ee795b17'92181a48_u128}, + {Sign::POS, -131, 0xbcc0ac23'98e00fd7'c40811f5'486aca6a_u128}, + {Sign::POS, -132, 0xfa756493'b381b917'6cdea268'e44dd2fd_u128}, + {Sign::POS, -132, 0xbe7fce1e'462b43c6'0537d6f7'138c87ac_u128}, + {Sign::POS, -132, 0x9d00958b'edc83095'b4cc907c'a92c30f1_u128}, + {Sign::POS, -132, 0x886a2440'ed93d825'333c19c2'6de36d73_u128}, + {Sign::POS, -133, 0xf616ebc0'4f576462'd9312544'e8fbe0fd_u128}, + {Sign::POS, -133, 0xe43f4c9d'ebb5d685'00903a00'7bd6ad39_u128}, + {Sign::POS, -133, 0xd8516eab'32337672'569b4e19'a44e795c_u128}, + {Sign::POS, -133, 0xd091fa04'954666ee'cc4da283'82e977c0_u128}, + {Sign::POS, -133, 0xcbf13442'c4c0f859'0449c2c4'2fc046fe_u128}, + {Sign::POS, -133, 0xc9c1d1b4'dea4c76c'd101e562'dc3af77f_u128}, + {Sign::POS, -133, 0xc9924d2a'b8ec37d9'80af1780'0fb63e4e_u128}, + {Sign::POS, -133, 0xcb24b252'1ff37e4a'41f35260'2b9ace95_u128}, + {Sign::POS, -133, 0xce2d87ac'194a6304'1658ed0e'4cdb8161_u128}, + }, + { + {Sign::POS, -127, 0x82d4310f'f58b570d'266275fc'1d085c87_u128}, + {Sign::POS, -130, 0xc048c361'72bee7b0'8d2ca7e5'afe4f335_u128}, + {Sign::POS, -131, 0xc2c3ecca'216e290e'b99c5c53'5d48595a_u128}, + {Sign::POS, -131, 0x83611e8f'3adf2217'be3c342a'dfb1c562_u128}, + {Sign::POS, -132, 0xcb481202'8b0ba9aa'e586f73d'faea68e4_u128}, + {Sign::POS, -132, 0xaa727c9a'4caba65d'c8dc13ef'8bed52e4_u128}, + {Sign::POS, -132, 0x96b05462'efac126e'db6871d0'0be1eff9_u128}, + {Sign::POS, -132, 0x8a4f8752'9b3c9232'63eb1596'a2c83eb4_u128}, + {Sign::POS, -132, 0x828be6f4'1b14e6e6'8efc1012'2afe425a_u128}, + {Sign::POS, -133, 0xfbd2f055'9d699ea9'b572008e'1fb08088_u128}, + {Sign::POS, -133, 0xf71b3c70'dc4610e6'bc1e581c'817b88bd_u128}, + {Sign::POS, -133, 0xf5e8ebf6'3b0aef3f'97ba4c8f'e49b6f0a_u128}, + {Sign::POS, -133, 0xf7986238'1eb8bd7a'73577ed0'c05e4abf_u128}, + {Sign::POS, -133, 0xfbc3832a'a903cd65'a46ee523'f342c621_u128}, + {Sign::POS, -132, 0x811ea5f3'7409245e'1777fdd1'59b29f80_u128}, + {Sign::POS, -132, 0x85619588'b83c90ef'67740d6a'd2f372a8_u128}, + }, + { + {Sign::POS, -127, 0x83351a49'8764656f'e1774024'a5e751a6_u128}, + {Sign::POS, -130, 0xc36057da'23d39c2b'336474e0'3a893914_u128}, + {Sign::POS, -131, 0xc913714c'a46cc0bf'3bdd68ba'53a309d4_u128}, + {Sign::POS, -131, 0x89f2254d'f1469d60'e1324bac'95db6742_u128}, + {Sign::POS, -132, 0xd92b27f6'38df6911'5842365c'c120cc63_u128}, + {Sign::POS, -132, 0xb94ff079'7848d391'486efffa'a6fbc37f_u128}, + {Sign::POS, -132, 0xa6c03919'862e8437'70f86a73'43da3a6e_u128}, + {Sign::POS, -132, 0x9bcb70c9'a378e97f'a59f25f3'ba202e33_u128}, + {Sign::POS, -132, 0x95b103b0'62aa9f64'ee2d6146'76020bc5_u128}, + {Sign::POS, -132, 0x92fa4a1c'7d7fd161'8f25aa4e'f65ca52f_u128}, + {Sign::POS, -132, 0x92d387a2'c5dd771d'4015ca29'e3eda1d9_u128}, + {Sign::POS, -132, 0x94c13c5c'997615c3'8a2f63c8'c314226f_u128}, + {Sign::POS, -132, 0x987b8c8f'5e9e7a5f'e8497909'd60d1194_u128}, + {Sign::POS, -132, 0x9ddb0978'da99e6ad'83d5eca2'9d079ef7_u128}, + {Sign::POS, -132, 0xa4d9aeee'4b512ed4'5ec95cd1'37ce3f22_u128}, + {Sign::POS, -132, 0xad602af3'1e14d681'8a267da2'57c030de_u128}, + }, + { + {Sign::POS, -127, 0x839795b7'8f3005a4'689f57cc'd201f7dc_u128}, + {Sign::POS, -130, 0xc691cb89'3d75d3d5'a1892f2a'bf54ec45_u128}, + {Sign::POS, -131, 0xcfb46fc4'6d28c32c'9ae5ad3d'a7749dc8_u128}, + {Sign::POS, -131, 0x90f71352'c806c830'20edb8b2'7594386b_u128}, + {Sign::POS, -132, 0xe8473840'd511dc77'd63def5d'7f4de9c0_u128}, + {Sign::POS, -132, 0xc9c6eb30'aaf2b63d'ec20f671'8689534a_u128}, + {Sign::POS, -132, 0xb8dcfa84'eb6cab93'3023ddcc'b8f68a2f_u128}, + {Sign::POS, -132, 0xafde4094'c1a14390'9609a3ea'847225a9_u128}, + {Sign::POS, -132, 0xac1254e7'5852a836'b2aca5e5'0cfc484f_u128}, + {Sign::POS, -132, 0xac0d3ffa'd6171016'b1a12557'858663c1_u128}, + {Sign::POS, -132, 0xaf0877f9'0ca5c52f'fc54b5af'b5cbc350_u128}, + {Sign::POS, -132, 0xb498574f'af349a2b'f391ff83'b3570919_u128}, + {Sign::POS, -132, 0xbc87c7bb'34182440'280647cd'976affb0_u128}, + {Sign::POS, -132, 0xc6c5688f'58a42593'4569de36'0855c393_u128}, + {Sign::POS, -132, 0xd368b088'5bb9496a'dd7c92df'8798aaf7_u128}, + {Sign::POS, -132, 0xe272168a'c8dbe668'381542bf'fc24c266_u128}, + }, + { + {Sign::POS, -127, 0x83fbb09c'fbb0ebf4'208c9037'70373f79_u128}, + {Sign::POS, -130, 0xc9de6f84'8e652b0b'3b2a2bb9'f7ce3de8_u128}, + {Sign::POS, -131, 0xd6ac93c7'6e215233'f184fdcc'e5872970_u128}, + {Sign::POS, -131, 0x987a35b9'87c02522'1927dee9'70fc6b18_u128}, + {Sign::POS, -132, 0xf8be450d'266409a9'2e534ffd'905f4424_u128}, + {Sign::POS, -132, 0xdc0c36d7'34415e3b'c5121c4d'4e28c17d_u128}, + {Sign::POS, -132, 0xcd551b98'81d982a8'1399d9ba'ddf55821_u128}, + {Sign::POS, -132, 0xc6f91e3f'428d6be3'646f3147'20445145_u128}, + {Sign::POS, -132, 0xc64f100c'85e1e8f1'6f501d1e'2155f872_u128}, + {Sign::POS, -132, 0xc9fe25ae'295f1f24'5924cf9a'036a31f2_u128}, + {Sign::POS, -132, 0xd157410e'fcc10fbb'fceb318a'b4990bd7_u128}, + {Sign::POS, -132, 0xdc0aeb56'ca679f92'3b3c44d8'99b1add7_u128}, + {Sign::POS, -132, 0xea05b383'bc339550'e5c5c34b'bfa416a1_u128}, + {Sign::POS, -132, 0xfb5e3897'5a5c8f62'280a90dc'9ebe9107_u128}, + {Sign::POS, -131, 0x88301d81'b38f225d'2226ab7e'df342d90_u128}, + {Sign::POS, -131, 0x949e3465'e4a8aef7'46311182'5fc3fde8_u128}, + }, + { + {Sign::POS, -127, 0x846178eb'1c7260da'3e0aca9a'51e68d84_u128}, + {Sign::POS, -130, 0xcd47ac90'3c311c2b'98dd7493'4656d210_u128}, + {Sign::POS, -131, 0xde020b2d'abd5628c'b88634e5'73f312fc_u128}, + {Sign::POS, -131, 0xa086fafa'c220fb73'9939cae3'2d69683f_u128}, + {Sign::POS, -131, 0x855b5efa'f6963d73'e4664cb1'd43f03a9_u128}, + {Sign::POS, -132, 0xf05c9774'fe0de25c'ccf1c1df'd2ed9941_u128}, + {Sign::POS, -132, 0xe484a941'19639229'f06ae955'f8edc7d1_u128}, + {Sign::POS, -132, 0xe1a32bb2'52ca122c'bf2f0904'cfc476cb_u128}, + {Sign::POS, -132, 0xe528e091'7bb8a01a'9218ce3e'1e85af60_u128}, + {Sign::POS, -132, 0xeddd556a'faa2d46f'e91c61fa'adf12aec_u128}, + {Sign::POS, -132, 0xfb390fa3'15e9d55f'5683c0c4'c7719f81_u128}, + {Sign::POS, -131, 0x868e5fa4'15597c8f'7c42a262'8f2d6332_u128}, + {Sign::POS, -131, 0x91d79767'a3d037f9'cd84ead5'c0714310_u128}, + {Sign::POS, -131, 0x9fa6a035'915bc052'377a8abb'faf4e3c6_u128}, + {Sign::POS, -131, 0xb04edefd'6ac2a93e'ec33e6f6'3d53e7c2_u128}, + {Sign::POS, -131, 0xc416980d'dc5c186b'7bdcded6'97ea5844_u128}, + }, + { + {Sign::POS, -127, 0x84c8fd4d'ffdf9fc6'bdd7ebca'88183d7b_u128}, + {Sign::POS, -130, 0xd0cf0544'11dbf845'cb6eeae5'bc980e2f_u128}, + {Sign::POS, -131, 0xe5bb9480'7ce0eaca'74300a46'8398e944_u128}, + {Sign::POS, -131, 0xa92a18f8'd611860b'5f2ef8c6'8e8ca002_u128}, + {Sign::POS, -131, 0x8f2e1684'17eb4e6c'1ec44b9b'e4b1c3e5_u128}, + {Sign::POS, -131, 0x837f1764'0ee8f416'8694b4a1'c647af0c_u128}, + {Sign::POS, -132, 0xfed7e2a9'05a5190e'b7d70a61'a24ad801_u128}, + {Sign::POS, -131, 0x803f29ff'dc6fd2bc'3c3c4b50'a9dc860c_u128}, + {Sign::POS, -131, 0x84c61e09'b8aa35e4'96239f9c'b1d00b3c_u128}, + {Sign::POS, -131, 0x8c7ed311'f77980d6'842ddf90'6a68a0bc_u128}, + {Sign::POS, -131, 0x9746077b'd397c2d1'038a4744'a76f5fb5_u128}, + {Sign::POS, -131, 0xa5341277'c4185ace'54f26328'322158e8_u128}, + {Sign::POS, -131, 0xb68d78f5'0972f6de'9189aa23'd3ecefc2_u128}, + {Sign::POS, -131, 0xcbbcefc2'15bade4e'f1d36947'c8b6e460_u128}, + {Sign::POS, -131, 0xe564a459'c851390d'd45a4748'f29f182b_u128}, + {Sign::POS, -130, 0x820ea28b'c89662c3'2a64ccdc'efb2b259_u128}, + }, + { + {Sign::POS, -127, 0x85324d39'f30f9174'ac0d817e'9c744b0b_u128}, + {Sign::POS, -130, 0xd476186e'49c47f3a'a71f8886'7f9f21c4_u128}, + {Sign::POS, -131, 0xede08f54'a830e87b'07881700'65e57b6c_u128}, + {Sign::POS, -131, 0xb271b8eb'309963ee'89187c73'0b92f7d5_u128}, + {Sign::POS, -131, 0x99f0011d'95d3a6dd'282bd00a'db808151_u128}, + {Sign::POS, -131, 0x9021134e'02b479e7'3aabf9bb'b7ab6cf3_u128}, + {Sign::POS, -131, 0x8e673bf2'f11db54a'909c4c72'6389499f_u128}, + {Sign::POS, -131, 0x9226a371'88dd55f7'bfe21777'4a42a7ae_u128}, + {Sign::POS, -131, 0x9a4d78fc'9df79d9a'44609c02'a625808a_u128}, + {Sign::POS, -131, 0xa68335fb'41d2d91c'e7bbd2a3'31a1d17b_u128}, + {Sign::POS, -131, 0xb6d89c39'28d0cb26'809d4df6'e55cba1a_u128}, + {Sign::POS, -131, 0xcba71468'9177fc2d'7f23df2f'37226488_u128}, + {Sign::POS, -131, 0xe5846de8'44833ae9'34416c87'0315eb9e_u128}, + {Sign::POS, -130, 0x82a07032'64e6226b'200d94a1'66fc7951_u128}, + {Sign::POS, -130, 0x9602695c'b6fa8886'68ca0cba'b59ea683_u128}, + {Sign::POS, -130, 0xad7d185a'ab3d14dd'd908a7b1'c57352bb_u128}, + }, + { + {Sign::POS, -127, 0x859d78fa'4405d8fa'287dbc69'95d0975e_u128}, + {Sign::POS, -130, 0xd83ea3bc'131d6baa'67c51d88'4c4dae01_u128}, + {Sign::POS, -131, 0xf6790edb'df07342b'aad85870'167af128_u128}, + {Sign::POS, -131, 0xbc6daa33'12be0f85'bc7fa753'52b10a83_u128}, + {Sign::POS, -131, 0xa5bd41bc'9c986b13'1af2542e'92aacb59_u128}, + {Sign::POS, -131, 0x9e4358bc'24e04364'b4539b76'e444b790_u128}, + {Sign::POS, -131, 0x9f7fc21b'dca1f2b5'f3f6d44b'c5a37626_u128}, + {Sign::POS, -131, 0xa6fd793c'0b9c44c1'30a518cc'66b5e511_u128}, + {Sign::POS, -131, 0xb3dccfac'cd1592b3'bcd6b7c0'9749993d_u128}, + {Sign::POS, -131, 0xc6056c3a'4a5f329a'48f1429d'27f930fc_u128}, + {Sign::POS, -131, 0xddd9e529'858a4502'6e7f3d1c'1e7dcb89_u128}, + {Sign::POS, -131, 0xfc1bccee'dc8d2567'1721c468'6f7f53ec_u128}, + {Sign::POS, -130, 0x90f2bb21'5cdbe7e2'f9ef8e12'059cc66a_u128}, + {Sign::POS, -130, 0xa857d5df'5b4da940'15ce4e95'7201fc79_u128}, + {Sign::POS, -130, 0xc54119c0'10c02bf4'd87ece17'1ef85c5f_u128}, + {Sign::POS, -130, 0xe8c50ebc'880356de'2c1f4c42'9ee9748f_u128}, + }, + { + {Sign::POS, -127, 0x860a91c1'6b9b2c23'2dd99707'ab3d688b_u128}, + {Sign::POS, -130, 0xdc2a86b1'5fdb645d'ea2781dd'25555f49_u128}, + {Sign::POS, -131, 0xff8def07'd1e514d7'b2e8ebb6'5c3afe5e_u128}, + {Sign::POS, -131, 0xc72f9d5b'4fb559e3'20db92e3'a5ae3f73_u128}, + {Sign::POS, -131, 0xb2b5f45b'1d26f4dd'0b210309'fb68914f_u128}, + {Sign::POS, -131, 0xae1cbaae'c7b55465'4da858f5'47e62a37_u128}, + {Sign::POS, -131, 0xb30f3998'10202a0d'a52ec085'a7d63289_u128}, + {Sign::POS, -131, 0xbf51f27f'b7aff89d'dc24e2aa'208d2054_u128}, + {Sign::POS, -131, 0xd250735e'87d0b527'6f99bcc9'bd6fc717_u128}, + {Sign::POS, -131, 0xec543ec2'bddb2efb'36d9ce81'a7c84336_u128}, + {Sign::POS, -130, 0x871f73e3'298ef45c'eed83998'2bc731b9_u128}, + {Sign::POS, -130, 0x9cbb5447'af8574f1'21fa4cda'93d82b7e_u128}, + {Sign::POS, -130, 0xb7f5a6c0'430a347f'11b22cde'91de0885_u128}, + {Sign::POS, -130, 0xda153cc4'14abdb96'840df7c2'3299fec0_u128}, + {Sign::POS, -129, 0x826c129b'3e4a2612'b2cd11f1'4d2ba60c_u128}, + {Sign::POS, -129, 0x9d19c289'fc0e8aa4'f351418b'b760ce90_u128}, + }, +}; + +constexpr Float128 PI_OVER_TWO_F128 = { + Sign::POS, -127, 0xc90fdaa2'2168c234'c4c6628b'80dc1cd1_u128}; + +constexpr Float128 PI_F128 = {Sign::POS, -126, + 0xc90fdaa2'2168c234'c4c6628b'80dc1cd1_u128}; + +LIBC_INLINE Float128 asin_eval(const Float128 &u, unsigned idx) { + return fputil::polyeval(u, ASIN_COEFFS_F128[idx][0], ASIN_COEFFS_F128[idx][1], + ASIN_COEFFS_F128[idx][2], ASIN_COEFFS_F128[idx][3], + ASIN_COEFFS_F128[idx][4], ASIN_COEFFS_F128[idx][5], + ASIN_COEFFS_F128[idx][6], ASIN_COEFFS_F128[idx][7], + ASIN_COEFFS_F128[idx][8], ASIN_COEFFS_F128[idx][9], + ASIN_COEFFS_F128[idx][10], ASIN_COEFFS_F128[idx][11], + ASIN_COEFFS_F128[idx][12], ASIN_COEFFS_F128[idx][13], + ASIN_COEFFS_F128[idx][14], ASIN_COEFFS_F128[idx][15]); +} + +#endif // LIBC_MATH_HAS_SKIP_ACCURATE_PASS + +} // anonymous namespace + +} // namespace LIBC_NAMESPACE_DECL + +#endif // LLVM_LIBC_SRC_MATH_GENERIC_ASIN_UTILS_H diff --git a/system/lib/llvm-libc/src/math/generic/asinf.cpp b/system/lib/llvm-libc/src/math/generic/asinf.cpp index 3a89def8f6e0c..12383bf6dacae 100644 --- a/system/lib/llvm-libc/src/math/generic/asinf.cpp +++ b/system/lib/llvm-libc/src/math/generic/asinf.cpp @@ -21,6 +21,7 @@ namespace LIBC_NAMESPACE_DECL { +#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS static constexpr size_t N_EXCEPTS = 2; // Exceptional values when |x| <= 0.5 @@ -40,6 +41,7 @@ static constexpr fputil::ExceptValues ASINF_EXCEPTS_HI = {{ // x = 0x1.ee836cp-1, asinf(x) = 0x1.4f0654p0 (RZ) {0x3f7741b6, 0x3fa7832a, 1, 0, 0}, }}; +#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS LLVM_LIBC_FUNCTION(float, asinf, (float x)) { using FPBits = typename fputil::FPBits; @@ -74,18 +76,20 @@ LLVM_LIBC_FUNCTION(float, asinf, (float x)) { // |x| < 2^-125. For targets without FMA instructions, we simply use // double for intermediate results as it is more efficient than using an // emulated version of FMA. -#if defined(LIBC_TARGET_CPU_HAS_FMA) +#if defined(LIBC_TARGET_CPU_HAS_FMA_FLOAT) return fputil::multiply_add(x, 0x1.0p-25f, x); #else double xd = static_cast(x); return static_cast(fputil::multiply_add(xd, 0x1.0p-25, xd)); -#endif // LIBC_TARGET_CPU_HAS_FMA +#endif // LIBC_TARGET_CPU_HAS_FMA_FLOAT } +#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS // Check for exceptional values if (auto r = ASINF_EXCEPTS_LO.lookup_odd(x_abs, x_sign); LIBC_UNLIKELY(r.has_value())) return r.value(); +#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS // For |x| <= 0.5, we approximate asinf(x) by: // asin(x) = x * P(x^2) @@ -104,17 +108,25 @@ LLVM_LIBC_FUNCTION(float, asinf, (float x)) { // |x| > 1, return NaNs. if (LIBC_UNLIKELY(x_abs > 0x3f80'0000U)) { + if (xbits.is_signaling_nan()) { + fputil::raise_except_if_required(FE_INVALID); + return FPBits::quiet_nan().get_val(); + } + if (x_abs <= 0x7f80'0000U) { fputil::set_errno_if_required(EDOM); fputil::raise_except_if_required(FE_INVALID); } + return FPBits::quiet_nan().get_val(); } +#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS // Check for exceptional values if (auto r = ASINF_EXCEPTS_HI.lookup_odd(x_abs, x_sign); LIBC_UNLIKELY(r.has_value())) return r.value(); +#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS // When |x| > 0.5, we perform range reduction as follow: // diff --git a/system/lib/llvm-libc/src/math/generic/asinhf.cpp b/system/lib/llvm-libc/src/math/generic/asinhf.cpp index 1d68ac9ea13bc..0bb7065eb1cfe 100644 --- a/system/lib/llvm-libc/src/math/generic/asinhf.cpp +++ b/system/lib/llvm-libc/src/math/generic/asinhf.cpp @@ -36,7 +36,7 @@ LLVM_LIBC_FUNCTION(float, asinhf, (float x)) { double x_d = x; double x_sq = x_d * x_d; // Generated by Sollya with: - // > P = fpminimax(asinh(x)/x, [|0, 2, 4, 6, 8, 10, 12, 14, 16], [|D...|], + // > P = fpminimax(asinh(x)/x, [|0, 2, 4, 6, 8, 10, 12, 14, 16|], [|D...|], // [0, 2^-2]); double p = fputil::polyeval( x_sq, 0.0, -0x1.555555555551ep-3, 0x1.3333333325495p-4, @@ -49,6 +49,7 @@ LLVM_LIBC_FUNCTION(float, asinhf, (float x)) { double x_sign = SIGN[x_u >> 31]; double x_d = x; +#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS // Helper functions to set results for exceptional cases. auto round_result_slightly_down = [x_sign](float r) -> float { return fputil::multiply_add(static_cast(x_sign), r, @@ -60,8 +61,14 @@ LLVM_LIBC_FUNCTION(float, asinhf, (float x)) { }; if (LIBC_UNLIKELY(x_abs >= 0x4bdd'65a5U)) { - if (LIBC_UNLIKELY(xbits.is_inf_or_nan())) + if (LIBC_UNLIKELY(xbits.is_inf_or_nan())) { + if (xbits.is_signaling_nan()) { + fputil::raise_except_if_required(FE_INVALID); + return FPBits_t::quiet_nan().get_val(); + } + return x; + } // Exceptional cases when x > 2^24. switch (x_abs) { @@ -95,6 +102,10 @@ LLVM_LIBC_FUNCTION(float, asinhf, (float x)) { return round_result_slightly_down(0x1.e1b92p3f); } } +#else + if (LIBC_UNLIKELY(xbits.is_inf_or_nan())) + return x; +#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS // asinh(x) = log(x + sqrt(x^2 + 1)) return static_cast( diff --git a/system/lib/llvm-libc/src/math/generic/atan.cpp b/system/lib/llvm-libc/src/math/generic/atan.cpp new file mode 100644 index 0000000000000..cbca60536a3fd --- /dev/null +++ b/system/lib/llvm-libc/src/math/generic/atan.cpp @@ -0,0 +1,179 @@ +//===-- Double-precision atan function ------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/atan.h" +#include "atan_utils.h" +#include "src/__support/FPUtil/FEnvImpl.h" +#include "src/__support/FPUtil/FPBits.h" +#include "src/__support/FPUtil/double_double.h" +#include "src/__support/FPUtil/multiply_add.h" +#include "src/__support/FPUtil/nearest_integer.h" +#include "src/__support/macros/config.h" +#include "src/__support/macros/optimization.h" // LIBC_UNLIKELY + +namespace LIBC_NAMESPACE_DECL { + +// To compute atan(x), we divided it into the following cases: +// * |x| < 2^-26: +// Since |x| > atan(|x|) > |x| - |x|^3/3, and |x|^3/3 < ulp(x)/2, we simply +// return atan(x) = x - sign(x) * epsilon. +// * 2^-26 <= |x| < 1: +// We perform range reduction mod 2^-6 = 1/64 as follow: +// Let k = 2^(-6) * round(|x| * 2^6), then +// atan(x) = sign(x) * atan(|x|) +// = sign(x) * (atan(k) + atan((|x| - k) / (1 + |x|*k)). +// We store atan(k) in a look up table, and perform intermediate steps in +// double-double. +// * 1 < |x| < 2^53: +// First we perform the transformation y = 1/|x|: +// atan(x) = sign(x) * (pi/2 - atan(1/|x|)) +// = sign(x) * (pi/2 - atan(y)). +// Then we compute atan(y) using range reduction mod 2^-6 = 1/64 as the +// previous case: +// Let k = 2^(-6) * round(y * 2^6), then +// atan(y) = atan(k) + atan((y - k) / (1 + y*k)) +// = atan(k) + atan((1/|x| - k) / (1 + k/|x|) +// = atan(k) + atan((1 - k*|x|) / (|x| + k)). +// * |x| >= 2^53: +// Using the reciprocal transformation: +// atan(x) = sign(x) * (pi/2 - atan(1/|x|)). +// We have that: +// atan(1/|x|) <= 1/|x| <= 2^-53, +// which is smaller than ulp(pi/2) / 2. +// So we can return: +// atan(x) = sign(x) * (pi/2 - epsilon) + +LLVM_LIBC_FUNCTION(double, atan, (double x)) { + using FPBits = fputil::FPBits; + + constexpr double IS_NEG[2] = {1.0, -1.0}; + constexpr DoubleDouble PI_OVER_2 = {0x1.1a62633145c07p-54, + 0x1.921fb54442d18p0}; + constexpr DoubleDouble MPI_OVER_2 = {-0x1.1a62633145c07p-54, + -0x1.921fb54442d18p0}; + + FPBits xbits(x); + bool x_sign = xbits.is_neg(); + xbits = xbits.abs(); + uint64_t x_abs = xbits.uintval(); + int x_exp = + static_cast(x_abs >> FPBits::FRACTION_LEN) - FPBits::EXP_BIAS; + + // |x| < 1. + if (x_exp < 0) { + if (LIBC_UNLIKELY(x_exp < -26)) { +#ifdef LIBC_MATH_HAS_SKIP_ACCURATE_PASS + return x; +#else + if (x == 0.0) + return x; + // |x| < 2^-26 + return fputil::multiply_add(-0x1.0p-54, x, x); +#endif // LIBC_MATH_HAS_SKIP_ACCURATE_PASS + } + + double x_d = xbits.get_val(); + // k = 2^-6 * round(2^6 * |x|) + double k = fputil::nearest_integer(0x1.0p6 * x_d); + unsigned idx = static_cast(k); + k *= 0x1.0p-6; + + // numerator = |x| - k + DoubleDouble num, den; + num.lo = 0.0; + num.hi = x_d - k; + + // denominator = 1 - k * |x| + den.hi = fputil::multiply_add(x_d, k, 1.0); + DoubleDouble prod = fputil::exact_mult(x_d, k); + // Using Dekker's 2SUM algorithm to compute the lower part. + den.lo = ((1.0 - den.hi) + prod.hi) + prod.lo; + + // x_r = (|x| - k) / (1 + k * |x|) + DoubleDouble x_r = fputil::div(num, den); + + // Approximating atan(x_r) using Taylor polynomial. + DoubleDouble p = atan_eval(x_r); + + // atan(x) = sign(x) * (atan(k) + atan(x_r)) + // = sign(x) * (atan(k) + atan( (|x| - k) / (1 + k * |x|) )) +#ifdef LIBC_MATH_HAS_SKIP_ACCURATE_PASS + return IS_NEG[x_sign] * (ATAN_I[idx].hi + (p.hi + (p.lo + ATAN_I[idx].lo))); +#else + + DoubleDouble c0 = fputil::exact_add(ATAN_I[idx].hi, p.hi); + double c1 = c0.lo + (ATAN_I[idx].lo + p.lo); + double r = IS_NEG[x_sign] * (c0.hi + c1); + + return r; +#endif // LIBC_MATH_HAS_SKIP_ACCURATE_PASS + } + + // |x| >= 2^53 or x is NaN. + if (LIBC_UNLIKELY(x_exp >= 53)) { + // x is nan + if (xbits.is_nan()) { + if (xbits.is_signaling_nan()) { + fputil::raise_except_if_required(FE_INVALID); + return FPBits::quiet_nan().get_val(); + } + return x; + } + // |x| >= 2^53 + // atan(x) ~ sign(x) * pi/2. + if (x_exp >= 53) +#ifdef LIBC_MATH_HAS_SKIP_ACCURATE_PASS + return IS_NEG[x_sign] * PI_OVER_2.hi; +#else + return fputil::multiply_add(IS_NEG[x_sign], PI_OVER_2.hi, + IS_NEG[x_sign] * PI_OVER_2.lo); +#endif // LIBC_MATH_HAS_SKIP_ACCURATE_PASS + } + + double x_d = xbits.get_val(); + double y = 1.0 / x_d; + + // k = 2^-6 * round(2^6 / |x|) + double k = fputil::nearest_integer(0x1.0p6 * y); + unsigned idx = static_cast(k); + k *= 0x1.0p-6; + + // denominator = |x| + k + DoubleDouble den = fputil::exact_add(x_d, k); + // numerator = 1 - k * |x| + DoubleDouble num; + num.hi = fputil::multiply_add(-x_d, k, 1.0); + DoubleDouble prod = fputil::exact_mult(x_d, k); + // Using Dekker's 2SUM algorithm to compute the lower part. + num.lo = ((1.0 - num.hi) - prod.hi) - prod.lo; + + // x_r = (1/|x| - k) / (1 - k/|x|) + // = (1 - k * |x|) / (|x| - k) + DoubleDouble x_r = fputil::div(num, den); + + // Approximating atan(x_r) using Taylor polynomial. + DoubleDouble p = atan_eval(x_r); + + // atan(x) = sign(x) * (pi/2 - atan(1/|x|)) + // = sign(x) * (pi/2 - atan(k) - atan(x_r)) + // = (-sign(x)) * (-pi/2 + atan(k) + atan((1 - k*|x|)/(|x| - k))) +#ifdef LIBC_MATH_HAS_SKIP_ACCURATE_PASS + double lo_part = p.lo + ATAN_I[idx].lo + MPI_OVER_2.lo; + return IS_NEG[!x_sign] * (MPI_OVER_2.hi + ATAN_I[idx].hi + (p.hi + lo_part)); +#else + DoubleDouble c0 = fputil::exact_add(MPI_OVER_2.hi, ATAN_I[idx].hi); + DoubleDouble c1 = fputil::exact_add(c0.hi, p.hi); + double c2 = c1.lo + (c0.lo + p.lo) + (ATAN_I[idx].lo + MPI_OVER_2.lo); + + double r = IS_NEG[!x_sign] * (c1.hi + c2); + + return r; +#endif +} + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/generic/atan2.cpp b/system/lib/llvm-libc/src/math/generic/atan2.cpp index 1b16e15d29d0b..aa770de33fb1f 100644 --- a/system/lib/llvm-libc/src/math/generic/atan2.cpp +++ b/system/lib/llvm-libc/src/math/generic/atan2.cpp @@ -7,133 +7,17 @@ //===----------------------------------------------------------------------===// #include "src/math/atan2.h" -#include "inv_trigf_utils.h" +#include "atan_utils.h" +#include "src/__support/FPUtil/FEnvImpl.h" #include "src/__support/FPUtil/FPBits.h" -#include "src/__support/FPUtil/PolyEval.h" #include "src/__support/FPUtil/double_double.h" #include "src/__support/FPUtil/multiply_add.h" #include "src/__support/FPUtil/nearest_integer.h" -#include "src/__support/FPUtil/rounding_mode.h" #include "src/__support/macros/config.h" #include "src/__support/macros/optimization.h" // LIBC_UNLIKELY namespace LIBC_NAMESPACE_DECL { -namespace { - -using DoubleDouble = fputil::DoubleDouble; - -// atan(i/64) with i = 0..64, generated by Sollya with: -// > for i from 0 to 64 do { -// a = round(atan(i/64), D, RN); -// b = round(atan(i/64) - a, D, RN); -// print("{", b, ",", a, "},"); -// }; -constexpr fputil::DoubleDouble ATAN_I[65] = { - {0.0, 0.0}, - {-0x1.220c39d4dff5p-61, 0x1.fff555bbb729bp-7}, - {-0x1.5ec431444912cp-60, 0x1.ffd55bba97625p-6}, - {-0x1.86ef8f794f105p-63, 0x1.7fb818430da2ap-5}, - {-0x1.c934d86d23f1dp-60, 0x1.ff55bb72cfdeap-5}, - {0x1.ac4ce285df847p-58, 0x1.3f59f0e7c559dp-4}, - {-0x1.cfb654c0c3d98p-58, 0x1.7ee182602f10fp-4}, - {0x1.f7b8f29a05987p-58, 0x1.be39ebe6f07c3p-4}, - {-0x1.cd37686760c17p-59, 0x1.fd5ba9aac2f6ep-4}, - {-0x1.b485914dacf8cp-59, 0x1.1e1fafb043727p-3}, - {0x1.61a3b0ce9281bp-57, 0x1.3d6eee8c6626cp-3}, - {-0x1.054ab2c010f3dp-58, 0x1.5c9811e3ec26ap-3}, - {0x1.347b0b4f881cap-58, 0x1.7b97b4bce5b02p-3}, - {0x1.cf601e7b4348ep-59, 0x1.9a6a8e96c8626p-3}, - {0x1.17b10d2e0e5abp-61, 0x1.b90d7529260a2p-3}, - {0x1.c648d1534597ep-57, 0x1.d77d5df205736p-3}, - {0x1.8ab6e3cf7afbdp-57, 0x1.f5b75f92c80ddp-3}, - {0x1.62e47390cb865p-56, 0x1.09dc597d86362p-2}, - {0x1.30ca4748b1bf9p-57, 0x1.18bf5a30bf178p-2}, - {-0x1.077cdd36dfc81p-56, 0x1.278372057ef46p-2}, - {-0x1.963a544b672d8p-57, 0x1.362773707ebccp-2}, - {-0x1.5d5e43c55b3bap-56, 0x1.44aa436c2af0ap-2}, - {-0x1.2566480884082p-57, 0x1.530ad9951cd4ap-2}, - {-0x1.a725715711fp-56, 0x1.614840309cfe2p-2}, - {-0x1.c63aae6f6e918p-56, 0x1.6f61941e4def1p-2}, - {0x1.69c885c2b249ap-56, 0x1.7d5604b63b3f7p-2}, - {0x1.b6d0ba3748fa8p-56, 0x1.8b24d394a1b25p-2}, - {0x1.9e6c988fd0a77p-56, 0x1.98cd5454d6b18p-2}, - {-0x1.24dec1b50b7ffp-56, 0x1.a64eec3cc23fdp-2}, - {0x1.ae187b1ca504p-56, 0x1.b3a911da65c6cp-2}, - {-0x1.cc1ce70934c34p-56, 0x1.c0db4c94ec9fp-2}, - {-0x1.a2cfa4418f1adp-56, 0x1.cde53432c1351p-2}, - {0x1.a2b7f222f65e2p-56, 0x1.dac670561bb4fp-2}, - {0x1.0e53dc1bf3435p-56, 0x1.e77eb7f175a34p-2}, - {-0x1.a3992dc382a23p-57, 0x1.f40dd0b541418p-2}, - {-0x1.b32c949c9d593p-55, 0x1.0039c73c1a40cp-1}, - {-0x1.d5b495f6349e6p-56, 0x1.0657e94db30dp-1}, - {0x1.974fa13b5404fp-58, 0x1.0c6145b5b43dap-1}, - {-0x1.2bdaee1c0ee35p-58, 0x1.1255d9bfbd2a9p-1}, - {0x1.c621cec00c301p-55, 0x1.1835a88be7c13p-1}, - {-0x1.928df287a668fp-58, 0x1.1e00babdefeb4p-1}, - {0x1.c421c9f38224ep-57, 0x1.23b71e2cc9e6ap-1}, - {-0x1.09e73b0c6c087p-56, 0x1.2958e59308e31p-1}, - {0x1.c5d5e9ff0cf8dp-55, 0x1.2ee628406cbcap-1}, - {0x1.1021137c71102p-55, 0x1.345f01cce37bbp-1}, - {-0x1.2304331d8bf46p-55, 0x1.39c391cd4171ap-1}, - {0x1.ecf8b492644fp-56, 0x1.3f13fb89e96f4p-1}, - {-0x1.f76d0163f79c8p-56, 0x1.445065b795b56p-1}, - {0x1.2419a87f2a458p-56, 0x1.4978fa3269ee1p-1}, - {0x1.4a33dbeb3796cp-55, 0x1.4e8de5bb6ec04p-1}, - {-0x1.1bb74abda520cp-55, 0x1.538f57b89061fp-1}, - {-0x1.5e5c9d8c5a95p-56, 0x1.587d81f732fbbp-1}, - {0x1.0028e4bc5e7cap-57, 0x1.5d58987169b18p-1}, - {-0x1.2b785350ee8c1p-57, 0x1.6220d115d7b8ep-1}, - {-0x1.6ea6febe8bbbap-56, 0x1.66d663923e087p-1}, - {-0x1.a80386188c50ep-55, 0x1.6b798920b3d99p-1}, - {-0x1.8c34d25aadef6p-56, 0x1.700a7c5784634p-1}, - {0x1.7b2a6165884a1p-59, 0x1.748978fba8e0fp-1}, - {0x1.406a08980374p-55, 0x1.78f6bbd5d315ep-1}, - {0x1.560821e2f3aa9p-55, 0x1.7d528289fa093p-1}, - {-0x1.bf76229d3b917p-56, 0x1.819d0b7158a4dp-1}, - {0x1.6b66e7fc8b8c3p-57, 0x1.85d69576cc2c5p-1}, - {-0x1.55b9a5e177a1bp-55, 0x1.89ff5ff57f1f8p-1}, - {-0x1.ec182ab042f61p-56, 0x1.8e17aa99cc05ep-1}, - {0x1.1a62633145c07p-55, 0x1.921fb54442d18p-1}, -}; - -// Approximate atan(x) for |x| <= 2^-7. -// Using degree-9 Taylor polynomial: -// P = x - x^3/3 + x^5/5 -x^7/7 + x^9/9; -// Then the absolute error is bounded by: -// |atan(x) - P(x)| < |x|^11/11 < 2^(-7*11) / 11 < 2^-80. -// And the relative error is bounded by: -// |(atan(x) - P(x))/atan(x)| < |x|^10 / 10 < 2^-73. -// For x = x_hi + x_lo, fully expand the polynomial and drop any terms less than -// ulp(x_hi^3 / 3) gives us: -// P(x) ~ x_hi - x_hi^3/3 + x_hi^5/5 - x_hi^7/7 + x_hi^9/9 + -// + x_lo * (1 - x_hi^2 + x_hi^4) -DoubleDouble atan_eval(const DoubleDouble &x) { - DoubleDouble p; - p.hi = x.hi; - double x_hi_sq = x.hi * x.hi; - // c0 ~ x_hi^2 * 1/5 - 1/3 - double c0 = fputil::multiply_add(x_hi_sq, 0x1.999999999999ap-3, - -0x1.5555555555555p-2); - // c1 ~ x_hi^2 * 1/9 - 1/7 - double c1 = fputil::multiply_add(x_hi_sq, 0x1.c71c71c71c71cp-4, - -0x1.2492492492492p-3); - // x_hi^3 - double x_hi_3 = x_hi_sq * x.hi; - // x_hi^4 - double x_hi_4 = x_hi_sq * x_hi_sq; - // d0 ~ 1/3 - x_hi^2 / 5 + x_hi^4 / 7 - x_hi^6 / 9 - double d0 = fputil::multiply_add(x_hi_4, c1, c0); - // x_lo - x_lo * x_hi^2 + x_lo * x_hi^4 - double d1 = fputil::multiply_add(x_hi_4 - x_hi_sq, x.lo, x.lo); - // p.lo ~ -x_hi^3/3 + x_hi^5/5 - x_hi^7/7 + x_hi^9/9 + - // + x_lo * (1 - x_hi^2 + x_hi^4) - p.lo = fputil::multiply_add(x_hi_3, d0, d1); - return p; -} - -} // anonymous namespace - // There are several range reduction steps we can take for atan2(y, x) as // follow: @@ -228,8 +112,11 @@ LLVM_LIBC_FUNCTION(double, atan2, (double y, double x)) { // Check for exceptional cases, whether inputs are 0, inf, nan, or close to // overflow, or close to underflow. if (LIBC_UNLIKELY(max_exp > 0x7ffU - 128U || min_exp < 128U)) { - if (x_bits.is_nan() || y_bits.is_nan()) + if (x_bits.is_nan() || y_bits.is_nan()) { + if (x_bits.is_signaling_nan() || y_bits.is_signaling_nan()) + fputil::raise_except_if_required(FE_INVALID); return FPBits::quiet_nan().get_val(); + } unsigned x_except = x == 0.0 ? 0 : (FPBits(x_abs).is_inf() ? 2 : 1); unsigned y_except = y == 0.0 ? 0 : (FPBits(y_abs).is_inf() ? 2 : 1); diff --git a/system/lib/llvm-libc/src/math/generic/atan2f.cpp b/system/lib/llvm-libc/src/math/generic/atan2f.cpp index db7639396cdd7..c04b0eb1cc589 100644 --- a/system/lib/llvm-libc/src/math/generic/atan2f.cpp +++ b/system/lib/llvm-libc/src/math/generic/atan2f.cpp @@ -7,7 +7,9 @@ //===----------------------------------------------------------------------===// #include "src/math/atan2f.h" +#include "hdr/fenv_macros.h" #include "inv_trigf_utils.h" +#include "src/__support/FPUtil/FEnvImpl.h" #include "src/__support/FPUtil/FPBits.h" #include "src/__support/FPUtil/PolyEval.h" #include "src/__support/FPUtil/double_double.h" @@ -17,6 +19,14 @@ #include "src/__support/macros/config.h" #include "src/__support/macros/optimization.h" // LIBC_UNLIKELY +#if defined(LIBC_MATH_HAS_SKIP_ACCURATE_PASS) && \ + defined(LIBC_MATH_HAS_INTERMEDIATE_COMP_IN_FLOAT) + +// We use float-float implementation to reduce size. +#include "src/math/generic/atan2f_float.h" + +#else + namespace LIBC_NAMESPACE_DECL { namespace { @@ -123,7 +133,7 @@ float atan2f_double_double(double num_d, double den_d, double q_d, int idx, num_r = num_d; den_r = den_d; } -#ifdef LIBC_TARGET_CPU_HAS_FMA +#ifdef LIBC_TARGET_CPU_HAS_FMA_DOUBLE q.lo = fputil::multiply_add(q.hi, -den_r, num_r) / den_r; #else // Compute `(num_r - q.hi * den_r) / den_r` accurately without FMA @@ -132,7 +142,7 @@ float atan2f_double_double(double num_d, double den_d, double q_d, int idx, double t1 = fputil::multiply_add(q_hi_dd.hi, -den_r, num_r); // Exact double t2 = fputil::multiply_add(q_hi_dd.lo, -den_r, t1); q.lo = t2 / den_r; -#endif // LIBC_TARGET_CPU_HAS_FMA +#endif // LIBC_TARGET_CPU_HAS_FMA_DOUBLE // Taylor polynomial, evaluating using Horner's scheme: // P = x - x^3/3 + x^5/5 -x^7/7 + x^9/9 - x^11/11 + x^13/13 - x^15/15 @@ -256,8 +266,11 @@ LLVM_LIBC_FUNCTION(float, atan2f, (float y, float x)) { double den_d = static_cast(den_f); if (LIBC_UNLIKELY(max_abs >= 0x7f80'0000U || num_d == 0.0)) { - if (x_bits.is_nan() || y_bits.is_nan()) + if (x_bits.is_nan() || y_bits.is_nan()) { + if (x_bits.is_signaling_nan() || y_bits.is_signaling_nan()) + fputil::raise_except_if_required(FE_INVALID); return FPBits::quiet_nan().get_val(); + } double x_d = static_cast(x); double y_d = static_cast(y); size_t x_except = (x_d == 0.0) ? 0 : (x_abs == 0x7f80'0000 ? 2 : 1); @@ -324,3 +337,5 @@ LLVM_LIBC_FUNCTION(float, atan2f, (float y, float x)) { } } // namespace LIBC_NAMESPACE_DECL + +#endif diff --git a/system/lib/llvm-libc/src/math/generic/atan2f128.cpp b/system/lib/llvm-libc/src/math/generic/atan2f128.cpp new file mode 100644 index 0000000000000..a3aba0bc7fa2a --- /dev/null +++ b/system/lib/llvm-libc/src/math/generic/atan2f128.cpp @@ -0,0 +1,203 @@ +//===-- Quad-precision atan2 function -------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/atan2f128.h" +#include "atan_utils.h" +#include "src/__support/FPUtil/FPBits.h" +#include "src/__support/FPUtil/dyadic_float.h" +#include "src/__support/FPUtil/multiply_add.h" +#include "src/__support/FPUtil/nearest_integer.h" +#include "src/__support/integer_literals.h" +#include "src/__support/macros/config.h" +#include "src/__support/macros/optimization.h" // LIBC_UNLIKELY +#include "src/__support/macros/properties/types.h" +#include "src/__support/uint128.h" + +namespace LIBC_NAMESPACE_DECL { + +namespace { + +using Float128 = fputil::DyadicFloat<128>; + +static constexpr Float128 ZERO = {Sign::POS, 0, 0_u128}; +static constexpr Float128 MZERO = {Sign::NEG, 0, 0_u128}; +static constexpr Float128 PI = {Sign::POS, -126, + 0xc90fdaa2'2168c234'c4c6628b'80dc1cd1_u128}; +static constexpr Float128 MPI = {Sign::NEG, -126, + 0xc90fdaa2'2168c234'c4c6628b'80dc1cd1_u128}; +static constexpr Float128 PI_OVER_2 = { + Sign::POS, -127, 0xc90fdaa2'2168c234'c4c6628b'80dc1cd1_u128}; +static constexpr Float128 MPI_OVER_2 = { + Sign::NEG, -127, 0xc90fdaa2'2168c234'c4c6628b'80dc1cd1_u128}; +static constexpr Float128 PI_OVER_4 = { + Sign::POS, -128, 0xc90fdaa2'2168c234'c4c6628b'80dc1cd1_u128}; +static constexpr Float128 THREE_PI_OVER_4 = { + Sign::POS, -128, 0x96cbe3f9'990e91a7'9394c9e8'a0a5159d_u128}; + +// Adjustment for constant term: +// CONST_ADJ[x_sign][y_sign][recip] +static constexpr Float128 CONST_ADJ[2][2][2] = { + {{ZERO, MPI_OVER_2}, {MZERO, MPI_OVER_2}}, + {{MPI, PI_OVER_2}, {MPI, PI_OVER_2}}}; + +} // anonymous namespace + +// There are several range reduction steps we can take for atan2(y, x) as +// follow: + +// * Range reduction 1: signness +// atan2(y, x) will return a number between -PI and PI representing the angle +// forming by the 0x axis and the vector (x, y) on the 0xy-plane. +// In particular, we have that: +// atan2(y, x) = atan( y/x ) if x >= 0 and y >= 0 (I-quadrant) +// = pi + atan( y/x ) if x < 0 and y >= 0 (II-quadrant) +// = -pi + atan( y/x ) if x < 0 and y < 0 (III-quadrant) +// = atan( y/x ) if x >= 0 and y < 0 (IV-quadrant) +// Since atan function is odd, we can use the formula: +// atan(-u) = -atan(u) +// to adjust the above conditions a bit further: +// atan2(y, x) = atan( |y|/|x| ) if x >= 0 and y >= 0 (I-quadrant) +// = pi - atan( |y|/|x| ) if x < 0 and y >= 0 (II-quadrant) +// = -pi + atan( |y|/|x| ) if x < 0 and y < 0 (III-quadrant) +// = -atan( |y|/|x| ) if x >= 0 and y < 0 (IV-quadrant) +// Which can be simplified to: +// atan2(y, x) = sign(y) * atan( |y|/|x| ) if x >= 0 +// = sign(y) * (pi - atan( |y|/|x| )) if x < 0 + +// * Range reduction 2: reciprocal +// Now that the argument inside atan is positive, we can use the formula: +// atan(1/x) = pi/2 - atan(x) +// to make the argument inside atan <= 1 as follow: +// atan2(y, x) = sign(y) * atan( |y|/|x|) if 0 <= |y| <= x +// = sign(y) * (pi/2 - atan( |x|/|y| ) if 0 <= x < |y| +// = sign(y) * (pi - atan( |y|/|x| )) if 0 <= |y| <= -x +// = sign(y) * (pi/2 + atan( |x|/|y| )) if 0 <= -x < |y| + +// * Range reduction 3: look up table. +// After the previous two range reduction steps, we reduce the problem to +// compute atan(u) with 0 <= u <= 1, or to be precise: +// atan( n / d ) where n = min(|x|, |y|) and d = max(|x|, |y|). +// An accurate polynomial approximation for the whole [0, 1] input range will +// require a very large degree. To make it more efficient, we reduce the input +// range further by finding an integer idx such that: +// | n/d - idx/64 | <= 1/128. +// In particular, +// idx := round(2^6 * n/d) +// Then for the fast pass, we find a polynomial approximation for: +// atan( n/d ) ~ atan( idx/64 ) + (n/d - idx/64) * Q(n/d - idx/64) +// For the accurate pass, we use the addition formula: +// atan( n/d ) - atan( idx/64 ) = atan( (n/d - idx/64)/(1 + (n*idx)/(64*d)) ) +// = atan( (n - d*(idx/64))/(d + n*(idx/64)) ) +// And for the fast pass, we use degree-13 minimax polynomial to compute the +// RHS: +// atan(u) ~ P(u) = u - c_3 * u^3 + c_5 * u^5 - c_7 * u^7 + c_9 *u^9 - +// - c_11 * u^11 + c_13 * u^13 +// with absolute errors bounded by: +// |atan(u) - P(u)| < 2^-121 +// and relative errors bounded by: +// |(atan(u) - P(u)) / P(u)| < 2^-114. + +LLVM_LIBC_FUNCTION(float128, atan2f128, (float128 y, float128 x)) { + using FPBits = fputil::FPBits; + using Float128 = fputil::DyadicFloat<128>; + + FPBits x_bits(x), y_bits(y); + bool x_sign = x_bits.sign().is_neg(); + bool y_sign = y_bits.sign().is_neg(); + x_bits = x_bits.abs(); + y_bits = y_bits.abs(); + UInt128 x_abs = x_bits.uintval(); + UInt128 y_abs = y_bits.uintval(); + bool recip = x_abs < y_abs; + UInt128 min_abs = recip ? x_abs : y_abs; + UInt128 max_abs = !recip ? x_abs : y_abs; + unsigned min_exp = static_cast(min_abs >> FPBits::FRACTION_LEN); + unsigned max_exp = static_cast(max_abs >> FPBits::FRACTION_LEN); + + Float128 num(FPBits(min_abs).get_val()); + Float128 den(FPBits(max_abs).get_val()); + + // Check for exceptional cases, whether inputs are 0, inf, nan, or close to + // overflow, or close to underflow. + if (LIBC_UNLIKELY(max_exp >= 0x7fffU || min_exp == 0U)) { + if (x_bits.is_nan() || y_bits.is_nan()) + return FPBits::quiet_nan().get_val(); + unsigned x_except = x == 0 ? 0 : (FPBits(x_abs).is_inf() ? 2 : 1); + unsigned y_except = y == 0 ? 0 : (FPBits(y_abs).is_inf() ? 2 : 1); + + // Exceptional cases: + // EXCEPT[y_except][x_except][x_is_neg] + // with x_except & y_except: + // 0: zero + // 1: finite, non-zero + // 2: infinity + constexpr Float128 EXCEPTS[3][3][2] = { + {{ZERO, PI}, {ZERO, PI}, {ZERO, PI}}, + {{PI_OVER_2, PI_OVER_2}, {ZERO, ZERO}, {ZERO, PI}}, + {{PI_OVER_2, PI_OVER_2}, + {PI_OVER_2, PI_OVER_2}, + {PI_OVER_4, THREE_PI_OVER_4}}, + }; + + if ((x_except != 1) || (y_except != 1)) { + Float128 r = EXCEPTS[y_except][x_except][x_sign]; + if (y_sign) + r.sign = r.sign.negate(); + return static_cast(r); + } + } + + bool final_sign = ((x_sign != y_sign) != recip); + Float128 const_term = CONST_ADJ[x_sign][y_sign][recip]; + int exp_diff = den.exponent - num.exponent; + // We have the following bound for normalized n and d: + // 2^(-exp_diff - 1) < n/d < 2^(-exp_diff + 1). + if (LIBC_UNLIKELY(exp_diff > FPBits::FRACTION_LEN + 2)) { + if (final_sign) + const_term.sign = const_term.sign.negate(); + return static_cast(const_term); + } + + // Take 24 leading bits of num and den to convert to float for fast division. + // We also multiply the numerator by 64 using integer addition directly to the + // exponent field. + float num_f = + cpp::bit_cast(static_cast(num.mantissa >> 104) + + (6U << fputil::FPBits::FRACTION_LEN)); + float den_f = cpp::bit_cast( + static_cast(den.mantissa >> 104) + + (static_cast(exp_diff) << fputil::FPBits::FRACTION_LEN)); + + float k = fputil::nearest_integer(num_f / den_f); + unsigned idx = static_cast(k); + + // k_f128 = idx / 64 + Float128 k_f128(Sign::POS, -6, Float128::MantissaType(idx)); + + // Range reduction: + // atan(n/d) - atan(k) = atan((n/d - k/64) / (1 + (n/d) * (k/64))) + // = atan((n - d * k/64)) / (d + n * k/64)) + // num_f128 = n - d * k/64 + Float128 num_f128 = fputil::multiply_add(den, -k_f128, num); + // den_f128 = d + n * k/64 + Float128 den_f128 = fputil::multiply_add(num, k_f128, den); + + // q = (n - d * k) / (d + n * k) + Float128 q = fputil::quick_mul(num_f128, fputil::approx_reciprocal(den_f128)); + // p ~ atan(q) + Float128 p = atan_eval(q); + + Float128 r = + fputil::quick_add(const_term, fputil::quick_add(ATAN_I_F128[idx], p)); + if (final_sign) + r.sign = r.sign.negate(); + + return static_cast(r); +} + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/generic/atan2f_float.h b/system/lib/llvm-libc/src/math/generic/atan2f_float.h new file mode 100644 index 0000000000000..1fd853d735950 --- /dev/null +++ b/system/lib/llvm-libc/src/math/generic/atan2f_float.h @@ -0,0 +1,237 @@ +//===-- Single-precision atan2f function ----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/__support/FPUtil/FPBits.h" +#include "src/__support/FPUtil/double_double.h" +#include "src/__support/FPUtil/multiply_add.h" +#include "src/__support/FPUtil/nearest_integer.h" +#include "src/__support/FPUtil/rounding_mode.h" +#include "src/__support/macros/config.h" +#include "src/__support/macros/optimization.h" // LIBC_UNLIKELY +#include "src/math/atan2f.h" + +namespace LIBC_NAMESPACE_DECL { + +namespace { + +using FloatFloat = fputil::FloatFloat; + +// atan(i/64) with i = 0..16, generated by Sollya with: +// > for i from 0 to 16 do { +// a = round(atan(i/16), SG, RN); +// b = round(atan(i/16) - a, SG, RN); +// print("{", b, ",", a, "},"); +// }; +constexpr FloatFloat ATAN_I[17] = { + {0.0f, 0.0f}, + {-0x1.1a6042p-30f, 0x1.ff55bcp-5f}, + {-0x1.54f424p-30f, 0x1.fd5baap-4f}, + {0x1.79cb6p-28f, 0x1.7b97b4p-3f}, + {-0x1.b4dfc8p-29f, 0x1.f5b76p-3f}, + {-0x1.1f0286p-27f, 0x1.362774p-2f}, + {0x1.e4defp-30f, 0x1.6f6194p-2f}, + {0x1.e611fep-29f, 0x1.a64eecp-2f}, + {0x1.586ed4p-28f, 0x1.dac67p-2f}, + {-0x1.6499e6p-26f, 0x1.0657eap-1f}, + {0x1.7bdfd6p-26f, 0x1.1e00bap-1f}, + {-0x1.98e422p-28f, 0x1.345f02p-1f}, + {0x1.934f7p-28f, 0x1.4978fap-1f}, + {0x1.c5a6c6p-27f, 0x1.5d5898p-1f}, + {0x1.5e118cp-27f, 0x1.700a7cp-1f}, + {-0x1.1d4eb6p-26f, 0x1.819d0cp-1f}, + {-0x1.777a5cp-26f, 0x1.921fb6p-1f}, +}; + +// Approximate atan(x) for |x| <= 2^-5. +// Using degree-3 Taylor polynomial: +// P = x - x^3/3 +// Then the absolute error is bounded by: +// |atan(x) - P(x)| < |x|^5/5 < 2^(-5*5) / 5 < 2^-27. +// And the relative error is bounded by: +// |(atan(x) - P(x))/atan(x)| < |x|^4 / 4 < 2^-22. +// For x = x_hi + x_lo, fully expand the polynomial and drop any terms less than +// ulp(x_hi^3 / 3) gives us: +// P(x) ~ x_hi - x_hi^3/3 + x_lo * (1 - x_hi^2) +FloatFloat atan_eval(const FloatFloat &x) { + FloatFloat p; + p.hi = x.hi; + float x_hi_sq = x.hi * x.hi; + // c0 ~ - x_hi^2 / 3 + float c0 = -0x1.555556p-2f * x_hi_sq; + // c1 ~ x_lo * (1 - x_hi^2) + float c1 = fputil::multiply_add(x_hi_sq, -x.lo, x.lo); + // p.lo ~ - x_hi^3 / 3 + x_lo * (1 - x_hi*2) + p.lo = fputil::multiply_add(x.hi, c0, c1); + return p; +} + +} // anonymous namespace + +// There are several range reduction steps we can take for atan2(y, x) as +// follow: + +// * Range reduction 1: signness +// atan2(y, x) will return a number between -PI and PI representing the angle +// forming by the 0x axis and the vector (x, y) on the 0xy-plane. +// In particular, we have that: +// atan2(y, x) = atan( y/x ) if x >= 0 and y >= 0 (I-quadrant) +// = pi + atan( y/x ) if x < 0 and y >= 0 (II-quadrant) +// = -pi + atan( y/x ) if x < 0 and y < 0 (III-quadrant) +// = atan( y/x ) if x >= 0 and y < 0 (IV-quadrant) +// Since atan function is odd, we can use the formula: +// atan(-u) = -atan(u) +// to adjust the above conditions a bit further: +// atan2(y, x) = atan( |y|/|x| ) if x >= 0 and y >= 0 (I-quadrant) +// = pi - atan( |y|/|x| ) if x < 0 and y >= 0 (II-quadrant) +// = -pi + atan( |y|/|x| ) if x < 0 and y < 0 (III-quadrant) +// = -atan( |y|/|x| ) if x >= 0 and y < 0 (IV-quadrant) +// Which can be simplified to: +// atan2(y, x) = sign(y) * atan( |y|/|x| ) if x >= 0 +// = sign(y) * (pi - atan( |y|/|x| )) if x < 0 + +// * Range reduction 2: reciprocal +// Now that the argument inside atan is positive, we can use the formula: +// atan(1/x) = pi/2 - atan(x) +// to make the argument inside atan <= 1 as follow: +// atan2(y, x) = sign(y) * atan( |y|/|x|) if 0 <= |y| <= x +// = sign(y) * (pi/2 - atan( |x|/|y| ) if 0 <= x < |y| +// = sign(y) * (pi - atan( |y|/|x| )) if 0 <= |y| <= -x +// = sign(y) * (pi/2 + atan( |x|/|y| )) if 0 <= -x < |y| + +// * Range reduction 3: look up table. +// After the previous two range reduction steps, we reduce the problem to +// compute atan(u) with 0 <= u <= 1, or to be precise: +// atan( n / d ) where n = min(|x|, |y|) and d = max(|x|, |y|). +// An accurate polynomial approximation for the whole [0, 1] input range will +// require a very large degree. To make it more efficient, we reduce the input +// range further by finding an integer idx such that: +// | n/d - idx/16 | <= 1/32. +// In particular, +// idx := 2^-4 * round(2^4 * n/d) +// Then for the fast pass, we find a polynomial approximation for: +// atan( n/d ) ~ atan( idx/16 ) + (n/d - idx/16) * Q(n/d - idx/16) +// with Q(x) = x - x^3/3 be the cubic Taylor polynomial of atan(x). +// It's error in float-float precision is estimated in Sollya to be: +// > P = x - x^3/3; +// > dirtyinfnorm(atan(x) - P, [-2^-5, 2^-5]); +// 0x1.995...p-28. + +LLVM_LIBC_FUNCTION(float, atan2f, (float y, float x)) { + using FPBits = typename fputil::FPBits; + constexpr float IS_NEG[2] = {1.0f, -1.0f}; + constexpr FloatFloat ZERO = {0.0f, 0.0f}; + constexpr FloatFloat MZERO = {-0.0f, -0.0f}; + constexpr FloatFloat PI = {-0x1.777a5cp-24f, 0x1.921fb6p1f}; + constexpr FloatFloat MPI = {0x1.777a5cp-24f, -0x1.921fb6p1f}; + constexpr FloatFloat PI_OVER_4 = {-0x1.777a5cp-26f, 0x1.921fb6p-1f}; + constexpr FloatFloat PI_OVER_2 = {-0x1.777a5cp-25f, 0x1.921fb6p0f}; + constexpr FloatFloat MPI_OVER_2 = {-0x1.777a5cp-25f, 0x1.921fb6p0f}; + constexpr FloatFloat THREE_PI_OVER_4 = {-0x1.99bc5cp-28f, 0x1.2d97c8p1f}; + // Adjustment for constant term: + // CONST_ADJ[x_sign][y_sign][recip] + constexpr FloatFloat CONST_ADJ[2][2][2] = { + {{ZERO, MPI_OVER_2}, {MZERO, MPI_OVER_2}}, + {{MPI, PI_OVER_2}, {MPI, PI_OVER_2}}}; + + FPBits x_bits(x), y_bits(y); + bool x_sign = x_bits.sign().is_neg(); + bool y_sign = y_bits.sign().is_neg(); + x_bits = x_bits.abs(); + y_bits = y_bits.abs(); + uint32_t x_abs = x_bits.uintval(); + uint32_t y_abs = y_bits.uintval(); + bool recip = x_abs < y_abs; + uint32_t min_abs = recip ? x_abs : y_abs; + uint32_t max_abs = !recip ? x_abs : y_abs; + auto min_exp = static_cast(min_abs >> FPBits::FRACTION_LEN); + auto max_exp = static_cast(max_abs >> FPBits::FRACTION_LEN); + + float num = FPBits(min_abs).get_val(); + float den = FPBits(max_abs).get_val(); + + // Check for exceptional cases, whether inputs are 0, inf, nan, or close to + // overflow, or close to underflow. + if (LIBC_UNLIKELY(max_exp > 0xffU - 64U || min_exp < 64U)) { + if (x_bits.is_nan() || y_bits.is_nan()) + return FPBits::quiet_nan().get_val(); + unsigned x_except = x == 0.0f ? 0 : (FPBits(x_abs).is_inf() ? 2 : 1); + unsigned y_except = y == 0.0f ? 0 : (FPBits(y_abs).is_inf() ? 2 : 1); + + // Exceptional cases: + // EXCEPT[y_except][x_except][x_is_neg] + // with x_except & y_except: + // 0: zero + // 1: finite, non-zero + // 2: infinity + constexpr FloatFloat EXCEPTS[3][3][2] = { + {{ZERO, PI}, {ZERO, PI}, {ZERO, PI}}, + {{PI_OVER_2, PI_OVER_2}, {ZERO, ZERO}, {ZERO, PI}}, + {{PI_OVER_2, PI_OVER_2}, + {PI_OVER_2, PI_OVER_2}, + {PI_OVER_4, THREE_PI_OVER_4}}, + }; + + if ((x_except != 1) || (y_except != 1)) { + FloatFloat r = EXCEPTS[y_except][x_except][x_sign]; + return fputil::multiply_add(IS_NEG[y_sign], r.hi, IS_NEG[y_sign] * r.lo); + } + bool scale_up = min_exp < 64U; + bool scale_down = max_exp > 0xffU - 64U; + // At least one input is denormal, multiply both numerator and denominator + // by some large enough power of 2 to normalize denormal inputs. + if (scale_up) { + num *= 0x1.0p32f; + if (!scale_down) + den *= 0x1.0p32f; + } else if (scale_down) { + den *= 0x1.0p-32f; + num *= 0x1.0p-32f; + } + + min_abs = FPBits(num).uintval(); + max_abs = FPBits(den).uintval(); + min_exp = static_cast(min_abs >> FPBits::FRACTION_LEN); + max_exp = static_cast(max_abs >> FPBits::FRACTION_LEN); + } + + float final_sign = IS_NEG[(x_sign != y_sign) != recip]; + FloatFloat const_term = CONST_ADJ[x_sign][y_sign][recip]; + unsigned exp_diff = max_exp - min_exp; + // We have the following bound for normalized n and d: + // 2^(-exp_diff - 1) < n/d < 2^(-exp_diff + 1). + if (LIBC_UNLIKELY(exp_diff > 25)) + return fputil::multiply_add(final_sign, const_term.hi, + final_sign * (const_term.lo + num / den)); + + float k = fputil::nearest_integer(16.0f * num / den); + unsigned idx = static_cast(k); + // k = idx / 16 + k *= 0x1.0p-4f; + + // Range reduction: + // atan(n/d) - atan(k/64) = atan((n/d - k/16) / (1 + (n/d) * (k/16))) + // = atan((n - d * k/16)) / (d + n * k/16)) + FloatFloat num_k = fputil::exact_mult(num, k); + FloatFloat den_k = fputil::exact_mult(den, k); + + // num_dd = n - d * k + FloatFloat num_ff = fputil::exact_add(num - den_k.hi, -den_k.lo); + // den_dd = d + n * k + FloatFloat den_ff = fputil::exact_add(den, num_k.hi); + den_ff.lo += num_k.lo; + + // q = (n - d * k) / (d + n * k) + FloatFloat q = fputil::div(num_ff, den_ff); + // p ~ atan(q) + FloatFloat p = atan_eval(q); + + FloatFloat r = fputil::add(const_term, fputil::add(ATAN_I[idx], p)); + return final_sign * r.hi; +} + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/generic/atan_utils.h b/system/lib/llvm-libc/src/math/generic/atan_utils.h new file mode 100644 index 0000000000000..24c7271b7e4ec --- /dev/null +++ b/system/lib/llvm-libc/src/math/generic/atan_utils.h @@ -0,0 +1,241 @@ +//===-- Collection of utils for atan/atan2 ----------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_MATH_GENERIC_ATAN_UTILS_H +#define LLVM_LIBC_SRC_MATH_GENERIC_ATAN_UTILS_H + +#include "src/__support/FPUtil/PolyEval.h" +#include "src/__support/FPUtil/double_double.h" +#include "src/__support/FPUtil/dyadic_float.h" +#include "src/__support/FPUtil/multiply_add.h" +#include "src/__support/integer_literals.h" +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +namespace { + +using DoubleDouble = fputil::DoubleDouble; +using Float128 = fputil::DyadicFloat<128>; + +// atan(i/64) with i = 0..64, generated by Sollya with: +// > for i from 0 to 64 do { +// a = round(atan(i/64), D, RN); +// b = round(atan(i/64) - a, D, RN); +// print("{", b, ",", a, "},"); +// }; +constexpr DoubleDouble ATAN_I[65] = { + {0.0, 0.0}, + {-0x1.220c39d4dff5p-61, 0x1.fff555bbb729bp-7}, + {-0x1.5ec431444912cp-60, 0x1.ffd55bba97625p-6}, + {-0x1.86ef8f794f105p-63, 0x1.7fb818430da2ap-5}, + {-0x1.c934d86d23f1dp-60, 0x1.ff55bb72cfdeap-5}, + {0x1.ac4ce285df847p-58, 0x1.3f59f0e7c559dp-4}, + {-0x1.cfb654c0c3d98p-58, 0x1.7ee182602f10fp-4}, + {0x1.f7b8f29a05987p-58, 0x1.be39ebe6f07c3p-4}, + {-0x1.cd37686760c17p-59, 0x1.fd5ba9aac2f6ep-4}, + {-0x1.b485914dacf8cp-59, 0x1.1e1fafb043727p-3}, + {0x1.61a3b0ce9281bp-57, 0x1.3d6eee8c6626cp-3}, + {-0x1.054ab2c010f3dp-58, 0x1.5c9811e3ec26ap-3}, + {0x1.347b0b4f881cap-58, 0x1.7b97b4bce5b02p-3}, + {0x1.cf601e7b4348ep-59, 0x1.9a6a8e96c8626p-3}, + {0x1.17b10d2e0e5abp-61, 0x1.b90d7529260a2p-3}, + {0x1.c648d1534597ep-57, 0x1.d77d5df205736p-3}, + {0x1.8ab6e3cf7afbdp-57, 0x1.f5b75f92c80ddp-3}, + {0x1.62e47390cb865p-56, 0x1.09dc597d86362p-2}, + {0x1.30ca4748b1bf9p-57, 0x1.18bf5a30bf178p-2}, + {-0x1.077cdd36dfc81p-56, 0x1.278372057ef46p-2}, + {-0x1.963a544b672d8p-57, 0x1.362773707ebccp-2}, + {-0x1.5d5e43c55b3bap-56, 0x1.44aa436c2af0ap-2}, + {-0x1.2566480884082p-57, 0x1.530ad9951cd4ap-2}, + {-0x1.a725715711fp-56, 0x1.614840309cfe2p-2}, + {-0x1.c63aae6f6e918p-56, 0x1.6f61941e4def1p-2}, + {0x1.69c885c2b249ap-56, 0x1.7d5604b63b3f7p-2}, + {0x1.b6d0ba3748fa8p-56, 0x1.8b24d394a1b25p-2}, + {0x1.9e6c988fd0a77p-56, 0x1.98cd5454d6b18p-2}, + {-0x1.24dec1b50b7ffp-56, 0x1.a64eec3cc23fdp-2}, + {0x1.ae187b1ca504p-56, 0x1.b3a911da65c6cp-2}, + {-0x1.cc1ce70934c34p-56, 0x1.c0db4c94ec9fp-2}, + {-0x1.a2cfa4418f1adp-56, 0x1.cde53432c1351p-2}, + {0x1.a2b7f222f65e2p-56, 0x1.dac670561bb4fp-2}, + {0x1.0e53dc1bf3435p-56, 0x1.e77eb7f175a34p-2}, + {-0x1.a3992dc382a23p-57, 0x1.f40dd0b541418p-2}, + {-0x1.b32c949c9d593p-55, 0x1.0039c73c1a40cp-1}, + {-0x1.d5b495f6349e6p-56, 0x1.0657e94db30dp-1}, + {0x1.974fa13b5404fp-58, 0x1.0c6145b5b43dap-1}, + {-0x1.2bdaee1c0ee35p-58, 0x1.1255d9bfbd2a9p-1}, + {0x1.c621cec00c301p-55, 0x1.1835a88be7c13p-1}, + {-0x1.928df287a668fp-58, 0x1.1e00babdefeb4p-1}, + {0x1.c421c9f38224ep-57, 0x1.23b71e2cc9e6ap-1}, + {-0x1.09e73b0c6c087p-56, 0x1.2958e59308e31p-1}, + {0x1.c5d5e9ff0cf8dp-55, 0x1.2ee628406cbcap-1}, + {0x1.1021137c71102p-55, 0x1.345f01cce37bbp-1}, + {-0x1.2304331d8bf46p-55, 0x1.39c391cd4171ap-1}, + {0x1.ecf8b492644fp-56, 0x1.3f13fb89e96f4p-1}, + {-0x1.f76d0163f79c8p-56, 0x1.445065b795b56p-1}, + {0x1.2419a87f2a458p-56, 0x1.4978fa3269ee1p-1}, + {0x1.4a33dbeb3796cp-55, 0x1.4e8de5bb6ec04p-1}, + {-0x1.1bb74abda520cp-55, 0x1.538f57b89061fp-1}, + {-0x1.5e5c9d8c5a95p-56, 0x1.587d81f732fbbp-1}, + {0x1.0028e4bc5e7cap-57, 0x1.5d58987169b18p-1}, + {-0x1.2b785350ee8c1p-57, 0x1.6220d115d7b8ep-1}, + {-0x1.6ea6febe8bbbap-56, 0x1.66d663923e087p-1}, + {-0x1.a80386188c50ep-55, 0x1.6b798920b3d99p-1}, + {-0x1.8c34d25aadef6p-56, 0x1.700a7c5784634p-1}, + {0x1.7b2a6165884a1p-59, 0x1.748978fba8e0fp-1}, + {0x1.406a08980374p-55, 0x1.78f6bbd5d315ep-1}, + {0x1.560821e2f3aa9p-55, 0x1.7d528289fa093p-1}, + {-0x1.bf76229d3b917p-56, 0x1.819d0b7158a4dp-1}, + {0x1.6b66e7fc8b8c3p-57, 0x1.85d69576cc2c5p-1}, + {-0x1.55b9a5e177a1bp-55, 0x1.89ff5ff57f1f8p-1}, + {-0x1.ec182ab042f61p-56, 0x1.8e17aa99cc05ep-1}, + {0x1.1a62633145c07p-55, 0x1.921fb54442d18p-1}, +}; + +// Approximate atan(x) for |x| <= 2^-7. +// Using degree-9 Taylor polynomial: +// P = x - x^3/3 + x^5/5 -x^7/7 + x^9/9; +// Then the absolute error is bounded by: +// |atan(x) - P(x)| < |x|^11/11 < 2^(-7*11) / 11 < 2^-80. +// And the relative error is bounded by: +// |(atan(x) - P(x))/atan(x)| < |x|^10 / 10 < 2^-73. +// For x = x_hi + x_lo, fully expand the polynomial and drop any terms less than +// ulp(x_hi^3 / 3) gives us: +// P(x) ~ x_hi - x_hi^3/3 + x_hi^5/5 - x_hi^7/7 + x_hi^9/9 + +// + x_lo * (1 - x_hi^2 + x_hi^4) +// Since p.lo is ~ x^3/3, the relative error from rounding is bounded by: +// |(atan(x) - P(x))/atan(x)| < ulp(x^2) <= 2^(-14-52) = 2^-66. +[[maybe_unused]] DoubleDouble atan_eval(const DoubleDouble &x) { + DoubleDouble p; + p.hi = x.hi; + double x_hi_sq = x.hi * x.hi; + // c0 ~ x_hi^2 * 1/5 - 1/3 + double c0 = fputil::multiply_add(x_hi_sq, 0x1.999999999999ap-3, + -0x1.5555555555555p-2); + // c1 ~ x_hi^2 * 1/9 - 1/7 + double c1 = fputil::multiply_add(x_hi_sq, 0x1.c71c71c71c71cp-4, + -0x1.2492492492492p-3); + // x_hi^3 + double x_hi_3 = x_hi_sq * x.hi; + // x_hi^4 + double x_hi_4 = x_hi_sq * x_hi_sq; + // d0 ~ 1/3 - x_hi^2 / 5 + x_hi^4 / 7 - x_hi^6 / 9 + double d0 = fputil::multiply_add(x_hi_4, c1, c0); + // x_lo - x_lo * x_hi^2 + x_lo * x_hi^4 + double d1 = fputil::multiply_add(x_hi_4 - x_hi_sq, x.lo, x.lo); + // p.lo ~ -x_hi^3/3 + x_hi^5/5 - x_hi^7/7 + x_hi^9/9 + + // + x_lo * (1 - x_hi^2 + x_hi^4) + p.lo = fputil::multiply_add(x_hi_3, d0, d1); + return p; +} + +// Float128 versions. +// atan(i/64) with i = 0..64, generated by Sollya with: +// > for i from 1 to 64 do { +// a = round(atan(i/64), 128, RN); +// ll = ceil(log2(a)); +// b = 2^ll + a; +// print("{Sign::POS, ", 2^(ll - 128), ",", b, "},"); +// }; +constexpr Float128 ATAN_I_F128[65] = { + {Sign::POS, 0, 0_u128}, + {Sign::POS, -134, 0xfffaaadd'db94d5bb'e78c5640'15f76048_u128}, + {Sign::POS, -133, 0xffeaaddd'4bb12542'779d776d'da8c6214_u128}, + {Sign::POS, -132, 0xbfdc0c21'86d14fcf'220e10d6'1df56ec7_u128}, + {Sign::POS, -132, 0xffaaddb9'67ef4e36'cb2792dc'0e2e0d51_u128}, + {Sign::POS, -131, 0x9facf873'e2aceb58'99c50bbf'08e6cdf6_u128}, + {Sign::POS, -131, 0xbf70c130'17887460'93567e78'4cf83676_u128}, + {Sign::POS, -131, 0xdf1cf5f3'783e1bef'71e5340b'30e5d9ef_u128}, + {Sign::POS, -131, 0xfeadd4d5'617b6e32'c897989f'3e888ef8_u128}, + {Sign::POS, -130, 0x8f0fd7d8'21b93725'bd375929'83a0af9a_u128}, + {Sign::POS, -130, 0x9eb77746'331362c3'47619d25'0360fe85_u128}, + {Sign::POS, -130, 0xae4c08f1'f6134efa'b54d3fef'0c2de994_u128}, + {Sign::POS, -130, 0xbdcbda5e'72d81134'7b0b4f88'1c9c7488_u128}, + {Sign::POS, -130, 0xcd35474b'643130e7'b00f3da1'a46eeb3b_u128}, + {Sign::POS, -130, 0xdc86ba94'93051022'f621a5c1'cb552f03_u128}, + {Sign::POS, -130, 0xebbeaef9'02b9b38c'91a2a68b'2fbd78e8_u128}, + {Sign::POS, -130, 0xfadbafc9'6406eb15'6dc79ef5'f7a217e6_u128}, + {Sign::POS, -129, 0x84ee2cbe'c31b12c5'c8e72197'0cabd3a3_u128}, + {Sign::POS, -129, 0x8c5fad18'5f8bc130'ca4748b1'bf88298d_u128}, + {Sign::POS, -129, 0x93c1b902'bf7a2df1'06459240'6fe1447a_u128}, + {Sign::POS, -129, 0x9b13b9b8'3f5e5e69'c5abb498'd27af328_u128}, + {Sign::POS, -129, 0xa25521b6'15784d45'43787549'88b8d9e3_u128}, + {Sign::POS, -129, 0xa9856cca'8e6a4eda'99b7f77b'f7d9e8c1_u128}, + {Sign::POS, -129, 0xb0a42018'4e7f0cb1'b51d51dc'200a0fc3_u128}, + {Sign::POS, -129, 0xb7b0ca0f'26f78473'8aa32122'dcfe4483_u128}, + {Sign::POS, -129, 0xbeab025b'1d9fbad3'910b8564'93411026_u128}, + {Sign::POS, -129, 0xc59269ca'50d92b6d'a1746e91'f50a28de_u128}, + {Sign::POS, -129, 0xcc66aa2a'6b58c33c'd9311fa1'4ed9b7c4_u128}, + {Sign::POS, -129, 0xd327761e'611fe5b6'427c95e9'001e7136_u128}, + {Sign::POS, -129, 0xd9d488ed'32e3635c'30f6394a'0806345d_u128}, + {Sign::POS, -129, 0xe06da64a'764f7c67'c631ed96'798cb804_u128}, + {Sign::POS, -129, 0xe6f29a19'609a84ba'60b77ce1'ca6dc2c8_u128}, + {Sign::POS, -129, 0xed63382b'0dda7b45'6fe445ec'bc3a8d03_u128}, + {Sign::POS, -129, 0xf3bf5bf8'bad1a21c'a7b837e6'86adf3fa_u128}, + {Sign::POS, -129, 0xfa06e85a'a0a0be5c'66d23c7d'5dc8ecc2_u128}, + {Sign::POS, -128, 0x801ce39e'0d205c99'a6d6c6c5'4d938596_u128}, + {Sign::POS, -128, 0x832bf4a6'd9867e2a'4b6a09cb'61a515c1_u128}, + {Sign::POS, -128, 0x8630a2da'da1ed065'd3e84ed5'013ca37e_u128}, + {Sign::POS, -128, 0x892aecdf'de9547b5'094478fc'472b4afc_u128}, + {Sign::POS, -128, 0x8c1ad445'f3e09b8c'439d8018'60205921_u128}, + {Sign::POS, -128, 0x8f005d5e'f7f59f9b'5c835e16'65c43748_u128}, + {Sign::POS, -128, 0x91db8f16'64f350e2'10e4f9c1'126e0220_u128}, + {Sign::POS, -128, 0x94ac72c9'847186f6'18c4f393'f78a32f9_u128}, + {Sign::POS, -128, 0x97731420'365e538b'abd3fe19'f1aeb6b3_u128}, + {Sign::POS, -128, 0x9a2f80e6'71bdda20'4226f8e2'204ff3bd_u128}, + {Sign::POS, -128, 0x9ce1c8e6'a0b8cdb9'f799c4e8'174cf11c_u128}, + {Sign::POS, -128, 0x9f89fdc4'f4b7a1ec'f8b49264'4f0701e0_u128}, + {Sign::POS, -128, 0xa22832db'cadaae08'92fe9c08'637af0e6_u128}, + {Sign::POS, -128, 0xa4bc7d19'34f70924'19a87f2a'457dac9f_u128}, + {Sign::POS, -128, 0xa746f2dd'b7602294'67b7d66f'2d74e019_u128}, + {Sign::POS, -128, 0xa9c7abdc'4830f5c8'916a84b5'be7933f6_u128}, + {Sign::POS, -128, 0xac3ec0fb'997dd6a1'a36273a5'6afa8ef4_u128}, + {Sign::POS, -128, 0xaeac4c38'b4d8c080'14725e2f'3e52070a_u128}, + {Sign::POS, -128, 0xb110688a'ebdc6f6a'43d65788'b9f6a7b5_u128}, + {Sign::POS, -128, 0xb36b31c9'1f043691'59014174'4462f93a_u128}, + {Sign::POS, -128, 0xb5bcc490'59ecc4af'f8f3cee7'5e3907d5_u128}, + {Sign::POS, -128, 0xb8053e2b'c2319e73'cb2da552'10a4443d_u128}, + {Sign::POS, -128, 0xba44bc7d'd470782f'654c2cb1'0942e386_u128}, + {Sign::POS, -128, 0xbc7b5dea'e98af280'd4113006'e80fb290_u128}, + {Sign::POS, -128, 0xbea94144'fd049aac'1043c5e7'55282e7d_u128}, + {Sign::POS, -128, 0xc0ce85b8'ac526640'89dd62c4'6e92fa25_u128}, + {Sign::POS, -128, 0xc2eb4abb'661628b5'b373fe45'c61bb9fb_u128}, + {Sign::POS, -128, 0xc4ffaffa'bf8fbd54'8cb43d10'bc9e0221_u128}, + {Sign::POS, -128, 0xc70bd54c'e602ee13'e7d54fbd'09f2be38_u128}, + {Sign::POS, -128, 0xc90fdaa2'2168c234'c4c6628b'80dc1cd1_u128}, +}; + +// Degree-13 minimax polynomial generated by Sollya with: +// > P = fpminimax(atan(x), [|1, 3, 5, 7, 9, 11, 13|], [|1, 128...|], +// [0, 2^-7]); +// > dirtyinfnorm(atan(x) - P, [0, 2^-7]); +// 0x1.26016ad97f323875760f869684c0898d7b7bb8bep-122 +constexpr Float128 ATAN_POLY_F128[] = { + {Sign::NEG, -129, 0xaaaaaaaa'aaaaaaaa'aaaaaaa6'003c5d1d_u128}, + {Sign::POS, -130, 0xcccccccc'cccccccc'cca00232'8776b063_u128}, + {Sign::NEG, -130, 0x92492492'49249201'27f5268a'cb24aec0_u128}, + {Sign::POS, -131, 0xe38e38e3'8dce3d96'626a1643'f8eb68f3_u128}, + {Sign::NEG, -131, 0xba2e8b7a'ea4ad00f'005a35c7'6ef609b1_u128}, + {Sign::POS, -131, 0x9d82765e'd22a7d92'ac09c405'c0a69214_u128}, +}; + +// Approximate atan for |x| <= 2^-7. +[[maybe_unused]] Float128 atan_eval(const Float128 &x) { + Float128 x_sq = fputil::quick_mul(x, x); + Float128 x3 = fputil::quick_mul(x, x_sq); + Float128 p = fputil::polyeval(x_sq, ATAN_POLY_F128[0], ATAN_POLY_F128[1], + ATAN_POLY_F128[2], ATAN_POLY_F128[3], + ATAN_POLY_F128[4], ATAN_POLY_F128[5]); + return fputil::multiply_add(x3, p, x); +} + +} // anonymous namespace + +} // namespace LIBC_NAMESPACE_DECL + +#endif // LLVM_LIBC_SRC_MATH_GENERIC_ATAN_UTILS_H diff --git a/system/lib/llvm-libc/src/math/generic/atanf.cpp b/system/lib/llvm-libc/src/math/generic/atanf.cpp index 5e0788efbeb88..46196dbe4162c 100644 --- a/system/lib/llvm-libc/src/math/generic/atanf.cpp +++ b/system/lib/llvm-libc/src/math/generic/atanf.cpp @@ -52,12 +52,12 @@ LLVM_LIBC_FUNCTION(float, atanf, (float x)) { return x; // x <= 2^-12; if (LIBC_UNLIKELY(x_abs < 0x3980'0000)) { -#if defined(LIBC_TARGET_CPU_HAS_FMA) +#if defined(LIBC_TARGET_CPU_HAS_FMA_FLOAT) return fputil::multiply_add(x, -0x1.0p-25f, x); #else double x_d = static_cast(x); return static_cast(fputil::multiply_add(x_d, -0x1.0p-25, x_d)); -#endif // LIBC_TARGET_CPU_HAS_FMA +#endif // LIBC_TARGET_CPU_HAS_FMA_FLOAT } // Use Taylor polynomial: // atan(x) ~ x * (1 - x^2 / 3 + x^4 / 5 - x^6 / 7 + x^8 / 9 - x^10 / 11). diff --git a/system/lib/llvm-libc/src/math/generic/atanhf.cpp b/system/lib/llvm-libc/src/math/generic/atanhf.cpp index a2051bd3e3e67..2149314d2f676 100644 --- a/system/lib/llvm-libc/src/math/generic/atanhf.cpp +++ b/system/lib/llvm-libc/src/math/generic/atanhf.cpp @@ -24,6 +24,10 @@ LLVM_LIBC_FUNCTION(float, atanhf, (float x)) { // |x| >= 1.0 if (LIBC_UNLIKELY(x_abs >= 0x3F80'0000U)) { if (xbits.is_nan()) { + if (xbits.is_signaling_nan()) { + fputil::raise_except_if_required(FE_INVALID); + return FPBits::quiet_nan().get_val(); + } return x; } // |x| == 1.0 diff --git a/system/lib/llvm-libc/src/math/generic/cbrt.cpp b/system/lib/llvm-libc/src/math/generic/cbrt.cpp index ee7d69b2c211f..ce227e6650c84 100644 --- a/system/lib/llvm-libc/src/math/generic/cbrt.cpp +++ b/system/lib/llvm-libc/src/math/generic/cbrt.cpp @@ -58,7 +58,7 @@ double intial_approximation(double x) { // Get the error term for Newton iteration: // h(x) = x^3 * a^2 - 1, -#ifdef LIBC_TARGET_CPU_HAS_FMA +#ifdef LIBC_TARGET_CPU_HAS_FMA_DOUBLE double get_error(const DoubleDouble &x_3, const DoubleDouble &a_sq) { return fputil::multiply_add(x_3.hi, a_sq.hi, -1.0) + fputil::multiply_add(x_3.lo, a_sq.hi, x_3.hi * a_sq.lo); diff --git a/system/lib/llvm-libc/src/math/generic/cbrtf.cpp b/system/lib/llvm-libc/src/math/generic/cbrtf.cpp index 0abbf6e879421..71b23c4a8c742 100644 --- a/system/lib/llvm-libc/src/math/generic/cbrtf.cpp +++ b/system/lib/llvm-libc/src/math/generic/cbrtf.cpp @@ -25,7 +25,7 @@ constexpr double CBRT2[3] = {1.0, 0x1.428a2f98d728bp0, 0x1.965fea53d6e3dp0}; // Degree-7 polynomials approximation of ((1 + x)^(1/3) - 1)/x for 0 <= x <= 1 // generated by Sollya with: // > for i from 0 to 15 do { -// P = fpminimax((1 + x)^(1/3) - 1)/x, 6, [|D...|], [i/16, (i + 1)/16]); +// P = fpminimax(((1 + x)^(1/3) - 1)/x, 6, [|D...|], [i/16, (i + 1)/16]); // print("{", coeff(P, 0), ",", coeff(P, 1), ",", coeff(P, 2), ",", // coeff(P, 3), ",", coeff(P, 4), ",", coeff(P, 5), ",", // coeff(P, 6), "},"); diff --git a/system/lib/llvm-libc/src/math/generic/common_constants.cpp b/system/lib/llvm-libc/src/math/generic/common_constants.cpp index e29c083fa474a..b2c1293c6326d 100644 --- a/system/lib/llvm-libc/src/math/generic/common_constants.cpp +++ b/system/lib/llvm-libc/src/math/generic/common_constants.cpp @@ -13,6 +13,45 @@ namespace LIBC_NAMESPACE_DECL { +// Lookup table for logf(f) = logf(1 + n*2^(-7)) where n = 0..127, +// computed and stored as float precision constants. +// Generated by Sollya with the following commands: +// display = hexadecimal; +// for n from 0 to 127 do { print(single(1 / (1 + n / 128.0))); }; +const float ONE_OVER_F_FLOAT[128] = { + 0x1p0f, 0x1.fc07fp-1f, 0x1.f81f82p-1f, 0x1.f4465ap-1f, + 0x1.f07c2p-1f, 0x1.ecc07cp-1f, 0x1.e9131ap-1f, 0x1.e573acp-1f, + 0x1.e1e1e2p-1f, 0x1.de5d6ep-1f, 0x1.dae608p-1f, 0x1.d77b66p-1f, + 0x1.d41d42p-1f, 0x1.d0cb58p-1f, 0x1.cd8568p-1f, 0x1.ca4b3p-1f, + 0x1.c71c72p-1f, 0x1.c3f8fp-1f, 0x1.c0e07p-1f, 0x1.bdd2b8p-1f, + 0x1.bacf92p-1f, 0x1.b7d6c4p-1f, 0x1.b4e81cp-1f, 0x1.b20364p-1f, + 0x1.af286cp-1f, 0x1.ac5702p-1f, 0x1.a98ef6p-1f, 0x1.a6d01ap-1f, + 0x1.a41a42p-1f, 0x1.a16d4p-1f, 0x1.9ec8eap-1f, 0x1.9c2d14p-1f, + 0x1.99999ap-1f, 0x1.970e5p-1f, 0x1.948b1p-1f, 0x1.920fb4p-1f, + 0x1.8f9c18p-1f, 0x1.8d3018p-1f, 0x1.8acb9p-1f, 0x1.886e6p-1f, + 0x1.861862p-1f, 0x1.83c978p-1f, 0x1.818182p-1f, 0x1.7f406p-1f, + 0x1.7d05f4p-1f, 0x1.7ad22p-1f, 0x1.78a4c8p-1f, 0x1.767dcep-1f, + 0x1.745d18p-1f, 0x1.724288p-1f, 0x1.702e06p-1f, 0x1.6e1f76p-1f, + 0x1.6c16c2p-1f, 0x1.6a13cep-1f, 0x1.681682p-1f, 0x1.661ec6p-1f, + 0x1.642c86p-1f, 0x1.623fa8p-1f, 0x1.605816p-1f, 0x1.5e75bcp-1f, + 0x1.5c9882p-1f, 0x1.5ac056p-1f, 0x1.58ed24p-1f, 0x1.571ed4p-1f, + 0x1.555556p-1f, 0x1.539094p-1f, 0x1.51d07ep-1f, 0x1.501502p-1f, + 0x1.4e5e0ap-1f, 0x1.4cab88p-1f, 0x1.4afd6ap-1f, 0x1.49539ep-1f, + 0x1.47ae14p-1f, 0x1.460cbcp-1f, 0x1.446f86p-1f, 0x1.42d662p-1f, + 0x1.414142p-1f, 0x1.3fb014p-1f, 0x1.3e22ccp-1f, 0x1.3c995ap-1f, + 0x1.3b13b2p-1f, 0x1.3991c2p-1f, 0x1.381382p-1f, 0x1.3698ep-1f, + 0x1.3521dp-1f, 0x1.33ae46p-1f, 0x1.323e34p-1f, 0x1.30d19p-1f, + 0x1.2f684cp-1f, 0x1.2e025cp-1f, 0x1.2c9fb4p-1f, 0x1.2b404ap-1f, + 0x1.29e412p-1f, 0x1.288b02p-1f, 0x1.27350cp-1f, 0x1.25e228p-1f, + 0x1.24924ap-1f, 0x1.234568p-1f, 0x1.21fb78p-1f, 0x1.20b47p-1f, + 0x1.1f7048p-1f, 0x1.1e2ef4p-1f, 0x1.1cf06ap-1f, 0x1.1bb4a4p-1f, + 0x1.1a7b96p-1f, 0x1.194538p-1f, 0x1.181182p-1f, 0x1.16e068p-1f, + 0x1.15b1e6p-1f, 0x1.1485fp-1f, 0x1.135c82p-1f, 0x1.12358ep-1f, + 0x1.111112p-1f, 0x1.0fef02p-1f, 0x1.0ecf56p-1f, 0x1.0db20ap-1f, + 0x1.0c9714p-1f, 0x1.0b7e6ep-1f, 0x1.0a681p-1f, 0x1.0953f4p-1f, + 0x1.08421p-1f, 0x1.07326p-1f, 0x1.0624dep-1f, 0x1.05198p-1f, + 0x1.041042p-1f, 0x1.03091cp-1f, 0x1.020408p-1f, 0x1.010102p-1f}; + // Lookup table for (1/f) where f = 1 + n*2^(-7), n = 0..127. const double ONE_OVER_F[128] = { 0x1.0000000000000p+0, 0x1.fc07f01fc07f0p-1, 0x1.f81f81f81f820p-1, @@ -59,6 +98,45 @@ const double ONE_OVER_F[128] = { 0x1.05197f7d73404p-1, 0x1.0410410410410p-1, 0x1.03091b51f5e1ap-1, 0x1.0204081020408p-1, 0x1.0101010101010p-1}; +// Lookup table for log(f) = log(1 + n*2^(-7)) where n = 0..127, +// computed and stored as float precision constants. +// Generated by Sollya with the following commands: +// display = hexadecimal; +// for n from 0 to 127 do { print(single(log(1 + n / 128.0))); }; +const float LOG_F_FLOAT[128] = { + 0.0f, 0x1.fe02a6p-8f, 0x1.fc0a8cp-7f, 0x1.7b91bp-6f, + 0x1.f829bp-6f, 0x1.39e87cp-5f, 0x1.77459p-5f, 0x1.b42dd8p-5f, + 0x1.f0a30cp-5f, 0x1.16536ep-4f, 0x1.341d7ap-4f, 0x1.51b074p-4f, + 0x1.6f0d28p-4f, 0x1.8c345ep-4f, 0x1.a926d4p-4f, 0x1.c5e548p-4f, + 0x1.e27076p-4f, 0x1.fec914p-4f, 0x1.0d77e8p-3f, 0x1.1b72aep-3f, + 0x1.29553p-3f, 0x1.371fc2p-3f, 0x1.44d2b6p-3f, 0x1.526e5ep-3f, + 0x1.5ff308p-3f, 0x1.6d60fep-3f, 0x1.7ab89p-3f, 0x1.87fa06p-3f, + 0x1.9525aap-3f, 0x1.a23bc2p-3f, 0x1.af3c94p-3f, 0x1.bc2868p-3f, + 0x1.c8ff7cp-3f, 0x1.d5c216p-3f, 0x1.e27076p-3f, 0x1.ef0adcp-3f, + 0x1.fb9186p-3f, 0x1.04025ap-2f, 0x1.0a324ep-2f, 0x1.1058cp-2f, + 0x1.1675cap-2f, 0x1.1c898cp-2f, 0x1.22942p-2f, 0x1.2895a2p-2f, + 0x1.2e8e2cp-2f, 0x1.347ddap-2f, 0x1.3a64c6p-2f, 0x1.404308p-2f, + 0x1.4618bcp-2f, 0x1.4be5fap-2f, 0x1.51aad8p-2f, 0x1.576772p-2f, + 0x1.5d1bdcp-2f, 0x1.62c83p-2f, 0x1.686c82p-2f, 0x1.6e08eap-2f, + 0x1.739d8p-2f, 0x1.792a56p-2f, 0x1.7eaf84p-2f, 0x1.842d1ep-2f, + 0x1.89a338p-2f, 0x1.8f11e8p-2f, 0x1.947942p-2f, 0x1.99d958p-2f, + 0x1.9f323ep-2f, 0x1.a4840ap-2f, 0x1.a9cecap-2f, 0x1.af1294p-2f, + 0x1.b44f78p-2f, 0x1.b9858ap-2f, 0x1.beb4dap-2f, 0x1.c3dd7ap-2f, + 0x1.c8ff7cp-2f, 0x1.ce1afp-2f, 0x1.d32fe8p-2f, 0x1.d83e72p-2f, + 0x1.dd46ap-2f, 0x1.e24882p-2f, 0x1.e74426p-2f, 0x1.ec399ep-2f, + 0x1.f128f6p-2f, 0x1.f6124p-2f, 0x1.faf588p-2f, 0x1.ffd2ep-2f, + 0x1.02552ap-1f, 0x1.04bdfap-1f, 0x1.0723e6p-1f, 0x1.0986f4p-1f, + 0x1.0be72ep-1f, 0x1.0e4498p-1f, 0x1.109f3ap-1f, 0x1.12f71ap-1f, + 0x1.154c3ep-1f, 0x1.179eacp-1f, 0x1.19ee6cp-1f, 0x1.1c3b82p-1f, + 0x1.1e85f6p-1f, 0x1.20cdcep-1f, 0x1.23130ep-1f, 0x1.2555bcp-1f, + 0x1.2795e2p-1f, 0x1.29d38p-1f, 0x1.2c0e9ep-1f, 0x1.2e4744p-1f, + 0x1.307d74p-1f, 0x1.32b134p-1f, 0x1.34e28ap-1f, 0x1.37117cp-1f, + 0x1.393e0ep-1f, 0x1.3b6844p-1f, 0x1.3d9026p-1f, 0x1.3fb5b8p-1f, + 0x1.41d8fep-1f, 0x1.43f9fep-1f, 0x1.4618bcp-1f, 0x1.48353ep-1f, + 0x1.4a4f86p-1f, 0x1.4c679ap-1f, 0x1.4e7d82p-1f, 0x1.50913cp-1f, + 0x1.52a2d2p-1f, 0x1.54b246p-1f, 0x1.56bf9ep-1f, 0x1.58cadcp-1f, + 0x1.5ad404p-1f, 0x1.5cdb1ep-1f, 0x1.5ee02ap-1f, 0x1.60e33p-1f}; + // Lookup table for log(f) = log(1 + n*2^(-7)) where n = 0..127. const double LOG_F[128] = { 0x0.0000000000000p+0, 0x1.fe02a6b106788p-8, 0x1.fc0a8b0fc03e3p-7, @@ -112,7 +190,7 @@ const double LOG_F[128] = { // precision, and -2^-8 <= v < 2^-7. // TODO(lntue): Add reference to how the constants are derived after the // resulting paper is ready. -alignas(32) const float R[128] = { +alignas(8) const float R[128] = { 0x1p0, 0x1.fcp-1, 0x1.f8p-1, 0x1.f4p-1, 0x1.fp-1, 0x1.ecp-1, 0x1.e8p-1, 0x1.e4p-1, 0x1.ep-1, 0x1.dep-1, 0x1.dap-1, 0x1.d6p-1, 0x1.d4p-1, 0x1.dp-1, 0x1.ccp-1, 0x1.cap-1, 0x1.c6p-1, 0x1.c4p-1, 0x1.cp-1, 0x1.bep-1, 0x1.bap-1, @@ -133,7 +211,7 @@ alignas(32) const float R[128] = { 0x1.0ap-1, 0x1.08p-1, 0x1.08p-1, 0x1.06p-1, 0x1.06p-1, 0x1.04p-1, 0x1.04p-1, 0x1.02p-1, 0x1.0p-1}; -alignas(64) const double RD[128] = { +const double RD[128] = { 0x1p0, 0x1.fcp-1, 0x1.f8p-1, 0x1.f4p-1, 0x1.fp-1, 0x1.ecp-1, 0x1.e8p-1, 0x1.e4p-1, 0x1.ep-1, 0x1.dep-1, 0x1.dap-1, 0x1.d6p-1, 0x1.d4p-1, 0x1.dp-1, 0x1.ccp-1, 0x1.cap-1, 0x1.c6p-1, 0x1.c4p-1, 0x1.cp-1, 0x1.bep-1, 0x1.bap-1, @@ -158,7 +236,7 @@ alignas(64) const double RD[128] = { // available. // Generated by Sollya with the formula: CD[i] = RD[i]*(1 + i*2^-7) - 1 // for RD[i] defined on the table above. -alignas(64) const double CD[128] = { +const double CD[128] = { 0.0, -0x1p-14, -0x1p-12, -0x1.2p-11, -0x1p-10, -0x1.9p-10, -0x1.2p-9, -0x1.88p-9, -0x1p-8, -0x1.9p-11, -0x1.fp-10, -0x1.9cp-9, -0x1p-12, -0x1.cp-10, -0x1.bp-9, -0x1.5p-11, -0x1.4p-9, 0x1p-14, @@ -183,7 +261,7 @@ alignas(64) const double CD[128] = { -0x1p-14, -0x1p-8, }; -alignas(64) const double LOG_R[128] = { +const double LOG_R[128] = { 0x0.0000000000000p0, 0x1.010157588de71p-7, 0x1.0205658935847p-6, 0x1.8492528c8cabfp-6, 0x1.0415d89e74444p-5, 0x1.466aed42de3eap-5, 0x1.894aa149fb343p-5, 0x1.ccb73cdddb2ccp-5, 0x1.08598b59e3a07p-4, @@ -228,7 +306,7 @@ alignas(64) const double LOG_R[128] = { 0x1.5707a26bb8c66p-1, 0x1.5af405c3649ep-1, 0x1.5af405c3649ep-1, 0x1.5ee82aa24192p-1, 0x0.000000000000p0}; -alignas(64) const double LOG2_R[128] = { +const double LOG2_R[128] = { 0x0.0000000000000p+0, 0x1.72c7ba20f7327p-7, 0x1.743ee861f3556p-6, 0x1.184b8e4c56af8p-5, 0x1.77394c9d958d5p-5, 0x1.d6ebd1f1febfep-5, 0x1.1bb32a600549dp-4, 0x1.4c560fe68af88p-4, 0x1.7d60496cfbb4cp-4, @@ -281,7 +359,7 @@ alignas(64) const double LOG2_R[128] = { // print("{", -c, ",", -b, "},"); // }; // We replace LOG_R[0] with log10(1.0) == 0.0 -alignas(64) const NumberPair LOG_R_DD[128] = { +alignas(16) const NumberPair LOG_R_DD[128] = { {0.0, 0.0}, {-0x1.0c76b999d2be8p-46, 0x1.010157589p-7}, {-0x1.3dc5b06e2f7d2p-45, 0x1.0205658938p-6}, @@ -417,7 +495,7 @@ alignas(64) const NumberPair LOG_R_DD[128] = { // Output range: // [-0x1.3ffcp-15, 0x1.3e3dp-15] // We store S2[i] = 2^16 (r(i - 2^6) - 1). -alignas(64) const int S2[193] = { +alignas(8) const int S2[193] = { 0x101, 0xfd, 0xf9, 0xf5, 0xf1, 0xed, 0xe9, 0xe5, 0xe1, 0xdd, 0xd9, 0xd5, 0xd1, 0xcd, 0xc9, 0xc5, 0xc1, 0xbd, 0xb9, 0xb4, 0xb0, 0xac, 0xa8, 0xa4, 0xa0, 0x9c, 0x98, @@ -441,7 +519,7 @@ alignas(64) const int S2[193] = { -0x1cd, -0x1d1, -0x1d5, -0x1d9, -0x1dd, -0x1e0, -0x1e4, -0x1e8, -0x1ec, -0x1f0, -0x1f4, -0x1f8, -0x1fc}; -alignas(64) const double R2[193] = { +const double R2[193] = { 0x1.0101p0, 0x1.00fdp0, 0x1.00f9p0, 0x1.00f5p0, 0x1.00f1p0, 0x1.00edp0, 0x1.00e9p0, 0x1.00e5p0, 0x1.00e1p0, 0x1.00ddp0, 0x1.00d9p0, 0x1.00d5p0, 0x1.00d1p0, 0x1.00cdp0, 0x1.00c9p0, @@ -488,7 +566,7 @@ alignas(64) const double R2[193] = { // Output range: // [-0x1.01928p-22 , 0x1p-22] // We store S[i] = 2^21 (r(i - 80) - 1). -alignas(64) const int S3[161] = { +alignas(8) const int S3[161] = { 0x50, 0x4f, 0x4e, 0x4d, 0x4c, 0x4b, 0x4a, 0x49, 0x48, 0x47, 0x46, 0x45, 0x44, 0x43, 0x42, 0x41, 0x40, 0x3f, 0x3e, 0x3d, 0x3c, 0x3b, 0x3a, 0x39, 0x38, 0x37, 0x36, 0x35, 0x34, 0x33, 0x32, 0x31, 0x30, @@ -511,7 +589,7 @@ alignas(64) const int S3[161] = { // Output range: // [-0x1.0002143p-29 , 0x1p-29] // We store S[i] = 2^28 (r(i - 65) - 1). -alignas(64) const int S4[130] = { +alignas(8) const int S4[130] = { 0x41, 0x40, 0x3f, 0x3e, 0x3d, 0x3c, 0x3b, 0x3a, 0x39, 0x38, 0x37, 0x36, 0x35, 0x34, 0x33, 0x32, 0x31, 0x30, 0x2f, 0x2e, 0x2d, 0x2c, 0x2b, 0x2a, 0x29, 0x28, 0x27, 0x26, 0x25, 0x24, 0x23, 0x22, 0x21, @@ -661,7 +739,7 @@ const double EXP_M2[128] = { // d = round(a - b - c, D, RN); // print("{", d, ",", c, ",", b, "},"); // }; -const fputil::TripleDouble EXP2_MID1[64] = { +alignas(16) const fputil::TripleDouble EXP2_MID1[64] = { {0, 0, 0x1p0}, {-0x1.9085b0a3d74d5p-110, -0x1.19083535b085dp-56, 0x1.02c9a3e778061p0}, {0x1.05ff94f8d257ep-110, 0x1.d73e2a475b465p-55, 0x1.059b0d3158574p0}, @@ -739,7 +817,7 @@ const fputil::TripleDouble EXP2_MID1[64] = { // d = round(a - b - c, D, RN); // print("{", d, ",", c, ",", b, "},"); // }; -const fputil::TripleDouble EXP2_MID2[64] = { +alignas(16) const fputil::TripleDouble EXP2_MID2[64] = { {0, 0, 0x1p0}, {0x1.39726694630e3p-108, 0x1.ae8e38c59c72ap-54, 0x1.000b175effdc7p0}, {0x1.e5e06ddd31156p-112, -0x1.7b5d0d58ea8f4p-58, 0x1.00162f3904052p0}, diff --git a/system/lib/llvm-libc/src/math/generic/common_constants.h b/system/lib/llvm-libc/src/math/generic/common_constants.h index dc1a90c1cb24a..e65f002845953 100644 --- a/system/lib/llvm-libc/src/math/generic/common_constants.h +++ b/system/lib/llvm-libc/src/math/generic/common_constants.h @@ -15,9 +15,17 @@ namespace LIBC_NAMESPACE_DECL { +// Lookup table for (1/f) where f = 1 + n*2^(-7), n = 0..127, +// computed and stored as float precision constants. +extern const float ONE_OVER_F_FLOAT[128]; + // Lookup table for (1/f) where f = 1 + n*2^(-7), n = 0..127. extern const double ONE_OVER_F[128]; +// Lookup table for log(f) = log(1 + n*2^(-7)) where n = 0..127, +// computed and stored as float precision constants. +extern const float LOG_F_FLOAT[128]; + // Lookup table for log(f) = log(1 + n*2^(-7)) where n = 0..127. extern const double LOG_F[128]; diff --git a/system/lib/llvm-libc/src/math/generic/cos.cpp b/system/lib/llvm-libc/src/math/generic/cos.cpp index 568b1254c6f02..5da0f86812a89 100644 --- a/system/lib/llvm-libc/src/math/generic/cos.cpp +++ b/system/lib/llvm-libc/src/math/generic/cos.cpp @@ -20,11 +20,11 @@ #include "src/math/generic/range_reduction_double_common.h" #include "src/math/generic/sincos_eval.h" -#ifdef LIBC_TARGET_CPU_HAS_FMA +#ifdef LIBC_TARGET_CPU_HAS_FMA_DOUBLE #include "range_reduction_double_fma.h" #else #include "range_reduction_double_nofma.h" -#endif // LIBC_TARGET_CPU_HAS_FMA +#endif // LIBC_TARGET_CPU_HAS_FMA_DOUBLE namespace LIBC_NAMESPACE_DECL { @@ -65,7 +65,11 @@ LLVM_LIBC_FUNCTION(double, cos, (double x)) { } else { // Inf or NaN if (LIBC_UNLIKELY(x_e > 2 * FPBits::EXP_BIAS)) { - // sin(+-Inf) = NaN + if (xbits.is_signaling_nan()) { + fputil::raise_except_if_required(FE_INVALID); + return FPBits::quiet_nan().get_val(); + } + // cos(+-Inf) = NaN if (xbits.get_mantissa() == 0) { fputil::set_errno_if_required(EDOM); fputil::raise_except_if_required(FE_INVALID); diff --git a/system/lib/llvm-libc/src/math/generic/cosf.cpp b/system/lib/llvm-libc/src/math/generic/cosf.cpp index 972ffa923aedf..7cdae09869588 100644 --- a/system/lib/llvm-libc/src/math/generic/cosf.cpp +++ b/system/lib/llvm-libc/src/math/generic/cosf.cpp @@ -20,6 +20,7 @@ namespace LIBC_NAMESPACE_DECL { +#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS // Exceptional cases for cosf. static constexpr size_t N_EXCEPTS = 6; @@ -38,6 +39,7 @@ static constexpr fputil::ExceptValues COSF_EXCEPTS{{ // x = 0x1.ddebdep120, cos(x) = 0x1.114438p-1 (RZ) {0x7beef5ef, 0x3f08a21c, 1, 0, 0}, }}; +#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS LLVM_LIBC_FUNCTION(float, cosf, (float x)) { using FPBits = typename fputil::FPBits; @@ -101,18 +103,25 @@ LLVM_LIBC_FUNCTION(float, cosf, (float x)) { // |x| < 2^-125. For targets without FMA instructions, we simply use // double for intermediate results as it is more efficient than using an // emulated version of FMA. -#if defined(LIBC_TARGET_CPU_HAS_FMA) +#if defined(LIBC_TARGET_CPU_HAS_FMA_FLOAT) return fputil::multiply_add(xbits.get_val(), -0x1.0p-25f, 1.0f); #else return static_cast(fputil::multiply_add(xd, -0x1.0p-25, 1.0)); -#endif // LIBC_TARGET_CPU_HAS_FMA +#endif // LIBC_TARGET_CPU_HAS_FMA_FLOAT } +#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS if (auto r = COSF_EXCEPTS.lookup(x_abs); LIBC_UNLIKELY(r.has_value())) return r.value(); +#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS // x is inf or nan. if (LIBC_UNLIKELY(x_abs >= 0x7f80'0000U)) { + if (xbits.is_signaling_nan()) { + fputil::raise_except_if_required(FE_INVALID); + return FPBits::quiet_nan().get_val(); + } + if (x_abs == 0x7f80'0000U) { fputil::set_errno_if_required(EDOM); fputil::raise_except_if_required(FE_INVALID); diff --git a/system/lib/llvm-libc/src/math/generic/cospif.cpp b/system/lib/llvm-libc/src/math/generic/cospif.cpp index 4ef1539539921..5b6880f853b26 100644 --- a/system/lib/llvm-libc/src/math/generic/cospif.cpp +++ b/system/lib/llvm-libc/src/math/generic/cospif.cpp @@ -50,11 +50,11 @@ LLVM_LIBC_FUNCTION(float, cospif, (float x)) { // The exhautive test passes for smaller values if (LIBC_UNLIKELY(x_abs < 0x38A2'F984U)) { -#if defined(LIBC_TARGET_CPU_HAS_FMA) +#if defined(LIBC_TARGET_CPU_HAS_FMA_FLOAT) return fputil::multiply_add(xbits.get_val(), -0x1.0p-25f, 1.0f); #else return static_cast(fputil::multiply_add(xd, -0x1.0p-25, 1.0)); -#endif // LIBC_TARGET_CPU_HAS_FMA +#endif // LIBC_TARGET_CPU_HAS_FMA_FLOAT } // Numbers greater or equal to 2^23 are always integers or NaN @@ -66,6 +66,11 @@ LLVM_LIBC_FUNCTION(float, cospif, (float x)) { // x is inf or nan. if (LIBC_UNLIKELY(x_abs >= 0x7f80'0000U)) { + if (xbits.is_signaling_nan()) { + fputil::raise_except_if_required(FE_INVALID); + return FPBits::quiet_nan().get_val(); + } + if (x_abs == 0x7f80'0000U) { fputil::set_errno_if_required(EDOM); fputil::raise_except_if_required(FE_INVALID); diff --git a/system/lib/llvm-libc/src/math/generic/erff.cpp b/system/lib/llvm-libc/src/math/generic/erff.cpp index 15357452759ea..44607a52a2e57 100644 --- a/system/lib/llvm-libc/src/math/generic/erff.cpp +++ b/system/lib/llvm-libc/src/math/generic/erff.cpp @@ -135,12 +135,17 @@ LLVM_LIBC_FUNCTION(float, erff, (float x)) { int sign = xbits.is_neg() ? 1 : 0; if (LIBC_UNLIKELY(x_abs >= 0x7f80'0000U)) { + if (xbits.is_signaling_nan()) { + fputil::raise_except_if_required(FE_INVALID); + return FPBits::quiet_nan().get_val(); + } return (x_abs > 0x7f80'0000) ? x : ONE[sign]; } return ONE[sign] + SMALL[sign]; } +#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS // Exceptional mask = common 0 bits of 2 exceptional values. constexpr uint32_t EXCEPT_MASK = 0x809a'6184U; @@ -155,6 +160,7 @@ LLVM_LIBC_FUNCTION(float, erff, (float x)) { if (x_abs == 0U) return x; } +#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS // Polynomial approximation: // erf(x) ~ x * (c0 + c1 * x^2 + c2 * x^4 + ... + c7 * x^14) diff --git a/system/lib/llvm-libc/src/math/generic/exp.cpp b/system/lib/llvm-libc/src/math/generic/exp.cpp index 38b683aa01166..143800ca078a6 100644 --- a/system/lib/llvm-libc/src/math/generic/exp.cpp +++ b/system/lib/llvm-libc/src/math/generic/exp.cpp @@ -39,8 +39,11 @@ constexpr double LOG2_E = 0x1.71547652b82fep+0; // Error bounds: // Errors when using double precision. constexpr double ERR_D = 0x1.8p-63; + +#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS // Errors when using double-double precision. constexpr double ERR_DD = 0x1.0p-99; +#endif // LIBC_MATH_HAS_SKIP_ACCURATE_PASS // -2^-12 * log(2) // > a = -2^-12 * log(2); @@ -50,8 +53,11 @@ constexpr double ERR_DD = 0x1.0p-99; // Errors < 1.5 * 2^-133 constexpr double MLOG_2_EXP2_M12_HI = -0x1.62e42ffp-13; constexpr double MLOG_2_EXP2_M12_MID = 0x1.718432a1b0e26p-47; + +#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS constexpr double MLOG_2_EXP2_M12_MID_30 = 0x1.718432ap-47; constexpr double MLOG_2_EXP2_M12_LO = 0x1.b0e2633fe0685p-79; +#endif // LIBC_MATH_HAS_SKIP_ACCURATE_PASS namespace { @@ -72,6 +78,7 @@ LIBC_INLINE double poly_approx_d(double dx) { return p; } +#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS // Polynomial approximation with double-double precision: // Return exp(dx) ~ 1 + dx + dx^2 / 2 + ... + dx^6 / 720 // For |dx| < 2^-13 + 2^-30: @@ -171,6 +178,7 @@ DoubleDouble exp_double_double(double x, double kd, return r; } +#endif // LIBC_MATH_HAS_SKIP_ACCURATE_PASS // Check for exceptional cases when // |x| <= 2^-53 or x < log(2^-1075) or x >= 0x1.6232bdd7abcd3p+9 @@ -373,6 +381,19 @@ LLVM_LIBC_FUNCTION(double, exp, (double x)) { double lo = fputil::multiply_add(p, mid_lo, exp_mid.lo); +#ifdef LIBC_MATH_HAS_SKIP_ACCURATE_PASS + if (LIBC_UNLIKELY(denorm)) { + return ziv_test_denorm(hi, exp_mid.hi, lo, ERR_D) + .value(); + } else { + // to multiply by 2^hi, a fast way is to simply add hi to the exponent + // field. + int64_t exp_hi = static_cast(hi) << FPBits::FRACTION_LEN; + double r = + cpp::bit_cast(exp_hi + cpp::bit_cast(exp_mid.hi + lo)); + return r; + } +#else if (LIBC_UNLIKELY(denorm)) { if (auto r = ziv_test_denorm(hi, exp_mid.hi, lo, ERR_D); LIBC_LIKELY(r.has_value())) @@ -413,6 +434,7 @@ LLVM_LIBC_FUNCTION(double, exp, (double x)) { Float128 r_f128 = exp_f128(x, kd, idx1, idx2); return static_cast(r_f128); +#endif // LIBC_MATH_HAS_SKIP_ACCURATE_PASS } } // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/generic/exp10.cpp b/system/lib/llvm-libc/src/math/generic/exp10.cpp index 748c8a22b2368..c464979b092c3 100644 --- a/system/lib/llvm-libc/src/math/generic/exp10.cpp +++ b/system/lib/llvm-libc/src/math/generic/exp10.cpp @@ -44,15 +44,20 @@ constexpr double LOG2_10 = 0x1.a934f0979a371p+1; // Errors < 1.5 * 2^-144 constexpr double MLOG10_2_EXP2_M12_HI = -0x1.3441350ap-14; constexpr double MLOG10_2_EXP2_M12_MID = 0x1.0c0219dc1da99p-51; + +#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS constexpr double MLOG10_2_EXP2_M12_MID_32 = 0x1.0c0219dcp-51; constexpr double MLOG10_2_EXP2_M12_LO = 0x1.da994fd20dba2p-87; +#endif // LIBC_MATH_HAS_SKIP_ACCURATE_PASS // Error bounds: // Errors when using double precision. constexpr double ERR_D = 0x1.8p-63; +#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS // Errors when using double-double precision. constexpr double ERR_DD = 0x1.8p-99; +#endif // LIBC_MATH_HAS_SKIP_ACCURATE_PASS namespace { @@ -72,6 +77,7 @@ LIBC_INLINE double poly_approx_d(double dx) { return p; } +#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS // Polynomial approximation with double-double precision. Generated by Solya // with: // > P = fpminimax((10^x - 1)/x, 5, [|DD...|], [-2^-14, 2^-14]); @@ -171,6 +177,7 @@ DoubleDouble exp10_double_double(double x, double kd, return r; } +#endif // LIBC_MATH_HAS_SKIP_ACCURATE_PASS // When output is denormal. double exp10_denorm(double x) { @@ -199,6 +206,10 @@ double exp10_denorm(double x) { double lo = fputil::multiply_add(p, mid_lo, exp_mid.lo); +#ifdef LIBC_MATH_HAS_SKIP_ACCURATE_PASS + return ziv_test_denorm(hi, exp_mid.hi, lo, ERR_D) + .value(); +#else if (auto r = ziv_test_denorm(hi, exp_mid.hi, lo, ERR_D); LIBC_LIKELY(r.has_value())) return r.value(); @@ -214,6 +225,7 @@ double exp10_denorm(double x) { Float128 r_f128 = exp10_f128(x, kd, idx1, idx2); return static_cast(r_f128); +#endif // LIBC_MATH_HAS_SKIP_ACCURATE_PASS } // Check for exceptional cases when: @@ -391,6 +403,12 @@ LLVM_LIBC_FUNCTION(double, exp10, (double x)) { double lo = fputil::multiply_add(p, mid_lo, exp_mid.lo); +#ifdef LIBC_MATH_HAS_SKIP_ACCURATE_PASS + int64_t exp_hi = static_cast(hi) << FPBits::FRACTION_LEN; + double r = + cpp::bit_cast(exp_hi + cpp::bit_cast(exp_mid.hi + lo)); + return r; +#else double upper = exp_mid.hi + (lo + ERR_D); double lower = exp_mid.hi + (lo - ERR_D); @@ -473,6 +491,7 @@ LLVM_LIBC_FUNCTION(double, exp10, (double x)) { Float128 r_f128 = exp10_f128(x, kd, idx1, idx2); return static_cast(r_f128); +#endif // LIBC_MATH_HAS_SKIP_ACCURATE_PASS } } // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/generic/exp10m1f.cpp b/system/lib/llvm-libc/src/math/generic/exp10m1f.cpp index c0e302eea7b08..27729104e038d 100644 --- a/system/lib/llvm-libc/src/math/generic/exp10m1f.cpp +++ b/system/lib/llvm-libc/src/math/generic/exp10m1f.cpp @@ -14,14 +14,15 @@ #include "src/__support/FPUtil/multiply_add.h" #include "src/__support/FPUtil/rounding_mode.h" #include "src/__support/common.h" +#include "src/__support/libc_errno.h" #include "src/__support/macros/config.h" #include "src/__support/macros/optimization.h" -#include "src/errno/libc_errno.h" #include "explogxf.h" namespace LIBC_NAMESPACE_DECL { +#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS static constexpr size_t N_EXCEPTS_LO = 11; static constexpr fputil::ExceptValues EXP10M1F_EXCEPTS_LO = @@ -94,6 +95,7 @@ static constexpr fputil::ExceptValues EXP10M1F_EXCEPTS_HI = // x = -0x1.ca4322p-5, exp10m1f(x) = -0x1.ef073p-4 (RZ) {0xbd65'2191U, 0xbdf7'8398U, 0U, 1U, 1U}, }}; +#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS LLVM_LIBC_FUNCTION(float, exp10m1f, (float x)) { using FPBits = fputil::FPBits; @@ -119,8 +121,10 @@ LLVM_LIBC_FUNCTION(float, exp10m1f, (float x)) { // When |x| <= log10(2) * 2^(-6) if (LIBC_UNLIKELY(x_abs <= 0x3b9a'209bU)) { +#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS if (auto r = EXP10M1F_EXCEPTS_LO.lookup(x_u); LIBC_UNLIKELY(r.has_value())) return r.value(); +#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS double dx = x; double dx_sq = dx * dx; @@ -192,8 +196,10 @@ LLVM_LIBC_FUNCTION(float, exp10m1f, (float x)) { } } +#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS if (auto r = EXP10M1F_EXCEPTS_HI.lookup(x_u); LIBC_UNLIKELY(r.has_value())) return r.value(); +#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS // Range reduction: 10^x = 2^(mid + hi) * 10^lo // rr = (2^(mid + hi), lo) diff --git a/system/lib/llvm-libc/src/math/generic/exp2.cpp b/system/lib/llvm-libc/src/math/generic/exp2.cpp index 935548b538b94..726f88b6457fc 100644 --- a/system/lib/llvm-libc/src/math/generic/exp2.cpp +++ b/system/lib/llvm-libc/src/math/generic/exp2.cpp @@ -35,14 +35,16 @@ using LIBC_NAMESPACE::operator""_u128; // Error bounds: // Errors when using double precision. -#ifdef LIBC_TARGET_CPU_HAS_FMA +#ifdef LIBC_TARGET_CPU_HAS_FMA_DOUBLE constexpr double ERR_D = 0x1.0p-63; #else constexpr double ERR_D = 0x1.8p-63; -#endif // LIBC_TARGET_CPU_HAS_FMA +#endif // LIBC_TARGET_CPU_HAS_FMA_DOUBLE +#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS // Errors when using double-double precision. constexpr double ERR_DD = 0x1.0p-100; +#endif // LIBC_MATH_HAS_SKIP_ACCURATE_PASS namespace { @@ -62,6 +64,7 @@ LIBC_INLINE double poly_approx_d(double dx) { return p; } +#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS // Polynomial approximation with double-double precision. Generated by Solya // with: // > P = fpminimax((2^x - 1)/x, 5, [|DD...|], [-2^-13 - 2^-30, 2^-13 + 2^-30]); @@ -147,6 +150,7 @@ DoubleDouble exp2_double_double(double x, const DoubleDouble &exp_mid) { return r; } +#endif // LIBC_MATH_HAS_SKIP_ACCURATE_PASS // When output is denormal. double exp2_denorm(double x) { @@ -174,6 +178,10 @@ double exp2_denorm(double x) { double lo = fputil::multiply_add(p, mid_lo, exp_mid.lo); +#ifdef LIBC_MATH_HAS_SKIP_ACCURATE_PASS + return ziv_test_denorm(hi, exp_mid.hi, lo, ERR_D) + .value(); +#else if (auto r = ziv_test_denorm(hi, exp_mid.hi, lo, ERR_D); LIBC_LIKELY(r.has_value())) return r.value(); @@ -189,6 +197,7 @@ double exp2_denorm(double x) { Float128 r_f128 = exp2_f128(dx, hi, idx1, idx2); return static_cast(r_f128); +#endif // LIBC_MATH_HAS_SKIP_ACCURATE_PASS } // Check for exceptional cases when: @@ -358,6 +367,14 @@ LLVM_LIBC_FUNCTION(double, exp2, (double x)) { double lo = fputil::multiply_add(p, mid_lo, exp_mid.lo); +#ifdef LIBC_MATH_HAS_SKIP_ACCURATE_PASS + // To multiply by 2^hi, a fast way is to simply add hi to the exponent + // field. + int64_t exp_hi = static_cast(hi) << FPBits::FRACTION_LEN; + double r = + cpp::bit_cast(exp_hi + cpp::bit_cast(exp_mid.hi + lo)); + return r; +#else double upper = exp_mid.hi + (lo + ERR_D); double lower = exp_mid.hi + (lo - ERR_D); @@ -387,6 +404,7 @@ LLVM_LIBC_FUNCTION(double, exp2, (double x)) { Float128 r_f128 = exp2_f128(dx, hi, idx1, idx2); return static_cast(r_f128); +#endif // LIBC_MATH_HAS_SKIP_ACCURATE_PASS } } // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/generic/exp2f_impl.h b/system/lib/llvm-libc/src/math/generic/exp2f_impl.h index ae2d0628c1205..5c6c2bd415188 100644 --- a/system/lib/llvm-libc/src/math/generic/exp2f_impl.h +++ b/system/lib/llvm-libc/src/math/generic/exp2f_impl.h @@ -27,10 +27,6 @@ namespace LIBC_NAMESPACE_DECL { namespace generic { LIBC_INLINE float exp2f(float x) { - constexpr uint32_t EXVAL1 = 0x3b42'9d37U; - constexpr uint32_t EXVAL2 = 0xbcf3'a937U; - constexpr uint32_t EXVAL_MASK = EXVAL1 & EXVAL2; - using FPBits = typename fputil::FPBits; FPBits xbits(x); @@ -46,6 +42,11 @@ LIBC_INLINE float exp2f(float x) { return 1.0f + x; } +#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS + constexpr uint32_t EXVAL1 = 0x3b42'9d37U; + constexpr uint32_t EXVAL2 = 0xbcf3'a937U; + constexpr uint32_t EXVAL_MASK = EXVAL1 & EXVAL2; + // Check exceptional values. if (LIBC_UNLIKELY((x_u & EXVAL_MASK) == EXVAL_MASK)) { if (LIBC_UNLIKELY(x_u == EXVAL1)) { // x = 0x1.853a6ep-9f @@ -54,6 +55,7 @@ LIBC_INLINE float exp2f(float x) { return fputil::round_result_slightly_down(0x1.f58d62p-1f); } } +#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS // Minimax polynomial generated by Sollya with: // > P = fpminimax((2^x - 1)/x, 5, [|D...|], [-2^-5, 2^-5]); diff --git a/system/lib/llvm-libc/src/math/generic/exp2m1f.cpp b/system/lib/llvm-libc/src/math/generic/exp2m1f.cpp index 2060dc34cc9bf..127c6eaa494d4 100644 --- a/system/lib/llvm-libc/src/math/generic/exp2m1f.cpp +++ b/system/lib/llvm-libc/src/math/generic/exp2m1f.cpp @@ -14,15 +14,16 @@ #include "src/__support/FPUtil/multiply_add.h" #include "src/__support/FPUtil/rounding_mode.h" #include "src/__support/common.h" +#include "src/__support/libc_errno.h" #include "src/__support/macros/config.h" #include "src/__support/macros/optimization.h" #include "src/__support/macros/properties/cpu_features.h" -#include "src/errno/libc_errno.h" #include "explogxf.h" namespace LIBC_NAMESPACE_DECL { +#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS static constexpr size_t N_EXCEPTS_LO = 8; static constexpr fputil::ExceptValues EXP2M1F_EXCEPTS_LO = @@ -58,6 +59,7 @@ static constexpr fputil::ExceptValues EXP2M1F_EXCEPTS_HI = // x = -0x1.de7b9cp-5, exp2m1f(x) = -0x1.4508f4p-5 (RZ) {0xbd6f'3dceU, 0xbd22'847aU, 0U, 1U, 1U}, }}; +#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS LLVM_LIBC_FUNCTION(float, exp2m1f, (float x)) { using FPBits = fputil::FPBits; @@ -70,8 +72,10 @@ LLVM_LIBC_FUNCTION(float, exp2m1f, (float x)) { if (LIBC_UNLIKELY(x_abs >= 0x4300'0000U || x_abs <= 0x3d00'0000U)) { // |x| <= 2^-5 if (x_abs <= 0x3d00'0000U) { +#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS if (auto r = EXP2M1F_EXCEPTS_LO.lookup(x_u); LIBC_UNLIKELY(r.has_value())) return r.value(); +#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS // Minimax polynomial generated by Sollya with: // > display = hexadecimal; @@ -121,8 +125,10 @@ LLVM_LIBC_FUNCTION(float, exp2m1f, (float x)) { return -1.0f; } +#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS if (auto r = EXP2M1F_EXCEPTS_HI.lookup(x_u); LIBC_UNLIKELY(r.has_value())) return r.value(); +#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS // For -25 < x < 128, to compute 2^x, we perform the following range // reduction: find hi, mid, lo such that: diff --git a/system/lib/llvm-libc/src/math/generic/exp_utils.cpp b/system/lib/llvm-libc/src/math/generic/exp_utils.cpp deleted file mode 100644 index cc21637e4c7c0..0000000000000 --- a/system/lib/llvm-libc/src/math/generic/exp_utils.cpp +++ /dev/null @@ -1,128 +0,0 @@ -//===-- Implemention of exp and friends' utils ----------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#include "exp_utils.h" -#include "src/__support/macros/config.h" - -namespace LIBC_NAMESPACE_DECL { - -const Exp2fDataTable exp2f_data = { - // :tab[i] = uint(2^(i/N)) - (i << 52-BITS) - // used for computing 2^(k/N) for an int |k| < 150 N as - // double(tab[k%N] + (k << 52-BITS)) - { -// tab -#if N == 8 - 0x3ff0000000000000, - 0x3fef72b83c7d517b, - 0x3fef06fe0a31b715, - 0x3feebfdad5362a27, - 0x3feea09e667f3bcd, - 0x3feeace5422aa0db, - 0x3feee89f995ad3ad, - 0x3fef5818dcfba487, -#elif N == 16 - 0x3ff0000000000000, - 0x3fefb5586cf9890f, - 0x3fef72b83c7d517b, - 0x3fef387a6e756238, - 0x3fef06fe0a31b715, - 0x3feedea64c123422, - 0x3feebfdad5362a27, - 0x3feeab07dd485429, - 0x3feea09e667f3bcd, - 0x3feea11473eb0187, - 0x3feeace5422aa0db, - 0x3feec49182a3f090, - 0x3feee89f995ad3ad, - 0x3fef199bdd85529c, - 0x3fef5818dcfba487, - 0x3fefa4afa2a490da, -#elif N == 32 - 0x3ff0000000000000, 0x3fefd9b0d3158574, 0x3fefb5586cf9890f, - 0x3fef9301d0125b51, 0x3fef72b83c7d517b, 0x3fef54873168b9aa, - 0x3fef387a6e756238, 0x3fef1e9df51fdee1, 0x3fef06fe0a31b715, - 0x3feef1a7373aa9cb, 0x3feedea64c123422, 0x3feece086061892d, - 0x3feebfdad5362a27, 0x3feeb42b569d4f82, 0x3feeab07dd485429, - 0x3feea47eb03a5585, 0x3feea09e667f3bcd, 0x3fee9f75e8ec5f74, - 0x3feea11473eb0187, 0x3feea589994cce13, 0x3feeace5422aa0db, - 0x3feeb737b0cdc5e5, 0x3feec49182a3f090, 0x3feed503b23e255d, - 0x3feee89f995ad3ad, 0x3feeff76f2fb5e47, 0x3fef199bdd85529c, - 0x3fef3720dcef9069, 0x3fef5818dcfba487, 0x3fef7c97337b9b5f, - 0x3fefa4afa2a490da, 0x3fefd0765b6e4540, -#elif N == 64 - 0x3ff0000000000000, 0x3fefec9a3e778061, 0x3fefd9b0d3158574, - 0x3fefc74518759bc8, 0x3fefb5586cf9890f, 0x3fefa3ec32d3d1a2, - 0x3fef9301d0125b51, 0x3fef829aaea92de0, 0x3fef72b83c7d517b, - 0x3fef635beb6fcb75, 0x3fef54873168b9aa, 0x3fef463b88628cd6, - 0x3fef387a6e756238, 0x3fef2b4565e27cdd, 0x3fef1e9df51fdee1, - 0x3fef1285a6e4030b, 0x3fef06fe0a31b715, 0x3feefc08b26416ff, - 0x3feef1a7373aa9cb, 0x3feee7db34e59ff7, 0x3feedea64c123422, - 0x3feed60a21f72e2a, 0x3feece086061892d, 0x3feec6a2b5c13cd0, - 0x3feebfdad5362a27, 0x3feeb9b2769d2ca7, 0x3feeb42b569d4f82, - 0x3feeaf4736b527da, 0x3feeab07dd485429, 0x3feea76f15ad2148, - 0x3feea47eb03a5585, 0x3feea23882552225, 0x3feea09e667f3bcd, - 0x3fee9fb23c651a2f, 0x3fee9f75e8ec5f74, 0x3fee9feb564267c9, - 0x3feea11473eb0187, 0x3feea2f336cf4e62, 0x3feea589994cce13, - 0x3feea8d99b4492ed, 0x3feeace5422aa0db, 0x3feeb1ae99157736, - 0x3feeb737b0cdc5e5, 0x3feebd829fde4e50, 0x3feec49182a3f090, - 0x3feecc667b5de565, 0x3feed503b23e255d, 0x3feede6b5579fdbf, - 0x3feee89f995ad3ad, 0x3feef3a2b84f15fb, 0x3feeff76f2fb5e47, - 0x3fef0c1e904bc1d2, 0x3fef199bdd85529c, 0x3fef27f12e57d14b, - 0x3fef3720dcef9069, 0x3fef472d4a07897c, 0x3fef5818dcfba487, - 0x3fef69e603db3285, 0x3fef7c97337b9b5f, 0x3fef902ee78b3ff6, - 0x3fefa4afa2a490da, 0x3fefba1bee615a27, 0x3fefd0765b6e4540, - 0x3fefe7c1819e90d8, -#endif - }, - as_double(0x4338000000000000) / N, // shift_scaled - { -// poly -#if N == 8 - as_double(0x3fac6a00335106e2), - as_double(0x3fcec0c313449f55), - as_double(0x3fe62e431111f69f), -#elif N == 16 - as_double(0x3fac6ac6aa313963), - as_double(0x3fcebfff4532d9ba), - as_double(0x3fe62e43001bc49f), -#elif N == 32 - as_double(0x3fac6af84b912394), - as_double(0x3fcebfce50fac4f3), - as_double(0x3fe62e42ff0c52d6), -#elif N == 64 - as_double(0x3fac6b04b4221b2a), - as_double(0x3fcebfc213e184d7), - as_double(0x3fe62e42fefb5b7f), -#endif - }, - as_double(0x4338000000000000), // shift - as_double(0x3ff71547652b82fe) * N, // invln2_scaled - { -// poly_scaled -#if N == 8 - as_double(0x3fac6a00335106e2) / N / N / N, - as_double(0x3fcec0c313449f55) / N / N, - as_double(0x3fe62e431111f69f) / N, -#elif N == 16 - as_double(0x3fac6ac6aa313963) / N / N / N, - as_double(0x3fcebfff4532d9ba) / N / N, - as_double(0x3fe62e43001bc49f) / N, -#elif N == 32 - as_double(0x3fac6af84b912394) / N / N / N, - as_double(0x3fcebfce50fac4f3) / N / N, - as_double(0x3fe62e42ff0c52d6) / N, -#elif N == 64 - as_double(0x3fac6b04b4221b2a) / N / N / N, - as_double(0x3fcebfc213e184d7) / N / N, - as_double(0x3fe62e42fefb5b7f) / N, -#endif - }, -}; - -} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/generic/exp_utils.h b/system/lib/llvm-libc/src/math/generic/exp_utils.h deleted file mode 100644 index dca9eb7c2d715..0000000000000 --- a/system/lib/llvm-libc/src/math/generic/exp_utils.h +++ /dev/null @@ -1,34 +0,0 @@ -//===-- Collection of utils for exp and friends -----------------*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_LIBC_SRC_MATH_GENERIC_EXP_UTILS_H -#define LLVM_LIBC_SRC_MATH_GENERIC_EXP_UTILS_H - -#include "src/__support/macros/config.h" -#include - -#define EXP2F_TABLE_BITS 5 -#define EXP2F_POLY_ORDER 3 -#define N (1 << EXP2F_TABLE_BITS) - -namespace LIBC_NAMESPACE_DECL { - -struct Exp2fDataTable { - uint64_t tab[1 << EXP2F_TABLE_BITS]; - double shift_scaled; - double poly[EXP2F_POLY_ORDER]; - double shift; - double invln2_scaled; - double poly_scaled[EXP2F_POLY_ORDER]; -}; - -extern const Exp2fDataTable exp2f_data; - -} // namespace LIBC_NAMESPACE_DECL - -#endif // LLVM_LIBC_SRC_MATH_GENERIC_EXP_UTILS_H diff --git a/system/lib/llvm-libc/src/math/generic/expf.cpp b/system/lib/llvm-libc/src/math/generic/expf.cpp index ee5c2a32b6c6b..de11f51ac64a0 100644 --- a/system/lib/llvm-libc/src/math/generic/expf.cpp +++ b/system/lib/llvm-libc/src/math/generic/expf.cpp @@ -7,101 +7,10 @@ //===----------------------------------------------------------------------===// #include "src/math/expf.h" -#include "common_constants.h" // Lookup tables EXP_M1 and EXP_M2. -#include "src/__support/FPUtil/BasicOperations.h" -#include "src/__support/FPUtil/FEnvImpl.h" -#include "src/__support/FPUtil/FPBits.h" -#include "src/__support/FPUtil/PolyEval.h" -#include "src/__support/FPUtil/multiply_add.h" -#include "src/__support/FPUtil/nearest_integer.h" -#include "src/__support/FPUtil/rounding_mode.h" -#include "src/__support/common.h" -#include "src/__support/macros/config.h" -#include "src/__support/macros/optimization.h" // LIBC_UNLIKELY +#include "src/__support/math/expf.h" namespace LIBC_NAMESPACE_DECL { -LLVM_LIBC_FUNCTION(float, expf, (float x)) { - using FPBits = typename fputil::FPBits; - FPBits xbits(x); - - uint32_t x_u = xbits.uintval(); - uint32_t x_abs = x_u & 0x7fff'ffffU; - - // Exceptional values - if (LIBC_UNLIKELY(x_u == 0xc236'bd8cU)) { // x = -0x1.6d7b18p+5f - return 0x1.108a58p-66f - x * 0x1.0p-95f; - } - - // When |x| >= 89, |x| < 2^-25, or x is nan - if (LIBC_UNLIKELY(x_abs >= 0x42b2'0000U || x_abs <= 0x3280'0000U)) { - // |x| < 2^-25 - if (xbits.get_biased_exponent() <= 101) { - return 1.0f + x; - } - - // When x < log(2^-150) or nan - if (xbits.uintval() >= 0xc2cf'f1b5U) { - // exp(-Inf) = 0 - if (xbits.is_inf()) - return 0.0f; - // exp(nan) = nan - if (xbits.is_nan()) - return x; - if (fputil::fenv_is_round_up()) - return FPBits::min_subnormal().get_val(); - fputil::set_errno_if_required(ERANGE); - fputil::raise_except_if_required(FE_UNDERFLOW); - return 0.0f; - } - // x >= 89 or nan - if (xbits.is_pos() && (xbits.uintval() >= 0x42b2'0000)) { - // x is finite - if (xbits.uintval() < 0x7f80'0000U) { - int rounding = fputil::quick_get_round(); - if (rounding == FE_DOWNWARD || rounding == FE_TOWARDZERO) - return FPBits::max_normal().get_val(); - - fputil::set_errno_if_required(ERANGE); - fputil::raise_except_if_required(FE_OVERFLOW); - } - // x is +inf or nan - return x + FPBits::inf().get_val(); - } - } - // For -104 < x < 89, to compute exp(x), we perform the following range - // reduction: find hi, mid, lo such that: - // x = hi + mid + lo, in which - // hi is an integer, - // mid * 2^7 is an integer - // -2^(-8) <= lo < 2^-8. - // In particular, - // hi + mid = round(x * 2^7) * 2^(-7). - // Then, - // exp(x) = exp(hi + mid + lo) = exp(hi) * exp(mid) * exp(lo). - // We store exp(hi) and exp(mid) in the lookup tables EXP_M1 and EXP_M2 - // respectively. exp(lo) is computed using a degree-4 minimax polynomial - // generated by Sollya. - - // x_hi = (hi + mid) * 2^7 = round(x * 2^7). - float kf = fputil::nearest_integer(x * 0x1.0p7f); - // Subtract (hi + mid) from x to get lo. - double xd = static_cast(fputil::multiply_add(kf, -0x1.0p-7f, x)); - int x_hi = static_cast(kf); - x_hi += 104 << 7; - // hi = x_hi >> 7 - double exp_hi = EXP_M1[x_hi >> 7]; - // mid * 2^7 = x_hi & 0x0000'007fU; - double exp_mid = EXP_M2[x_hi & 0x7f]; - // Degree-4 minimax polynomial generated by Sollya with the following - // commands: - // > display = hexadecimal; - // > Q = fpminimax(expm1(x)/x, 3, [|D...|], [-2^-8, 2^-8]); - // > Q; - double exp_lo = - fputil::polyeval(xd, 0x1p0, 0x1.ffffffffff777p-1, 0x1.000000000071cp-1, - 0x1.555566668e5e7p-3, 0x1.55555555ef243p-5); - return static_cast(exp_hi * exp_mid * exp_lo); -} +LLVM_LIBC_FUNCTION(float, expf, (float x)) { return math::expf(x); } } // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/generic/explogxf.cpp b/system/lib/llvm-libc/src/math/generic/explogxf.cpp index 9e945eca7aed6..d38efa0269693 100644 --- a/system/lib/llvm-libc/src/math/generic/explogxf.cpp +++ b/system/lib/llvm-libc/src/math/generic/explogxf.cpp @@ -12,7 +12,7 @@ namespace LIBC_NAMESPACE_DECL { // N[Table[Log[2, 1 + x], {x, 0/64, 63/64, 1/64}], 40] -alignas(64) const double LOG_P1_LOG2[LOG_P1_SIZE] = { +alignas(8) const double LOG_P1_LOG2[LOG_P1_SIZE] = { 0x0.0000000000000p+0, 0x1.6e79685c2d22ap-6, 0x1.6bad3758efd87p-5, 0x1.0eb389fa29f9bp-4, 0x1.663f6fac91316p-4, 0x1.bc84240adabbap-4, 0x1.08c588cda79e4p-3, 0x1.32ae9e278ae1ap-3, 0x1.5c01a39fbd688p-3, @@ -38,7 +38,7 @@ alignas(64) const double LOG_P1_LOG2[LOG_P1_SIZE] = { }; // N[Table[1/(1 + x), {x, 0/64, 63/64, 1/64}], 40] -alignas(64) const double LOG_P1_1_OVER[LOG_P1_SIZE] = { +alignas(8) const double LOG_P1_1_OVER[LOG_P1_SIZE] = { 0x1.0000000000000p+0, 0x1.f81f81f81f820p-1, 0x1.f07c1f07c1f08p-1, 0x1.e9131abf0b767p-1, 0x1.e1e1e1e1e1e1ep-1, 0x1.dae6076b981dbp-1, 0x1.d41d41d41d41dp-1, 0x1.cd85689039b0bp-1, 0x1.c71c71c71c71cp-1, @@ -64,11 +64,11 @@ alignas(64) const double LOG_P1_1_OVER[LOG_P1_SIZE] = { // Taylos series expansion for Log[2, 1 + x] splitted to EVEN AND ODD numbers // K_LOG2_ODD starts from x^3 -alignas(64) const +alignas(8) const double K_LOG2_ODD[4] = {0x1.ec709dc3a03fdp-2, 0x1.2776c50ef9bfep-2, 0x1.a61762a7aded9p-3, 0x1.484b13d7c02a9p-3}; -alignas(64) const +alignas(8) const double K_LOG2_EVEN[4] = {-0x1.71547652b82fep-1, -0x1.71547652b82fep-2, -0x1.ec709dc3a03fdp-3, -0x1.2776c50ef9bfep-3}; diff --git a/system/lib/llvm-libc/src/math/generic/explogxf.h b/system/lib/llvm-libc/src/math/generic/explogxf.h index 651524a165f03..212ede4758549 100644 --- a/system/lib/llvm-libc/src/math/generic/explogxf.h +++ b/system/lib/llvm-libc/src/math/generic/explogxf.h @@ -297,6 +297,49 @@ LIBC_INLINE static double log2_eval(double x) { return result; } +// x should be positive, normal finite value +// TODO: Simplify range reduction and polynomial degree for float16. +// See issue #137190. +LIBC_INLINE static float log_eval_f(float x) { + // For x = 2^ex * (1 + mx), logf(x) = ex * logf(2) + logf(1 + mx). + using FPBits = fputil::FPBits; + FPBits xbits(x); + + float ex = static_cast(xbits.get_exponent()); + // p1 is the leading 7 bits of mx, i.e. + // p1 * 2^(-7) <= m_x < (p1 + 1) * 2^(-7). + int p1 = static_cast(xbits.get_mantissa() >> (FPBits::FRACTION_LEN - 7)); + + // Set bits to (1 + (mx - p1*2^(-7))) + xbits.set_uintval(xbits.uintval() & (FPBits::FRACTION_MASK >> 7)); + xbits.set_biased_exponent(FPBits::EXP_BIAS); + // dx = (mx - p1*2^(-7)) / (1 + p1*2^(-7)). + float dx = (xbits.get_val() - 1.0f) * ONE_OVER_F_FLOAT[p1]; + + // Minimax polynomial for log(1 + dx), generated using Sollya: + // > P = fpminimax(log(1 + x)/x, 6, [|SG...|], [0, 2^-7]); + // > Q = (P - 1) / x; + // > for i from 0 to degree(Q) do print(coeff(Q, i)); + constexpr float COEFFS[6] = {-0x1p-1f, 0x1.555556p-2f, -0x1.00022ep-2f, + 0x1.9ea056p-3f, -0x1.e50324p-2f, 0x1.c018fp3f}; + + float dx2 = dx * dx; + + float c1 = fputil::multiply_add(dx, COEFFS[1], COEFFS[0]); + float c2 = fputil::multiply_add(dx, COEFFS[3], COEFFS[2]); + float c3 = fputil::multiply_add(dx, COEFFS[5], COEFFS[4]); + + float p = fputil::polyeval(dx2, dx, c1, c2, c3); + + // Generated by Sollya with the following commands: + // > display = hexadecimal; + // > round(log(2), SG, RN); + constexpr float LOGF_2 = 0x1.62e43p-1f; + + float result = fputil::multiply_add(ex, LOGF_2, LOG_F_FLOAT[p1] + p); + return result; +} + // x should be positive, normal finite value LIBC_INLINE static double log_eval(double x) { // For x = 2^ex * (1 + mx) @@ -338,8 +381,9 @@ LIBC_INLINE static double log_eval(double x) { // double(2^-1022 + x) - 2^-1022 = double(x). // So if we scale x up by 2^1022, we can use // double(1.0 + 2^1022 * x) - 1.0 to test how x is rounded in denormal range. -LIBC_INLINE cpp::optional ziv_test_denorm(int hi, double mid, double lo, - double err) { +template +LIBC_INLINE static cpp::optional +ziv_test_denorm(int hi, double mid, double lo, double err) { using FPBits = typename fputil::FPBits; // Scaling factor = 1/(min normal number) = 2^1022 @@ -360,22 +404,27 @@ LIBC_INLINE cpp::optional ziv_test_denorm(int hi, double mid, double lo, scale_down = 0x3FF0'0000'0000'0000; // 1023 in the exponent field. } - double err_scaled = - cpp::bit_cast(exp_hi + cpp::bit_cast(err)); - - double lo_u = lo_scaled + err_scaled; - double lo_l = lo_scaled - err_scaled; - // By adding 1.0, the results will have similar rounding points as denormal // outputs. - double upper = extra_factor + (mid_hi + lo_u); - double lower = extra_factor + (mid_hi + lo_l); + if constexpr (SKIP_ZIV_TEST) { + double r = extra_factor + (mid_hi + lo_scaled); + return cpp::bit_cast(cpp::bit_cast(r) - scale_down); + } else { + double err_scaled = + cpp::bit_cast(exp_hi + cpp::bit_cast(err)); - if (LIBC_LIKELY(upper == lower)) { - return cpp::bit_cast(cpp::bit_cast(upper) - scale_down); - } + double lo_u = lo_scaled + err_scaled; + double lo_l = lo_scaled - err_scaled; + + double upper = extra_factor + (mid_hi + lo_u); + double lower = extra_factor + (mid_hi + lo_l); - return cpp::nullopt; + if (LIBC_LIKELY(upper == lower)) { + return cpp::bit_cast(cpp::bit_cast(upper) - scale_down); + } + + return cpp::nullopt; + } } } // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/generic/expm1f.cpp b/system/lib/llvm-libc/src/math/generic/expm1f.cpp index d5e9e85ed4bd3..b2967e2516197 100644 --- a/system/lib/llvm-libc/src/math/generic/expm1f.cpp +++ b/system/lib/llvm-libc/src/math/generic/expm1f.cpp @@ -30,6 +30,7 @@ LLVM_LIBC_FUNCTION(float, expm1f, (float x)) { uint32_t x_u = xbits.uintval(); uint32_t x_abs = x_u & 0x7fff'ffffU; +#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS // Exceptional value if (LIBC_UNLIKELY(x_u == 0x3e35'bec5U)) { // x = 0x1.6b7d8ap-3f int round_mode = fputil::quick_get_round(); @@ -37,15 +38,15 @@ LLVM_LIBC_FUNCTION(float, expm1f, (float x)) { return 0x1.8dbe64p-3f; return 0x1.8dbe62p-3f; } - -#if !defined(LIBC_TARGET_CPU_HAS_FMA) +#if !defined(LIBC_TARGET_CPU_HAS_FMA_DOUBLE) if (LIBC_UNLIKELY(x_u == 0xbdc1'c6cbU)) { // x = -0x1.838d96p-4f int round_mode = fputil::quick_get_round(); if (round_mode == FE_TONEAREST || round_mode == FE_DOWNWARD) return -0x1.71c884p-4f; return -0x1.71c882p-4f; } -#endif // LIBC_TARGET_CPU_HAS_FMA +#endif // LIBC_TARGET_CPU_HAS_FMA_DOUBLE +#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS // When |x| > 25*log(2), or nan if (LIBC_UNLIKELY(x_abs >= 0x418a'a123U)) { @@ -102,12 +103,12 @@ LLVM_LIBC_FUNCTION(float, expm1f, (float x)) { // 2^-76. For targets without FMA instructions, we simply use double for // intermediate results as it is more efficient than using an emulated // version of FMA. -#if defined(LIBC_TARGET_CPU_HAS_FMA) - return fputil::fma(x, x, x); +#if defined(LIBC_TARGET_CPU_HAS_FMA_FLOAT) + return fputil::multiply_add(x, x, x); #else double xd = x; return static_cast(fputil::multiply_add(xd, xd, xd)); -#endif // LIBC_TARGET_CPU_HAS_FMA +#endif // LIBC_TARGET_CPU_HAS_FMA_FLOAT } constexpr double COEFFS[] = {0x1p-1, diff --git a/system/lib/llvm-libc/src/math/generic/expxf16.h b/system/lib/llvm-libc/src/math/generic/expxf16.h deleted file mode 100644 index 67bb248307519..0000000000000 --- a/system/lib/llvm-libc/src/math/generic/expxf16.h +++ /dev/null @@ -1,349 +0,0 @@ -//===-- Common utilities for half-precision exponential functions ---------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_LIBC_SRC_MATH_GENERIC_EXPXF16_H -#define LLVM_LIBC_SRC_MATH_GENERIC_EXPXF16_H - -#include "src/__support/CPP/array.h" -#include "src/__support/FPUtil/FPBits.h" -#include "src/__support/FPUtil/PolyEval.h" -#include "src/__support/FPUtil/cast.h" -#include "src/__support/FPUtil/multiply_add.h" -#include "src/__support/FPUtil/nearest_integer.h" -#include "src/__support/macros/attributes.h" -#include "src/__support/macros/config.h" -#include - -namespace LIBC_NAMESPACE_DECL { - -// Generated by Sollya with the following commands: -// > display = hexadecimal; -// > for i from -18 to 12 do print(round(exp(i), SG, RN)); -static constexpr cpp::array EXP_HI = { - 0x1.05a628p-26f, 0x1.639e32p-25f, 0x1.e355bcp-24f, 0x1.4875cap-22f, - 0x1.be6c7p-21f, 0x1.2f6054p-19f, 0x1.9c54c4p-18f, 0x1.183542p-16f, - 0x1.7cd79cp-15f, 0x1.02cf22p-13f, 0x1.5fc21p-12f, 0x1.de16bap-11f, - 0x1.44e52p-9f, 0x1.b993fep-8f, 0x1.2c155cp-6f, 0x1.97db0cp-5f, - 0x1.152aaap-3f, 0x1.78b564p-2f, 0x1p+0f, 0x1.5bf0a8p+1f, - 0x1.d8e64cp+2f, 0x1.415e5cp+4f, 0x1.b4c902p+5f, 0x1.28d38ap+7f, - 0x1.936dc6p+8f, 0x1.122886p+10f, 0x1.749ea8p+11f, 0x1.fa7158p+12f, - 0x1.5829dcp+14f, 0x1.d3c448p+15f, 0x1.3de166p+17f, -}; - -// Generated by Sollya with the following commands: -// > display = hexadecimal; -// > for i from 0 to 7 do print(round(exp(i * 2^-3), SG, RN)); -static constexpr cpp::array EXP_MID = { - 0x1p+0f, 0x1.221604p+0f, 0x1.48b5e4p+0f, 0x1.747a52p+0f, - 0x1.a61298p+0f, 0x1.de455ep+0f, 0x1.0ef9dcp+1f, 0x1.330e58p+1f, -}; - -struct ExpRangeReduction { - float exp_hi_mid; - float exp_lo; -}; - -LIBC_INLINE ExpRangeReduction exp_range_reduction(float16 x) { - // For -18 < x < 12, to compute exp(x), we perform the following range - // reduction: find hi, mid, lo, such that: - // x = hi + mid + lo, in which - // hi is an integer, - // mid * 2^3 is an integer, - // -2^(-4) <= lo < 2^(-4). - // In particular, - // hi + mid = round(x * 2^3) * 2^(-3). - // Then, - // exp(x) = exp(hi + mid + lo) = exp(hi) * exp(mid) * exp(lo). - // We store exp(hi) and exp(mid) in the lookup tables EXP_HI and EXP_MID - // respectively. exp(lo) is computed using a degree-3 minimax polynomial - // generated by Sollya. - - float xf = x; - float kf = fputil::nearest_integer(xf * 0x1.0p+3f); - int x_hi_mid = static_cast(kf); - int x_hi = x_hi_mid >> 3; - int x_mid = x_hi_mid & 0x7; - // lo = x - (hi + mid) = round(x * 2^3) * (-2^(-3)) + x - float lo = fputil::multiply_add(kf, -0x1.0p-3f, xf); - - float exp_hi = EXP_HI[x_hi + 18]; - float exp_mid = EXP_MID[x_mid]; - // Degree-3 minimax polynomial generated by Sollya with the following - // commands: - // > display = hexadecimal; - // > P = fpminimax(expm1(x)/x, 2, [|SG...|], [-2^-4, 2^-4]); - // > 1 + x * P; - float exp_lo = - fputil::polyeval(lo, 0x1p+0f, 0x1p+0f, 0x1.001p-1f, 0x1.555ddep-3f); - return {exp_hi * exp_mid, exp_lo}; -} - -// Generated by Sollya with the following commands: -// > display = hexadecimal; -// > for i from 0 to 7 do printsingle(round(2^(i * 2^-3), SG, RN)); -constexpr cpp::array EXP2_MID_BITS = { - 0x3f80'0000U, 0x3f8b'95c2U, 0x3f98'37f0U, 0x3fa5'fed7U, - 0x3fb5'04f3U, 0x3fc5'672aU, 0x3fd7'44fdU, 0x3fea'c0c7U, -}; - -LIBC_INLINE ExpRangeReduction exp2_range_reduction(float16 x) { - // For -25 < x < 16, to compute 2^x, we perform the following range reduction: - // find hi, mid, lo, such that: - // x = hi + mid + lo, in which - // hi is an integer, - // mid * 2^3 is an integer, - // -2^(-4) <= lo < 2^(-4). - // In particular, - // hi + mid = round(x * 2^3) * 2^(-3). - // Then, - // 2^x = 2^(hi + mid + lo) = 2^hi * 2^mid * 2^lo. - // We store 2^mid in the lookup table EXP2_MID_BITS, and compute 2^hi * 2^mid - // by adding hi to the exponent field of 2^mid. 2^lo is computed using a - // degree-3 minimax polynomial generated by Sollya. - - float xf = x; - float kf = fputil::nearest_integer(xf * 0x1.0p+3f); - int x_hi_mid = static_cast(kf); - unsigned x_hi = static_cast(x_hi_mid) >> 3; - unsigned x_mid = static_cast(x_hi_mid) & 0x7; - // lo = x - (hi + mid) = round(x * 2^3) * (-2^(-3)) + x - float lo = fputil::multiply_add(kf, -0x1.0p-3f, xf); - - uint32_t exp2_hi_mid_bits = - EXP2_MID_BITS[x_mid] + - static_cast(x_hi << fputil::FPBits::FRACTION_LEN); - float exp2_hi_mid = fputil::FPBits(exp2_hi_mid_bits).get_val(); - // Degree-3 minimax polynomial generated by Sollya with the following - // commands: - // > display = hexadecimal; - // > P = fpminimax((2^x - 1)/x, 2, [|SG...|], [-2^-4, 2^-4]); - // > 1 + x * P; - float exp2_lo = fputil::polyeval(lo, 0x1p+0f, 0x1.62e43p-1f, 0x1.ec0aa6p-3f, - 0x1.c6b4a6p-5f); - return {exp2_hi_mid, exp2_lo}; -} - -// Generated by Sollya with the following commands: -// > display = hexadecimal; -// > round(log2(10), SG, RN); -static constexpr float LOG2F_10 = 0x1.a934fp+1f; - -// Generated by Sollya with the following commands: -// > display = hexadecimal; -// > round(log10(2), SG, RN); -static constexpr float LOG10F_2 = 0x1.344136p-2f; - -LIBC_INLINE ExpRangeReduction exp10_range_reduction(float16 x) { - // For -8 < x < 5, to compute 10^x, we perform the following range reduction: - // find hi, mid, lo, such that: - // x = (hi + mid) * log2(10) + lo, in which - // hi is an integer, - // mid * 2^3 is an integer, - // -2^(-4) <= lo < 2^(-4). - // In particular, - // hi + mid = round(x * 2^3) * 2^(-3). - // Then, - // 10^x = 10^(hi + mid + lo) = 2^((hi + mid) * log2(10)) + 10^lo - // We store 2^mid in the lookup table EXP2_MID_BITS, and compute 2^hi * 2^mid - // by adding hi to the exponent field of 2^mid. 10^lo is computed using a - // degree-4 minimax polynomial generated by Sollya. - - float xf = x; - float kf = fputil::nearest_integer(xf * (LOG2F_10 * 0x1.0p+3f)); - int x_hi_mid = static_cast(kf); - unsigned x_hi = static_cast(x_hi_mid) >> 3; - unsigned x_mid = static_cast(x_hi_mid) & 0x7; - // lo = x - (hi + mid) = round(x * 2^3 * log2(10)) * log10(2) * (-2^(-3)) + x - float lo = fputil::multiply_add(kf, LOG10F_2 * -0x1.0p-3f, xf); - - uint32_t exp2_hi_mid_bits = - EXP2_MID_BITS[x_mid] + - static_cast(x_hi << fputil::FPBits::FRACTION_LEN); - float exp2_hi_mid = fputil::FPBits(exp2_hi_mid_bits).get_val(); - // Degree-4 minimax polynomial generated by Sollya with the following - // commands: - // > display = hexadecimal; - // > P = fpminimax((10^x - 1)/x, 3, [|SG...|], [-2^-4, 2^-4]); - // > 1 + x * P; - float exp10_lo = fputil::polyeval(lo, 0x1p+0f, 0x1.26bb14p+1f, 0x1.53526p+1f, - 0x1.04b434p+1f, 0x1.2bcf9ep+0f); - return {exp2_hi_mid, exp10_lo}; -} - -// Generated by Sollya with the following commands: -// > display = hexadecimal; -// > round(log2(exp(1)), SG, RN); -static constexpr float LOG2F_E = 0x1.715476p+0f; - -// Generated by Sollya with the following commands: -// > display = hexadecimal; -// > round(log(2), SG, RN); -static constexpr float LOGF_2 = 0x1.62e43p-1f; - -// Generated by Sollya with the following commands: -// > display = hexadecimal; -// > for i from 0 to 31 do printsingle(round(2^(i * 2^-5), SG, RN)); -static constexpr cpp::array EXP2_MID_5_BITS = { - 0x3f80'0000U, 0x3f82'cd87U, 0x3f85'aac3U, 0x3f88'980fU, 0x3f8b'95c2U, - 0x3f8e'a43aU, 0x3f91'c3d3U, 0x3f94'f4f0U, 0x3f98'37f0U, 0x3f9b'8d3aU, - 0x3f9e'f532U, 0x3fa2'7043U, 0x3fa5'fed7U, 0x3fa9'a15bU, 0x3fad'583fU, - 0x3fb1'23f6U, 0x3fb5'04f3U, 0x3fb8'fbafU, 0x3fbd'08a4U, 0x3fc1'2c4dU, - 0x3fc5'672aU, 0x3fc9'b9beU, 0x3fce'248cU, 0x3fd2'a81eU, 0x3fd7'44fdU, - 0x3fdb'fbb8U, 0x3fe0'ccdfU, 0x3fe5'b907U, 0x3fea'c0c7U, 0x3fef'e4baU, - 0x3ff5'257dU, 0x3ffa'83b3U, -}; - -// This function correctly calculates sinh(x) and cosh(x) by calculating exp(x) -// and exp(-x) simultaneously. -// To compute e^x, we perform the following range reduction: -// find hi, mid, lo such that: -// x = (hi + mid) * log(2) + lo, in which -// hi is an integer, -// 0 <= mid * 2^5 < 32 is an integer -// -2^(-5) <= lo * log2(e) <= 2^-5. -// In particular, -// hi + mid = round(x * log2(e) * 2^5) * 2^(-5). -// Then, -// e^x = 2^(hi + mid) * e^lo = 2^hi * 2^mid * e^lo. -// We store 2^mid in the lookup table EXP2_MID_5_BITS, and compute 2^hi * 2^mid -// by adding hi to the exponent field of 2^mid. -// e^lo is computed using a degree-3 minimax polynomial generated by Sollya: -// e^lo ~ P(lo) -// = 1 + lo + c2 * lo^2 + ... + c5 * lo^5 -// = (1 + c2*lo^2 + c4*lo^4) + lo * (1 + c3*lo^2 + c5*lo^4) -// = P_even + lo * P_odd -// To compute e^(-x), notice that: -// e^(-x) = 2^(-(hi + mid)) * e^(-lo) -// ~ 2^(-(hi + mid)) * P(-lo) -// = 2^(-(hi + mid)) * (P_even - lo * P_odd) -// So: -// sinh(x) = (e^x - e^(-x)) / 2 -// ~ 0.5 * (2^(hi + mid) * (P_even + lo * P_odd) - -// 2^(-(hi + mid)) * (P_even - lo * P_odd)) -// = 0.5 * (P_even * (2^(hi + mid) - 2^(-(hi + mid))) + -// lo * P_odd * (2^(hi + mid) + 2^(-(hi + mid)))) -// And similarly: -// cosh(x) = (e^x + e^(-x)) / 2 -// ~ 0.5 * (P_even * (2^(hi + mid) + 2^(-(hi + mid))) + -// lo * P_odd * (2^(hi + mid) - 2^(-(hi + mid)))) -// The main point of these formulas is that the expensive part of calculating -// the polynomials approximating lower parts of e^x and e^(-x) is shared and -// only done once. -template LIBC_INLINE float16 eval_sinh_or_cosh(float16 x) { - float xf = x; - float kf = fputil::nearest_integer(xf * (LOG2F_E * 0x1.0p+5f)); - int x_hi_mid_p = static_cast(kf); - int x_hi_mid_m = -x_hi_mid_p; - - unsigned x_hi_p = static_cast(x_hi_mid_p) >> 5; - unsigned x_hi_m = static_cast(x_hi_mid_m) >> 5; - unsigned x_mid_p = static_cast(x_hi_mid_p) & 0x1f; - unsigned x_mid_m = static_cast(x_hi_mid_m) & 0x1f; - - uint32_t exp2_hi_mid_bits_p = - EXP2_MID_5_BITS[x_mid_p] + - static_cast(x_hi_p << fputil::FPBits::FRACTION_LEN); - uint32_t exp2_hi_mid_bits_m = - EXP2_MID_5_BITS[x_mid_m] + - static_cast(x_hi_m << fputil::FPBits::FRACTION_LEN); - // exp2_hi_mid_p = 2^(hi + mid) - float exp2_hi_mid_p = fputil::FPBits(exp2_hi_mid_bits_p).get_val(); - // exp2_hi_mid_m = 2^(-(hi + mid)) - float exp2_hi_mid_m = fputil::FPBits(exp2_hi_mid_bits_m).get_val(); - - // exp2_hi_mid_sum = 2^(hi + mid) + 2^(-(hi + mid)) - float exp2_hi_mid_sum = exp2_hi_mid_p + exp2_hi_mid_m; - // exp2_hi_mid_diff = 2^(hi + mid) - 2^(-(hi + mid)) - float exp2_hi_mid_diff = exp2_hi_mid_p - exp2_hi_mid_m; - - // lo = x - (hi + mid) = round(x * log2(e) * 2^5) * log(2) * (-2^(-5)) + x - float lo = fputil::multiply_add(kf, LOGF_2 * -0x1.0p-5f, xf); - float lo_sq = lo * lo; - - // Degree-3 minimax polynomial generated by Sollya with the following - // commands: - // > display = hexadecimal; - // > P = fpminimax(expm1(x)/x, 2, [|SG...|], [-2^-5, 2^-5]); - // > 1 + x * P; - constexpr cpp::array COEFFS = {0x1p+0f, 0x1p+0f, 0x1.0004p-1f, - 0x1.555778p-3f}; - float half_p_odd = - fputil::polyeval(lo_sq, COEFFS[1] * 0.5f, COEFFS[3] * 0.5f); - float half_p_even = - fputil::polyeval(lo_sq, COEFFS[0] * 0.5f, COEFFS[2] * 0.5f); - - // sinh(x) = lo * (0.5 * P_odd * (2^(hi + mid) + 2^(-(hi + mid)))) + - // (0.5 * P_even * (2^(hi + mid) - 2^(-(hi + mid)))) - if constexpr (IsSinh) - return fputil::cast(fputil::multiply_add( - lo, half_p_odd * exp2_hi_mid_sum, half_p_even * exp2_hi_mid_diff)); - // cosh(x) = lo * (0.5 * P_odd * (2^(hi + mid) - 2^(-(hi + mid)))) + - // (0.5 * P_even * (2^(hi + mid) + 2^(-(hi + mid)))) - return fputil::cast(fputil::multiply_add( - lo, half_p_odd * exp2_hi_mid_diff, half_p_even * exp2_hi_mid_sum)); -} - -// Generated by Sollya with the following commands: -// > display = hexadecimal; -// > for i from 0 to 31 do print(round(log(1 + i * 2^-5), SG, RN)); -constexpr cpp::array LOGF_F = { - 0x0p+0f, 0x1.f829bp-6f, 0x1.f0a30cp-5f, 0x1.6f0d28p-4f, - 0x1.e27076p-4f, 0x1.29553p-3f, 0x1.5ff308p-3f, 0x1.9525aap-3f, - 0x1.c8ff7cp-3f, 0x1.fb9186p-3f, 0x1.1675cap-2f, 0x1.2e8e2cp-2f, - 0x1.4618bcp-2f, 0x1.5d1bdcp-2f, 0x1.739d8p-2f, 0x1.89a338p-2f, - 0x1.9f323ep-2f, 0x1.b44f78p-2f, 0x1.c8ff7cp-2f, 0x1.dd46ap-2f, - 0x1.f128f6p-2f, 0x1.02552ap-1f, 0x1.0be72ep-1f, 0x1.154c3ep-1f, - 0x1.1e85f6p-1f, 0x1.2795e2p-1f, 0x1.307d74p-1f, 0x1.393e0ep-1f, - 0x1.41d8fep-1f, 0x1.4a4f86p-1f, 0x1.52a2d2p-1f, 0x1.5ad404p-1f, -}; - -// Generated by Sollya with the following commands: -// > display = hexadecimal; -// > for i from 0 to 31 do print(round(log2(1 + i * 2^-5), SG, RN)); -constexpr cpp::array LOG2F_F = { - 0x0p+0f, 0x1.6bad38p-5f, 0x1.663f7p-4f, 0x1.08c588p-3f, - 0x1.5c01a4p-3f, 0x1.acf5e2p-3f, 0x1.fbc16cp-3f, 0x1.24407ap-2f, - 0x1.49a784p-2f, 0x1.6e221cp-2f, 0x1.91bba8p-2f, 0x1.b47ecp-2f, - 0x1.d6753ep-2f, 0x1.f7a856p-2f, 0x1.0c105p-1f, 0x1.1bf312p-1f, - 0x1.2b8034p-1f, 0x1.3abb4p-1f, 0x1.49a784p-1f, 0x1.584822p-1f, - 0x1.66a008p-1f, 0x1.74b1fep-1f, 0x1.82809ep-1f, 0x1.900e62p-1f, - 0x1.9d5dap-1f, 0x1.aa709p-1f, 0x1.b74948p-1f, 0x1.c3e9cap-1f, - 0x1.d053f6p-1f, 0x1.dc899ap-1f, 0x1.e88c6cp-1f, 0x1.f45e08p-1f, -}; - -// Generated by Sollya with the following commands: -// > display = hexadecimal; -// > for i from 0 to 31 do print(round(log10(1 + i * 2^-5), SG, RN)); -constexpr cpp::array LOG10F_F = { - 0x0p+0f, 0x1.b5e908p-7f, 0x1.af5f92p-6f, 0x1.3ed11ap-5f, - 0x1.a30a9ep-5f, 0x1.02428cp-4f, 0x1.31b306p-4f, 0x1.5fe804p-4f, - 0x1.8cf184p-4f, 0x1.b8de4ep-4f, 0x1.e3bc1ap-4f, 0x1.06cbd6p-3f, - 0x1.1b3e72p-3f, 0x1.2f3b6ap-3f, 0x1.42c7e8p-3f, 0x1.55e8c6p-3f, - 0x1.68a288p-3f, 0x1.7af974p-3f, 0x1.8cf184p-3f, 0x1.9e8e7cp-3f, - 0x1.afd3e4p-3f, 0x1.c0c514p-3f, 0x1.d1653p-3f, 0x1.e1b734p-3f, - 0x1.f1bdeep-3f, 0x1.00be06p-2f, 0x1.087a08p-2f, 0x1.101432p-2f, - 0x1.178da6p-2f, 0x1.1ee778p-2f, 0x1.2622bp-2f, 0x1.2d404cp-2f, -}; - -// Generated by Sollya with the following commands: -// > display = hexadecimal; -// > for i from 0 to 31 do print(round(1 / (1 + i * 2^-5), SG, RN)); -constexpr cpp::array ONE_OVER_F_F = { - 0x1p+0f, 0x1.f07c2p-1f, 0x1.e1e1e2p-1f, 0x1.d41d42p-1f, - 0x1.c71c72p-1f, 0x1.bacf92p-1f, 0x1.af286cp-1f, 0x1.a41a42p-1f, - 0x1.99999ap-1f, 0x1.8f9c18p-1f, 0x1.861862p-1f, 0x1.7d05f4p-1f, - 0x1.745d18p-1f, 0x1.6c16c2p-1f, 0x1.642c86p-1f, 0x1.5c9882p-1f, - 0x1.555556p-1f, 0x1.4e5e0ap-1f, 0x1.47ae14p-1f, 0x1.414142p-1f, - 0x1.3b13b2p-1f, 0x1.3521dp-1f, 0x1.2f684cp-1f, 0x1.29e412p-1f, - 0x1.24924ap-1f, 0x1.1f7048p-1f, 0x1.1a7b96p-1f, 0x1.15b1e6p-1f, - 0x1.111112p-1f, 0x1.0c9714p-1f, 0x1.08421p-1f, 0x1.041042p-1f, -}; - -} // namespace LIBC_NAMESPACE_DECL - -#endif // LLVM_LIBC_SRC_MATH_GENERIC_EXPXF16_H diff --git a/system/lib/llvm-libc/src/math/generic/fmul.cpp b/system/lib/llvm-libc/src/math/generic/fmul.cpp index e759e48cd6989..daad64873f27a 100644 --- a/system/lib/llvm-libc/src/math/generic/fmul.cpp +++ b/system/lib/llvm-libc/src/math/generic/fmul.cpp @@ -21,7 +21,7 @@ LLVM_LIBC_FUNCTION(float, fmul, (double x, double y)) { // correctly rounded for all rounding modes, so we fall // back to the generic `fmul` implementation -#ifndef LIBC_TARGET_CPU_HAS_FMA +#ifndef LIBC_TARGET_CPU_HAS_FMA_DOUBLE return fputil::generic::mul(x, y); #else fputil::DoubleDouble prod = fputil::exact_mult(x, y); diff --git a/system/lib/llvm-libc/src/math/generic/hypotf.cpp b/system/lib/llvm-libc/src/math/generic/hypotf.cpp index 959c0420ae149..ec48f62163a48 100644 --- a/system/lib/llvm-libc/src/math/generic/hypotf.cpp +++ b/system/lib/llvm-libc/src/math/generic/hypotf.cpp @@ -55,7 +55,7 @@ LLVM_LIBC_FUNCTION(float, hypotf, (float x, float y)) { // These squares are exact. double a_sq = ad * ad; -#ifdef LIBC_TARGET_CPU_HAS_FMA +#ifdef LIBC_TARGET_CPU_HAS_FMA_DOUBLE double sum_sq = fputil::multiply_add(bd, bd, a_sq); #else double b_sq = bd * bd; @@ -72,7 +72,7 @@ LLVM_LIBC_FUNCTION(float, hypotf, (float x, float y)) { double r_d = result.get_val(); // Perform rounding correction. -#ifdef LIBC_TARGET_CPU_HAS_FMA +#ifdef LIBC_TARGET_CPU_HAS_FMA_DOUBLE double sum_sq_lo = fputil::multiply_add(bd, bd, a_sq - sum_sq); double err = sum_sq_lo - fputil::multiply_add(r_d, r_d, -sum_sq); #else diff --git a/system/lib/llvm-libc/src/math/generic/log.cpp b/system/lib/llvm-libc/src/math/generic/log.cpp index 4302c64c8abac..0cd4424ee0baf 100644 --- a/system/lib/llvm-libc/src/math/generic/log.cpp +++ b/system/lib/llvm-libc/src/math/generic/log.cpp @@ -30,6 +30,7 @@ using LIBC_NAMESPACE::operator""_u128; namespace { +#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS // A simple upper bound for the error of e_x * log(2) - log(r). constexpr double HI_ERR = 0x1.0p-85; @@ -47,7 +48,7 @@ constexpr double P_ERR = 0x1.0p-50; constexpr Float128 LOG_2(Sign::POS, /*exponent=*/-128, /*mantissa=*/ 0xb17217f7'd1cf79ab'c9e3b398'03f2f6af_u128); -alignas(64) constexpr LogRR LOG_TABLE = { +alignas(16) constexpr LogRR LOG_TABLE = { // -log(r) with 128-bit precision generated by SageMath with: // for i in range(128): // r = 2^-8 * ceil( 2^8 * (1 - 2^(-8)) / (1 + i*2^(-7)) ); @@ -731,6 +732,7 @@ double log_accurate(int e_x, int index, double m_x) { return static_cast(r); } +#endif // LIBC_MATH_HAS_SKIP_ACCURATE_PASS } // namespace @@ -794,17 +796,17 @@ LLVM_LIBC_FUNCTION(double, log, (double x)) { uint64_t x_m = (x_u & 0x000F'FFFF'FFFF'FFFFULL) | 0x3FF0'0000'0000'0000ULL; double m = FPBits_t(x_m).get_val(); - double u, u_sq, err; + double u, u_sq; fputil::DoubleDouble r1; // Perform exact range reduction -#ifdef LIBC_TARGET_CPU_HAS_FMA +#ifdef LIBC_TARGET_CPU_HAS_FMA_DOUBLE u = fputil::multiply_add(r, m, -1.0); // exact #else uint64_t c_m = x_m & 0x3FFF'E000'0000'0000ULL; double c = FPBits_t(c_m).get_val(); u = fputil::multiply_add(r, m - c, CD[index]); // exact -#endif // LIBC_TARGET_CPU_HAS_FMA +#endif // LIBC_TARGET_CPU_HAS_FMA_DOUBLE // Exact sum: // r1.hi + r1.lo = e_x * log(2)_hi - log(r)_hi + u @@ -819,12 +821,15 @@ LLVM_LIBC_FUNCTION(double, log, (double x)) { double p2 = fputil::multiply_add(u, LOG_COEFFS[5], LOG_COEFFS[4]); double p = fputil::polyeval(u_sq, lo + r1.lo, p0, p1, p2); +#ifdef LIBC_MATH_HAS_SKIP_ACCURATE_PASS + return r1.hi + p; +#else // Technicallly error of r1.lo is bounded by: // hi*ulp(log(2)_lo) + C*ulp(u^2) // To simplify the error computation a bit, we replace |hi|*ulp(log(2)_lo) // with the upper bound: 2^11 * ulp(log(2)_lo) = 2^-85. // Total error is bounded by ~ C * ulp(u^2) + 2^-85. - err = fputil::multiply_add(u_sq, P_ERR, HI_ERR); + double err = fputil::multiply_add(u_sq, P_ERR, HI_ERR); // Lower bound from the result double left = r1.hi + (p - err); @@ -836,6 +841,7 @@ LLVM_LIBC_FUNCTION(double, log, (double x)) { return left; return log_accurate(x_e, index, u); +#endif // LIBC_MATH_HAS_SKIP_ACCURATE_PASS } } // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/generic/log10.cpp b/system/lib/llvm-libc/src/math/generic/log10.cpp index 7df57ef85b81b..1c4e559ba083c 100644 --- a/system/lib/llvm-libc/src/math/generic/log10.cpp +++ b/system/lib/llvm-libc/src/math/generic/log10.cpp @@ -33,6 +33,7 @@ namespace { constexpr fputil::DoubleDouble LOG10_E = {0x1.95355baaafad3p-57, 0x1.bcb7b1526e50ep-2}; +#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS // A simple upper bound for the error of e_x * log(2) - log(r). constexpr double HI_ERR = 0x1.0p-85; @@ -50,7 +51,7 @@ constexpr double P_ERR = 0x1.0p-51; constexpr Float128 LOG10_2(Sign::POS, /*exponent=*/-129, /*mantissa=*/ 0x9a209a84'fbcff798'8f8959ac'0b7c9178_u128); -alignas(64) constexpr LogRR LOG10_TABLE = { +alignas(16) constexpr LogRR LOG10_TABLE = { // -log10(r) with 128-bit precision generated by SageMath with: // // for i in range(128): @@ -733,6 +734,7 @@ double log10_accurate(int e_x, int index, double m_x) { return static_cast(r); } +#endif // LIBC_MATH_HAS_SKIP_ACCURATE_PASS } // namespace @@ -796,17 +798,17 @@ LLVM_LIBC_FUNCTION(double, log10, (double x)) { uint64_t x_m = (x_u & 0x000F'FFFF'FFFF'FFFFULL) | 0x3FF0'0000'0000'0000ULL; double m = FPBits_t(x_m).get_val(); - double u, u_sq, err; + double u, u_sq; fputil::DoubleDouble r1; // Perform exact range reduction -#ifdef LIBC_TARGET_CPU_HAS_FMA +#ifdef LIBC_TARGET_CPU_HAS_FMA_DOUBLE u = fputil::multiply_add(r, m, -1.0); // exact #else uint64_t c_m = x_m & 0x3FFF'E000'0000'0000ULL; double c = FPBits_t(c_m).get_val(); u = fputil::multiply_add(r, m - c, CD[index]); // exact -#endif // LIBC_TARGET_CPU_HAS_FMA +#endif // LIBC_TARGET_CPU_HAS_FMA_DOUBLE // Error of u_sq = ulp(u^2); u_sq = u * u; @@ -827,12 +829,15 @@ LLVM_LIBC_FUNCTION(double, log10, (double x)) { // 4*ulp( ulp(r2.hi) ) fputil::DoubleDouble r2 = fputil::quick_mult(r1, LOG10_E); +#ifdef LIBC_MATH_HAS_SKIP_ACCURATE_PASS + return r2.hi + r2.lo; +#else // Technicallly error of r1.lo is bounded by: // |hi|*ulp(log(2)_lo) + C*ulp(u^2) // To simplify the error computation a bit, we replace |hi|*ulp(log(2)_lo) // with the upper bound: 2^11 * ulp(log(2)_lo) = 2^-85. // Total error is bounded by ~ C * ulp(u^2) + 2^-85. - err = fputil::multiply_add(u_sq, P_ERR, HI_ERR); + double err = fputil::multiply_add(u_sq, P_ERR, HI_ERR); // Lower bound from the result double left = r2.hi + (r2.lo - err); @@ -897,6 +902,7 @@ LLVM_LIBC_FUNCTION(double, log10, (double x)) { } return log10_accurate(x_e, index, u); +#endif // LIBC_MATH_HAS_SKIP_ACCURATE_PASS } } // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/generic/log10f.cpp b/system/lib/llvm-libc/src/math/generic/log10f.cpp index c635fa4ef9b63..81e7cdbcfe5a1 100644 --- a/system/lib/llvm-libc/src/math/generic/log10f.cpp +++ b/system/lib/llvm-libc/src/math/generic/log10f.cpp @@ -139,13 +139,14 @@ LLVM_LIBC_FUNCTION(float, log10f, (float x)) { return 9.0f; case 0x5015'02f9U: // x = 10,000,000,000 return 10.0f; +#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS case 0x0efe'ee7aU: // x = 0x1.fddcf4p-98f return fputil::round_result_slightly_up(-0x1.d33a46p+4f); case 0x3f5f'de1bU: // x = 0x1.bfbc36p-1f return fputil::round_result_slightly_up(-0x1.dd2c6ep-5f); case 0x3f80'70d8U: // x = 0x1.00e1bp0f return fputil::round_result_slightly_up(0x1.8762c4p-10f); -#ifndef LIBC_TARGET_CPU_HAS_FMA +#ifndef LIBC_TARGET_CPU_HAS_FMA_DOUBLE case 0x08ae'a356U: // x = 0x1.5d46acp-110f return fputil::round_result_slightly_up(-0x1.07d3b4p+5f); case 0x120b'93dcU: // x = 0x1.1727b8p-91f @@ -156,7 +157,8 @@ LLVM_LIBC_FUNCTION(float, log10f, (float x)) { return fputil::round_result_slightly_down(0x1.2c9314p+3f); case 0x7956'ba5eU: // x = 69683218960000541503257137270226944.0 return fputil::round_result_slightly_up(0x1.16bebap+5f); -#endif // LIBC_TARGET_CPU_HAS_FMA +#endif // LIBC_TARGET_CPU_HAS_FMA_DOUBLE +#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS } } @@ -194,12 +196,12 @@ LLVM_LIBC_FUNCTION(float, log10f, (float x)) { float u = xbits.get_val(); double v; -#ifdef LIBC_TARGET_CPU_HAS_FMA +#ifdef LIBC_TARGET_CPU_HAS_FMA_FLOAT v = static_cast(fputil::multiply_add(u, R[index], -1.0f)); // Exact. #else v = fputil::multiply_add(static_cast(u), static_cast(R[index]), -1.0); // Exact -#endif // LIBC_TARGET_CPU_HAS_FMA +#endif // LIBC_TARGET_CPU_HAS_FMA_FLOAT // Degree-5 polynomial approximation of log10 generated by: // > P = fpminimax(log10(1 + x)/x, 4, [|D...|], [-2^-8, 2^-7]); diff --git a/system/lib/llvm-libc/src/math/generic/log1p.cpp b/system/lib/llvm-libc/src/math/generic/log1p.cpp index b9c58b843a240..09f465a6ba774 100644 --- a/system/lib/llvm-libc/src/math/generic/log1p.cpp +++ b/system/lib/llvm-libc/src/math/generic/log1p.cpp @@ -29,21 +29,6 @@ using LIBC_NAMESPACE::operator""_u128; namespace { -// Extra errors from P is from using x^2 to reduce evaluation latency and -// directional rounding. -constexpr double P_ERR = 0x1.0p-49; - -// log(2) with 128-bit precision generated by SageMath with: -// def format_hex(value): -// l = hex(value)[2:] -// n = 8 -// x = [l[i:i + n] for i in range(0, len(l), n)] -// return "0x" + "'".join(x) + "_u128" -// (s, m, e) = RealField(128)(2).log().sign_mantissa_exponent(); -// print(format_hex(m)); -constexpr Float128 LOG_2(Sign::POS, /*exponent=*/-128, /*mantissa=*/ - 0xb17217f7'd1cf79ab'c9e3b398'03f2f6af_u128); - // R1[i] = 2^-8 * nearestint( 2^8 / (1 + i * 2^-7) ) constexpr double R1[129] = { 0x1p0, 0x1.fcp-1, 0x1.f8p-1, 0x1.f4p-1, 0x1.fp-1, 0x1.ecp-1, 0x1.eap-1, @@ -105,7 +90,7 @@ constexpr double R1[129] = { // print("{", -c, ",", -b, "},"); // }; // We replace LOG_R1_DD[128] with log(1.0) == 0.0 -constexpr fputil::DoubleDouble LOG_R1_DD[129] = { +alignas(16) constexpr fputil::DoubleDouble LOG_R1_DD[129] = { {0.0, 0.0}, {-0x1.0c76b999d2be8p-46, 0x1.010157589p-7}, {-0x1.3dc5b06e2f7d2p-45, 0x1.0205658938p-6}, @@ -248,6 +233,22 @@ constexpr double P_COEFFS[6] = {-0x1p-1, -0x1.555874ce8ce22p-3, 0x1.24335555ddbe5p-3}; +#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS +// Extra errors from P is from using x^2 to reduce evaluation latency and +// directional rounding. +constexpr double P_ERR = 0x1.0p-49; + +// log(2) with 128-bit precision generated by SageMath with: +// def format_hex(value): +// l = hex(value)[2:] +// n = 8 +// x = [l[i:i + n] for i in range(0, len(l), n)] +// return "0x" + "'".join(x) + "_u128" +// (s, m, e) = RealField(128)(2).log().sign_mantissa_exponent(); +// print(format_hex(m)); +constexpr Float128 LOG_2(Sign::POS, /*exponent=*/-128, /*mantissa=*/ + 0xb17217f7'd1cf79ab'c9e3b398'03f2f6af_u128); + // -log(r1) with 128-bit precision generated by SageMath with: // // for i in range(129): @@ -874,6 +875,7 @@ constexpr Float128 BIG_COEFFS[4]{ return static_cast(r); } +#endif // LIBC_MATH_HAS_SKIP_ACCURATE_PASS } // namespace @@ -908,7 +910,12 @@ LLVM_LIBC_FUNCTION(double, log1p, (double x)) { return FPBits_t::quiet_nan().get_val(); } // x is +Inf or NaN - return x; + if (xbits.is_inf() && xbits.is_pos()) + return x; + + if (xbits.is_signaling_nan()) + fputil::raise_except_if_required(FE_INVALID); + return FPBits_t::quiet_nan().get_val(); } x_dd.hi = x; } else { @@ -969,9 +976,11 @@ LLVM_LIBC_FUNCTION(double, log1p, (double x)) { // <= 2^11 * 2^(-43-53) = 2^-85 double lo = fputil::multiply_add(e_x, LOG_2_LO, LOG_R1_DD[idx].lo); +#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS // Error bound of e_x * log(2) - log(r1) constexpr double ERR_HI[2] = {0x1.0p-85, 0.0}; double err_hi = ERR_HI[hi == 0.0]; +#endif // LIBC_MATH_HAS_SKIP_ACCURATE_PASS // Scale x_dd by 2^(-xh_bits.get_exponent()). int64_t s_u = static_cast(x_u & FPBits_t::EXP_MASK) - @@ -1005,7 +1014,7 @@ LLVM_LIBC_FUNCTION(double, log1p, (double x)) { fputil::DoubleDouble v_lo = fputil::exact_mult(m_dd.lo, r); // Perform exact range reduction -#ifdef LIBC_TARGET_CPU_HAS_FMA +#ifdef LIBC_TARGET_CPU_HAS_FMA_DOUBLE v_hi = fputil::multiply_add(r, m_dd.hi, -1.0); // Exact. #else // c = 1 + idx * 2^-7. @@ -1013,7 +1022,7 @@ LLVM_LIBC_FUNCTION(double, log1p, (double x)) { uint64_t(0x3FF0'0000'0000'0000ULL)) .get_val(); v_hi = fputil::multiply_add(r, m_dd.hi - c, RCM1[idx]); // Exact -#endif // LIBC_TARGET_CPU_HAS_FMA +#endif // LIBC_TARGET_CPU_HAS_FMA_DOUBLE // Range reduction output: // -0x1.69000000000edp-8 < v_hi + v_lo < 0x1.7f00000000081p-8 @@ -1033,6 +1042,9 @@ LLVM_LIBC_FUNCTION(double, log1p, (double x)) { double p2 = fputil::multiply_add(v_dd.hi, P_COEFFS[5], P_COEFFS[4]); double p = fputil::polyeval(v_sq, (v_dd.lo + r1.lo) + lo, p0, p1, p2); +#ifdef LIBC_MATH_HAS_SKIP_ACCURATE_PASS + return r1.hi + p; +#else double err = fputil::multiply_add(v_sq, P_ERR, err_hi); double left = r1.hi + (p - err); @@ -1043,6 +1055,7 @@ LLVM_LIBC_FUNCTION(double, log1p, (double x)) { return left; return log1p_accurate(x_e, idx, v_dd); +#endif // LIBC_MATH_HAS_SKIP_ACCURATE_PASS } } // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/generic/log1pf.cpp b/system/lib/llvm-libc/src/math/generic/log1pf.cpp index 869cb077cc434..7f614293029de 100644 --- a/system/lib/llvm-libc/src/math/generic/log1pf.cpp +++ b/system/lib/llvm-libc/src/math/generic/log1pf.cpp @@ -90,6 +90,7 @@ LLVM_LIBC_FUNCTION(float, log1pf, (float x)) { // Use log1p(x) = log(1 + x) for |x| > 2^-6; if (x_a > 0x3c80'0000U) { +#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS // Hard-to-round cases. switch (x_u) { case 0x41078febU: // x = 0x1.0f1fd6p3 @@ -108,7 +109,7 @@ LLVM_LIBC_FUNCTION(float, log1pf, (float x)) { fputil::set_errno_if_required(ERANGE); fputil::raise_except_if_required(FE_DIVBYZERO); return FPBits::inf(Sign::NEG).get_val(); -#ifndef LIBC_TARGET_CPU_HAS_FMA +#ifndef LIBC_TARGET_CPU_HAS_FMA_DOUBLE case 0x4cc1c80bU: // x = 0x1.839016p+26f return fputil::round_result_slightly_down(0x1.26fc04p+4f); case 0x5ee8984eU: // x = 0x1.d1309cp+62f @@ -117,13 +118,21 @@ LLVM_LIBC_FUNCTION(float, log1pf, (float x)) { return fputil::round_result_slightly_up(0x1.af66cp+5f); case 0x79e7ec37U: // x = 0x1.cfd86ep+116f return fputil::round_result_slightly_up(0x1.43ff6ep+6f); -#endif // LIBC_TARGET_CPU_HAS_FMA +#endif // LIBC_TARGET_CPU_HAS_FMA_DOUBLE } +#else + if (x == -1.0f) { + fputil::set_errno_if_required(ERANGE); + fputil::raise_except_if_required(FE_DIVBYZERO); + return FPBits::inf(Sign::NEG).get_val(); + } +#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS return internal::log(xd + 1.0); } // |x| <= 2^-6. +#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS // Hard-to round cases. switch (x_u) { case 0x35400003U: // x = 0x1.800006p-21f @@ -137,6 +146,7 @@ LLVM_LIBC_FUNCTION(float, log1pf, (float x)) { case 0xbb0ec8c4U: // x = -0x1.1d9188p-9 return fputil::round_result_slightly_up(-0x1.1de14ap-9f); } +#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS // Polymial generated by Sollya with: // > fpminimax(log(1 + x)/x, 7, [|D...|], [-2^-6; 2^-6]); diff --git a/system/lib/llvm-libc/src/math/generic/log2.cpp b/system/lib/llvm-libc/src/math/generic/log2.cpp index 37ea0c8f13431..27ca2fc350f17 100644 --- a/system/lib/llvm-libc/src/math/generic/log2.cpp +++ b/system/lib/llvm-libc/src/math/generic/log2.cpp @@ -33,10 +33,7 @@ namespace { constexpr fputil::DoubleDouble LOG2_E = {0x1.777d0ffda0d24p-56, 0x1.71547652b82fep0}; -// Extra errors from P is from using x^2 to reduce evaluation latency. -constexpr double P_ERR = 0x1.0p-49; - -const fputil::DoubleDouble LOG_R1[128] = { +alignas(16) const fputil::DoubleDouble LOG_R1[128] = { {0.0, 0.0}, {0x1.46662d417cedp-62, 0x1.010157588de71p-7}, {0x1.27c8e8416e71fp-60, 0x1.0205658935847p-6}, @@ -167,7 +164,11 @@ const fputil::DoubleDouble LOG_R1[128] = { {0.0, 0.0}, }; -alignas(64) constexpr LogRR LOG2_TABLE = { +#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS +// Extra errors from P is from using x^2 to reduce evaluation latency. +constexpr double P_ERR = 0x1.0p-49; + +alignas(16) constexpr LogRR LOG2_TABLE = { // -log2(r) with 128-bit precision generated by SageMath with: // def format_hex(value): // l = hex(value)[2:] @@ -853,6 +854,7 @@ double log2_accurate(int e_x, int index, double m_x) { return static_cast(r); } +#endif // LIBC_MATH_HAS_SKIP_ACCURATE_PASS } // namespace @@ -909,17 +911,17 @@ LLVM_LIBC_FUNCTION(double, log2, (double x)) { uint64_t x_m = (x_u & 0x000F'FFFF'FFFF'FFFFULL) | 0x3FF0'0000'0000'0000ULL; double m = FPBits_t(x_m).get_val(); - double u, u_sq, err; + double u, u_sq; fputil::DoubleDouble r1; // Perform exact range reduction -#ifdef LIBC_TARGET_CPU_HAS_FMA +#ifdef LIBC_TARGET_CPU_HAS_FMA_DOUBLE u = fputil::multiply_add(r, m, -1.0); // exact #else uint64_t c_m = x_m & 0x3FFF'E000'0000'0000ULL; double c = FPBits_t(c_m).get_val(); u = fputil::multiply_add(r, m - c, CD[index]); // exact -#endif // LIBC_TARGET_CPU_HAS_FMA +#endif // LIBC_TARGET_CPU_HAS_FMA_DOUBLE // Exact sum: // r1.hi + r1.lo = e_x * log(2)_hi - log(r)_hi + u @@ -927,8 +929,6 @@ LLVM_LIBC_FUNCTION(double, log2, (double x)) { // Error of u_sq = ulp(u^2); u_sq = u * u; - // Total error is bounded by ~ C * ulp(u^2). - err = u_sq * P_ERR; // Degree-7 minimax polynomial double p0 = fputil::multiply_add(u, LOG_COEFFS[1], LOG_COEFFS[0]); double p1 = fputil::multiply_add(u, LOG_COEFFS[3], LOG_COEFFS[2]); @@ -948,6 +948,11 @@ LLVM_LIBC_FUNCTION(double, log2, (double x)) { // Overall, if we choose sufficiently large constant C, the total error is // bounded by (C * ulp(u^2)). +#ifdef LIBC_MATH_HAS_SKIP_ACCURATE_PASS + return r3.hi + r3.lo; +#else + // Total error is bounded by ~ C * ulp(u^2). + double err = u_sq * P_ERR; // Lower bound from the result double left = r3.hi + (r3.lo - err); // Upper bound from the result @@ -958,6 +963,7 @@ LLVM_LIBC_FUNCTION(double, log2, (double x)) { return left; return log2_accurate(x_e, index, u); +#endif // LIBC_MATH_HAS_SKIP_ACCURATE_PASS } } // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/generic/log2f.cpp b/system/lib/llvm-libc/src/math/generic/log2f.cpp index 111f3f130bcab..b25ec41f277b6 100644 --- a/system/lib/llvm-libc/src/math/generic/log2f.cpp +++ b/system/lib/llvm-libc/src/math/generic/log2f.cpp @@ -97,11 +97,11 @@ LLVM_LIBC_FUNCTION(float, log2f, (float x)) { float u = xbits.get_val(); double v; -#ifdef LIBC_TARGET_CPU_HAS_FMA +#ifdef LIBC_TARGET_CPU_HAS_FMA_FLOAT v = static_cast(fputil::multiply_add(u, R[index], -1.0f)); // Exact. #else v = fputil::multiply_add(static_cast(u), RD[index], -1.0); // Exact -#endif // LIBC_TARGET_CPU_HAS_FMA +#endif // LIBC_TARGET_CPU_HAS_FMA_FLOAT double extra_factor = static_cast(m) + LOG2_R[index]; diff --git a/system/lib/llvm-libc/src/math/generic/logf.cpp b/system/lib/llvm-libc/src/math/generic/logf.cpp index 30c00edafe21d..e8d2ba2cfe175 100644 --- a/system/lib/llvm-libc/src/math/generic/logf.cpp +++ b/system/lib/llvm-libc/src/math/generic/logf.cpp @@ -66,6 +66,7 @@ LLVM_LIBC_FUNCTION(float, logf, (float x)) { // Small inputs if (x_u < 0x4c5d65a5U) { +#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS // Hard-to-round cases. switch (x_u) { case 0x3f7f4d6fU: // x = 0x1.fe9adep-1f @@ -80,6 +81,7 @@ LLVM_LIBC_FUNCTION(float, logf, (float x)) { return round_result_slightly_up(-0x1.6d7b18p+5f); #endif // LIBC_TARGET_CPU_HAS_FMA } +#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS // Subnormal inputs. if (LIBC_UNLIKELY(x_u < FPBits::min_normal().uintval())) { if (x == 0.0f) { @@ -94,6 +96,7 @@ LLVM_LIBC_FUNCTION(float, logf, (float x)) { x_u = xbits.uintval(); } } else { +#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS // Hard-to-round cases. switch (x_u) { case 0x4c5d65a5U: // x = 0x1.bacb4ap+25f @@ -104,15 +107,16 @@ LLVM_LIBC_FUNCTION(float, logf, (float x)) { return round_result_slightly_down(0x1.08b512p+6f); case 0x7a17f30aU: // x = 0x1.2fe614p+117f return round_result_slightly_up(0x1.451436p+6f); -#ifndef LIBC_TARGET_CPU_HAS_FMA +#ifndef LIBC_TARGET_CPU_HAS_FMA_DOUBLE case 0x500ffb03U: // x = 0x1.1ff606p+33f return round_result_slightly_up(0x1.6fdd34p+4f); case 0x5cd69e88U: // x = 0x1.ad3d1p+58f return round_result_slightly_up(0x1.45c146p+5f); case 0x5ee8984eU: // x = 0x1.d1309cp+62f; return round_result_slightly_up(0x1.5c9442p+5f); -#endif // LIBC_TARGET_CPU_HAS_FMA +#endif // LIBC_TARGET_CPU_HAS_FMA_DOUBLE } +#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS // Exceptional inputs. if (LIBC_UNLIKELY(x_u > FPBits::max_normal().uintval())) { if (x_u == 0x8000'0000U) { @@ -128,6 +132,11 @@ LLVM_LIBC_FUNCTION(float, logf, (float x)) { return FPBits::quiet_nan().get_val(); } // x is +inf or nan + if (xbits.is_signaling_nan()) { + fputil::raise_except_if_required(FE_INVALID); + return FPBits::quiet_nan().get_val(); + } + return x; } } @@ -152,11 +161,11 @@ LLVM_LIBC_FUNCTION(float, logf, (float x)) { float u = xbits.get_val(); double v; -#ifdef LIBC_TARGET_CPU_HAS_FMA +#ifdef LIBC_TARGET_CPU_HAS_FMA_FLOAT v = static_cast(fputil::multiply_add(u, R[index], -1.0f)); // Exact. #else v = fputil::multiply_add(static_cast(u), RD[index], -1.0); // Exact -#endif // LIBC_TARGET_CPU_HAS_FMA +#endif // LIBC_TARGET_CPU_HAS_FMA_FLOAT // Degree-5 polynomial approximation of log generated by Sollya with: // > P = fpminimax(log(1 + x)/x, 4, [|1, D...|], [-2^-8, 2^-7]); diff --git a/system/lib/llvm-libc/src/math/generic/nan.cpp b/system/lib/llvm-libc/src/math/generic/nan.cpp index f92cd3ff5eb50..829a2ea435ac0 100644 --- a/system/lib/llvm-libc/src/math/generic/nan.cpp +++ b/system/lib/llvm-libc/src/math/generic/nan.cpp @@ -8,9 +8,9 @@ #include "src/math/nan.h" #include "src/__support/common.h" +#include "src/__support/libc_errno.h" #include "src/__support/macros/config.h" #include "src/__support/str_to_float.h" -#include "src/errno/libc_errno.h" namespace LIBC_NAMESPACE_DECL { diff --git a/system/lib/llvm-libc/src/math/generic/nanf.cpp b/system/lib/llvm-libc/src/math/generic/nanf.cpp index 7287182406acd..1cb66160e736e 100644 --- a/system/lib/llvm-libc/src/math/generic/nanf.cpp +++ b/system/lib/llvm-libc/src/math/generic/nanf.cpp @@ -8,9 +8,9 @@ #include "src/math/nanf.h" #include "src/__support/common.h" +#include "src/__support/libc_errno.h" #include "src/__support/macros/config.h" #include "src/__support/str_to_float.h" -#include "src/errno/libc_errno.h" namespace LIBC_NAMESPACE_DECL { diff --git a/system/lib/llvm-libc/src/math/generic/nanf128.cpp b/system/lib/llvm-libc/src/math/generic/nanf128.cpp index 3d8581afa0371..4155c5333a9c2 100644 --- a/system/lib/llvm-libc/src/math/generic/nanf128.cpp +++ b/system/lib/llvm-libc/src/math/generic/nanf128.cpp @@ -8,9 +8,9 @@ #include "src/math/nanf128.h" #include "src/__support/common.h" +#include "src/__support/libc_errno.h" #include "src/__support/macros/config.h" #include "src/__support/str_to_float.h" -#include "src/errno/libc_errno.h" namespace LIBC_NAMESPACE_DECL { diff --git a/system/lib/llvm-libc/src/math/generic/nanl.cpp b/system/lib/llvm-libc/src/math/generic/nanl.cpp index 4f698cb3c88d0..58d638c4b531d 100644 --- a/system/lib/llvm-libc/src/math/generic/nanl.cpp +++ b/system/lib/llvm-libc/src/math/generic/nanl.cpp @@ -8,9 +8,9 @@ #include "src/math/nanl.h" #include "src/__support/common.h" +#include "src/__support/libc_errno.h" #include "src/__support/macros/config.h" #include "src/__support/str_to_float.h" -#include "src/errno/libc_errno.h" namespace LIBC_NAMESPACE_DECL { diff --git a/system/lib/llvm-libc/src/math/generic/pow.cpp b/system/lib/llvm-libc/src/math/generic/pow.cpp index 213dbd959039c..43e99a7acf690 100644 --- a/system/lib/llvm-libc/src/math/generic/pow.cpp +++ b/system/lib/llvm-libc/src/math/generic/pow.cpp @@ -217,6 +217,11 @@ LLVM_LIBC_FUNCTION(double, pow, (double x, double y)) { uint64_t sign = 0; ///////// BEGIN - Check exceptional cases //////////////////////////////////// + // If x or y is signaling NaN + if (x_abs.is_signaling_nan() || y_abs.is_signaling_nan()) { + fputil::raise_except_if_required(FE_INVALID); + return FPBits::quiet_nan().get_val(); + } // The double precision number that is closest to 1 is (1 - 2^-53), which has // log2(1 - 2^-53) ~ -1.715...p-53. @@ -394,14 +399,14 @@ LLVM_LIBC_FUNCTION(double, pow, (double x, double y)) { DoubleDouble dx_c0; // Perform exact range reduction and exact product dx * c0. -#ifdef LIBC_TARGET_CPU_HAS_FMA +#ifdef LIBC_TARGET_CPU_HAS_FMA_DOUBLE dx = fputil::multiply_add(RD[idx_x], m_x.get_val(), -1.0); // Exact dx_c0 = fputil::exact_mult(COEFFS[0], dx); #else double c = FPBits(m_x.uintval() & 0x3fff'e000'0000'0000).get_val(); dx = fputil::multiply_add(RD[idx_x], m_x.get_val() - c, CD[idx_x]); // Exact - dx_c0 = fputil::exact_mult<28>(dx, COEFFS[0]); // Exact -#endif // LIBC_TARGET_CPU_HAS_FMA + dx_c0 = fputil::exact_mult(dx, COEFFS[0]); // Exact +#endif // LIBC_TARGET_CPU_HAS_FMA_DOUBLE double dx2 = dx * dx; double c0 = fputil::multiply_add(dx, COEFFS[2], COEFFS[1]); diff --git a/system/lib/llvm-libc/src/math/generic/powf.cpp b/system/lib/llvm-libc/src/math/generic/powf.cpp index c84ce0da34b10..dfdfd5d6d5760 100644 --- a/system/lib/llvm-libc/src/math/generic/powf.cpp +++ b/system/lib/llvm-libc/src/math/generic/powf.cpp @@ -32,11 +32,144 @@ using fputil::TripleDouble; namespace { -#ifdef LIBC_TARGET_CPU_HAS_FMA +#ifdef LIBC_MATH_HAS_SKIP_ACCURATE_PASS +alignas(16) constexpr DoubleDouble LOG2_R_DD[128] = { + {0.0, 0.0}, + {-0x1.177c23362928cp-25, 0x1.72c8p-7}, + {-0x1.179e0caa9c9abp-22, 0x1.744p-6}, + {-0x1.c6cea541f5b7p-23, 0x1.184cp-5}, + {-0x1.66c4d4e554434p-22, 0x1.773ap-5}, + {-0x1.70700a00fdd55p-24, 0x1.d6ecp-5}, + {0x1.53002a4e86631p-23, 0x1.1bb3p-4}, + {0x1.fcd15f101c142p-25, 0x1.4c56p-4}, + {0x1.25b3eed319cedp-22, 0x1.7d6p-4}, + {-0x1.4195120d8486fp-22, 0x1.960dp-4}, + {0x1.45b878e27d0d9p-23, 0x1.c7b5p-4}, + {0x1.770744593a4cbp-22, 0x1.f9c9p-4}, + {0x1.c673032495d24p-22, 0x1.097ep-3}, + {-0x1.1eaa65b49696ep-22, 0x1.22dbp-3}, + {0x1.b2866f2850b22p-22, 0x1.3c6f8p-3}, + {0x1.8ee37cd2ea9d3p-25, 0x1.494f8p-3}, + {0x1.7e86f9c2154fbp-24, 0x1.633a8p-3}, + {0x1.8e3cfc25f0ce6p-26, 0x1.7046p-3}, + {0x1.57f7a64ccd537p-28, 0x1.8a898p-3}, + {-0x1.a761c09fbd2aep-22, 0x1.97c2p-3}, + {0x1.24bea9a2c66f3p-22, 0x1.b26p-3}, + {-0x1.60002ccfe43f5p-25, 0x1.bfc68p-3}, + {0x1.69f220e97f22cp-22, 0x1.dac2p-3}, + {-0x1.6164f64c210ep-22, 0x1.e858p-3}, + {-0x1.0c1678ae89767p-24, 0x1.01d9cp-2}, + {-0x1.f26a05c813d57p-22, 0x1.08bdp-2}, + {0x1.4d8fc561c8d44p-24, 0x1.169cp-2}, + {-0x1.362ad8f7ca2dp-22, 0x1.1d984p-2}, + {0x1.2b13cd6c4d042p-22, 0x1.249ccp-2}, + {-0x1.1c8f11979a5dbp-22, 0x1.32cp-2}, + {0x1.c2ab3edefe569p-23, 0x1.39de8p-2}, + {0x1.7c3eca28e69cap-26, 0x1.4106p-2}, + {-0x1.34c4e99e1c6c6p-24, 0x1.4f6fcp-2}, + {-0x1.194a871b63619p-22, 0x1.56b24p-2}, + {0x1.e3dd5c1c885aep-23, 0x1.5dfdcp-2}, + {-0x1.6ccf3b1129b7cp-23, 0x1.6552cp-2}, + {-0x1.2f346e2bf924bp-23, 0x1.6cb1p-2}, + {-0x1.fa61aaa59c1d8p-23, 0x1.7b8ap-2}, + {0x1.90c11fd32a3abp-22, 0x1.8304cp-2}, + {0x1.57f7a64ccd537p-27, 0x1.8a898p-2}, + {0x1.249ba76fee235p-27, 0x1.9218p-2}, + {-0x1.aad2729b21ae5p-23, 0x1.99b08p-2}, + {0x1.71810a5e1818p-22, 0x1.a8ff8p-2}, + {-0x1.6172fe015e13cp-27, 0x1.b0b68p-2}, + {0x1.5ec6c1bfbf89ap-24, 0x1.b877cp-2}, + {0x1.678bf6cdedf51p-24, 0x1.c0438p-2}, + {0x1.c2d45fe43895ep-22, 0x1.c819cp-2}, + {-0x1.9ee52ed49d71dp-22, 0x1.cffbp-2}, + {0x1.5786af187a96bp-27, 0x1.d7e6cp-2}, + {0x1.3ab0dc56138c9p-23, 0x1.dfdd8p-2}, + {0x1.fe538ab34efb5p-22, 0x1.e7df4p-2}, + {-0x1.e4fee07aa4b68p-22, 0x1.efec8p-2}, + {-0x1.172f32fe67287p-22, 0x1.f804cp-2}, + {-0x1.9a83ff9ab9cc8p-22, 0x1.00144p-1}, + {-0x1.68cb06cece193p-22, 0x1.042bep-1}, + {0x1.8cd71ddf82e2p-22, 0x1.08494p-1}, + {0x1.5e18ab2df3ae6p-22, 0x1.0c6cap-1}, + {0x1.5dee4d9d8a273p-25, 0x1.1096p-1}, + {0x1.fcd15f101c142p-26, 0x1.14c56p-1}, + {-0x1.2474b0f992ba1p-23, 0x1.18faep-1}, + {0x1.4b5a92a606047p-24, 0x1.1d368p-1}, + {0x1.16186fcf54bbdp-22, 0x1.21786p-1}, + {0x1.18efabeb7d722p-27, 0x1.25c0ap-1}, + {-0x1.e5fc7d238691dp-24, 0x1.2a0f4p-1}, + {0x1.f5809faf6283cp-22, 0x1.2e644p-1}, + {0x1.f5809faf6283cp-22, 0x1.2e644p-1}, + {0x1.c6e1dcd0cb449p-22, 0x1.32bfep-1}, + {0x1.76e0e8f74b4d5p-22, 0x1.37222p-1}, + {-0x1.cb82c89692d99p-24, 0x1.3b8b2p-1}, + {-0x1.63161c5432aebp-22, 0x1.3ffaep-1}, + {0x1.458104c41b901p-22, 0x1.44716p-1}, + {0x1.458104c41b901p-22, 0x1.44716p-1}, + {-0x1.cd9d0cde578d5p-22, 0x1.48efp-1}, + {0x1.b9884591add87p-26, 0x1.4d738p-1}, + {0x1.c6042978605ffp-22, 0x1.51ff2p-1}, + {-0x1.fc4c96b37dcf6p-22, 0x1.56922p-1}, + {-0x1.2f346e2bf924bp-24, 0x1.5b2c4p-1}, + {-0x1.2f346e2bf924bp-24, 0x1.5b2c4p-1}, + {0x1.c4e4fbb68a4d1p-22, 0x1.5fcdcp-1}, + {-0x1.9d499bd9b3226p-23, 0x1.6476ep-1}, + {-0x1.f89b355ede26fp-23, 0x1.69278p-1}, + {-0x1.f89b355ede26fp-23, 0x1.69278p-1}, + {0x1.53c7e319f6e92p-24, 0x1.6ddfcp-1}, + {-0x1.b291f070528c7p-22, 0x1.729fep-1}, + {0x1.2967a451a7b48p-25, 0x1.7767cp-1}, + {0x1.2967a451a7b48p-25, 0x1.7767cp-1}, + {0x1.244fcff690fcep-22, 0x1.7c37ap-1}, + {0x1.46fd97f5dc572p-23, 0x1.810fap-1}, + {0x1.46fd97f5dc572p-23, 0x1.810fap-1}, + {-0x1.f3a7352663e5p-22, 0x1.85efep-1}, + {0x1.b3cda690370b5p-23, 0x1.8ad84p-1}, + {0x1.b3cda690370b5p-23, 0x1.8ad84p-1}, + {0x1.3226b211bf1d9p-23, 0x1.8fc92p-1}, + {0x1.d24b136c101eep-23, 0x1.94c28p-1}, + {0x1.d24b136c101eep-23, 0x1.94c28p-1}, + {0x1.7c40c7907e82ap-22, 0x1.99c48p-1}, + {-0x1.e81781d97ee91p-22, 0x1.9ecf6p-1}, + {-0x1.e81781d97ee91p-22, 0x1.9ecf6p-1}, + {-0x1.6a77813f94e01p-22, 0x1.a3e3p-1}, + {-0x1.1cfdeb43cfdp-22, 0x1.a8ffap-1}, + {-0x1.1cfdeb43cfdp-22, 0x1.a8ffap-1}, + {-0x1.f983f74d3138fp-23, 0x1.ae256p-1}, + {-0x1.e278ae1a1f51fp-23, 0x1.b3546p-1}, + {-0x1.e278ae1a1f51fp-23, 0x1.b3546p-1}, + {-0x1.97552b7b5ea45p-23, 0x1.b88ccp-1}, + {-0x1.97552b7b5ea45p-23, 0x1.b88ccp-1}, + {-0x1.19b4f3c72c4f8p-24, 0x1.bdceap-1}, + {0x1.f7402d26f1a12p-23, 0x1.c31a2p-1}, + {0x1.f7402d26f1a12p-23, 0x1.c31a2p-1}, + {-0x1.2056d5dd31d96p-23, 0x1.c86f8p-1}, + {-0x1.2056d5dd31d96p-23, 0x1.c86f8p-1}, + {-0x1.6e46335aae723p-24, 0x1.cdcecp-1}, + {-0x1.beb244c59f331p-22, 0x1.d3382p-1}, + {-0x1.beb244c59f331p-22, 0x1.d3382p-1}, + {0x1.16c071e93fd97p-27, 0x1.d8abap-1}, + {0x1.16c071e93fd97p-27, 0x1.d8abap-1}, + {0x1.d8175819530c2p-22, 0x1.de298p-1}, + {0x1.d8175819530c2p-22, 0x1.de298p-1}, + {0x1.51bd552842c1cp-23, 0x1.e3b2p-1}, + {0x1.51bd552842c1cp-23, 0x1.e3b2p-1}, + {0x1.914e204f19d94p-22, 0x1.e9452p-1}, + {0x1.914e204f19d94p-22, 0x1.e9452p-1}, + {0x1.c55d997da24fdp-22, 0x1.eee32p-1}, + {0x1.c55d997da24fdp-22, 0x1.eee32p-1}, + {-0x1.685c2d2298a6ep-22, 0x1.f48c4p-1}, + {-0x1.685c2d2298a6ep-22, 0x1.f48c4p-1}, + {0x1.7a4887bd74039p-22, 0x1.fa406p-1}, + {0.0, 1.0}, +}; +#else + +#ifdef LIBC_TARGET_CPU_HAS_FMA_DOUBLE constexpr uint64_t ERR = 64; #else constexpr uint64_t ERR = 128; -#endif // LIBC_TARGET_CPU_HAS_FMA +#endif // LIBC_TARGET_CPU_HAS_FMA_DOUBLE // We choose the precision of the high part to be 53 - 24 - 8, so that when // y * (e_x + LOG2_R_DD[i].hi) is exact. @@ -384,6 +517,7 @@ static constexpr DoubleDouble LOG2_R2_DD[] = { {0x1.3d979ddf4746cp-61, 0x1.6cf6ddd2611d4p-7}, {-0x1.dc930484501f8p-63, 0x1.6fdf461d2e4f8p-7}, }; +#endif // LIBC_MATH_HAS_SKIP_ACCURATE_PASS LIBC_INLINE bool is_odd_integer(float x) { using FPBits = typename fputil::FPBits; @@ -407,6 +541,7 @@ LIBC_INLINE bool is_integer(float x) { return (x_e + lsb >= UNIT_EXPONENT); } +#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS LIBC_INLINE bool larger_exponent(double a, double b) { using DoubleBits = typename fputil::FPBits; return DoubleBits(a).get_biased_exponent() >= @@ -506,6 +641,7 @@ double powf_double_double(int idx_x, double dx, double y6, double lo6_hi, return cpp::bit_cast(r_bits); } +#endif // LIBC_MATH_HAS_SKIP_ACCURATE_PASS } // namespace @@ -528,6 +664,12 @@ LLVM_LIBC_FUNCTION(float, powf, (float x, float y)) { // |y * log2(x)| = 0 or > 151. // Hence x^y will either overflow or underflow if x is not zero. if (LIBC_UNLIKELY((y_abs & 0x0007'ffff) == 0) || (y_abs > 0x4f170000)) { + // y is signaling NaN + if (xbits.is_signaling_nan() || ybits.is_signaling_nan()) { + fputil::raise_except_if_required(FE_INVALID); + return FloatBits::quiet_nan().get_val(); + } + // Exceptional exponents. if (y == 0.0f) return 1.0f; @@ -600,8 +742,8 @@ LLVM_LIBC_FUNCTION(float, powf, (float x, float y)) { } } if (y_abs > 0x4f17'0000) { + // if y is NaN if (y_abs > 0x7f80'0000) { - // y is NaN if (x_u == 0x3f80'0000) { // x = 1.0f // pow(1, NaN) = 1 return 1.0f; @@ -623,16 +765,24 @@ LLVM_LIBC_FUNCTION(float, powf, (float x, float y)) { // y is finite and non-zero. if (LIBC_UNLIKELY(((x_u & 0x801f'ffffU) == 0) || x_u >= 0x7f80'0000U || x_u < 0x0080'0000U)) { + // if x is signaling NaN + if (xbits.is_signaling_nan()) { + fputil::raise_except_if_required(FE_INVALID); + return FloatBits::quiet_nan().get_val(); + } + switch (x_u) { case 0x3f80'0000: // x = 1.0f return 1.0f; // TODO: Put these 2 entrypoint dependency under control flag. +#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS case 0x4000'0000: // x = 2.0f // pow(2, y) = exp2(y) return generic::exp2f(y); case 0x4120'0000: // x = 10.0f // pow(10, y) = exp10(y) return generic::exp10f(y); +#endif // LIBC_MATH_HAS_SKIP_ACCURATE_PASS } const bool x_is_neg = x_u >= FloatBits::SIGN_MASK; @@ -713,11 +863,11 @@ LLVM_LIBC_FUNCTION(float, powf, (float x, float y)) { // log2(m_x) = log2( (1 + dx) / r ) // = log2(1 + dx) - log2(r). double dx; -#ifdef LIBC_TARGET_CPU_HAS_FMA +#ifdef LIBC_TARGET_CPU_HAS_FMA_FLOAT dx = static_cast(fputil::multiply_add(m_x, R[idx_x], -1.0f)); // Exact #else dx = fputil::multiply_add(static_cast(m_x), RD[idx_x], -1.0); // Exact -#endif // LIBC_TARGET_CPU_HAS_FMA +#endif // LIBC_TARGET_CPU_HAS_FMA_FLOAT // Degree-5 polynomial approximation: // dx * P(dx) ~ log2(1 + dx) @@ -782,6 +932,16 @@ LLVM_LIBC_FUNCTION(float, powf, (float x, float y)) { // lo6 = 2^6 * lo = 2^6 * (y - (hi + mid)) = y6 * log2(x) - hm. double y6 = static_cast(y * 0x1.0p6f); // Exact. double hm = fputil::nearest_integer(s * y6); +#ifdef LIBC_MATH_HAS_SKIP_ACCURATE_PASS + // lo6 = 2^6 * lo. + double lo6_hi = + fputil::multiply_add(y6, e_x + LOG2_R_DD[idx_x].hi, -hm); // Exact + // Error bounds: + // | (y*log2(x) - hm * 2^-6 - lo) / y| < err(dx * p) + err(LOG2_R_DD.lo) + // < 2^-51 + 2^-75 + double lo6 = fputil::multiply_add( + y6, fputil::multiply_add(dx, p, LOG2_R_DD[idx_x].lo), lo6_hi); +#else // lo6 = 2^6 * lo. double lo6_hi = fputil::multiply_add(y6, e_x + LOG2_R_TD[idx_x].hi, -hm); // Exact @@ -790,6 +950,7 @@ LLVM_LIBC_FUNCTION(float, powf, (float x, float y)) { // < 2^-51 + 2^-75 double lo6 = fputil::multiply_add( y6, fputil::multiply_add(dx, p, LOG2_R_TD[idx_x].mid), lo6_hi); +#endif // |2^(hi + mid) - exp2_hi_mid| <= ulp(exp2_hi_mid) / 2 // Clamp the exponent part into smaller range that fits double precision. @@ -830,6 +991,9 @@ LLVM_LIBC_FUNCTION(float, powf, (float x, float y)) { double r = pp * exp2_hi_mid; +#ifdef LIBC_MATH_HAS_SKIP_ACCURATE_PASS + return static_cast(r); +#else // Ziv accuracy test. uint64_t r_u = cpp::bit_cast(r); float r_upper = static_cast(cpp::bit_cast(r_u + ERR)); @@ -861,6 +1025,7 @@ LLVM_LIBC_FUNCTION(float, powf, (float x, float y)) { double r_dd = powf_double_double(idx_x, dx, y6, lo6_hi, exp2_hi_mid_dd); return static_cast(r_dd); +#endif // LIBC_MATH_HAS_SKIP_ACCURATE_PASS } } // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/generic/range_reduction_double_common.h b/system/lib/llvm-libc/src/math/generic/range_reduction_double_common.h index 06aeb49495ad2..f3dcdb937333c 100644 --- a/system/lib/llvm-libc/src/math/generic/range_reduction_double_common.h +++ b/system/lib/llvm-libc/src/math/generic/range_reduction_double_common.h @@ -20,14 +20,14 @@ namespace LIBC_NAMESPACE_DECL { -#ifdef LIBC_TARGET_CPU_HAS_FMA -static constexpr unsigned SPLIT = DEFAULT_DOUBLE_SPLIT; +#ifdef LIBC_TARGET_CPU_HAS_FMA_DOUBLE +static constexpr unsigned SPLIT = fputil::DefaultSplit::VALUE; #else // When there is no-FMA instructions, in order to have exact product of 2 double // precision with directional roundings, we need to lower the precision of the // constants by at least 1 bit, and use a different splitting constant. static constexpr unsigned SPLIT = 28; -#endif // LIBC_TARGET_CPU_HAS_FMA +#endif // LIBC_TARGET_CPU_HAS_FMA_DOUBLE using LIBC_NAMESPACE::fputil::DoubleDouble; using Float128 = LIBC_NAMESPACE::fputil::DyadicFloat<128>; diff --git a/system/lib/llvm-libc/src/math/generic/range_reduction_double_fma.h b/system/lib/llvm-libc/src/math/generic/range_reduction_double_fma.h index cab031c28baa1..160fb2461fe21 100644 --- a/system/lib/llvm-libc/src/math/generic/range_reduction_double_fma.h +++ b/system/lib/llvm-libc/src/math/generic/range_reduction_double_fma.h @@ -33,14 +33,14 @@ LIBC_INLINE unsigned LargeRangeReduction::fast(double x, DoubleDouble &u) { // 2^62 <= |x_reduced| < 2^(62 + 16) = 2^78 x_reduced = xbits.get_val(); // x * c_hi = ph.hi + ph.lo exactly. - DoubleDouble ph = - fputil::exact_mult(x_reduced, ONE_TWENTY_EIGHT_OVER_PI[idx][0]); + DoubleDouble ph = fputil::exact_mult( + x_reduced, ONE_TWENTY_EIGHT_OVER_PI[idx][0]); // x * c_mid = pm.hi + pm.lo exactly. - DoubleDouble pm = - fputil::exact_mult(x_reduced, ONE_TWENTY_EIGHT_OVER_PI[idx][1]); + DoubleDouble pm = fputil::exact_mult( + x_reduced, ONE_TWENTY_EIGHT_OVER_PI[idx][1]); // x * c_lo = pl.hi + pl.lo exactly. - DoubleDouble pl = - fputil::exact_mult(x_reduced, ONE_TWENTY_EIGHT_OVER_PI[idx][2]); + DoubleDouble pl = fputil::exact_mult( + x_reduced, ONE_TWENTY_EIGHT_OVER_PI[idx][2]); // Extract integral parts and fractional parts of (ph.lo + pm.hi). double sum_hi = ph.lo + pm.hi; double kd = fputil::nearest_integer(sum_hi); @@ -80,7 +80,7 @@ LIBC_INLINE unsigned LargeRangeReduction::fast(double x, DoubleDouble &u) { // b = D(sin(k * pi/128) - a); // print("{", b, ",", a, "},"); // }; -LIBC_INLINE constexpr DoubleDouble SIN_K_PI_OVER_128[256] = { +LIBC_INLINE constexpr DoubleDouble SIN_K_PI_OVER_128[] = { {0, 0}, {-0x1.b1d63091a013p-64, 0x1.92155f7a3667ep-6}, {-0x1.912bd0d569a9p-61, 0x1.91f65f10dd814p-5}, diff --git a/system/lib/llvm-libc/src/math/generic/range_reduction_double_nofma.h b/system/lib/llvm-libc/src/math/generic/range_reduction_double_nofma.h index 5640732947798..9d13d246ce91f 100644 --- a/system/lib/llvm-libc/src/math/generic/range_reduction_double_nofma.h +++ b/system/lib/llvm-libc/src/math/generic/range_reduction_double_nofma.h @@ -34,14 +34,14 @@ LIBC_INLINE unsigned LargeRangeReduction::fast(double x, DoubleDouble &u) { x_reduced = xbits.get_val(); // x * c_hi = ph.hi + ph.lo exactly. DoubleDouble x_split = fputil::split(x_reduced); - DoubleDouble ph = fputil::exact_mult(x_split, x_reduced, - ONE_TWENTY_EIGHT_OVER_PI[idx][0]); + DoubleDouble ph = fputil::exact_mult( + x_split, x_reduced, ONE_TWENTY_EIGHT_OVER_PI[idx][0]); // x * c_mid = pm.hi + pm.lo exactly. - DoubleDouble pm = fputil::exact_mult(x_split, x_reduced, - ONE_TWENTY_EIGHT_OVER_PI[idx][1]); + DoubleDouble pm = fputil::exact_mult( + x_split, x_reduced, ONE_TWENTY_EIGHT_OVER_PI[idx][1]); // x * c_lo = pl.hi + pl.lo exactly. - DoubleDouble pl = fputil::exact_mult(x_split, x_reduced, - ONE_TWENTY_EIGHT_OVER_PI[idx][2]); + DoubleDouble pl = fputil::exact_mult( + x_split, x_reduced, ONE_TWENTY_EIGHT_OVER_PI[idx][2]); // Extract integral parts and fractional parts of (ph.lo + pm.hi). double sum_hi = ph.lo + pm.hi; double kd = fputil::nearest_integer(sum_hi); @@ -81,7 +81,7 @@ LIBC_INLINE unsigned LargeRangeReduction::fast(double x, DoubleDouble &u) { // b = round(sin(k * pi/128) - a, D, RN); // print("{", b, ",", a, "},"); // }; -LIBC_INLINE constexpr DoubleDouble SIN_K_PI_OVER_128[256] = { +LIBC_INLINE constexpr DoubleDouble SIN_K_PI_OVER_128[] = { {0, 0}, {0x1.f938a73db97fbp-58, 0x1.92155f7a3667cp-6}, {-0x1.912bd0d569a9p-61, 0x1.91f65f10dd814p-5}, diff --git a/system/lib/llvm-libc/src/math/generic/sin.cpp b/system/lib/llvm-libc/src/math/generic/sin.cpp index b32486dff487c..a614427bd7ee3 100644 --- a/system/lib/llvm-libc/src/math/generic/sin.cpp +++ b/system/lib/llvm-libc/src/math/generic/sin.cpp @@ -21,11 +21,11 @@ #include "src/math/generic/range_reduction_double_common.h" #include "src/math/generic/sincos_eval.h" -#ifdef LIBC_TARGET_CPU_HAS_FMA +#ifdef LIBC_TARGET_CPU_HAS_FMA_DOUBLE #include "range_reduction_double_fma.h" #else #include "range_reduction_double_nofma.h" -#endif // LIBC_TARGET_CPU_HAS_FMA +#endif // LIBC_TARGET_CPU_HAS_FMA_DOUBLE namespace LIBC_NAMESPACE_DECL { @@ -52,7 +52,7 @@ LLVM_LIBC_FUNCTION(double, sin, (double x)) { if (LIBC_UNLIKELY(x == 0.0)) return x + x; // Make sure it works with FTZ/DAZ. -#ifdef LIBC_TARGET_CPU_HAS_FMA +#ifdef LIBC_TARGET_CPU_HAS_FMA_DOUBLE return fputil::multiply_add(x, -0x1.0p-54, x); #else if (LIBC_UNLIKELY(x_e < 4)) { @@ -63,7 +63,7 @@ LLVM_LIBC_FUNCTION(double, sin, (double x)) { return FPBits(xbits.uintval() - 1).get_val(); } return fputil::multiply_add(x, -0x1.0p-54, x); -#endif // LIBC_TARGET_CPU_HAS_FMA +#endif // LIBC_TARGET_CPU_HAS_FMA_DOUBLE } // No range reduction needed. k = 0; @@ -77,6 +77,11 @@ LLVM_LIBC_FUNCTION(double, sin, (double x)) { // Inf or NaN if (LIBC_UNLIKELY(x_e > 2 * FPBits::EXP_BIAS)) { // sin(+-Inf) = NaN + if (xbits.is_signaling_nan()) { + fputil::raise_except_if_required(FE_INVALID); + return FPBits::quiet_nan().get_val(); + } + if (xbits.get_mantissa() == 0) { fputil::set_errno_if_required(EDOM); fputil::raise_except_if_required(FE_INVALID); @@ -158,7 +163,7 @@ LLVM_LIBC_FUNCTION(double, sin, (double x)) { Float128 sin_k_f128 = get_sin_k(k); Float128 cos_k_f128 = get_sin_k(k + 64); - // sin(x) = sin((k * pi/128 + u) + // sin(x) = sin(k * pi/128 + u) // = sin(u) * cos(k*pi/128) + cos(u) * sin(k*pi/128) Float128 r = fputil::quick_add(fputil::quick_mul(sin_k_f128, cos_u), fputil::quick_mul(cos_k_f128, sin_u)); diff --git a/system/lib/llvm-libc/src/math/generic/sincos.cpp b/system/lib/llvm-libc/src/math/generic/sincos.cpp index 166ce46603140..08c8a8298f029 100644 --- a/system/lib/llvm-libc/src/math/generic/sincos.cpp +++ b/system/lib/llvm-libc/src/math/generic/sincos.cpp @@ -22,11 +22,11 @@ #include "src/math/generic/range_reduction_double_common.h" #include "src/math/generic/sincos_eval.h" -#ifdef LIBC_TARGET_CPU_HAS_FMA +#ifdef LIBC_TARGET_CPU_HAS_FMA_DOUBLE #include "range_reduction_double_fma.h" #else #include "range_reduction_double_nofma.h" -#endif // LIBC_TARGET_CPU_HAS_FMA +#endif // LIBC_TARGET_CPU_HAS_FMA_DOUBLE namespace LIBC_NAMESPACE_DECL { @@ -57,7 +57,7 @@ LLVM_LIBC_FUNCTION(void, sincos, (double x, double *sin_x, double *cos_x)) { } // For |x| < 2^-27, max(|sin(x) - x|, |cos(x) - 1|) < ulp(x)/2. -#ifdef LIBC_TARGET_CPU_HAS_FMA +#ifdef LIBC_TARGET_CPU_HAS_FMA_DOUBLE *sin_x = fputil::multiply_add(x, -0x1.0p-54, x); *cos_x = fputil::multiply_add(x, -x, 1.0); #else @@ -71,7 +71,7 @@ LLVM_LIBC_FUNCTION(void, sincos, (double x, double *sin_x, double *cos_x)) { *sin_x = FPBits(xbits.uintval() - 1).get_val(); } *sin_x = fputil::multiply_add(x, -0x1.0p-54, x); -#endif // LIBC_TARGET_CPU_HAS_FMA +#endif // LIBC_TARGET_CPU_HAS_FMA_DOUBLE return; } // No range reduction needed. @@ -85,6 +85,12 @@ LLVM_LIBC_FUNCTION(void, sincos, (double x, double *sin_x, double *cos_x)) { } else { // Inf or NaN if (LIBC_UNLIKELY(x_e > 2 * FPBits::EXP_BIAS)) { + if (xbits.is_signaling_nan()) { + fputil::raise_except_if_required(FE_INVALID); + *sin_x = *cos_x = FPBits::quiet_nan().get_val(); + return; + } + // sin(+-Inf) = NaN if (xbits.get_mantissa() == 0) { fputil::set_errno_if_required(EDOM); diff --git a/system/lib/llvm-libc/src/math/generic/sincos_eval.h b/system/lib/llvm-libc/src/math/generic/sincos_eval.h index 6cd1da4721bf5..41a4c75849ff4 100644 --- a/system/lib/llvm-libc/src/math/generic/sincos_eval.h +++ b/system/lib/llvm-libc/src/math/generic/sincos_eval.h @@ -65,7 +65,7 @@ LIBC_INLINE double sincos_eval(const DoubleDouble &u, DoubleDouble &sin_u, double u_hi_neg_half = (-0.5) * u.hi; DoubleDouble v; -#ifdef LIBC_TARGET_CPU_HAS_FMA +#ifdef LIBC_TARGET_CPU_HAS_FMA_DOUBLE v.hi = fputil::multiply_add(u.hi, u_hi_neg_half, 1.0); v.lo = 1.0 - v.hi; // Exact v.lo = fputil::multiply_add(u.hi, u_hi_neg_half, v.lo); @@ -73,7 +73,7 @@ LIBC_INLINE double sincos_eval(const DoubleDouble &u, DoubleDouble &sin_u, DoubleDouble u_hi_sq_neg_half = fputil::exact_mult(u.hi, u_hi_neg_half); v = fputil::exact_add(1.0, u_hi_sq_neg_half.hi); v.lo += u_hi_sq_neg_half.lo; -#endif // LIBC_TARGET_CPU_HAS_FMA +#endif // LIBC_TARGET_CPU_HAS_FMA_DOUBLE // r1 ~ -1/720 + u_hi^2 / 40320 double r1 = fputil::multiply_add(u_hi_sq, 0x1.a01a01a01a01ap-16, diff --git a/system/lib/llvm-libc/src/math/generic/sincosf.cpp b/system/lib/llvm-libc/src/math/generic/sincosf.cpp index ccaa29c10c4c6..9c7bf181e485e 100644 --- a/system/lib/llvm-libc/src/math/generic/sincosf.cpp +++ b/system/lib/llvm-libc/src/math/generic/sincosf.cpp @@ -19,6 +19,7 @@ namespace LIBC_NAMESPACE_DECL { +#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS // Exceptional values static constexpr int N_EXCEPTS = 6; @@ -48,6 +49,7 @@ static constexpr uint32_t EXCEPT_OUTPUTS_COS[N_EXCEPTS][4] = { {0x3f78142e, 1, 0, 1}, // x = 0x1.2b9622p67, cos(x) = 0x1.f0285cp-1 (RZ) {0x3f08a21c, 1, 0, 0}, // x = 0x1.ddebdep120, cos(x) = 0x1.114438p-1 (RZ) }; +#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS LLVM_LIBC_FUNCTION(void, sincosf, (float x, float *sinp, float *cosp)) { using FPBits = typename fputil::FPBits; @@ -130,19 +132,25 @@ LLVM_LIBC_FUNCTION(void, sincosf, (float x, float *sinp, float *cosp)) { // |x| < 2^-125. For targets without FMA instructions, we simply use // double for intermediate results as it is more efficient than using an // emulated version of FMA. -#if defined(LIBC_TARGET_CPU_HAS_FMA) +#if defined(LIBC_TARGET_CPU_HAS_FMA_FLOAT) *sinp = fputil::multiply_add(x, -0x1.0p-25f, x); *cosp = fputil::multiply_add(FPBits(x_abs).get_val(), -0x1.0p-25f, 1.0f); #else *sinp = static_cast(fputil::multiply_add(xd, -0x1.0p-25, xd)); *cosp = static_cast(fputil::multiply_add( static_cast(FPBits(x_abs).get_val()), -0x1.0p-25, 1.0)); -#endif // LIBC_TARGET_CPU_HAS_FMA +#endif // LIBC_TARGET_CPU_HAS_FMA_FLOAT return; } // x is inf or nan. if (LIBC_UNLIKELY(x_abs >= 0x7f80'0000U)) { + if (xbits.is_signaling_nan()) { + fputil::raise_except_if_required(FE_INVALID); + *sinp = *cosp = FPBits::quiet_nan().get_val(); + return; + } + if (x_abs == 0x7f80'0000U) { fputil::set_errno_if_required(EDOM); fputil::raise_except_if_required(FE_INVALID); @@ -152,6 +160,7 @@ LLVM_LIBC_FUNCTION(void, sincosf, (float x, float *sinp, float *cosp)) { return; } +#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS // Check exceptional values. for (int i = 0; i < N_EXCEPTS; ++i) { if (LIBC_UNLIKELY(x_abs == EXCEPT_INPUTS[i])) { @@ -178,6 +187,7 @@ LLVM_LIBC_FUNCTION(void, sincosf, (float x, float *sinp, float *cosp)) { return; } } +#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS // Combine the results with the sine and cosine of sum formulas: // sin(x) = sin((k + y)*pi/32) diff --git a/system/lib/llvm-libc/src/math/generic/sincosf16_utils.h b/system/lib/llvm-libc/src/math/generic/sincosf16_utils.h deleted file mode 100644 index 133896b5de7a3..0000000000000 --- a/system/lib/llvm-libc/src/math/generic/sincosf16_utils.h +++ /dev/null @@ -1,112 +0,0 @@ -//===-- Collection of utils for sinf16/cosf16 -------------------*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_LIBC_SRC_MATH_GENERIC_SINCOSF16_UTILS_H -#define LLVM_LIBC_SRC_MATH_GENERIC_SINCOSF16_UTILS_H - -#include "src/__support/FPUtil/PolyEval.h" -#include "src/__support/FPUtil/nearest_integer.h" -#include "src/__support/common.h" -#include "src/__support/macros/config.h" - -namespace LIBC_NAMESPACE_DECL { - -// Lookup table for sin(k * pi / 32) with k = 0, ..., 63. -// Table is generated with Sollya as follows: -// > display = hexadecimmal; -// > for k from 0 to 63 do { round(sin(k * pi/32), SG, RN); }; -constexpr float SIN_K_PI_OVER_32[64] = { - 0x0.0p0, 0x1.917a6cp-4, 0x1.8f8b84p-3, 0x1.294062p-2, - 0x1.87de2ap-2, 0x1.e2b5d4p-2, 0x1.1c73b4p-1, 0x1.44cf32p-1, - 0x1.6a09e6p-1, 0x1.8bc806p-1, 0x1.a9b662p-1, 0x1.c38b3p-1, - 0x1.d906bcp-1, 0x1.e9f416p-1, 0x1.f6297cp-1, 0x1.fd88dap-1, - 0x1p0, 0x1.fd88dap-1, 0x1.f6297cp-1, 0x1.e9f416p-1, - 0x1.d906bcp-1, 0x1.c38b3p-1, 0x1.a9b662p-1, 0x1.8bc806p-1, - 0x1.6a09e6p-1, 0x1.44cf32p-1, 0x1.1c73b4p-1, 0x1.e2b5d4p-2, - 0x1.87de2ap-2, 0x1.294062p-2, 0x1.8f8b84p-3, 0x1.917a6cp-4, - 0x0.0p0, -0x1.917a6cp-4, -0x1.8f8b84p-3, -0x1.294062p-2, - -0x1.87de2ap-2, -0x1.e2b5d4p-2, -0x1.1c73b4p-1, -0x1.44cf32p-1, - -0x1.6a09e6p-1, -0x1.8bc806p-1, -0x1.a9b662p-1, -0x1.c38b3p-1, - -0x1.d906bcp-1, -0x1.e9f416p-1, -0x1.f6297ep-1, -0x1.fd88dap-1, - -0x1p0, -0x1.fd88dap-1, -0x1.f6297cp-1, -0x1.e9f416p-1, - -0x1.d906bcp-1, -0x1.c38b3p-1, -0x1.a9b662p-1, -0x1.8bc806p-1, - -0x1.6a09e6p-1, -0x1.44cf32p-1, -0x1.1c73b4p-1, -0x1.e2b5d4p-2, - -0x1.87de2ap-2, -0x1.294062p-2, -0x1.8f8b84p-3, -0x1.917a6cp-4}; - -LIBC_INLINE int32_t range_reduction_sincospif16(float x, float &y) { - float kf = fputil::nearest_integer(x * 32); - y = fputil::multiply_add(x, 32.0, -kf); - - return static_cast(kf); -} - -// Recall, range reduction: -// k = round(x * 32/pi) -// -// The precision choice of 'double' in the following function is to minimize -// rounding errors in this initial scaling step, -// preserving enough bits so errors accumulated while computing the subtraction: -// y = x * 32/pi - round(x * 32/pi) -// are beyond the least-significant bit of single-precision used during -// further intermediate computation. -LIBC_INLINE int32_t range_reduction_sincosf16(float x, float &y) { - // Generated by Sollya with: - // > D(32/pi); - constexpr double THIRTYTWO_OVER_PI = 0x1.45f306dc9c883p3; - - double prod = x * THIRTYTWO_OVER_PI; - double kd = fputil::nearest_integer(prod); - y = static_cast(prod - kd); - - return static_cast(kd); -} - -static LIBC_INLINE void sincosf16_poly_eval(int32_t k, float y, float &sin_k, - float &cos_k, float &sin_y, - float &cosm1_y) { - - sin_k = SIN_K_PI_OVER_32[k & 63]; - cos_k = SIN_K_PI_OVER_32[(k + 16) & 63]; - - // Recall, after range reduction, -0.5 <= y <= 0.5. For very small values of - // y, calculating sin(y * p/32) can be inaccurate. Generating a polynomial for - // sin(y * p/32)/y instead significantly reduces the relative errors. - float ysq = y * y; - - // Degree-6 minimax even polynomial for sin(y*pi/32)/y generated by Sollya - // with: - // > Q = fpminimax(sin(y * pi/32)/y, [|0, 2, 4, 6|], [|SG...|], [0, 0.5]); - sin_y = y * fputil::polyeval(ysq, 0x1.921fb6p-4f, -0x1.4aeabcp-13f, - 0x1.a03354p-21f, -0x1.ad02d2p-20f); - - // Degree-6 minimax even polynomial for cos(y*pi/32) generated by Sollya - // with: - // > P = fpminimax(cos(y * pi/32), [|0, 2, 4, 6|],[|1, SG...|], [0, 0.5]); - cosm1_y = ysq * fputil::polyeval(ysq, -0x1.3bd3ccp-8f, 0x1.03a61ap-18f, - 0x1.a6f7a2p-29f); -} - -LIBC_INLINE void sincosf16_eval(float xf, float &sin_k, float &cos_k, - float &sin_y, float &cosm1_y) { - float y; - int32_t k = range_reduction_sincosf16(xf, y); - - sincosf16_poly_eval(k, y, sin_k, cos_k, sin_y, cosm1_y); -} - -LIBC_INLINE void sincospif16_eval(float xf, float &sin_k, float &cos_k, - float &sin_y, float &cosm1_y) { - float y; - int32_t k = range_reduction_sincospif16(xf, y); - - sincosf16_poly_eval(k, y, sin_k, cos_k, sin_y, cosm1_y); -} - -} // namespace LIBC_NAMESPACE_DECL - -#endif // LLVM_LIBC_SRC_MATH_GENERIC_SINCOSF16_UTILS_H diff --git a/system/lib/llvm-libc/src/math/generic/sincosf_utils.h b/system/lib/llvm-libc/src/math/generic/sincosf_utils.h index 726a5ab9b64be..6eaf820e5c1b0 100644 --- a/system/lib/llvm-libc/src/math/generic/sincosf_utils.h +++ b/system/lib/llvm-libc/src/math/generic/sincosf_utils.h @@ -14,7 +14,7 @@ #include "src/__support/macros/config.h" #include "src/__support/macros/properties/cpu_features.h" // LIBC_TARGET_CPU_HAS_FMA -#if defined(LIBC_TARGET_CPU_HAS_FMA) +#if defined(LIBC_TARGET_CPU_HAS_FMA_DOUBLE) #include "range_reduction_fma.h" // using namespace LIBC_NAMESPACE::fma; using LIBC_NAMESPACE::fma::FAST_PASS_BOUND; @@ -27,7 +27,7 @@ using LIBC_NAMESPACE::fma::small_range_reduction; using LIBC_NAMESPACE::generic::FAST_PASS_BOUND; using LIBC_NAMESPACE::generic::large_range_reduction; using LIBC_NAMESPACE::generic::small_range_reduction; -#endif // LIBC_TARGET_CPU_HAS_FMA +#endif // LIBC_TARGET_CPU_HAS_FMA_DOUBLE namespace LIBC_NAMESPACE_DECL { @@ -108,7 +108,7 @@ LIBC_INLINE void sincosf_eval(double xd, uint32_t x_abs, double &sin_k, // => pi * x = (k + y) * pi / 32 static LIBC_INLINE int64_t range_reduction_sincospi(double x, double &y) { double kd = fputil::nearest_integer(x * 32); - y = fputil::multiply_add(x, 32.0, -kd); + y = fputil::multiply_add(x, 32.0, -kd); return static_cast(kd); } diff --git a/system/lib/llvm-libc/src/math/generic/sinf.cpp b/system/lib/llvm-libc/src/math/generic/sinf.cpp index cea267d4c683e..38ea56f5f28c6 100644 --- a/system/lib/llvm-libc/src/math/generic/sinf.cpp +++ b/system/lib/llvm-libc/src/math/generic/sinf.cpp @@ -19,7 +19,7 @@ #include "src/__support/macros/optimization.h" // LIBC_UNLIKELY #include "src/__support/macros/properties/cpu_features.h" // LIBC_TARGET_CPU_HAS_FMA -#if defined(LIBC_TARGET_CPU_HAS_FMA) +#if defined(LIBC_TARGET_CPU_HAS_FMA_DOUBLE) #include "range_reduction_fma.h" #else #include "range_reduction.h" @@ -101,11 +101,11 @@ LLVM_LIBC_FUNCTION(float, sinf, (float x)) { // |x| < 2^-125. For targets without FMA instructions, we simply use // double for intermediate results as it is more efficient than using an // emulated version of FMA. -#if defined(LIBC_TARGET_CPU_HAS_FMA) +#if defined(LIBC_TARGET_CPU_HAS_FMA_FLOAT) return fputil::multiply_add(x, -0x1.0p-25f, x); #else return static_cast(fputil::multiply_add(xd, -0x1.0p-25, xd)); -#endif // LIBC_TARGET_CPU_HAS_FMA +#endif // LIBC_TARGET_CPU_HAS_FMA_FLOAT } // |x| < pi/16. @@ -124,6 +124,7 @@ LLVM_LIBC_FUNCTION(float, sinf, (float x)) { return static_cast(xd * result); } +#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS if (LIBC_UNLIKELY(x_abs == 0x4619'9998U)) { // x = 0x1.33333p13 float r = -0x1.63f4bap-2f; int rounding = fputil::quick_get_round(); @@ -132,8 +133,14 @@ LLVM_LIBC_FUNCTION(float, sinf, (float x)) { r = -0x1.63f4bcp-2f; return xbits.is_neg() ? -r : r; } +#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS if (LIBC_UNLIKELY(x_abs >= 0x7f80'0000U)) { + if (xbits.is_signaling_nan()) { + fputil::raise_except_if_required(FE_INVALID); + return FPBits::quiet_nan().get_val(); + } + if (x_abs == 0x7f80'0000U) { fputil::set_errno_if_required(EDOM); fputil::raise_except_if_required(FE_INVALID); diff --git a/system/lib/llvm-libc/src/math/generic/sinhf.cpp b/system/lib/llvm-libc/src/math/generic/sinhf.cpp index 371dd6e67e66e..d6158fd302536 100644 --- a/system/lib/llvm-libc/src/math/generic/sinhf.cpp +++ b/system/lib/llvm-libc/src/math/generic/sinhf.cpp @@ -24,11 +24,13 @@ LLVM_LIBC_FUNCTION(float, sinhf, (float x)) { if (LIBC_UNLIKELY(x_abs >= 0x42b4'0000U || x_abs <= 0x3da0'0000U)) { // |x| <= 0.078125 if (x_abs <= 0x3da0'0000U) { +#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS // |x| = 0.0005589424981735646724700927734375 if (LIBC_UNLIKELY(x_abs == 0x3a12'85ffU)) { if (fputil::fenv_is_round_to_nearest()) return x; } +#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS // |x| <= 2^-26 if (LIBC_UNLIKELY(x_abs <= 0x3280'0000U)) { diff --git a/system/lib/llvm-libc/src/math/generic/sinpif.cpp b/system/lib/llvm-libc/src/math/generic/sinpif.cpp index f572ded06b25a..492689d594d90 100644 --- a/system/lib/llvm-libc/src/math/generic/sinpif.cpp +++ b/system/lib/llvm-libc/src/math/generic/sinpif.cpp @@ -83,6 +83,11 @@ LLVM_LIBC_FUNCTION(float, sinpif, (float x)) { // check for NaN values if (LIBC_UNLIKELY(x_abs >= 0x7f80'0000U)) { + if (xbits.is_signaling_nan()) { + fputil::raise_except_if_required(FE_INVALID); + return FPBits::quiet_nan().get_val(); + } + if (x_abs == 0x7f80'0000U) { fputil::set_errno_if_required(EDOM); fputil::raise_except_if_required(FE_INVALID); diff --git a/system/lib/llvm-libc/src/math/generic/sqrtf128.cpp b/system/lib/llvm-libc/src/math/generic/sqrtf128.cpp index f87066b6f6403..3aa7db8362734 100644 --- a/system/lib/llvm-libc/src/math/generic/sqrtf128.cpp +++ b/system/lib/llvm-libc/src/math/generic/sqrtf128.cpp @@ -7,14 +7,432 @@ //===----------------------------------------------------------------------===// #include "src/math/sqrtf128.h" -#include "src/__support/FPUtil/sqrt.h" +#include "src/__support/CPP/bit.h" +#include "src/__support/FPUtil/FEnvImpl.h" +#include "src/__support/FPUtil/FPBits.h" +#include "src/__support/FPUtil/rounding_mode.h" #include "src/__support/common.h" -#include "src/__support/macros/config.h" +#include "src/__support/macros/optimization.h" +#include "src/__support/uint128.h" + +// Compute sqrtf128 with correct rounding for all rounding modes using integer +// arithmetic by Alexei Sibidanov (sibid@uvic.ca): +// https://github.com/sibidanov/llvm-project/tree/as_sqrt_v2 +// https://github.com/sibidanov/llvm-project/tree/as_sqrt_v3 +// TODO: Update the reference once Alexei's implementation is in the CORE-MATH +// project. https://github.com/llvm/llvm-project/issues/126794 + +// Let the input be expressed as x = 2^e * m_x, +// - Step 1: Range reduction +// Let x_reduced = 2^(e % 2) * m_x, +// Then sqrt(x) = 2^(e / 2) * sqrt(x_reduced), with +// 1 <= x_reduced < 4. +// - Step 2: Polynomial approximation +// Approximate 1/sqrt(x_reduced) using polynomial approximation with the +// result errors bounded by: +// |r0 - 1/sqrt(x_reduced)| < 2^-32. +// The computations are done in uint64_t. +// - Step 3: First Newton iteration +// Let the scaled error defined by: +// h0 = r0^2 * x_reduced - 1. +// Then we compute the first Newton iteration: +// r1 = r0 - r0 * h0 / 2. +// The result is then bounded by: +// |r1 - 1 / sqrt(x_reduced)| < 2^-62. +// - Step 4: Second Newton iteration +// We calculate the scaled error from Step 3: +// h1 = r1^2 * x_reduced - 1. +// Then the second Newton iteration is computed by: +// r2 = x_reduced * (r1 - r1 * h0 / 2) +// ~ x_reduced * (1/sqrt(x_reduced)) = sqrt(x_reduced) +// - Step 5: Perform rounding test and correction if needed. +// Rounding correction is done by computing the exact rounding errors: +// x_reduced - r2^2. namespace LIBC_NAMESPACE_DECL { +using FPBits = fputil::FPBits; + +namespace { + +template static inline constexpr T prod_hi(T, U); + +// Get high part of integer multiplications. +// Use template to prevent implicit conversion. +template <> +inline constexpr uint64_t prod_hi(uint64_t x, uint64_t y) { + return static_cast( + (static_cast(x) * static_cast(y)) >> 64); +} + +// Get high part of unsigned 128x64 bit multiplication. +template <> +inline constexpr UInt128 prod_hi(UInt128 x, uint64_t y) { + uint64_t x_lo = static_cast(x); + uint64_t x_hi = static_cast(x >> 64); + UInt128 xyl = static_cast(x_lo) * static_cast(y); + UInt128 xyh = static_cast(x_hi) * static_cast(y); + return xyh + (xyl >> 64); +} + +// Get high part of signed 64x64 bit multiplication. +template <> inline constexpr int64_t prod_hi(int64_t x, int64_t y) { + return static_cast( + (static_cast(x) * static_cast(y)) >> 64); +} + +// Get high 128-bit part of unsigned 128x128 bit multiplication. +template <> inline constexpr UInt128 prod_hi(UInt128 x, UInt128 y) { + uint64_t x_lo = static_cast(x); + uint64_t x_hi = static_cast(x >> 64); + uint64_t y_lo = static_cast(y); + uint64_t y_hi = static_cast(y >> 64); + + UInt128 xh_yh = static_cast(x_hi) * static_cast(y_hi); + UInt128 xh_yl = static_cast(x_hi) * static_cast(y_lo); + UInt128 xl_yh = static_cast(x_lo) * static_cast(y_hi); + + xh_yh += xh_yl >> 64; + + return xh_yh + (xl_yh >> 64); +} + +// Get high 128-bit part of mixed sign 128x128 bit multiplication. +template <> +inline constexpr Int128 prod_hi(Int128 x, UInt128 y) { + UInt128 mask = static_cast(x >> 127); + UInt128 negative_part = y & mask; + UInt128 prod = prod_hi(static_cast(x), y); + return static_cast(prod - negative_part); +} + +// Newton-Raphson first order step to improve accuracy of the result. +// For the initial approximation r0 ~ 1/sqrt(x), let +// h = r0^2 * x - 1 +// be its scaled error. Then the first-order Newton-Raphson iteration is: +// r1 = r0 - r0 * h / 2 +// which has error bounded by: +// |r1 - 1/sqrt(x)| < h^2 / 2. +LIBC_INLINE uint64_t rsqrt_newton_raphson(uint64_t m, uint64_t r) { + uint64_t r2 = prod_hi(r, r); + // h = r0^2*x - 1. + int64_t h = static_cast(prod_hi(m, r2) + r2); + // hr = r * h / 2 + int64_t hr = prod_hi(h, static_cast(r >> 1)); + return r - hr; +} + +#ifdef LIBC_MATH_HAS_SMALL_TABLES +// Degree-12 minimax polynomials for 1/sqrt(x) on [1, 2]. +constexpr uint32_t RSQRT_COEFFS[12] = { + 0xb5947a4a, 0x2d651e32, 0x9ad50532, 0x2d28d093, 0x0d8be653, 0x04239014, + 0x01492449, 0x0066ff7d, 0x001e74a1, 0x000984cc, 0x00049abc, 0x00018340, +}; + +LIBC_INLINE uint64_t rsqrt_approx(uint64_t m) { + int64_t x = static_cast(m) ^ (uint64_t(1) << 63); + int64_t x_26 = x >> 2; + int64_t z = x >> 31; + + if (LIBC_UNLIKELY(z <= -4294967296)) + return ~(m >> 1); + + uint64_t x2 = static_cast(z) * static_cast(z); + uint64_t x2_26 = x2 >> 5; + x2 >>= 32; + // Calculate the odd part of the polynomial using Horner's method. + uint64_t c0 = RSQRT_COEFFS[8] + ((x2 * RSQRT_COEFFS[10]) >> 32); + uint64_t c1 = RSQRT_COEFFS[6] + ((x2 * c0) >> 32); + uint64_t c2 = RSQRT_COEFFS[4] + ((x2 * c1) >> 32); + uint64_t c3 = RSQRT_COEFFS[2] + ((x2 * c2) >> 32); + uint64_t c4 = RSQRT_COEFFS[0] + ((x2 * c3) >> 32); + uint64_t odd = + static_cast((x >> 34) * static_cast(c4 >> 3)) + x_26; + // Calculate the even part of the polynomial using Horner's method. + uint64_t d0 = RSQRT_COEFFS[9] + ((x2 * RSQRT_COEFFS[11]) >> 32); + uint64_t d1 = RSQRT_COEFFS[7] + ((x2 * d0) >> 32); + uint64_t d2 = RSQRT_COEFFS[5] + ((x2 * d1) >> 32); + uint64_t d3 = RSQRT_COEFFS[3] + ((x2 * d2) >> 32); + uint64_t d4 = RSQRT_COEFFS[1] + ((x2 * d3) >> 32); + uint64_t even = 0xd105eb806655d608ul + ((x2 * d4) >> 6) + x2_26; + + uint64_t r = even - odd; // error < 1.5e-10 + // Newton-Raphson first order step to improve accuracy of the result to almost + // 64 bits. + return rsqrt_newton_raphson(m, r); +} + +#else +// Cubic minimax polynomials for 1/sqrt(x) on [1 + k/64, 1 + (k + 1)/64] +// for k = 0..63. +constexpr uint32_t RSQRT_COEFFS[64][4] = { + {0xffffffff, 0xfffff780, 0xbff55815, 0x9bb5b6e7}, + {0xfc0bd889, 0xfa1d6e7d, 0xb8a95a89, 0x938bf8f0}, + {0xf82ec882, 0xf473bea9, 0xb1bf4705, 0x8bed0079}, + {0xf467f280, 0xeefff2a1, 0xab309d4a, 0x84cdb431}, + {0xf0b6848c, 0xe9bf46f4, 0xa4f76232, 0x7e24037b}, + {0xed19b75e, 0xe4af2628, 0x9f0e1340, 0x77e6ca62}, + {0xe990cdad, 0xdfcd2521, 0x996f9b96, 0x720db8df}, + {0xe61b138e, 0xdb16ffde, 0x94174a00, 0x6c913cff}, + {0xe2b7dddf, 0xd68a967b, 0x8f00c812, 0x676a6f92}, + {0xdf6689b7, 0xd225ea80, 0x8a281226, 0x62930308}, + {0xdc267bea, 0xcde71c63, 0x8589702c, 0x5e05343e}, + {0xd8f7208e, 0xc9cc6948, 0x81216f2e, 0x59bbbcf8}, + {0xd5d7ea91, 0xc5d428ee, 0x7cecdb76, 0x55b1c7d6}, + {0xd2c8534e, 0xc1fccbc9, 0x78e8bb45, 0x51e2e592}, + {0xcfc7da32, 0xbe44d94a, 0x75124a0a, 0x4e4b0369}, + {0xccd6045f, 0xbaaaee41, 0x7166f40f, 0x4ae66284}, + {0xc9f25c5c, 0xb72dbb69, 0x6de45288, 0x47b19045}, + {0xc71c71c7, 0xb3cc040f, 0x6a882804, 0x44a95f5f}, + {0xc453d90f, 0xb0849cd4, 0x67505d2a, 0x41cae1a0}, + {0xc1982b2e, 0xad566a85, 0x643afdc8, 0x3f13625c}, + {0xbee9056f, 0xaa406113, 0x6146361f, 0x3c806169}, + {0xbc46092e, 0xa7418293, 0x5e70506d, 0x3a0f8e8e}, + {0xb9aedba5, 0xa458de58, 0x5bb7b2b1, 0x37bec572}, + {0xb72325b7, 0xa1859022, 0x591adc9a, 0x358c09e2}, + {0xb4a293c2, 0x9ec6bf52, 0x569865a7, 0x33758476}, + {0xb22cd56d, 0x9c1b9e36, 0x542efb6a, 0x31797f8a}, + {0xafc19d86, 0x9983695c, 0x51dd5ffb, 0x2f96647a}, + {0xad60a1d1, 0x96fd66f7, 0x4fa2687c, 0x2dcab91f}, + {0xab099ae9, 0x9488e64b, 0x4d7cfbc9, 0x2c151d8a}, + {0xa8bc441a, 0x92253f20, 0x4b6c1139, 0x2a7449ef}, + {0xa6785b42, 0x8fd1d14a, 0x496eaf82, 0x28e70cc3}, + {0xa43da0ae, 0x8d8e042a, 0x4783eba7, 0x276c4900}, + {0xa20bd701, 0x8b594648, 0x45aae80a, 0x2602f493}, + {0x9fe2c315, 0x89330ce4, 0x43e2d382, 0x24aa16ec}, + {0x9dc22be4, 0x871ad399, 0x422ae88c, 0x2360c7af}, + {0x9ba9da6c, 0x85101c05, 0x40826c88, 0x22262d7b}, + {0x99999999, 0x83126d70, 0x3ee8af07, 0x20f97cd2}, + {0x97913630, 0x81215480, 0x3d5d0922, 0x1fd9f714}, + {0x95907eb8, 0x7f3c62ef, 0x3bdedce0, 0x1ec6e994}, + {0x93974369, 0x7d632f45, 0x3a6d94a9, 0x1dbfacbb}, + {0x91a55615, 0x7b955498, 0x3908a2be, 0x1cc3a33b}, + {0x8fba8a1c, 0x79d2724e, 0x37af80bf, 0x1bd23960}, + {0x8dd6b456, 0x781a2be4, 0x3661af39, 0x1aeae458}, + {0x8bf9ab07, 0x766c28ba, 0x351eb539, 0x1a0d21a2}, + {0x8a2345cc, 0x74c813dd, 0x33e61feb, 0x19387676}, + {0x88535d90, 0x732d9bdc, 0x32b7823a, 0x186c6f3e}, + {0x8689cc7e, 0x719c7297, 0x3192747d, 0x17a89f21}, + {0x84c66df1, 0x70144d19, 0x30769424, 0x16ec9f89}, + {0x83091e6a, 0x6e94e36c, 0x2f63836f, 0x16380fbf}, + {0x8151bb87, 0x6d1df079, 0x2e58e925, 0x158a9484}, + {0x7fa023f1, 0x6baf31de, 0x2d567053, 0x14e3d7ba}, + {0x7df43758, 0x6a4867d3, 0x2c5bc811, 0x1443880e}, + {0x7c4dd664, 0x68e95508, 0x2b68a346, 0x13a958ab}, + {0x7aace2b0, 0x6791be86, 0x2a7cb871, 0x131500ee}, + {0x79113ebc, 0x66416b95, 0x2997c17a, 0x12863c29}, + {0x777acde8, 0x64f825a1, 0x28b97b82, 0x11fcc95c}, + {0x75e9746a, 0x63b5b822, 0x27e1a6b4, 0x11786b03}, + {0x745d1746, 0x6279f081, 0x2710061d, 0x10f8e6da}, + {0x72d59c46, 0x61449e06, 0x26445f86, 0x107e05ac}, + {0x7152e9f4, 0x601591be, 0x257e7b4d, 0x10079327}, + {0x6fd4e793, 0x5eec9e6b, 0x24be2445, 0x0f955da9}, + {0x6e5b7d16, 0x5dc9986e, 0x24032795, 0x0f273620}, + {0x6ce6931d, 0x5cac55b7, 0x234d5496, 0x0ebcefdb}, + {0x6b7612ec, 0x5b94adb2, 0x229c7cbc, 0x0e56606e}, +}; + +// Approximate rsqrt with cubic polynomials. +// The range [1,2] is splitted into 64 equal sub-ranges and the reciprocal +// square root is approximated by a cubic polynomial by the minimax method in +// each subrange. The approximation accuracy fits into 32-33 bits and thus it is +// natural to round coefficients into 32 bit. The constant coefficient can be +// rounded to 33 bits since the most significant bit is always 1 and implicitly +// assumed in the table. +LIBC_INLINE uint64_t rsqrt_approx(uint64_t m) { + // ULP(m) = 2^-64. + // Use the top 6 bits as index for looking up polynomial coeffs. + uint64_t indx = m >> 58; + + uint64_t c0 = static_cast(RSQRT_COEFFS[indx][0]); + c0 <<= 31; // to 64 bit with the space for the implicit bit + c0 |= 1ull << 63; // add implicit bit + + uint64_t c1 = static_cast(RSQRT_COEFFS[indx][1]); + c1 <<= 25; // to 64 bit format + + uint64_t c2 = static_cast(RSQRT_COEFFS[indx][2]); + uint64_t c3 = static_cast(RSQRT_COEFFS[indx][3]); + + uint64_t d = (m << 6) >> 32; // local coordinate in the subrange [0, 2^32] + uint64_t d2 = (d * d) >> 32; // square of the local coordinate + uint64_t re = c0 + (d2 * c2 >> 13); // even part of the polynomial (positive) + uint64_t ro = d * ((c1 + ((d2 * c3) >> 19)) >> 26) >> + 6; // odd part of the polynomial (negative) + uint64_t r = re - ro; // maximal error < 1.55e-10 and it is less than 2^-32 + // Newton-Raphson first order step to improve accuracy of the result to almost + // 64 bits. + r = rsqrt_newton_raphson(m, r); + // Adjust in the unlucky case x~1; + if (LIBC_UNLIKELY(!r)) + --r; + return r; +} +#endif // LIBC_MATH_HAS_SMALL_TABLES + +} // anonymous namespace + LLVM_LIBC_FUNCTION(float128, sqrtf128, (float128 x)) { - return fputil::sqrt(x); + using FPBits = fputil::FPBits; + // Get rounding mode. + uint32_t rm = fputil::get_round(); + + FPBits xbits(x); + UInt128 x_u = xbits.uintval(); + // Bring leading bit of the mantissa to the highest bit. + // ulp(x_frac) = 2^-128. + UInt128 x_frac = xbits.get_mantissa() << (FPBits::EXP_LEN + 1); + + int sign_exp = static_cast(x_u >> FPBits::FRACTION_LEN); + + if (LIBC_UNLIKELY(sign_exp == 0 || sign_exp >= 0x7fff)) { + // Special cases: NAN, inf, negative numbers + if (sign_exp >= 0x7fff) { + // x = -0 or x = inf + if (xbits.is_zero() || xbits == xbits.inf()) + return x; + // x is nan + if (xbits.is_nan()) { + // pass through quiet nan + if (xbits.is_quiet_nan()) + return x; + // transform signaling nan to quiet and return + return xbits.quiet_nan().get_val(); + } + // x < 0 or x = -inf + fputil::set_errno_if_required(EDOM); + fputil::raise_except_if_required(FE_INVALID); + return xbits.quiet_nan().get_val(); + } + // Now x is subnormal or x = +0. + + // x is +0. + if (x_frac == 0) + return x; + + // Normalize subnormal inputs. + sign_exp = -cpp::countl_zero(x_frac); + int normal_shifts = 1 - sign_exp; + x_frac <<= normal_shifts; + } + + // For sign_exp = biased exponent of x = real_exponent + 16383, + // let f be the real exponent of the output: + // f = floor(real_exponent / 2) + // Then: + // floor((sign_exp + 1) / 2) = f + 8192 + // Hence, the biased exponent of the final result is: + // f + 16383 = floor((sign_exp + 1) / 2) + 8191. + // Since the output mantissa will include the hidden bit, we can define the + // output exponent part: + // e2 = floor((sign_exp + 1) / 2) + 8190 + unsigned i = static_cast(1 - (sign_exp & 1)); + uint32_t q2 = (sign_exp + 1) >> 1; + // Exponent of the final result + uint32_t e2 = q2 + 8190; + + constexpr uint64_t RSQRT_2[2] = {~0ull, + 0xb504f333f9de6484 /* 2^64/sqrt(2) */}; + + // Approximate 1/sqrt(1 + x_frac) + // Error: |r_1 - 1/sqrt(x)| < 2^-62. + uint64_t r1 = rsqrt_approx(static_cast(x_frac >> 64)); + // Adjust for the even/odd exponent. + uint64_t r2 = prod_hi(r1, RSQRT_2[i]); + unsigned shift = 2 - i; + + // Normalized input: + // 1 <= x_reduced < 4 + UInt128 x_reduced = (x_frac >> shift) | (UInt128(1) << (126 + i)); + // With r2 ~ 1/sqrt(x) up to 2^-63, we perform another round of Newton-Raphson + // iteration: + // r3 = r2 - r2 * h / 2, + // for h = r2^2 * x - 1. + // Then: + // sqrt(x) = x * (1 / sqrt(x)) + // ~ x * r3 + // = x * (r2 - r2 * h / 2) + // = (x * r2) - (x * r2) * h / 2 + UInt128 sx = prod_hi(x_reduced, r2); + UInt128 h = prod_hi(sx, r2) << 2; + UInt128 ds = static_cast(prod_hi(static_cast(h), sx)); + UInt128 v = (sx << 1) - ds; + + uint32_t nrst = rm == FE_TONEAREST; + // The result lies within (-2,5) of true square root so we now + // test that we can correctly round the result taking into account + // the rounding mode. + // Check the lowest 14 bits (by clearing and sign-extending the top + // 32 - 14 = 18 bits). + int dd = (static_cast(v) << 18) >> 18; + + if (LIBC_UNLIKELY(dd < 4 && dd >= -8)) { // can round correctly? + // m is almost the final result it can be only 1 ulp off so we + // just need to test both possibilities. We square it and + // compare with the initial argument. + UInt128 m = v >> 15; + UInt128 m2 = m * m; + // The difference of the squared result and the argument + Int128 t0 = static_cast(m2 - (x_reduced << 98)); + if (t0 == 0) { + // the square root is exact + v = m << 15; + } else { + // Add +-1 ulp to m depend on the sign of the difference. Here + // we do not need to square again since (m+1)^2 = m^2 + 2*m + + // 1 so just need to add shifted m and 1. + Int128 t1 = t0; + Int128 sgn = t0 >> 127; // sign of the difference + Int128 m_xor_sgn = static_cast(m << 1) ^ sgn; + t1 -= m_xor_sgn; + t1 += Int128(1) + sgn; + + Int128 sgn1 = t1 >> 127; + if (LIBC_UNLIKELY(sgn == sgn1)) { + t0 = t1; + v -= sgn << 15; + t1 -= m_xor_sgn; + t1 += Int128(1) + sgn; + } + + if (t1 == 0) { + // 1 ulp offset brings again an exact root + v = (m - static_cast((sgn << 1) + 1)) << 15; + } else { + t1 += t0; + Int128 side = t1 >> 127; // select what is closer m or m+-1 + v &= ~UInt128(0) << 15; // wipe the fractional bits + v -= ((sgn & side) | (~sgn & 1)) << (15 + static_cast(side)); + v |= 1; // add sticky bit since we cannot have an exact mid-point + // situation + } + } + } + + unsigned frac = static_cast(v) & 0x7fff; // fractional part + unsigned rnd; // round bit + if (LIBC_LIKELY(nrst != 0)) { + rnd = frac >> 14; // round to nearest tie to even + } else if (rm == FE_UPWARD) { + rnd = !!frac; // round up + } else { + rnd = 0; // round down or round to zero + } + + v >>= 15; // position mantissa + v += rnd; // round + + // Set inexact flag only if square root is inexact + // TODO: We will have to raise FE_INEXACT most of the time, but this + // operation is very costly, especially in x86-64, since technically, it + // needs to synchronize both SSE and x87 flags. Need to investigate + // further to see how we can make this performant. + // https://github.com/llvm/llvm-project/issues/126753 + + // if(frac) fputil::raise_except_if_required(FE_INEXACT); + + v += static_cast(e2) << FPBits::FRACTION_LEN; // place exponent + return cpp::bit_cast(v); } } // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/generic/tan.cpp b/system/lib/llvm-libc/src/math/generic/tan.cpp index 19d31a8441efb..89b812cfc23a0 100644 --- a/system/lib/llvm-libc/src/math/generic/tan.cpp +++ b/system/lib/llvm-libc/src/math/generic/tan.cpp @@ -22,11 +22,11 @@ #include "src/__support/macros/properties/cpu_features.h" // LIBC_TARGET_CPU_HAS_FMA #include "src/math/generic/range_reduction_double_common.h" -#ifdef LIBC_TARGET_CPU_HAS_FMA +#ifdef LIBC_TARGET_CPU_HAS_FMA_DOUBLE #include "range_reduction_double_fma.h" #else #include "range_reduction_double_nofma.h" -#endif // LIBC_TARGET_CPU_HAS_FMA +#endif // LIBC_TARGET_CPU_HAS_FMA_DOUBLE namespace LIBC_NAMESPACE_DECL { @@ -140,7 +140,7 @@ LLVM_LIBC_FUNCTION(double, tan, (double x)) { if (LIBC_UNLIKELY(x == 0.0)) return x + x; // Make sure it works with FTZ/DAZ. -#ifdef LIBC_TARGET_CPU_HAS_FMA +#ifdef LIBC_TARGET_CPU_HAS_FMA_DOUBLE return fputil::multiply_add(x, 0x1.0p-54, x); #else if (LIBC_UNLIKELY(x_e < 4)) { @@ -150,7 +150,7 @@ LLVM_LIBC_FUNCTION(double, tan, (double x)) { return FPBits(xbits.uintval() + 1).get_val(); } return fputil::multiply_add(x, 0x1.0p-54, x); -#endif // LIBC_TARGET_CPU_HAS_FMA +#endif // LIBC_TARGET_CPU_HAS_FMA_DOUBLE } // No range reduction needed. k = 0; @@ -163,6 +163,10 @@ LLVM_LIBC_FUNCTION(double, tan, (double x)) { } else { // Inf or NaN if (LIBC_UNLIKELY(x_e > 2 * FPBits::EXP_BIAS)) { + if (xbits.is_signaling_nan()) { + fputil::raise_except_if_required(FE_INVALID); + return FPBits::quiet_nan().get_val(); + } // tan(+-Inf) = NaN if (xbits.get_mantissa() == 0) { fputil::set_errno_if_required(EDOM); diff --git a/system/lib/llvm-libc/src/math/generic/tanf.cpp b/system/lib/llvm-libc/src/math/generic/tanf.cpp index 6fd5f9a103676..ca5e35dca4c91 100644 --- a/system/lib/llvm-libc/src/math/generic/tanf.cpp +++ b/system/lib/llvm-libc/src/math/generic/tanf.cpp @@ -21,6 +21,7 @@ namespace LIBC_NAMESPACE_DECL { +#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS // Exceptional cases for tanf. constexpr size_t N_EXCEPTS = 6; @@ -39,11 +40,11 @@ constexpr fputil::ExceptValues TANF_EXCEPTS{{ // x = 0x1.a6ce12p86, tan(x) = -0x1.c5612ep-1 (RZ) {0x6ad36709, 0xbf62b097, 0, 1, 0}, }}; +#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS LLVM_LIBC_FUNCTION(float, tanf, (float x)) { using FPBits = typename fputil::FPBits; FPBits xbits(x); - bool x_sign = xbits.uintval() >> 31; uint32_t x_abs = xbits.uintval() & 0x7fff'ffffU; // |x| < pi/32 @@ -74,11 +75,11 @@ LLVM_LIBC_FUNCTION(float, tanf, (float x)) { // |x| < 2^-125. For targets without FMA instructions, we simply use // double for intermediate results as it is more efficient than using an // emulated version of FMA. -#if defined(LIBC_TARGET_CPU_HAS_FMA) +#if defined(LIBC_TARGET_CPU_HAS_FMA_FLOAT) return fputil::multiply_add(x, 0x1.0p-25f, x); #else return static_cast(fputil::multiply_add(xd, 0x1.0p-25, xd)); -#endif // LIBC_TARGET_CPU_HAS_FMA +#endif // LIBC_TARGET_CPU_HAS_FMA_FLOAT } // |x| < pi/32 @@ -92,6 +93,8 @@ LLVM_LIBC_FUNCTION(float, tanf, (float x)) { return static_cast(xd * result); } +#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS + bool x_sign = xbits.uintval() >> 31; // Check for exceptional values if (LIBC_UNLIKELY(x_abs == 0x3f8a1f62U)) { // |x| = 0x1.143ec4p0 @@ -104,21 +107,29 @@ LLVM_LIBC_FUNCTION(float, tanf, (float x)) { return tmp; } +#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS // |x| > 0x1.ada6a8p+27f if (LIBC_UNLIKELY(x_abs > 0x4d56'd354U)) { // Inf or NaN if (LIBC_UNLIKELY(x_abs >= 0x7f80'0000U)) { + if (xbits.is_signaling_nan()) { + fputil::raise_except_if_required(FE_INVALID); + return FPBits::quiet_nan().get_val(); + } + if (x_abs == 0x7f80'0000U) { fputil::set_errno_if_required(EDOM); fputil::raise_except_if_required(FE_INVALID); } return x + FPBits::quiet_nan().get_val(); } +#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS // Other large exceptional values if (auto r = TANF_EXCEPTS.lookup_odd(x_abs, x_sign); LIBC_UNLIKELY(r.has_value())) return r.value(); +#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS } // For |x| >= pi/32, we use the definition of tan(x) function: diff --git a/system/lib/llvm-libc/src/math/hypotf16.h b/system/lib/llvm-libc/src/math/hypotf16.h new file mode 100644 index 0000000000000..2d37c61b4ee7b --- /dev/null +++ b/system/lib/llvm-libc/src/math/hypotf16.h @@ -0,0 +1,21 @@ +//===-- Implementation header for hypotf16 ----------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_MATH_HYPOTF16_H +#define LLVM_LIBC_SRC_MATH_HYPOTF16_H + +#include "src/__support/macros/config.h" +#include "src/__support/macros/properties/types.h" + +namespace LIBC_NAMESPACE_DECL { + +float16 hypotf16(float16 x, float16 y); + +} // namespace LIBC_NAMESPACE_DECL + +#endif // LLVM_LIBC_SRC_MATH_HYPOTF16_H diff --git a/system/lib/llvm-libc/src/math/nvptx/ceil.cpp b/system/lib/llvm-libc/src/math/nvptx/ceil.cpp new file mode 100644 index 0000000000000..8834c7b560a1b --- /dev/null +++ b/system/lib/llvm-libc/src/math/nvptx/ceil.cpp @@ -0,0 +1,17 @@ +//===-- Implementation of the ceil function for GPU -----------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/ceil.h" +#include "src/__support/common.h" +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(double, ceil, (double x)) { return __builtin_ceil(x); } + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/nvptx/ceilf.cpp b/system/lib/llvm-libc/src/math/nvptx/ceilf.cpp new file mode 100644 index 0000000000000..5d26a30c849cd --- /dev/null +++ b/system/lib/llvm-libc/src/math/nvptx/ceilf.cpp @@ -0,0 +1,17 @@ +//===-- Implementation of the ceilf function for GPU ----------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/ceilf.h" +#include "src/__support/common.h" +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(float, ceilf, (float x)) { return __builtin_ceilf(x); } + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/nvptx/copysign.cpp b/system/lib/llvm-libc/src/math/nvptx/copysign.cpp new file mode 100644 index 0000000000000..06ef36fb3595f --- /dev/null +++ b/system/lib/llvm-libc/src/math/nvptx/copysign.cpp @@ -0,0 +1,19 @@ +//===-- Implementation of the copysign function for GPU -------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/copysign.h" +#include "src/__support/common.h" +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(double, copysign, (double x, double y)) { + return __builtin_copysign(x, y); +} + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/nvptx/copysignf.cpp b/system/lib/llvm-libc/src/math/nvptx/copysignf.cpp new file mode 100644 index 0000000000000..aea94f3577d8f --- /dev/null +++ b/system/lib/llvm-libc/src/math/nvptx/copysignf.cpp @@ -0,0 +1,19 @@ +//===-- Implementation of the copysignf function for GPU ------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/copysignf.h" +#include "src/__support/common.h" +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(float, copysignf, (float x, float y)) { + return __builtin_copysignf(x, y); +} + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/nvptx/fabs.cpp b/system/lib/llvm-libc/src/math/nvptx/fabs.cpp new file mode 100644 index 0000000000000..bb37596b9d563 --- /dev/null +++ b/system/lib/llvm-libc/src/math/nvptx/fabs.cpp @@ -0,0 +1,17 @@ +//===-- Implementation of the fabs function for GPU -----------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/fabs.h" +#include "src/__support/common.h" +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(double, fabs, (double x)) { return __builtin_fabs(x); } + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/nvptx/fabsf.cpp b/system/lib/llvm-libc/src/math/nvptx/fabsf.cpp new file mode 100644 index 0000000000000..2698618f3f1e1 --- /dev/null +++ b/system/lib/llvm-libc/src/math/nvptx/fabsf.cpp @@ -0,0 +1,17 @@ +//===-- Implementation of the fabsf function for GPU ----------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/fabsf.h" +#include "src/__support/common.h" +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(float, fabsf, (float x)) { return __builtin_fabsf(x); } + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/nvptx/floor.cpp b/system/lib/llvm-libc/src/math/nvptx/floor.cpp new file mode 100644 index 0000000000000..564efa9a7da38 --- /dev/null +++ b/system/lib/llvm-libc/src/math/nvptx/floor.cpp @@ -0,0 +1,17 @@ +//===-- Implementation of the floor function for GPU ----------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/floor.h" +#include "src/__support/common.h" +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(double, floor, (double x)) { return __builtin_floor(x); } + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/nvptx/floorf.cpp b/system/lib/llvm-libc/src/math/nvptx/floorf.cpp new file mode 100644 index 0000000000000..6717c8f60c992 --- /dev/null +++ b/system/lib/llvm-libc/src/math/nvptx/floorf.cpp @@ -0,0 +1,17 @@ +//===-- Implementation of the floorf function for GPU ---------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/floorf.h" +#include "src/__support/common.h" +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(float, floorf, (float x)) { return __builtin_floorf(x); } + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/nvptx/fma.cpp b/system/lib/llvm-libc/src/math/nvptx/fma.cpp new file mode 100644 index 0000000000000..c4a117e42a3aa --- /dev/null +++ b/system/lib/llvm-libc/src/math/nvptx/fma.cpp @@ -0,0 +1,19 @@ +//===-- Implementation of the fma function for GPU ------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/fma.h" +#include "src/__support/common.h" +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(double, fma, (double x, double y, double z)) { + return __builtin_fma(x, y, z); +} + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/nvptx/fmaf.cpp b/system/lib/llvm-libc/src/math/nvptx/fmaf.cpp new file mode 100644 index 0000000000000..c088bd5b30fea --- /dev/null +++ b/system/lib/llvm-libc/src/math/nvptx/fmaf.cpp @@ -0,0 +1,19 @@ +//===-- Implementation of the fmaf function for GPU -----------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/fmaf.h" +#include "src/__support/common.h" +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(float, fmaf, (float x, float y, float z)) { + return __builtin_fmaf(x, y, z); +} + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/nvptx/fmax.cpp b/system/lib/llvm-libc/src/math/nvptx/fmax.cpp new file mode 100644 index 0000000000000..a2496fecb01c4 --- /dev/null +++ b/system/lib/llvm-libc/src/math/nvptx/fmax.cpp @@ -0,0 +1,20 @@ +//===-- Implementation of the fmax function for GPU -----------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/fmax.h" + +#include "src/__support/common.h" +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(double, fmax, (double x, double y)) { + return __builtin_fmax(x, y); +} + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/nvptx/fmaxf.cpp b/system/lib/llvm-libc/src/math/nvptx/fmaxf.cpp new file mode 100644 index 0000000000000..586fd32fee59b --- /dev/null +++ b/system/lib/llvm-libc/src/math/nvptx/fmaxf.cpp @@ -0,0 +1,22 @@ +//===-- Implementation of the fmaxf function for GPU ----------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/fmaxf.h" + +#include "src/__support/CPP/bit.h" +#include "src/__support/common.h" +#include "src/__support/macros/config.h" +#include "src/__support/macros/optimization.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(float, fmaxf, (float x, float y)) { + return __builtin_fmaxf(x, y); +} + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/nvptx/fmin.cpp b/system/lib/llvm-libc/src/math/nvptx/fmin.cpp new file mode 100644 index 0000000000000..694eb664b599b --- /dev/null +++ b/system/lib/llvm-libc/src/math/nvptx/fmin.cpp @@ -0,0 +1,20 @@ +//===-- Implementation of the fmin function for GPU -----------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/fmin.h" + +#include "src/__support/common.h" +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(double, fmin, (double x, double y)) { + return __builtin_fmin(x, y); +} + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/nvptx/fminf.cpp b/system/lib/llvm-libc/src/math/nvptx/fminf.cpp new file mode 100644 index 0000000000000..2060b71b0841f --- /dev/null +++ b/system/lib/llvm-libc/src/math/nvptx/fminf.cpp @@ -0,0 +1,20 @@ +//===-- Implementation of the fminf function for GPU ----------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/fminf.h" + +#include "src/__support/common.h" +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(float, fminf, (float x, float y)) { + return __builtin_fminf(x, y); +} + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/nvptx/fmod.cpp b/system/lib/llvm-libc/src/math/nvptx/fmod.cpp new file mode 100644 index 0000000000000..49d19c4decb96 --- /dev/null +++ b/system/lib/llvm-libc/src/math/nvptx/fmod.cpp @@ -0,0 +1,19 @@ +//===-- Implementation of the fmod function for GPU -----------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/fmod.h" +#include "src/__support/common.h" +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(double, fmod, (double x, double y)) { + return __builtin_fmod(x, y); +} + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/nvptx/fmodf.cpp b/system/lib/llvm-libc/src/math/nvptx/fmodf.cpp new file mode 100644 index 0000000000000..8fbcb0cc2ad9e --- /dev/null +++ b/system/lib/llvm-libc/src/math/nvptx/fmodf.cpp @@ -0,0 +1,19 @@ +//===-- Implementation of the fmodf function for GPU ----------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/fmodf.h" +#include "src/__support/common.h" +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(float, fmodf, (float x, float y)) { + return __builtin_fmodf(x, y); +} + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/nvptx/lgamma.cpp b/system/lib/llvm-libc/src/math/nvptx/lgamma.cpp new file mode 100644 index 0000000000000..0dc3111c6ae12 --- /dev/null +++ b/system/lib/llvm-libc/src/math/nvptx/lgamma.cpp @@ -0,0 +1,18 @@ +//===-- Implementation of the lgamma function for GPU ---------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/lgamma.h" +#include "src/__support/common.h" + +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(double, lgamma, (double)) { return 0.0; } + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/nvptx/lgamma_r.cpp b/system/lib/llvm-libc/src/math/nvptx/lgamma_r.cpp new file mode 100644 index 0000000000000..92018da4f860f --- /dev/null +++ b/system/lib/llvm-libc/src/math/nvptx/lgamma_r.cpp @@ -0,0 +1,21 @@ +//===-- Implementation of the lgamma_r function for GPU -------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/lgamma_r.h" +#include "src/__support/common.h" + +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(double, lgamma_r, (double, int *signp)) { + *signp = 0.0; + return 0.0; +} + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/nvptx/llrint.cpp b/system/lib/llvm-libc/src/math/nvptx/llrint.cpp new file mode 100644 index 0000000000000..6e0f57a5a0387 --- /dev/null +++ b/system/lib/llvm-libc/src/math/nvptx/llrint.cpp @@ -0,0 +1,20 @@ +//===-- Implementation of the llrint function for GPU ---------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/llrint.h" +#include "src/__support/common.h" + +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(long long, llrint, (double x)) { + return static_cast(__builtin_rint(x)); +} + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/nvptx/llrintf.cpp b/system/lib/llvm-libc/src/math/nvptx/llrintf.cpp new file mode 100644 index 0000000000000..d8de23fac3c8b --- /dev/null +++ b/system/lib/llvm-libc/src/math/nvptx/llrintf.cpp @@ -0,0 +1,20 @@ +//===-- Implementation of the llrintf function for GPU --------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/llrintf.h" +#include "src/__support/common.h" + +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(long long, llrintf, (float x)) { + return static_cast(__builtin_rintf(x)); +} + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/nvptx/lrint.cpp b/system/lib/llvm-libc/src/math/nvptx/lrint.cpp new file mode 100644 index 0000000000000..5ba70ec890b2c --- /dev/null +++ b/system/lib/llvm-libc/src/math/nvptx/lrint.cpp @@ -0,0 +1,20 @@ +//===-- Implementation of the lrint function for GPU ----------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/lrint.h" +#include "src/__support/common.h" + +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(long, lrint, (double x)) { + return static_cast(__builtin_rint(x)); +} + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/nvptx/nearbyint.cpp b/system/lib/llvm-libc/src/math/nvptx/nearbyint.cpp new file mode 100644 index 0000000000000..7d78c7241d023 --- /dev/null +++ b/system/lib/llvm-libc/src/math/nvptx/nearbyint.cpp @@ -0,0 +1,19 @@ +//===-- Implementation of the GPU nearbyint function ----------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/nearbyint.h" +#include "src/__support/common.h" +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(double, nearbyint, (double x)) { + return __builtin_nearbyint(x); +} + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/nvptx/nearbyintf.cpp b/system/lib/llvm-libc/src/math/nvptx/nearbyintf.cpp new file mode 100644 index 0000000000000..4bd20dc58fb89 --- /dev/null +++ b/system/lib/llvm-libc/src/math/nvptx/nearbyintf.cpp @@ -0,0 +1,19 @@ +//===-- Implementation of the GPU nearbyintf function ---------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/nearbyintf.h" +#include "src/__support/common.h" +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(float, nearbyintf, (float x)) { + return __builtin_nearbyintf(x); +} + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/nvptx/remainder.cpp b/system/lib/llvm-libc/src/math/nvptx/remainder.cpp new file mode 100644 index 0000000000000..9027204312e00 --- /dev/null +++ b/system/lib/llvm-libc/src/math/nvptx/remainder.cpp @@ -0,0 +1,19 @@ +//===-- Implementation of the GPU remainder function ----------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/remainder.h" +#include "src/__support/common.h" +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(double, remainder, (double x, double y)) { + return __builtin_remainder(x, y); +} + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/nvptx/remainderf.cpp b/system/lib/llvm-libc/src/math/nvptx/remainderf.cpp new file mode 100644 index 0000000000000..50df3b2ce25c1 --- /dev/null +++ b/system/lib/llvm-libc/src/math/nvptx/remainderf.cpp @@ -0,0 +1,19 @@ +//===-- Implementation of the GPU remainderf function ---------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/remainderf.h" +#include "src/__support/common.h" +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(float, remainderf, (float x, float y)) { + return __builtin_remainderf(x, y); +} + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/nvptx/rint.cpp b/system/lib/llvm-libc/src/math/nvptx/rint.cpp new file mode 100644 index 0000000000000..ac6837a4abc37 --- /dev/null +++ b/system/lib/llvm-libc/src/math/nvptx/rint.cpp @@ -0,0 +1,17 @@ +//===-- Implementation of the GPU rint function ---------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/rint.h" +#include "src/__support/common.h" +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(double, rint, (double x)) { return __builtin_rint(x); } + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/nvptx/rintf.cpp b/system/lib/llvm-libc/src/math/nvptx/rintf.cpp new file mode 100644 index 0000000000000..94093471a8d92 --- /dev/null +++ b/system/lib/llvm-libc/src/math/nvptx/rintf.cpp @@ -0,0 +1,17 @@ +//===-- Implementation of the GPU rintf function --------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/rintf.h" +#include "src/__support/common.h" +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(float, rintf, (float x)) { return __builtin_rintf(x); } + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/nvptx/round.cpp b/system/lib/llvm-libc/src/math/nvptx/round.cpp new file mode 100644 index 0000000000000..0d2765f2e959f --- /dev/null +++ b/system/lib/llvm-libc/src/math/nvptx/round.cpp @@ -0,0 +1,17 @@ +//===-- Implementation of the GPU round function --------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/round.h" +#include "src/__support/common.h" +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(double, round, (double x)) { return __builtin_round(x); } + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/nvptx/roundf.cpp b/system/lib/llvm-libc/src/math/nvptx/roundf.cpp new file mode 100644 index 0000000000000..86e8ba3ac83d6 --- /dev/null +++ b/system/lib/llvm-libc/src/math/nvptx/roundf.cpp @@ -0,0 +1,17 @@ +//===-- Implementation of the GPU roundf function -------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/roundf.h" +#include "src/__support/common.h" +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(float, roundf, (float x)) { return __builtin_roundf(x); } + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/nvptx/sqrt.cpp b/system/lib/llvm-libc/src/math/nvptx/sqrt.cpp new file mode 100644 index 0000000000000..ed83b6e5c6cae --- /dev/null +++ b/system/lib/llvm-libc/src/math/nvptx/sqrt.cpp @@ -0,0 +1,17 @@ +//===-- Implementation of the GPU sqrt function ---------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/sqrt.h" +#include "src/__support/common.h" +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(double, sqrt, (double x)) { return __builtin_sqrt(x); } + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/nvptx/sqrtf.cpp b/system/lib/llvm-libc/src/math/nvptx/sqrtf.cpp new file mode 100644 index 0000000000000..851922c316452 --- /dev/null +++ b/system/lib/llvm-libc/src/math/nvptx/sqrtf.cpp @@ -0,0 +1,17 @@ +//===-- Implementation of the GPU sqrtf function --------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/sqrtf.h" +#include "src/__support/common.h" +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(float, sqrtf, (float x)) { return __builtin_sqrtf(x); } + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/nvptx/tgamma.cpp b/system/lib/llvm-libc/src/math/nvptx/tgamma.cpp new file mode 100644 index 0000000000000..0dfb0fd9ee13c --- /dev/null +++ b/system/lib/llvm-libc/src/math/nvptx/tgamma.cpp @@ -0,0 +1,18 @@ +//===-- Implementation of the GPU tgamma function -------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/tgamma.h" +#include "src/__support/common.h" + +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(double, tgamma, (double)) { return 0.0; } + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/nvptx/tgammaf.cpp b/system/lib/llvm-libc/src/math/nvptx/tgammaf.cpp new file mode 100644 index 0000000000000..bb77cf56600b4 --- /dev/null +++ b/system/lib/llvm-libc/src/math/nvptx/tgammaf.cpp @@ -0,0 +1,18 @@ +//===-- Implementation of the GPU tgammaf function ------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/tgammaf.h" +#include "src/__support/common.h" + +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(float, tgammaf, (float)) { return 0.0; } + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/nvptx/trunc.cpp b/system/lib/llvm-libc/src/math/nvptx/trunc.cpp new file mode 100644 index 0000000000000..f60caa2a71d78 --- /dev/null +++ b/system/lib/llvm-libc/src/math/nvptx/trunc.cpp @@ -0,0 +1,17 @@ +//===-- Implementation of the GPU trunc function --------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/trunc.h" +#include "src/__support/common.h" +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(double, trunc, (double x)) { return __builtin_trunc(x); } + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/math/nvptx/truncf.cpp b/system/lib/llvm-libc/src/math/nvptx/truncf.cpp new file mode 100644 index 0000000000000..a6c9b8f188f02 --- /dev/null +++ b/system/lib/llvm-libc/src/math/nvptx/truncf.cpp @@ -0,0 +1,17 @@ +//===-- Implementation of the GPU truncf function -------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/math/truncf.h" +#include "src/__support/common.h" +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(float, truncf, (float x)) { return __builtin_truncf(x); } + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/stdio/printf_core/char_converter.h b/system/lib/llvm-libc/src/stdio/printf_core/char_converter.h index 2596cba813c2e..fd2eb2553887a 100644 --- a/system/lib/llvm-libc/src/stdio/printf_core/char_converter.h +++ b/system/lib/llvm-libc/src/stdio/printf_core/char_converter.h @@ -17,7 +17,9 @@ namespace LIBC_NAMESPACE_DECL { namespace printf_core { -LIBC_INLINE int convert_char(Writer *writer, const FormatSection &to_conv) { +template +LIBC_INLINE int convert_char(Writer *writer, + const FormatSection &to_conv) { char c = static_cast(to_conv.conv_val_raw); constexpr int STRING_LEN = 1; diff --git a/system/lib/llvm-libc/src/stdio/printf_core/converter.cpp b/system/lib/llvm-libc/src/stdio/printf_core/converter.cpp deleted file mode 100644 index b1c66451f53f0..0000000000000 --- a/system/lib/llvm-libc/src/stdio/printf_core/converter.cpp +++ /dev/null @@ -1,105 +0,0 @@ -//===-- Format specifier converter implmentation for printf -----*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#include "src/stdio/printf_core/converter.h" - -#include "src/__support/macros/config.h" -#include "src/stdio/printf_core/core_structs.h" -#include "src/stdio/printf_core/printf_config.h" -#include "src/stdio/printf_core/strerror_converter.h" -#include "src/stdio/printf_core/writer.h" - -// This option allows for replacing all of the conversion functions with custom -// replacements. This allows conversions to be replaced at compile time. -#ifndef LIBC_COPT_PRINTF_CONV_ATLAS -#include "src/stdio/printf_core/converter_atlas.h" -#else -#include LIBC_COPT_PRINTF_CONV_ATLAS -#endif - -#include - -namespace LIBC_NAMESPACE_DECL { -namespace printf_core { - -int convert(Writer *writer, const FormatSection &to_conv) { - if (!to_conv.has_conv) - return writer->write(to_conv.raw_string); - -#if !defined(LIBC_COPT_PRINTF_DISABLE_FLOAT) && \ - defined(LIBC_COPT_PRINTF_HEX_LONG_DOUBLE) - if (to_conv.length_modifier == LengthModifier::L) { - switch (to_conv.conv_name) { - case 'f': - case 'F': - case 'e': - case 'E': - case 'g': - case 'G': - return convert_float_hex_exp(writer, to_conv); - default: - break; - } - } -#endif // LIBC_COPT_PRINTF_DISABLE_FLOAT - - switch (to_conv.conv_name) { - case '%': - return writer->write("%"); - case 'c': - return convert_char(writer, to_conv); - case 's': - return convert_string(writer, to_conv); - case 'd': - case 'i': - case 'u': - case 'o': - case 'x': - case 'X': - case 'b': - case 'B': - return convert_int(writer, to_conv); -#ifndef LIBC_COPT_PRINTF_DISABLE_FLOAT - case 'f': - case 'F': - return convert_float_decimal(writer, to_conv); - case 'e': - case 'E': - return convert_float_dec_exp(writer, to_conv); - case 'a': - case 'A': - return convert_float_hex_exp(writer, to_conv); - case 'g': - case 'G': - return convert_float_dec_auto(writer, to_conv); -#endif // LIBC_COPT_PRINTF_DISABLE_FLOAT -#ifdef LIBC_INTERNAL_PRINTF_HAS_FIXED_POINT - case 'r': - case 'R': - case 'k': - case 'K': - return convert_fixed(writer, to_conv); -#endif // LIBC_INTERNAL_PRINTF_HAS_FIXED_POINT -#ifndef LIBC_COPT_PRINTF_DISABLE_STRERROR - case 'm': - return convert_strerror(writer, to_conv); -#endif // LIBC_COPT_PRINTF_DISABLE_STRERROR -#ifndef LIBC_COPT_PRINTF_DISABLE_WRITE_INT - case 'n': - return convert_write_int(writer, to_conv); -#endif // LIBC_COPT_PRINTF_DISABLE_WRITE_INT - case 'p': - return convert_pointer(writer, to_conv); - default: - return writer->write(to_conv.raw_string); - } - return -1; -} - -} // namespace printf_core -} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/stdio/printf_core/converter.h b/system/lib/llvm-libc/src/stdio/printf_core/converter.h index 2b3f06d0aa7a3..f26ed727f05f4 100644 --- a/system/lib/llvm-libc/src/stdio/printf_core/converter.h +++ b/system/lib/llvm-libc/src/stdio/printf_core/converter.h @@ -11,8 +11,18 @@ #include "src/__support/macros/config.h" #include "src/stdio/printf_core/core_structs.h" +#include "src/stdio/printf_core/printf_config.h" +#include "src/stdio/printf_core/strerror_converter.h" #include "src/stdio/printf_core/writer.h" +// This option allows for replacing all of the conversion functions with custom +// replacements. This allows conversions to be replaced at compile time. +#ifndef LIBC_COPT_PRINTF_CONV_ATLAS +#include "src/stdio/printf_core/converter_atlas.h" +#else +#include LIBC_COPT_PRINTF_CONV_ATLAS +#endif + #include namespace LIBC_NAMESPACE_DECL { @@ -21,7 +31,80 @@ namespace printf_core { // convert will call a conversion function to convert the FormatSection into // its string representation, and then that will write the result to the // writer. -int convert(Writer *writer, const FormatSection &to_conv); +template +int convert(Writer *writer, const FormatSection &to_conv) { + if (!to_conv.has_conv) + return writer->write(to_conv.raw_string); + +#if !defined(LIBC_COPT_PRINTF_DISABLE_FLOAT) && \ + defined(LIBC_COPT_PRINTF_HEX_LONG_DOUBLE) + if (to_conv.length_modifier == LengthModifier::L) { + switch (to_conv.conv_name) { + case 'f': + case 'F': + case 'e': + case 'E': + case 'g': + case 'G': + return convert_float_hex_exp(writer, to_conv); + default: + break; + } + } +#endif // LIBC_COPT_PRINTF_DISABLE_FLOAT + + switch (to_conv.conv_name) { + case '%': + return writer->write("%"); + case 'c': + return convert_char(writer, to_conv); + case 's': + return convert_string(writer, to_conv); + case 'd': + case 'i': + case 'u': + case 'o': + case 'x': + case 'X': + case 'b': + case 'B': + return convert_int(writer, to_conv); +#ifndef LIBC_COPT_PRINTF_DISABLE_FLOAT + case 'f': + case 'F': + return convert_float_decimal(writer, to_conv); + case 'e': + case 'E': + return convert_float_dec_exp(writer, to_conv); + case 'a': + case 'A': + return convert_float_hex_exp(writer, to_conv); + case 'g': + case 'G': + return convert_float_dec_auto(writer, to_conv); +#endif // LIBC_COPT_PRINTF_DISABLE_FLOAT +#ifdef LIBC_INTERNAL_PRINTF_HAS_FIXED_POINT + case 'r': + case 'R': + case 'k': + case 'K': + return convert_fixed(writer, to_conv); +#endif // LIBC_INTERNAL_PRINTF_HAS_FIXED_POINT +#ifndef LIBC_COPT_PRINTF_DISABLE_STRERROR + case 'm': + return convert_strerror(writer, to_conv); +#endif // LIBC_COPT_PRINTF_DISABLE_STRERROR +#ifndef LIBC_COPT_PRINTF_DISABLE_WRITE_INT + case 'n': + return convert_write_int(writer, to_conv); +#endif // LIBC_COPT_PRINTF_DISABLE_WRITE_INT + case 'p': + return convert_pointer(writer, to_conv); + default: + return writer->write(to_conv.raw_string); + } + return -1; +} } // namespace printf_core } // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/stdio/printf_core/converter_atlas.h b/system/lib/llvm-libc/src/stdio/printf_core/converter_atlas.h index 18cfe1e717cbe..dfb91b30e80f8 100644 --- a/system/lib/llvm-libc/src/stdio/printf_core/converter_atlas.h +++ b/system/lib/llvm-libc/src/stdio/printf_core/converter_atlas.h @@ -26,7 +26,11 @@ // defines convert_float_decimal // defines convert_float_dec_exp // defines convert_float_dec_auto +#ifdef LIBC_COPT_FLOAT_TO_STR_USE_FLOAT320 +#include "src/stdio/printf_core/float_dec_converter_limited.h" +#else #include "src/stdio/printf_core/float_dec_converter.h" +#endif // defines convert_float_hex_exp #include "src/stdio/printf_core/float_hex_converter.h" #endif // LIBC_COPT_PRINTF_DISABLE_FLOAT diff --git a/system/lib/llvm-libc/src/stdio/printf_core/core_structs.h b/system/lib/llvm-libc/src/stdio/printf_core/core_structs.h index 4c3b81ff018ab..e27f77b6b594a 100644 --- a/system/lib/llvm-libc/src/stdio/printf_core/core_structs.h +++ b/system/lib/llvm-libc/src/stdio/printf_core/core_structs.h @@ -55,8 +55,13 @@ struct FormatSection { int min_width = 0; int precision = -1; - // Needs to be large enough to hold a long double. + // Needs to be large enough to hold a long double. Special case handling for + // the PowerPC double double type because it has no FPBits interface. +#ifdef LIBC_TYPES_LONG_DOUBLE_IS_DOUBLE_DOUBLE + UInt128 conv_val_raw; +#else fputil::FPBits::StorageType conv_val_raw; +#endif // LIBC_TYPES_LONG_DOUBLE_IS_DOUBLE_DOUBLE void *conv_val_ptr; char conv_name; diff --git a/system/lib/llvm-libc/src/stdio/printf_core/fixed_converter.h b/system/lib/llvm-libc/src/stdio/printf_core/fixed_converter.h index ba0a62d9fcb87..e8a3314967562 100644 --- a/system/lib/llvm-libc/src/stdio/printf_core/fixed_converter.h +++ b/system/lib/llvm-libc/src/stdio/printf_core/fixed_converter.h @@ -63,7 +63,9 @@ LIBC_INLINE constexpr uint32_t const_ten_exp(uint32_t exponent) { } \ } while (false) -LIBC_INLINE int convert_fixed(Writer *writer, const FormatSection &to_conv) { +template +LIBC_INLINE int convert_fixed(Writer *writer, + const FormatSection &to_conv) { // Long accum should be the largest type, so we can store all the smaller // numbers in things sized for it. using LARep = fixed_point::FXRep; diff --git a/system/lib/llvm-libc/src/stdio/printf_core/float_dec_converter.h b/system/lib/llvm-libc/src/stdio/printf_core/float_dec_converter.h index d93457fcafd7f..ed004f9a26a13 100644 --- a/system/lib/llvm-libc/src/stdio/printf_core/float_dec_converter.h +++ b/system/lib/llvm-libc/src/stdio/printf_core/float_dec_converter.h @@ -93,7 +93,7 @@ zero_after_digits(int32_t base_2_exp, int32_t digits_after_point, T mantissa, return has_trailing_zeros; } -class PaddingWriter { +template class PaddingWriter { bool left_justified = false; bool leading_zeroes = false; char sign_char = 0; @@ -107,7 +107,8 @@ class PaddingWriter { sign_char(init_sign_char), min_width(to_conv.min_width > 0 ? to_conv.min_width : 0) {} - LIBC_INLINE int write_left_padding(Writer *writer, size_t total_digits) { + LIBC_INLINE int write_left_padding(Writer *writer, + size_t total_digits) { // The pattern is (spaces) (sign) (zeroes), but only one of spaces and // zeroes can be written, and only if the padding amount is positive. int padding_amount = @@ -130,7 +131,8 @@ class PaddingWriter { return 0; } - LIBC_INLINE int write_right_padding(Writer *writer, size_t total_digits) { + LIBC_INLINE int write_right_padding(Writer *writer, + size_t total_digits) { // If and only if the conversion is left justified, there may be trailing // spaces. int padding_amount = @@ -155,7 +157,7 @@ class PaddingWriter { This FloatWriter class does the buffering and counting, and writes to the output when necessary. */ -class FloatWriter { +template class FloatWriter { char block_buffer[BLOCK_SIZE]; // The buffer that holds a block. size_t buffered_digits = 0; // The number of digits held in the buffer. bool has_written = false; // True once any digits have been output. @@ -164,8 +166,9 @@ class FloatWriter { size_t digits_before_decimal = 0; // The # of digits to write before the '.' size_t total_digits_written = 0; // The # of digits that have been output. bool has_decimal_point; // True if the number has a decimal point. - Writer *writer; // Writes to the final output. - PaddingWriter padding_writer; // Handles prefixes/padding, uses total_digits. + Writer *writer; // Writes to the final output. + PaddingWriter + padding_writer; // Handles prefixes/padding, uses total_digits. LIBC_INLINE int flush_buffer(bool round_up_max_blocks = false) { const char MAX_BLOCK_DIGIT = (round_up_max_blocks ? '0' : '9'); @@ -183,13 +186,12 @@ class FloatWriter { if (total_digits_written < digits_before_decimal && total_digits_written + buffered_digits >= digits_before_decimal && has_decimal_point) { + // digits_to_write > 0 guaranteed by outer if size_t digits_to_write = digits_before_decimal - total_digits_written; - if (digits_to_write > 0) { - // Write the digits before the decimal point. - RET_IF_RESULT_NEGATIVE(writer->write({block_buffer, digits_to_write})); - } + // Write the digits before the decimal point. + RET_IF_RESULT_NEGATIVE(writer->write({block_buffer, digits_to_write})); RET_IF_RESULT_NEGATIVE(writer->write(DECIMAL_POINT)); - if (buffered_digits - digits_to_write > 0) { + if (buffered_digits > digits_to_write) { // Write the digits after the decimal point. RET_IF_RESULT_NEGATIVE( writer->write({block_buffer + digits_to_write, @@ -214,12 +216,11 @@ class FloatWriter { total_digits_written + BLOCK_SIZE * max_block_count >= digits_before_decimal && has_decimal_point) { + // digits_to_write > 0 guaranteed by outer if size_t digits_to_write = digits_before_decimal - total_digits_written; - if (digits_to_write > 0) { - RET_IF_RESULT_NEGATIVE(writer->write(MAX_BLOCK_DIGIT, digits_to_write)); - } + RET_IF_RESULT_NEGATIVE(writer->write(MAX_BLOCK_DIGIT, digits_to_write)); RET_IF_RESULT_NEGATIVE(writer->write(DECIMAL_POINT)); - if ((BLOCK_SIZE * max_block_count) - digits_to_write > 0) { + if ((BLOCK_SIZE * max_block_count) > digits_to_write) { RET_IF_RESULT_NEGATIVE(writer->write( MAX_BLOCK_DIGIT, (BLOCK_SIZE * max_block_count) - digits_to_write)); } @@ -245,8 +246,9 @@ class FloatWriter { static_assert(fputil::FPBits::EXP_LEN < (sizeof(int) * 8)); public: - LIBC_INLINE FloatWriter(Writer *init_writer, bool init_has_decimal_point, - const PaddingWriter &init_padding_writer) + LIBC_INLINE FloatWriter(Writer *init_writer, + bool init_has_decimal_point, + const PaddingWriter &init_padding_writer) : has_decimal_point(init_has_decimal_point), writer(init_writer), padding_writer(init_padding_writer) {} @@ -466,12 +468,24 @@ class FloatWriter { } }; +// Class-template auto deduction helpers, add more if needed. +FloatWriter(Writer, bool, + const PaddingWriter) + -> FloatWriter; +FloatWriter(Writer, bool, + const PaddingWriter) + -> FloatWriter; +FloatWriter(Writer, bool, + const PaddingWriter) + -> FloatWriter; + // This implementation is based on the Ryu Printf algorithm by Ulf Adams: // Ulf Adams. 2019. Ryū revisited: printf floating point conversion. // Proc. ACM Program. Lang. 3, OOPSLA, Article 169 (October 2019), 23 pages. // https://doi.org/10.1145/3360595 -template , int> = 0> -LIBC_INLINE int convert_float_decimal_typed(Writer *writer, +template , int> = 0> +LIBC_INLINE int convert_float_decimal_typed(Writer *writer, const FormatSection &to_conv, fputil::FPBits float_bits) { // signed because later we use -FRACTION_LEN @@ -498,7 +512,7 @@ LIBC_INLINE int convert_float_decimal_typed(Writer *writer, // ignored. bool nonzero = false; - PaddingWriter padding_writer(to_conv, sign_char); + PaddingWriter padding_writer(to_conv, sign_char); FloatWriter float_writer(writer, has_decimal_point, padding_writer); FloatToString float_converter(float_bits.get_val()); @@ -579,8 +593,9 @@ LIBC_INLINE int convert_float_decimal_typed(Writer *writer, return WRITE_OK; } -template , int> = 0> -LIBC_INLINE int convert_float_dec_exp_typed(Writer *writer, +template , int> = 0> +LIBC_INLINE int convert_float_dec_exp_typed(Writer *writer, const FormatSection &to_conv, fputil::FPBits float_bits) { // signed because later we use -FRACTION_LEN @@ -603,7 +618,7 @@ LIBC_INLINE int convert_float_dec_exp_typed(Writer *writer, bool has_decimal_point = (precision > 0) || ((to_conv.flags & FormatFlags::ALTERNATE_FORM) != 0); - PaddingWriter padding_writer(to_conv, sign_char); + PaddingWriter padding_writer(to_conv, sign_char); FloatWriter float_writer(writer, has_decimal_point, padding_writer); FloatToString float_converter(float_bits.get_val()); @@ -740,8 +755,9 @@ LIBC_INLINE int convert_float_dec_exp_typed(Writer *writer, return WRITE_OK; } -template , int> = 0> -LIBC_INLINE int convert_float_dec_auto_typed(Writer *writer, +template , int> = 0> +LIBC_INLINE int convert_float_dec_auto_typed(Writer *writer, const FormatSection &to_conv, fputil::FPBits float_bits) { // signed because later we use -FRACTION_LEN @@ -1107,7 +1123,9 @@ LIBC_INLINE int convert_float_dec_auto_typed(Writer *writer, } // TODO: unify the float converters to remove the duplicated checks for inf/nan. -LIBC_INLINE int convert_float_decimal(Writer *writer, + +template +LIBC_INLINE int convert_float_decimal(Writer *writer, const FormatSection &to_conv) { if (to_conv.length_modifier == LengthModifier::L) { fputil::FPBits::StorageType float_raw = to_conv.conv_val_raw; @@ -1128,7 +1146,8 @@ LIBC_INLINE int convert_float_decimal(Writer *writer, return convert_inf_nan(writer, to_conv); } -LIBC_INLINE int convert_float_dec_exp(Writer *writer, +template +LIBC_INLINE int convert_float_dec_exp(Writer *writer, const FormatSection &to_conv) { if (to_conv.length_modifier == LengthModifier::L) { fputil::FPBits::StorageType float_raw = to_conv.conv_val_raw; @@ -1149,7 +1168,8 @@ LIBC_INLINE int convert_float_dec_exp(Writer *writer, return convert_inf_nan(writer, to_conv); } -LIBC_INLINE int convert_float_dec_auto(Writer *writer, +template +LIBC_INLINE int convert_float_dec_auto(Writer *writer, const FormatSection &to_conv) { if (to_conv.length_modifier == LengthModifier::L) { fputil::FPBits::StorageType float_raw = to_conv.conv_val_raw; diff --git a/system/lib/llvm-libc/src/stdio/printf_core/float_dec_converter_limited.h b/system/lib/llvm-libc/src/stdio/printf_core/float_dec_converter_limited.h new file mode 100644 index 0000000000000..f468dbc8e2ae8 --- /dev/null +++ b/system/lib/llvm-libc/src/stdio/printf_core/float_dec_converter_limited.h @@ -0,0 +1,699 @@ +//===-- Decimal Float Converter for printf (320-bit float) ------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements an alternative to the Ryū printf algorithm in +// float_dec_converter.h. Instead of generating output digits 9 at a time on +// demand, in this implementation, a float is converted to decimal by computing +// just one power of 10 and multiplying/dividing the entire input by it, +// generating the whole string of decimal output digits in one go. +// +// This avoids the large constant lookup table of Ryū, making it more suitable +// for low-memory embedded contexts; but it's also faster than the fallback +// version of Ryū which computes table entries on demand using DyadicFloat, +// because those must calculate a potentially large power of 10 per 9-digit +// output block, whereas this computes just one, which does the whole job. +// +// The calculation is done in 320-bit DyadicFloat, which provides enough +// precision to generate 39 correct digits of output from any floating-point +// size up to and including 128-bit long double, because the rounding errors in +// computing the largest necessary power of 10 are still smaller than the +// distance (in the 320-bit float format) between adjacent 39-decimal-digit +// outputs. +// +// No further digits beyond the 39th are generated: if the printf format string +// asks for more precision than that, the answer is padded with 0s. This is a +// permitted option in IEEE 754-2019 (section 5.12.2): you're allowed to define +// a limit H on the number of decimal digits you can generate, and pad with 0s +// if asked for more than that, subject to the constraint that H must be +// consistent across all float formats you support (you can't use a smaller H +// for single precision than double or long double), and must be large enough +// that even in the largest supported precision the only numbers misrounded are +// ones extremely close to a rounding boundary. 39 digits is the smallest +// permitted value for an implementation supporting binary128. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_STDIO_PRINTF_CORE_FLOAT_DEC_CONVERTER_LIMITED_H +#define LLVM_LIBC_SRC_STDIO_PRINTF_CORE_FLOAT_DEC_CONVERTER_LIMITED_H + +#include "src/__support/CPP/algorithm.h" +#include "src/__support/CPP/string.h" +#include "src/__support/CPP/string_view.h" +#include "src/__support/FPUtil/FPBits.h" +#include "src/__support/FPUtil/dyadic_float.h" +#include "src/__support/FPUtil/rounding_mode.h" +#include "src/__support/integer_to_string.h" +#include "src/__support/libc_assert.h" +#include "src/__support/macros/config.h" +#include "src/stdio/printf_core/core_structs.h" +#include "src/stdio/printf_core/float_inf_nan_converter.h" +#include "src/stdio/printf_core/writer.h" + +namespace LIBC_NAMESPACE_DECL { +namespace printf_core { + +enum class ConversionType { E, F, G }; +using StorageType = fputil::FPBits::StorageType; + +constexpr unsigned MAX_DIGITS = 39; +constexpr size_t DF_BITS = 320; +constexpr char DECIMAL_POINT = '.'; + +struct DigitsInput { + // Input mantissa, stored with the explicit leading 1 bit (if any) at the + // top. So either it has a value in the range [2^127,2^128) representing a + // real number in [1,2), or it has the value 0, representing 0. + UInt128 mantissa; + + // Input exponent, as a power of 2 to multiply into mantissa. + int exponent; + + // Input sign. + Sign sign; + + // Constructor which accepts a mantissa direct from a floating-point format, + // and shifts it up to the top of the UInt128 so that a function consuming + // this struct afterwards doesn't have to remember which format it came from. + DigitsInput(int32_t fraction_len, StorageType mantissa_, int exponent_, + Sign sign) + : mantissa(UInt128(mantissa_) << (127 - fraction_len)), + exponent(exponent_), sign(sign) { + if (!(mantissa & (UInt128(1) << 127)) && mantissa != 0) { + // Normalize a denormalized input. + int shift = cpp::countl_zero(mantissa); + mantissa <<= shift; + exponent -= shift; + } + } +}; + +struct DigitsOutput { + // Output from decimal_digits(). + // + // `digits` is a buffer containing nothing but ASCII digits. Even if the + // decimal point needs to appear somewhere in the final output string, it + // isn't represented in _this_ string; the client of this object will insert + // it in an appropriate place. `ndigits` gives the buffer size. + // + // `exponent` represents the exponent you would display if the decimal point + // comes after the first digit of decimal_digits, e.g. if digits == "1234" + // and exponent = 3 then this represents 1.234e3, or just the integer 1234. + size_t ndigits; + int exponent; + char digits[MAX_DIGITS + 1]; +}; + +// Estimate log10 of a power of 2, by multiplying its exponent by +// 1292913986/2^32. That is a rounded-down approximation to log10(2), accurate +// enough that for any binary exponent in the range of float128 it will give +// the correct value of floor(log10(2^n)). +LIBC_INLINE int estimate_log10(int exponent_of_2) { + return static_cast((exponent_of_2 * 1292913986LL) >> 32); +} + +// Calculate the actual digits of a decimal representation of an FP number. +// +// If `e_mode` is true, then `precision` indicates the desired number of output +// decimal digits. On return, `decimal_digits` will be a string of length +// exactly `precision` starting with a nonzero digit; `decimal_exponent` will +// be filled in to indicate the exponent as shown above. +// +// If `e_mode` is false, then `precision` indicates the desired number of +// digits after the decimal point. On return, the last digit in the string +// `decimal_digits` has a place value of _at least_ 10^-precision. But also, at +// most `MAX_DIGITS` digits are returned, so the caller may need to pad it at +// the end with the appropriate number of extra 0s. +LIBC_INLINE +DigitsOutput decimal_digits(DigitsInput input, int precision, bool e_mode) { + if (input.mantissa == 0) { + // Special-case zero, by manually generating the right number of zero + // digits and setting an appropriate exponent. + DigitsOutput output; + if (!e_mode) { + // In F mode, it's enough to return an empty string of digits. That's the + // same thing we do when given a nonzero number that rounds down to 0. + output.ndigits = 0; + output.exponent = -precision - 1; + } else { + // In E mode, generate a string containing the expected number of 0s. + __builtin_memset(output.digits, '0', precision); + output.ndigits = precision; + output.exponent = 0; + } + return output; + } + + // Calculate bounds on log10 of the input value. Its binary exponent bounds + // the value between two powers of 2, and we use estimate_log10 to determine + // log10 of each of those. + // + // If a power of 10 falls in the interval between those powers of 2, then + // log10_input_min and log10_input_max will differ by 1, and the correct + // decimal exponent of the output will be one of those two values. If no + // power of 10 is in the interval, then these two values will be equal and + // there is only one choice for the decimal exponent. + int log10_input_min = estimate_log10(input.exponent - 1); + int log10_input_max = estimate_log10(input.exponent); + + // Make a DyadicFloat containing the value 10, to use as the base for + // exponentiation. + fputil::DyadicFloat ten(Sign::POS, 1, 5); + + // Compute the exponent of the lowest-order digit we want as output. In F + // mode this depends only on the desired precision. In E mode it's based on + // log10_input, which is (an estimate of) the exponent corresponding to the + // _high_-order decimal digit of the number. + int log10_low_digit = e_mode ? log10_input_min + 1 - precision : -precision; + + // The general plan is to calculate an integer whose decimal representation + // is precisely the string of output digits, by doing a DyadicFloat + // computation of (input_mantissa / 10^(log10_low_digit)) and then rounding + // that to an integer. + // + // The number of output decimal digits (if the mathematical result of this + // operation were computed without overflow) will be one of these: + // (log10_input_min - log10_low_digit + 1) + // (log10_input_max - log10_low_digit + 1) + // + // In E mode, this means we'll either get the correct number of output digits + // immediately, or else one too many (in which case we can correct for that + // at the rounding stage). But in F mode, if the number is very large + // compared to the number of decimal places the user asked for, we might be + // about to generate far too many digits and overflow our float format. In + // that case, reset to E mode immediately, to avoid having to detect the + // overflow _after_ the multiplication and retry. So if even the smaller + // number of possible output digits is too many, we might as well change our + // mind right now and switch into E mode. + if (log10_input_max - log10_low_digit + 1 > int(MAX_DIGITS)) { + precision = MAX_DIGITS; + e_mode = true; + log10_low_digit = log10_input_min + 1 - precision; + } + + // Now actually calculate (input_mantissa / 10^(log10_low_digit)). + // + // If log10_low_digit < 0, then we calculate 10^(-log10_low_digit) and + // multiply by it instead, so that the exponent is non-negative in all cases. + // This ensures that the power of 10 is always mathematically speaking an + // integer, so that it can be represented exactly in binary (without a + // recurring fraction), and when it's small enough to fit in DF_BITS, + // fputil::pow_n should return the exact answer, and then + // fputil::rounded_{div,mul} will introduce only the unavoidable rounding + // error of up to 1/2 ULP. + // + // Beyond that point, pow_n will be imprecise. But DF_BITS is set high enough + // that even for the most difficult cases in 128-bit long double, the extra + // precision in the calculation is enough to ensure we still get the right + // answer. + // + // If the output integer doesn't fit in DF_BITS, we set the `overflow` flag. + + // Calculate the power of 10 to divide or multiply by. + fputil::DyadicFloat power_of_10 = + fputil::pow_n(ten, cpp::abs(log10_low_digit)); + + // Convert the mantissa into a DyadicFloat, making sure it has the right + // sign, so that directed rounding will go in the right direction, if + // enabled. + fputil::DyadicFloat flt_mantissa( + input.sign, + input.exponent - + (cpp::numeric_limits::digits - 1), + input.mantissa); + + // Divide or multiply, depending on whether log10_low_digit was positive + // or negative. + fputil::DyadicFloat flt_quotient = + log10_low_digit > 0 ? fputil::rounded_div(flt_mantissa, power_of_10) + : fputil::rounded_mul(flt_mantissa, power_of_10); + + // Convert to an integer. + int round_dir; + UInt integer = flt_quotient.as_mantissa_type_rounded(&round_dir); + + // And take the absolute value. + if (flt_quotient.sign.is_neg()) + integer = -integer; + + // Convert the mantissa integer into a string of decimal digits, and check + // to see if it's the right size. + const IntegerToString buf{integer}; + cpp::string_view view = buf.view(); + + // Start making the output struct, by copying in the digits from the above + // object. At this stage we may also have one digit too many (but that's OK, + // there's space for it in the DigitsOutput buffer). + DigitsOutput output; + output.ndigits = view.size(); + __builtin_memcpy(output.digits, view.data(), output.ndigits); + + // Set up the output exponent, which is done differently depending on mode. + // Also, figure out whether we have one digit too many, and if so, set the + // `need_reround` flag and adjust the exponent appropriately. + bool need_reround = false; + if (e_mode) { + // In E mode, the output exponent is the exponent of the first decimal + // digit, which we already calculated. + output.exponent = log10_input_min; + + // In E mode, we're returning a fixed number of digits, given by + // `precision`, so if we have more than that, then we must shorten the + // buffer by one digit. + // + // If this happens, it's because the actual log10 of the input is + // log10_input_min + 1. Equivalently, we guessed we'd see something like + // X.YZe+NN and instead got WX.YZe+NN. So when we shorten the digit string + // by one, we'll also need to increment the output exponent. + if (output.ndigits > size_t(precision)) { + LIBC_ASSERT(output.ndigits == size_t(precision) + 1); + need_reround = true; + output.exponent++; + } + } else { + // In F mode, the output exponent is based on the place value of the _last_ + // digit, so we must recover the exponent of the first digit by adding + // the number of digits. + // + // Because this takes the length of the buffer into account, it sets the + // correct decimal exponent even if this digit string is one too long. So + // we don't need to adjust the exponent if we reround. + output.exponent = int(output.ndigits) - precision - 1; + + // In F mode, the number of returned digits isn't based on `precision`: + // it's variable, and we don't mind how many digits we get as long as it + // isn't beyond the limit MAX_DIGITS. If it is, we expect that it's only + // one digit too long, or else we'd have spotted the problem in advance and + // flipped into E mode already. + if (output.ndigits > MAX_DIGITS) { + LIBC_ASSERT(output.ndigits == MAX_DIGITS + 1); + need_reround = true; + } + } + + if (need_reround) { + // If either of the branches above decided that we had one digit too many, + // we must now shorten the digit buffer by one. But we can't just truncate: + // we need to make sure the remaining n-1 digits are correctly rounded, as + // if we'd rounded just once from the original `flt_quotient`. + // + // In directed rounding modes this can't go wrong. If you had a real number + // x, and the first rounding produced floor(x), then the second rounding + // wants floor(x/10), and it doesn't matter if you actually compute + // floor(floor(x)/10): the result is the same, because each rounding + // boundary in the second rounding aligns with one in the first rounding, + // which nothing could have crossed. Similarly for rounding away from zero, + // with 'floor' replaced with 'ceil' throughout. + // + // In rounding to nearest, the danger is in the boundary case where the + // final digit of the original output is 5. Then if we just rerounded the + // digit string to remove the last digit, it would look like an exact + // halfway case, and we'd break the tie by choosing the even one of the two + // outputs. But if the original value before the first rounding was on one + // side or the other of 5, then that supersedes the 'round to even' tie + // break. So we need to consult `round_dir` from above, which tells us + // which way (if either) the value was adjusted during the first rounding. + // Effectively, we treat the last digit as 5+ε or 5-ε. + // + // To make this work in both directed modes and round-to-nearest mode + // without having to look up the rounding direction, a simple rule is: take + // account of round_dir if and only if the round digit (the one we're + // removing when shortening the buffer) is 5. In directed rounding modes + // this makes no difference. + + // Extract the two relevant digits. round_digit is the one we're removing; + // new_low_digit is the last one we're keeping, so we need to know if it's + // even or odd to handle exact tie cases (when round_dir == 0). + --output.ndigits; + int round_digit = internal::b36_char_to_int(output.digits[output.ndigits]); + int new_low_digit = + output.ndigits == 0 + ? 0 + : internal::b36_char_to_int(output.digits[output.ndigits - 1]); + + // Make a binary number that we can pass to `fputil::rounding_direction`. + // We put new_low_digit at bit 8, and imagine that we're rounding away the + // bottom 8 bits. Therefore round_digit must be "just below" bit 8, in the + // sense that we set the bottom 8 bits to (256/10 * round_digit) so that + // round_digit==5 corresponds to the binary half-way case of 0x80. + // + // Then we adjust by +1 or -1 based on round_dir if the round digit is 5, + // as described above. + // + // The subexpression `(round_digit * 0x19a) >> 4` is computing the + // expression (256/10 * round_digit) mentioned above, accurately enough to + // map 5 to exactly 128 but avoiding an integer division (for platforms + // where it's slow, e.g. not in hardware). + LIBC_NAMESPACE::UInt<64> round_word = (new_low_digit * 256) + + ((round_digit * 0x19a) >> 4) + + (round_digit == 5 ? -round_dir : 0); + + // Now we can call the existing binary rounding helper function, which + // takes account of the rounding mode. + if (fputil::rounding_direction(round_word, 8, flt_quotient.sign) > 0) { + // If that returned a positive answer, we must round the number up. + // + // The number is already in decimal, so we need to increment it one digit + // at a time. (A bit painful, but better than going back to the integer + // we made it from and doing the decimal conversion all over again.) + for (size_t i = output.ndigits; i-- > 0;) { + if (output.digits[i] != '9') { + output.digits[i] = static_cast(internal::int_to_b36_char( + internal::b36_char_to_int(output.digits[i]) + 1)); + break; + } else { + output.digits[i] = '0'; + } + } + } + } + + return output; +} + +template +LIBC_INLINE int +convert_float_inner(Writer *writer, const FormatSection &to_conv, + int32_t fraction_len, int exponent, StorageType mantissa, + Sign sign, ConversionType ctype) { + // If to_conv doesn't specify a precision, the precision defaults to 6. + unsigned precision = to_conv.precision < 0 ? 6 : to_conv.precision; + + // Decide if we're displaying a sign character, depending on the format flags + // and whether the input is negative. + char sign_char = 0; + if (sign.is_neg()) + sign_char = '-'; + else if ((to_conv.flags & FormatFlags::FORCE_SIGN) == FormatFlags::FORCE_SIGN) + sign_char = '+'; // FORCE_SIGN has precedence over SPACE_PREFIX + else if ((to_conv.flags & FormatFlags::SPACE_PREFIX) == + FormatFlags::SPACE_PREFIX) + sign_char = ' '; + + // Prepare the input to decimal_digits(). + DigitsInput input(fraction_len, mantissa, exponent, sign); + + // Call decimal_digits() in a different way, based on whether the format + // character is 'e', 'f', or 'g'. After this loop we expect to have filled + // in the following variables: + + // The decimal digits, and the exponent of the topmost one. + DigitsOutput output; + // The start and end of the digit string we're displaying, as indices into + // `output.digits`. The indices may be out of bounds in either direction, in + // which case digits beyond the bounds of the buffer should be displayed as + // zeroes. + // + // As usual, the index 'start' is included, and 'limit' is not. + int start, limit; + // The index of the digit that we display a decimal point immediately after. + // Again, represented as an index in `output.digits`, and may be out of + // bounds. + int pointpos; + // Whether we need to display an "e+NNN" exponent suffix at all. + bool show_exponent; + + switch (ctype) { + case ConversionType::E: + // In E mode, we display one digit more than the specified precision + // (`%.6e` means six digits _after_ the decimal point, like 1.123456e+00). + // + // Also, bound the number of digits we request at MAX_DIGITS. + output = decimal_digits(input, cpp::min(precision + 1, MAX_DIGITS), true); + + // We display digits from the start of the buffer, and always output + // `precision+1` of them (which will append zeroes if the user requested + // more than MAX_DIGITS). + start = 0; + limit = precision + 1; + + // The decimal point is always after the first digit of the buffer. + pointpos = start; + + // The exponent is always displayed explicitly. + show_exponent = true; + break; + case ConversionType::F: + // In F mode, we provide decimal_digits() with the unmodified input + // precision, and let it give us as many digits as we can. + output = decimal_digits(input, precision, false); + + // Initialize (start, limit) to display everything from the first nonzero + // digit (necessarily at the start of the output buffer) to the digit at + // the correct distance after the decimal point. + start = 0; + limit = 1 + output.exponent + precision; + + // But we must display at least one digit _before_ the decimal point, i.e. + // at least precision+1 digits in total. So if we're not already doing + // that, we must correct those values. + if (limit <= int(precision)) + start -= precision + 1 - limit; + + // The decimal point appears precisely 'precision' digits before the end of + // the digits we output. + pointpos = limit - 1 - precision; + + // The exponent is never displayed. + show_exponent = false; + break; + case ConversionType::G: + // In G mode, the precision says exactly how many significant digits you + // want. (In that respect it's subtly unlike E mode: %.6g means six digits + // _including_ the one before the point, whereas %.6e means six digits + // _excluding_ that one.) + // + // Also, a precision of 0 is treated the same as 1. + precision = cpp::max(precision, 1u); + output = decimal_digits(input, cpp::min(precision, MAX_DIGITS), true); + + // As in E mode, we default to displaying precisely the digits in the + // output buffer. + start = 0; + limit = precision; + + // If we're not in ALTERNATE_FORM mode, trailing zeroes on the mantissa are + // removed (although not to the extent of leaving no digits at all - if the + // entire output mantissa is all 0 then we keep a single zero digit). + if (!(to_conv.flags & FormatFlags::ALTERNATE_FORM)) { + // Start by removing trailing zeroes that were outside the buffer + // entirely. + limit = cpp::min(limit, int(output.ndigits)); + + // Then check the digits in the buffer and remove as many as possible. + while (limit > 1 && output.digits[limit - 1] == '0') + limit--; + } + + // Decide whether to display in %e style with an explicit exponent, or %f + // style with the decimal point after the units place. + // + // %e mode is used to avoid an excessive number of leading zeroes after the + // decimal point but before the first nonzero digit (specifically, 0.0001 + // is fine as it is, but 0.00001 prints as 1e-5), and also to avoid adding + // trailing zeroes if the last digit in the buffer is still higher than the + // units place. + // + // output.exponent is an int whereas precision is unsigned, so we must + // check output.exponent >= 0 before comparing it against precision to + // prevent a negative exponent from wrapping round to a large unsigned int. + if ((output.exponent >= 0 && output.exponent >= int(precision)) || + output.exponent < -4) { + // Display in %e style, so the point goes after the first digit and the + // exponent is shown. + pointpos = start; + show_exponent = true; + } else { + // Display in %f style, so the point goes at its true mathematical + // location and the exponent is not shown. + pointpos = output.exponent; + show_exponent = false; + + if (output.exponent < 0) { + // If the first digit is below the decimal point, add leading zeroes. + // (This _decreases_ start, because output.exponent is negative here.) + start += output.exponent; + } else if (limit <= output.exponent) { + // If the last digit is above the decimal point, add trailing zeroes. + // (This may involve putting back some zeroes that we trimmed in the + // loop above!) + limit = output.exponent + 1; + } + } + break; + } + + // Find out for sure whether we're displaying the decimal point, so that we + // can include it in the calculation of the total string length for padding. + // + // We never expect pointpos to be _before_ the start of the displayed range + // of digits. (If it had been, we'd have added leading zeroes.) But it might + // be beyond the end. + // + // We don't display the point if it appears immediately after the _last_ + // digit we display, except in ALTERNATE_FORM mode. + int last_point_digit = + (to_conv.flags & FormatFlags::ALTERNATE_FORM) ? limit - 1 : limit - 2; + bool show_point = pointpos <= last_point_digit; + + // Format the exponent suffix (e+NN, e-NN) into a buffer, or leave the buffer + // empty if we're not displaying one. + char expbuf[16]; // more than enough space for e+NNNN + size_t explen = 0; + if (show_exponent) { + const IntegerToString::WithSign> + expcvt{output.exponent}; + cpp::string_view expview = expcvt.view(); + expbuf[0] = internal::islower(to_conv.conv_name) ? 'e' : 'E'; + explen = expview.size() + 1; + __builtin_memcpy(expbuf + 1, expview.data(), expview.size()); + } + + // Now we know enough to work out the length of the unpadded output: + // * whether to write a sign + // * how many mantissa digits to write + // * whether to write a decimal point + // * the length of the trailing exponent string. + size_t unpadded_len = + (sign_char != 0) + (limit - start) + show_point + explen; + + // Work out how much padding is needed. + size_t min_width = to_conv.min_width > 0 ? to_conv.min_width : 0; + size_t padding_amount = cpp::max(min_width, unpadded_len) - unpadded_len; + + // Work out what the padding looks like and where it appears. + enum class Padding { + LeadingSpace, // spaces at the start of the string + Zero, // zeroes between sign and mantissa + TrailingSpace, // spaces at the end of the string + } padding = Padding::LeadingSpace; + // The '-' flag for left-justification takes priority over the '0' flag + if (to_conv.flags & FormatFlags::LEFT_JUSTIFIED) + padding = Padding::TrailingSpace; + else if (to_conv.flags & FormatFlags::LEADING_ZEROES) + padding = Padding::Zero; + + // Finally, write all the output! + + // Leading-space padding, if any + if (padding == Padding::LeadingSpace) + RET_IF_RESULT_NEGATIVE(writer->write(' ', padding_amount)); + + // Sign, if any + if (sign_char) + RET_IF_RESULT_NEGATIVE(writer->write(sign_char)); + + // Zero padding, if any + if (padding == Padding::Zero) + RET_IF_RESULT_NEGATIVE(writer->write('0', padding_amount)); + + // Mantissa digits, maybe with a decimal point + for (int pos = start; pos < limit; ++pos) { + if (pos >= 0 && pos < int(output.ndigits)) { + // Fetch a digit from the buffer + RET_IF_RESULT_NEGATIVE(writer->write(output.digits[pos])); + } else { + // This digit is outside the buffer, so write a zero + RET_IF_RESULT_NEGATIVE(writer->write('0')); + } + + // Show the decimal point, if this is the digit it comes after + if (show_point && pos == pointpos) + RET_IF_RESULT_NEGATIVE(writer->write(DECIMAL_POINT)); + } + + // Exponent + RET_IF_RESULT_NEGATIVE(writer->write(cpp::string_view(expbuf, explen))); + + // Trailing-space padding, if any + if (padding == Padding::TrailingSpace) + RET_IF_RESULT_NEGATIVE(writer->write(' ', padding_amount)); + + return WRITE_OK; +} + +template , int> = 0> +LIBC_INLINE int +convert_float_typed(Writer *writer, const FormatSection &to_conv, + fputil::FPBits float_bits, ConversionType ctype) { + return convert_float_inner(writer, to_conv, float_bits.FRACTION_LEN, + float_bits.get_explicit_exponent(), + float_bits.get_explicit_mantissa(), + float_bits.sign(), ctype); +} + +template +LIBC_INLINE int convert_float_outer(Writer *writer, + const FormatSection &to_conv, + ConversionType ctype) { + if (to_conv.length_modifier == LengthModifier::L) { + fputil::FPBits::StorageType float_raw = to_conv.conv_val_raw; + fputil::FPBits float_bits(float_raw); + if (!float_bits.is_inf_or_nan()) { + return convert_float_typed(writer, to_conv, float_bits, + ctype); + } + } else { + fputil::FPBits::StorageType float_raw = + static_cast::StorageType>(to_conv.conv_val_raw); + fputil::FPBits float_bits(float_raw); + if (!float_bits.is_inf_or_nan()) { + return convert_float_typed(writer, to_conv, float_bits, ctype); + } + } + + return convert_inf_nan(writer, to_conv); +} + +template , int> = 0> +LIBC_INLINE int convert_float_decimal_typed(Writer *writer, + const FormatSection &to_conv, + fputil::FPBits float_bits) { + return convert_float_typed(writer, to_conv, float_bits, ConversionType::F); +} + +template , int> = 0> +LIBC_INLINE int convert_float_dec_exp_typed(Writer *writer, + const FormatSection &to_conv, + fputil::FPBits float_bits) { + return convert_float_typed(writer, to_conv, float_bits, ConversionType::E); +} + +template , int> = 0> +LIBC_INLINE int convert_float_dec_auto_typed(Writer *writer, + const FormatSection &to_conv, + fputil::FPBits float_bits) { + return convert_float_typed(writer, to_conv, float_bits, ConversionType::G); +} + +template +LIBC_INLINE int convert_float_decimal(Writer *writer, + const FormatSection &to_conv) { + return convert_float_outer(writer, to_conv, ConversionType::F); +} + +template +LIBC_INLINE int convert_float_dec_exp(Writer *writer, + const FormatSection &to_conv) { + return convert_float_outer(writer, to_conv, ConversionType::E); +} + +template +LIBC_INLINE int convert_float_dec_auto(Writer *writer, + const FormatSection &to_conv) { + return convert_float_outer(writer, to_conv, ConversionType::G); +} + +} // namespace printf_core +} // namespace LIBC_NAMESPACE_DECL + +#endif // LLVM_LIBC_SRC_STDIO_PRINTF_CORE_FLOAT_DEC_CONVERTER_LIMITED_H diff --git a/system/lib/llvm-libc/src/stdio/printf_core/float_hex_converter.h b/system/lib/llvm-libc/src/stdio/printf_core/float_hex_converter.h index b264b5cf20728..16592e7bac932 100644 --- a/system/lib/llvm-libc/src/stdio/printf_core/float_hex_converter.h +++ b/system/lib/llvm-libc/src/stdio/printf_core/float_hex_converter.h @@ -25,7 +25,8 @@ namespace LIBC_NAMESPACE_DECL { namespace printf_core { -LIBC_INLINE int convert_float_hex_exp(Writer *writer, +template +LIBC_INLINE int convert_float_hex_exp(Writer *writer, const FormatSection &to_conv) { using LDBits = fputil::FPBits; using StorageType = LDBits::StorageType; diff --git a/system/lib/llvm-libc/src/stdio/printf_core/float_inf_nan_converter.h b/system/lib/llvm-libc/src/stdio/printf_core/float_inf_nan_converter.h index 3e41612e21c9f..ce31d7ae55499 100644 --- a/system/lib/llvm-libc/src/stdio/printf_core/float_inf_nan_converter.h +++ b/system/lib/llvm-libc/src/stdio/printf_core/float_inf_nan_converter.h @@ -24,7 +24,9 @@ namespace printf_core { using StorageType = fputil::FPBits::StorageType; -LIBC_INLINE int convert_inf_nan(Writer *writer, const FormatSection &to_conv) { +template +LIBC_INLINE int convert_inf_nan(Writer *writer, + const FormatSection &to_conv) { // All of the letters will be defined relative to variable a, which will be // the appropriate case based on the case of the conversion. bool is_negative; diff --git a/system/lib/llvm-libc/src/stdio/printf_core/int_converter.h b/system/lib/llvm-libc/src/stdio/printf_core/int_converter.h index d0af229f89be5..11234c32ce997 100644 --- a/system/lib/llvm-libc/src/stdio/printf_core/int_converter.h +++ b/system/lib/llvm-libc/src/stdio/printf_core/int_converter.h @@ -61,7 +61,9 @@ num_to_strview(uintmax_t num, cpp::span bufref, char conv_name) { } // namespace details -LIBC_INLINE int convert_int(Writer *writer, const FormatSection &to_conv) { +template +LIBC_INLINE int convert_int(Writer *writer, + const FormatSection &to_conv) { static constexpr size_t BITS_IN_BYTE = 8; static constexpr size_t BITS_IN_NUM = sizeof(uintmax_t) * BITS_IN_BYTE; diff --git a/system/lib/llvm-libc/src/stdio/printf_core/parser.h b/system/lib/llvm-libc/src/stdio/printf_core/parser.h index acbbaa25b1c9d..cef9b1ae58fa0 100644 --- a/system/lib/llvm-libc/src/stdio/printf_core/parser.h +++ b/system/lib/llvm-libc/src/stdio/printf_core/parser.h @@ -25,7 +25,7 @@ #include "src/__support/fixed_point/fx_rep.h" #endif // LIBC_INTERNAL_PRINTF_HAS_FIXED_POINT #ifndef LIBC_COPT_PRINTF_DISABLE_STRERROR -#include "src/errno/libc_errno.h" +#include "src/__support/libc_errno.h" #endif // LIBC_COPT_PRINTF_DISABLE_STRERROR namespace LIBC_NAMESPACE_DECL { @@ -57,7 +57,8 @@ template using int_type_of_v = typename int_type_of::type; if (!temp.has_value()) { \ section.has_conv = false; \ } else { \ - dst = cpp::bit_cast>(temp.value()); \ + dst = static_cast( \ + cpp::bit_cast>(temp.value())); \ } \ } #else @@ -130,7 +131,7 @@ template class Parser { } else if (internal::isdigit(str[cur_pos])) { auto result = internal::strtointeger(str + cur_pos, 10); section.min_width = result.value; - cur_pos = cur_pos + result.parsed_len; + cur_pos = cur_pos + static_cast(result.parsed_len); } if (section.min_width < 0) { section.min_width = @@ -153,7 +154,7 @@ template class Parser { } else if (internal::isdigit(str[cur_pos])) { auto result = internal::strtointeger(str + cur_pos, 10); section.precision = result.value; - cur_pos = cur_pos + result.parsed_len; + cur_pos = cur_pos + static_cast(result.parsed_len); } } @@ -265,7 +266,8 @@ template class Parser { case ('m'): // %m is an odd conversion in that it doesn't consume an argument, it // just takes the current value of errno as its argument. - section.conv_val_raw = static_cast(libc_errno); + section.conv_val_raw = + static_cast::StorageType>(libc_errno); break; #endif // LIBC_COPT_PRINTF_DISABLE_STRERROR #ifndef LIBC_COPT_PRINTF_DISABLE_WRITE_INT @@ -356,7 +358,7 @@ template class Parser { } if (internal::isdigit(str[*local_pos])) { const auto result = internal::strtointeger(str + *local_pos, 10); - *local_pos += result.parsed_len; + *local_pos += static_cast(result.parsed_len); return {lm, static_cast(cpp::max(0, result.value))}; } return {lm, 0}; @@ -405,10 +407,10 @@ template class Parser { LIBC_INLINE size_t parse_index(size_t *local_pos) { if (internal::isdigit(str[*local_pos])) { auto result = internal::strtointeger(str + *local_pos, 10); - size_t index = result.value; - if (str[*local_pos + result.parsed_len] != '$') + size_t index = static_cast(result.value); + if (str[*local_pos + static_cast(result.parsed_len)] != '$') return 0; - *local_pos = 1 + result.parsed_len + *local_pos; + *local_pos = static_cast(1 + result.parsed_len) + *local_pos; return index; } return 0; diff --git a/system/lib/llvm-libc/src/stdio/printf_core/printf_main.cpp b/system/lib/llvm-libc/src/stdio/printf_core/printf_main.cpp deleted file mode 100644 index bd4a5a168bd23..0000000000000 --- a/system/lib/llvm-libc/src/stdio/printf_core/printf_main.cpp +++ /dev/null @@ -1,43 +0,0 @@ -//===-- Starting point for printf -------------------------------*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#include "src/stdio/printf_core/printf_main.h" - -#include "src/__support/arg_list.h" -#include "src/__support/macros/config.h" -#include "src/stdio/printf_core/converter.h" -#include "src/stdio/printf_core/core_structs.h" -#include "src/stdio/printf_core/parser.h" -#include "src/stdio/printf_core/writer.h" - -#include - -namespace LIBC_NAMESPACE_DECL { -namespace printf_core { - -int printf_main(Writer *writer, const char *__restrict str, - internal::ArgList &args) { - Parser parser(str, args); - int result = 0; - for (FormatSection cur_section = parser.get_next_section(); - !cur_section.raw_string.empty(); - cur_section = parser.get_next_section()) { - if (cur_section.has_conv) - result = convert(writer, cur_section); - else - result = writer->write(cur_section.raw_string); - - if (result < 0) - return result; - } - - return writer->get_chars_written(); -} - -} // namespace printf_core -} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/stdio/printf_core/printf_main.h b/system/lib/llvm-libc/src/stdio/printf_core/printf_main.h index 3e73bf36e0e30..57f29858d5298 100644 --- a/system/lib/llvm-libc/src/stdio/printf_core/printf_main.h +++ b/system/lib/llvm-libc/src/stdio/printf_core/printf_main.h @@ -11,6 +11,9 @@ #include "src/__support/arg_list.h" #include "src/__support/macros/config.h" +#include "src/stdio/printf_core/converter.h" +#include "src/stdio/printf_core/core_structs.h" +#include "src/stdio/printf_core/parser.h" #include "src/stdio/printf_core/writer.h" #include @@ -18,8 +21,25 @@ namespace LIBC_NAMESPACE_DECL { namespace printf_core { -int printf_main(Writer *writer, const char *__restrict str, - internal::ArgList &args); +template +int printf_main(Writer *writer, const char *__restrict str, + internal::ArgList &args) { + Parser parser(str, args); + int result = 0; + for (FormatSection cur_section = parser.get_next_section(); + !cur_section.raw_string.empty(); + cur_section = parser.get_next_section()) { + if (cur_section.has_conv) + result = convert(writer, cur_section); + else + result = writer->write(cur_section.raw_string); + + if (result < 0) + return result; + } + + return writer->get_chars_written(); +} } // namespace printf_core } // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/stdio/printf_core/ptr_converter.h b/system/lib/llvm-libc/src/stdio/printf_core/ptr_converter.h index bf84718dfe6a8..c2a74e3043e6f 100644 --- a/system/lib/llvm-libc/src/stdio/printf_core/ptr_converter.h +++ b/system/lib/llvm-libc/src/stdio/printf_core/ptr_converter.h @@ -18,7 +18,9 @@ namespace LIBC_NAMESPACE_DECL { namespace printf_core { -LIBC_INLINE int convert_pointer(Writer *writer, const FormatSection &to_conv) { +template +LIBC_INLINE int convert_pointer(Writer *writer, + const FormatSection &to_conv) { FormatSection new_conv = to_conv; if (to_conv.conv_val_ptr == nullptr) { diff --git a/system/lib/llvm-libc/src/stdio/printf_core/strerror_converter.h b/system/lib/llvm-libc/src/stdio/printf_core/strerror_converter.h index 2902fd37c31ae..2cd6df0c01d45 100644 --- a/system/lib/llvm-libc/src/stdio/printf_core/strerror_converter.h +++ b/system/lib/llvm-libc/src/stdio/printf_core/strerror_converter.h @@ -19,7 +19,9 @@ namespace LIBC_NAMESPACE_DECL { namespace printf_core { -LIBC_INLINE int convert_strerror(Writer *writer, const FormatSection &to_conv) { +template +LIBC_INLINE int convert_strerror(Writer *writer, + const FormatSection &to_conv) { FormatSection new_conv = to_conv; const int error_num = static_cast(to_conv.conv_val_raw); diff --git a/system/lib/llvm-libc/src/stdio/printf_core/string_converter.h b/system/lib/llvm-libc/src/stdio/printf_core/string_converter.h index 1f36d51124107..74c9f598210f7 100644 --- a/system/lib/llvm-libc/src/stdio/printf_core/string_converter.h +++ b/system/lib/llvm-libc/src/stdio/printf_core/string_converter.h @@ -20,7 +20,9 @@ namespace LIBC_NAMESPACE_DECL { namespace printf_core { -LIBC_INLINE int convert_string(Writer *writer, const FormatSection &to_conv) { +template +LIBC_INLINE int convert_string(Writer *writer, + const FormatSection &to_conv) { size_t string_len = 0; const char *str_ptr = reinterpret_cast(to_conv.conv_val_ptr); diff --git a/system/lib/llvm-libc/src/stdio/printf_core/vasprintf_internal.h b/system/lib/llvm-libc/src/stdio/printf_core/vasprintf_internal.h index 0e446f856e438..9d46617da7751 100644 --- a/system/lib/llvm-libc/src/stdio/printf_core/vasprintf_internal.h +++ b/system/lib/llvm-libc/src/stdio/printf_core/vasprintf_internal.h @@ -19,8 +19,9 @@ namespace LIBC_NAMESPACE_DECL { namespace printf_core { LIBC_INLINE int resize_overflow_hook(cpp::string_view new_str, void *target) { - printf_core::WriteBuffer *wb = - reinterpret_cast(target); + WriteBuffer::value> *wb = + reinterpret_cast< + WriteBuffer::value> *>(target); size_t new_size = new_str.size() + wb->buff_cur; const bool isBuffOnStack = (wb->buff == wb->init_buff); char *new_buff = static_cast( @@ -45,9 +46,9 @@ constexpr size_t DEFAULT_BUFFER_SIZE = 200; LIBC_INLINE int vasprintf_internal(char **ret, const char *__restrict format, internal::ArgList args) { char init_buff_on_stack[DEFAULT_BUFFER_SIZE]; - printf_core::WriteBuffer wb(init_buff_on_stack, DEFAULT_BUFFER_SIZE, - resize_overflow_hook); - printf_core::Writer writer(&wb); + printf_core::WriteBuffer::value> wb( + init_buff_on_stack, DEFAULT_BUFFER_SIZE, resize_overflow_hook); + printf_core::Writer writer(wb); auto ret_val = printf_core::printf_main(&writer, format, args); if (ret_val < 0) { diff --git a/system/lib/llvm-libc/src/stdio/printf_core/vfprintf_internal.h b/system/lib/llvm-libc/src/stdio/printf_core/vfprintf_internal.h index 3becfee71dd27..630de9d9d43dd 100644 --- a/system/lib/llvm-libc/src/stdio/printf_core/vfprintf_internal.h +++ b/system/lib/llvm-libc/src/stdio/printf_core/vfprintf_internal.h @@ -72,9 +72,9 @@ LIBC_INLINE int vfprintf_internal(::FILE *__restrict stream, internal::ArgList &args) { constexpr size_t BUFF_SIZE = 1024; char buffer[BUFF_SIZE]; - printf_core::WriteBuffer wb(buffer, BUFF_SIZE, &file_write_hook, - reinterpret_cast(stream)); - Writer writer(&wb); + printf_core::WriteBuffer::value> wb( + buffer, BUFF_SIZE, &file_write_hook, reinterpret_cast(stream)); + Writer writer(wb); internal::flockfile(stream); int retval = printf_main(&writer, format, args); int flushval = wb.overflow_write(""); diff --git a/system/lib/llvm-libc/src/stdio/printf_core/write_int_converter.h b/system/lib/llvm-libc/src/stdio/printf_core/write_int_converter.h index a47cb41cb3287..efcff278bd284 100644 --- a/system/lib/llvm-libc/src/stdio/printf_core/write_int_converter.h +++ b/system/lib/llvm-libc/src/stdio/printf_core/write_int_converter.h @@ -19,7 +19,8 @@ namespace LIBC_NAMESPACE_DECL { namespace printf_core { -LIBC_INLINE int convert_write_int(Writer *writer, +template +LIBC_INLINE int convert_write_int(Writer *writer, const FormatSection &to_conv) { #ifndef LIBC_COPT_PRINTF_NO_NULLPTR_CHECKS diff --git a/system/lib/llvm-libc/src/stdio/printf_core/writer.cpp b/system/lib/llvm-libc/src/stdio/printf_core/writer.cpp deleted file mode 100644 index d1cf85df1c8f8..0000000000000 --- a/system/lib/llvm-libc/src/stdio/printf_core/writer.cpp +++ /dev/null @@ -1,46 +0,0 @@ -//===-- Writer definition for printf ----------------------------*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#include "writer.h" -#include "src/__support/CPP/string_view.h" -#include "src/__support/macros/config.h" -#include "src/stdio/printf_core/core_structs.h" -#include "src/string/memory_utils/inline_memset.h" -#include - -namespace LIBC_NAMESPACE_DECL { -namespace printf_core { - -int Writer::pad(char new_char, size_t length) { - // First, fill as much of the buffer as possible with the padding char. - size_t written = 0; - const size_t buff_space = wb->buff_len - wb->buff_cur; - // ASSERT: length > buff_space - if (buff_space > 0) { - inline_memset(wb->buff + wb->buff_cur, new_char, buff_space); - wb->buff_cur += buff_space; - written = buff_space; - } - - // Next, overflow write the rest of length using the mini_buff. - constexpr size_t MINI_BUFF_SIZE = 64; - char mini_buff[MINI_BUFF_SIZE]; - inline_memset(mini_buff, new_char, MINI_BUFF_SIZE); - cpp::string_view mb_string_view(mini_buff, MINI_BUFF_SIZE); - while (written + MINI_BUFF_SIZE < length) { - int result = wb->overflow_write(mb_string_view); - if (result != WRITE_OK) - return result; - written += MINI_BUFF_SIZE; - } - cpp::string_view mb_substr = mb_string_view.substr(0, length - written); - return wb->overflow_write(mb_substr); -} - -} // namespace printf_core -} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/stdio/printf_core/writer.h b/system/lib/llvm-libc/src/stdio/printf_core/writer.h index 5526a478ea620..1d4734a51b9b8 100644 --- a/system/lib/llvm-libc/src/stdio/printf_core/writer.h +++ b/system/lib/llvm-libc/src/stdio/printf_core/writer.h @@ -21,12 +21,24 @@ namespace LIBC_NAMESPACE_DECL { namespace printf_core { -struct WriteBuffer { - enum class WriteMode { - FILL_BUFF_AND_DROP_OVERFLOW, - FLUSH_TO_STREAM, - RESIZE_AND_FILL_BUFF, - }; +enum class WriteMode { + FILL_BUFF_AND_DROP_OVERFLOW, + FLUSH_TO_STREAM, + RESIZE_AND_FILL_BUFF, + RUNTIME_DISPATCH, +}; + +// Helper to omit the template argument if we are using runtime dispatch and +// avoid multiple copies of the converter functions. +template struct Mode { +#ifdef LIBC_COPT_PRINTF_RUNTIME_DISPATCH + static constexpr WriteMode value = WriteMode::RUNTIME_DISPATCH; +#else + static constexpr WriteMode value = write_mode; +#endif +}; + +template struct WriteBuffer { using StreamWriter = int (*)(cpp::string_view, void *); char *buff; const char *init_buff; // for checking when resize. @@ -35,23 +47,26 @@ struct WriteBuffer { // The stream writer will be called when the buffer is full. It will be passed // string_views to write to the stream. - StreamWriter stream_writer; + const StreamWriter stream_writer; void *output_target; - WriteMode write_mode; - LIBC_INLINE WriteBuffer(char *Buff, size_t Buff_len, StreamWriter hook, + // The current writing mode in case the user wants runtime dispatch of the + // stream writer with function pointers. + [[maybe_unused]] WriteMode write_mode_; + + LIBC_INLINE WriteBuffer(char *buff, size_t buff_len, StreamWriter hook, void *target) - : buff(Buff), init_buff(Buff), buff_len(Buff_len), stream_writer(hook), - output_target(target), write_mode(WriteMode::FLUSH_TO_STREAM) {} + : buff(buff), init_buff(buff), buff_len(buff_len), stream_writer(hook), + output_target(target), write_mode_(WriteMode::FLUSH_TO_STREAM) {} - LIBC_INLINE WriteBuffer(char *Buff, size_t Buff_len) - : buff(Buff), init_buff(Buff), buff_len(Buff_len), stream_writer(nullptr), + LIBC_INLINE WriteBuffer(char *buff, size_t buff_len) + : buff(buff), init_buff(buff), buff_len(buff_len), stream_writer(nullptr), output_target(nullptr), - write_mode(WriteMode::FILL_BUFF_AND_DROP_OVERFLOW) {} + write_mode_(WriteMode::FILL_BUFF_AND_DROP_OVERFLOW) {} - LIBC_INLINE WriteBuffer(char *Buff, size_t Buff_len, StreamWriter hook) - : buff(Buff), init_buff(Buff), buff_len(Buff_len), stream_writer(hook), - output_target(this), write_mode(WriteMode::RESIZE_AND_FILL_BUFF) {} + LIBC_INLINE WriteBuffer(char *buff, size_t buff_len, StreamWriter hook) + : buff(buff), init_buff(buff), buff_len(buff_len), stream_writer(hook), + output_target(this), write_mode_(WriteMode::RESIZE_AND_FILL_BUFF) {} LIBC_INLINE int flush_to_stream(cpp::string_view new_str) { if (buff_cur > 0) { @@ -92,40 +107,68 @@ struct WriteBuffer { // this with an empty string will flush the buffer if relevant. LIBC_INLINE int overflow_write(cpp::string_view new_str) { - switch (write_mode) { - case WriteMode::FILL_BUFF_AND_DROP_OVERFLOW: + if constexpr (write_mode == WriteMode::RUNTIME_DISPATCH) { + if (write_mode_ == WriteMode::FILL_BUFF_AND_DROP_OVERFLOW) + return fill_remaining_to_buff(new_str); + else if (write_mode_ == WriteMode::FLUSH_TO_STREAM) + return flush_to_stream(new_str); + else if (write_mode_ == WriteMode::RESIZE_AND_FILL_BUFF) + return resize_and_write(new_str); + } else if constexpr (write_mode == WriteMode::FILL_BUFF_AND_DROP_OVERFLOW) { return fill_remaining_to_buff(new_str); - case WriteMode::FLUSH_TO_STREAM: + } else if constexpr (write_mode == WriteMode::FLUSH_TO_STREAM) { return flush_to_stream(new_str); - case WriteMode::RESIZE_AND_FILL_BUFF: + } else if constexpr (write_mode == WriteMode::RESIZE_AND_FILL_BUFF) { return resize_and_write(new_str); } __builtin_unreachable(); } }; -class Writer final { - WriteBuffer *wb; +template class Writer final { + WriteBuffer &wb; int chars_written = 0; - // This is a separate, non-inlined function so that the inlined part of the - // write function is shorter. - int pad(char new_char, size_t length); + LIBC_INLINE int pad(char new_char, size_t length) { + // First, fill as much of the buffer as possible with the padding char. + size_t written = 0; + const size_t buff_space = wb.buff_len - wb.buff_cur; + // ASSERT: length > buff_space + if (buff_space > 0) { + inline_memset(wb.buff + wb.buff_cur, new_char, buff_space); + wb.buff_cur += buff_space; + written = buff_space; + } + + // Next, overflow write the rest of length using the mini_buff. + constexpr size_t MINI_BUFF_SIZE = 64; + char mini_buff[MINI_BUFF_SIZE]; + inline_memset(mini_buff, new_char, MINI_BUFF_SIZE); + cpp::string_view mb_string_view(mini_buff, MINI_BUFF_SIZE); + while (written + MINI_BUFF_SIZE < length) { + int result = wb.overflow_write(mb_string_view); + if (result != WRITE_OK) + return result; + written += MINI_BUFF_SIZE; + } + cpp::string_view mb_substr = mb_string_view.substr(0, length - written); + return wb.overflow_write(mb_substr); + } public: - LIBC_INLINE Writer(WriteBuffer *WB) : wb(WB) {} + LIBC_INLINE Writer(WriteBuffer &wb) : wb(wb) {} // Takes a string, copies it into the buffer if there is space, else passes it // to the overflow mechanism to be handled separately. LIBC_INLINE int write(cpp::string_view new_string) { chars_written += static_cast(new_string.size()); - if (LIBC_LIKELY(wb->buff_cur + new_string.size() <= wb->buff_len)) { - inline_memcpy(wb->buff + wb->buff_cur, new_string.data(), + if (LIBC_LIKELY(wb.buff_cur + new_string.size() <= wb.buff_len)) { + inline_memcpy(wb.buff + wb.buff_cur, new_string.data(), new_string.size()); - wb->buff_cur += new_string.size(); + wb.buff_cur += new_string.size(); return WRITE_OK; } - return wb->overflow_write(new_string); + return wb.overflow_write(new_string); } // Takes a char and a length, memsets the next length characters of the buffer @@ -134,9 +177,10 @@ class Writer final { LIBC_INLINE int write(char new_char, size_t length) { chars_written += static_cast(length); - if (LIBC_LIKELY(wb->buff_cur + length <= wb->buff_len)) { - inline_memset(wb->buff + wb->buff_cur, new_char, length); - wb->buff_cur += length; + if (LIBC_LIKELY(wb.buff_cur + length <= wb.buff_len)) { + inline_memset(wb.buff + wb.buff_cur, static_cast(new_char), + length); + wb.buff_cur += length; return WRITE_OK; } return pad(new_char, length); @@ -146,18 +190,26 @@ class Writer final { // to the overflow mechanism to be handled separately. LIBC_INLINE int write(char new_char) { chars_written += 1; - if (LIBC_LIKELY(wb->buff_cur + 1 <= wb->buff_len)) { - wb->buff[wb->buff_cur] = new_char; - wb->buff_cur += 1; + if (LIBC_LIKELY(wb.buff_cur + 1 <= wb.buff_len)) { + wb.buff[wb.buff_cur] = new_char; + wb.buff_cur += 1; return WRITE_OK; } cpp::string_view char_string_view(&new_char, 1); - return wb->overflow_write(char_string_view); + return wb.overflow_write(char_string_view); } LIBC_INLINE int get_chars_written() { return chars_written; } }; +// Class-template auto deduction helpers. +Writer(WriteBuffer) + -> Writer; +Writer(WriteBuffer) + -> Writer; +Writer(WriteBuffer) + -> Writer; + } // namespace printf_core } // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/stdlib/a64l.cpp b/system/lib/llvm-libc/src/stdlib/a64l.cpp new file mode 100644 index 0000000000000..84be2d208f7d7 --- /dev/null +++ b/system/lib/llvm-libc/src/stdlib/a64l.cpp @@ -0,0 +1,65 @@ +//===-- Implementation of a64l --------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/stdlib/a64l.h" +#include "hdr/types/size_t.h" +#include "src/__support/common.h" +#include "src/__support/ctype_utils.h" +#include "src/__support/macros/config.h" + +#include + +namespace LIBC_NAMESPACE_DECL { + +// I'm not sure this should go in ctype_utils since the specific ordering of +// base64 is so very implementation specific, and also this set is unusual. +// Returns -1 on any char without a specified value. +constexpr static int32_t b64_char_to_int(char ch) { + // from the standard: "The characters used to represent digits are '.' (dot) + // for 0, '/' for 1, '0' through '9' for [2,11], 'A' through 'Z' for [12,37], + // and 'a' through 'z' for [38,63]." + if (ch == '.') + return 0; + if (ch == '/') + return 1; + + // handle the case of an unspecified char. + if (!internal::isalnum(ch)) + return -1; + + bool is_lower = internal::islower(ch); + // add 2 to account for '.' and '/', then b36_char_to_int is case insensitive + // so add case sensitivity back. + return internal::b36_char_to_int(ch) + 2 + (is_lower ? 26 : 0); +} + +// This function takes a base 64 string and writes it to the low 32 bits of a +// long. +// TODO: use LIBC_ADD_NULL_CHECKS for checking if the input is a null pointer. +LLVM_LIBC_FUNCTION(long, a64l, (const char *s)) { + // the standard says to only use up to 6 characters. + constexpr size_t MAX_LENGTH = 6; + int32_t result = 0; + + for (size_t i = 0; i < MAX_LENGTH && s[i] != '\0'; ++i) { + int32_t cur_val = b64_char_to_int(s[i]); + // The standard says what happens on an unspecified character is undefined, + // here we treat it as the end of the string. + if (cur_val == -1) + break; + + // the first digit is the least significant, so for each subsequent digit we + // shift it more. 6 bits since 2^6 = 64 + result += (cur_val << (6 * i)); + } + + // standard says to sign extend from 32 bits. + return static_cast(result); +} + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/stdlib/a64l.h b/system/lib/llvm-libc/src/stdlib/a64l.h new file mode 100644 index 0000000000000..024be058f756c --- /dev/null +++ b/system/lib/llvm-libc/src/stdlib/a64l.h @@ -0,0 +1,20 @@ +//===-- Implementation header for a64l --------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_STDLIB_A64L_H +#define LLVM_LIBC_SRC_STDLIB_A64L_H + +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +long a64l(const char *s); + +} // namespace LIBC_NAMESPACE_DECL + +#endif // LLVM_LIBC_SRC_STDLIB_A64L_H diff --git a/system/lib/llvm-libc/src/stdlib/atof.cpp b/system/lib/llvm-libc/src/stdlib/atof.cpp index 18a65c67705d3..d0d8d211dea8c 100644 --- a/system/lib/llvm-libc/src/stdlib/atof.cpp +++ b/system/lib/llvm-libc/src/stdlib/atof.cpp @@ -8,9 +8,9 @@ #include "src/stdlib/atof.h" #include "src/__support/common.h" +#include "src/__support/libc_errno.h" #include "src/__support/macros/config.h" #include "src/__support/str_to_float.h" -#include "src/errno/libc_errno.h" namespace LIBC_NAMESPACE_DECL { diff --git a/system/lib/llvm-libc/src/stdlib/atoi.cpp b/system/lib/llvm-libc/src/stdlib/atoi.cpp index 9e46b53b1aa0b..420bbc8143d55 100644 --- a/system/lib/llvm-libc/src/stdlib/atoi.cpp +++ b/system/lib/llvm-libc/src/stdlib/atoi.cpp @@ -8,9 +8,9 @@ #include "src/stdlib/atoi.h" #include "src/__support/common.h" +#include "src/__support/libc_errno.h" #include "src/__support/macros/config.h" #include "src/__support/str_to_integer.h" -#include "src/errno/libc_errno.h" namespace LIBC_NAMESPACE_DECL { diff --git a/system/lib/llvm-libc/src/stdlib/atol.cpp b/system/lib/llvm-libc/src/stdlib/atol.cpp index 7f3414a4afdd2..e1110ffa449b0 100644 --- a/system/lib/llvm-libc/src/stdlib/atol.cpp +++ b/system/lib/llvm-libc/src/stdlib/atol.cpp @@ -8,9 +8,9 @@ #include "src/stdlib/atol.h" #include "src/__support/common.h" +#include "src/__support/libc_errno.h" #include "src/__support/macros/config.h" #include "src/__support/str_to_integer.h" -#include "src/errno/libc_errno.h" namespace LIBC_NAMESPACE_DECL { diff --git a/system/lib/llvm-libc/src/stdlib/atoll.cpp b/system/lib/llvm-libc/src/stdlib/atoll.cpp index 4f1a02ad8315b..063e817f9b790 100644 --- a/system/lib/llvm-libc/src/stdlib/atoll.cpp +++ b/system/lib/llvm-libc/src/stdlib/atoll.cpp @@ -8,9 +8,9 @@ #include "src/stdlib/atoll.h" #include "src/__support/common.h" +#include "src/__support/libc_errno.h" #include "src/__support/macros/config.h" #include "src/__support/str_to_integer.h" -#include "src/errno/libc_errno.h" namespace LIBC_NAMESPACE_DECL { diff --git a/system/lib/llvm-libc/src/stdlib/l64a.cpp b/system/lib/llvm-libc/src/stdlib/l64a.cpp new file mode 100644 index 0000000000000..b5506c3e40c35 --- /dev/null +++ b/system/lib/llvm-libc/src/stdlib/l64a.cpp @@ -0,0 +1,64 @@ +//===-- Implementation of l64a --------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/stdlib/l64a.h" +#include "hdr/types/size_t.h" +#include "src/__support/common.h" +#include "src/__support/ctype_utils.h" +#include "src/__support/libc_assert.h" +#include "src/__support/macros/config.h" + +#include + +namespace LIBC_NAMESPACE_DECL { + +// the standard says to only use up to 6 characters. Null terminator is +// unnecessary, but we'll add it for ease-of-use. Also going from 48 -> 56 bits +// probably won't matter since it's likely 32-bit aligned anyways. +constexpr size_t MAX_BASE64_LENGTH = 6; +LIBC_THREAD_LOCAL char BASE64_BUFFER[MAX_BASE64_LENGTH + 1]; + +constexpr static char b64_int_to_char(uint32_t num) { + // from the standard: "The characters used to represent digits are '.' (dot) + // for 0, '/' for 1, '0' through '9' for [2,11], 'A' through 'Z' for [12,37], + // and 'a' through 'z' for [38,63]." + LIBC_ASSERT(num < 64); + if (num == 0) + return '.'; + if (num == 1) + return '/'; + if (num < 38) + return static_cast( + internal::toupper(internal::int_to_b36_char(num - 2))); + + // this tolower is technically unnecessary, but it provides safety if we + // change the default behavior of int_to_b36_char. Also the compiler + // completely elides it so there's no performance penalty, see: + // https://godbolt.org/z/o5ennv7fc + return static_cast( + internal::tolower(internal::int_to_b36_char(num - 2 - 26))); +} + +// This function takes a long and converts the low 32 bits of it into at most 6 +// characters. It's returned as a pointer to a static buffer. +LLVM_LIBC_FUNCTION(char *, l64a, (long value)) { + // static cast to uint32_t to get just the low 32 bits in a consistent way. + // The standard says negative values are undefined, so I'm just defining them + // to be treated as unsigned. + uint32_t cur_value = static_cast(value); + for (size_t i = 0; i < MAX_BASE64_LENGTH; ++i) { + uint32_t cur_char = cur_value % 64; + BASE64_BUFFER[i] = b64_int_to_char(cur_char); + cur_value /= 64; + } + + BASE64_BUFFER[MAX_BASE64_LENGTH] = '\0'; // force null termination. + return BASE64_BUFFER; +} + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/stdlib/l64a.h b/system/lib/llvm-libc/src/stdlib/l64a.h new file mode 100644 index 0000000000000..be01f04f2c3b5 --- /dev/null +++ b/system/lib/llvm-libc/src/stdlib/l64a.h @@ -0,0 +1,20 @@ +//===-- Implementation header for l64a --------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_STDLIB_l64a_H +#define LLVM_LIBC_SRC_STDLIB_l64a_H + +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +char *l64a(long value); + +} // namespace LIBC_NAMESPACE_DECL + +#endif // LLVM_LIBC_SRC_STDLIB_l64a_H diff --git a/system/lib/llvm-libc/src/stdlib/memalignment.cpp b/system/lib/llvm-libc/src/stdlib/memalignment.cpp new file mode 100644 index 0000000000000..f06e7bf0f2d0d --- /dev/null +++ b/system/lib/llvm-libc/src/stdlib/memalignment.cpp @@ -0,0 +1,25 @@ +//===-- Implementation for memalignment -------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/stdlib/memalignment.h" +#include "src/__support/CPP/bit.h" +#include "src/__support/common.h" +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(size_t, memalignment, (const void *p)) { + if (p == nullptr) + return 0; + + uintptr_t addr = reinterpret_cast(p); + + return size_t(1) << cpp::countr_zero(addr); +} + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/stdlib/memalignment.h b/system/lib/llvm-libc/src/stdlib/memalignment.h new file mode 100644 index 0000000000000..b7c7430af8c8e --- /dev/null +++ b/system/lib/llvm-libc/src/stdlib/memalignment.h @@ -0,0 +1,21 @@ +//===-- Implementation header for memalignment ------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_STDLIB_MEM_ALIGNMENT_H +#define LLVM_LIBC_SRC_STDLIB_MEM_ALIGNMENT_H + +#include "hdr/types/size_t.h" +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +size_t memalignment(const void *p); + +} // namespace LIBC_NAMESPACE_DECL + +#endif // LLVM_LIBC_SRC_STDLIB_LDIV_H diff --git a/system/lib/llvm-libc/src/stdlib/qsort_pivot.h b/system/lib/llvm-libc/src/stdlib/qsort_pivot.h index b27e74663d901..6507f956a74b0 100644 --- a/system/lib/llvm-libc/src/stdlib/qsort_pivot.h +++ b/system/lib/llvm-libc/src/stdlib/qsort_pivot.h @@ -9,7 +9,9 @@ #ifndef LLVM_LIBC_SRC_STDLIB_QSORT_PIVOT_H #define LLVM_LIBC_SRC_STDLIB_QSORT_PIVOT_H -#include // For size_t +#include "src/__support/macros/attributes.h" + +#include // For size_t namespace LIBC_NAMESPACE_DECL { namespace internal { @@ -22,7 +24,7 @@ constexpr size_t PSEUDO_MEDIAN_REC_THRESHOLD = 64; // This chooses a pivot by sampling an adaptive amount of points, approximating // the quality of a median of sqrt(n) elements. template -size_t choose_pivot(const A &array, const F &is_less) { +LIBC_INLINE size_t choose_pivot(const A &array, const F &is_less) { const size_t len = array.len(); if (len < 8) { @@ -47,8 +49,8 @@ size_t choose_pivot(const A &array, const F &is_less) { // recursion depth and overall sample from f(n) = 3*f(n/8) -> f(n) = // O(n^(log(3)/log(8))) ~= O(n^0.528) elements. template -size_t median3_rec(const A &array, size_t a, size_t b, size_t c, size_t n, - const F &is_less) { +LIBC_INLINE size_t median3_rec(const A &array, size_t a, size_t b, size_t c, + size_t n, const F &is_less) { if (n * 8 >= PSEUDO_MEDIAN_REC_THRESHOLD) { const size_t n8 = n / 8; a = median3_rec(array, a, a + (n8 * 4), a + (n8 * 7), n8, is_less); @@ -60,7 +62,8 @@ size_t median3_rec(const A &array, size_t a, size_t b, size_t c, size_t n, /// Calculates the median of 3 elements. template -size_t median3(const A &array, size_t a, size_t b, size_t c, const F &is_less) { +LIBC_INLINE size_t median3(const A &array, size_t a, size_t b, size_t c, + const F &is_less) { const void *a_ptr = array.get(a); const void *b_ptr = array.get(b); const void *c_ptr = array.get(c); diff --git a/system/lib/llvm-libc/src/stdlib/qsort_r.h b/system/lib/llvm-libc/src/stdlib/qsort_r.h index 51a459c3f7633..b0d8c57c75c78 100644 --- a/system/lib/llvm-libc/src/stdlib/qsort_r.h +++ b/system/lib/llvm-libc/src/stdlib/qsort_r.h @@ -14,10 +14,9 @@ namespace LIBC_NAMESPACE_DECL { -// This qsort_r uses the glibc argument ordering instead of the BSD argument -// ordering (which puts arg before the function pointer). Putting arg after the -// function pointer more closely matches the ordering for qsort_s, which is the -// standardized equivalent of qsort_r. +// This qsort_r uses the POSIX 1003.1-2024 argument ordering instead of the +// historical BSD argument ordering (which put arg before the function pointer). +// https://www.austingroupbugs.net/view.php?id=900 void qsort_r(void *array, size_t array_size, size_t elem_size, int (*compare)(const void *, const void *, void *), void *arg); diff --git a/system/lib/llvm-libc/src/stdlib/quick_sort.h b/system/lib/llvm-libc/src/stdlib/quick_sort.h index 9ab2830250018..8ba0098854d19 100644 --- a/system/lib/llvm-libc/src/stdlib/quick_sort.h +++ b/system/lib/llvm-libc/src/stdlib/quick_sort.h @@ -167,7 +167,9 @@ LIBC_INLINE void quick_sort_impl(A &array, const void *ancestor_pivot, } } -constexpr size_t ilog2(size_t n) { return cpp::bit_width(n) - 1; } +constexpr size_t ilog2(size_t n) { + return static_cast(cpp::bit_width(n)) - 1; +} template LIBC_INLINE void quick_sort(A &array, const F &is_less) { diff --git a/system/lib/llvm-libc/src/stdlib/str_from_util.h b/system/lib/llvm-libc/src/stdlib/str_from_util.h index 7f54bdf71a018..61e6ba24b3817 100644 --- a/system/lib/llvm-libc/src/stdlib/str_from_util.h +++ b/system/lib/llvm-libc/src/stdlib/str_from_util.h @@ -104,8 +104,8 @@ printf_core::FormatSection parse_format_string(const char *__restrict format, return section; } -template -int strfromfloat_convert(printf_core::Writer *writer, +template +int strfromfloat_convert(printf_core::Writer *writer, const printf_core::FormatSection §ion) { if (!section.has_conv) return writer->write(section.raw_string); diff --git a/system/lib/llvm-libc/src/stdlib/strfromd.cpp b/system/lib/llvm-libc/src/stdlib/strfromd.cpp index 4c51e4c5c8a01..f51e6d4c7f1df 100644 --- a/system/lib/llvm-libc/src/stdlib/strfromd.cpp +++ b/system/lib/llvm-libc/src/stdlib/strfromd.cpp @@ -19,8 +19,10 @@ LLVM_LIBC_FUNCTION(int, strfromd, printf_core::FormatSection section = internal::parse_format_string(format, fp); - printf_core::WriteBuffer wb(s, (n > 0 ? n - 1 : 0)); - printf_core::Writer writer(&wb); + printf_core::WriteBuffer::value> + wb(s, (n > 0 ? n - 1 : 0)); + printf_core::Writer writer(wb); int result = 0; if (section.has_conv) diff --git a/system/lib/llvm-libc/src/stdlib/strfromf.cpp b/system/lib/llvm-libc/src/stdlib/strfromf.cpp index ea98a69ee4d60..14dbfdb25bab6 100644 --- a/system/lib/llvm-libc/src/stdlib/strfromf.cpp +++ b/system/lib/llvm-libc/src/stdlib/strfromf.cpp @@ -19,8 +19,10 @@ LLVM_LIBC_FUNCTION(int, strfromf, printf_core::FormatSection section = internal::parse_format_string(format, fp); - printf_core::WriteBuffer wb(s, (n > 0 ? n - 1 : 0)); - printf_core::Writer writer(&wb); + printf_core::WriteBuffer::value> + wb(s, (n > 0 ? n - 1 : 0)); + printf_core::Writer writer(wb); int result = 0; if (section.has_conv) diff --git a/system/lib/llvm-libc/src/stdlib/strfroml.cpp b/system/lib/llvm-libc/src/stdlib/strfroml.cpp index d5bee7609f69c..12f22a8a2fb65 100644 --- a/system/lib/llvm-libc/src/stdlib/strfroml.cpp +++ b/system/lib/llvm-libc/src/stdlib/strfroml.cpp @@ -24,8 +24,10 @@ LLVM_LIBC_FUNCTION(int, strfroml, // the length modifier has to be set to LenghtModifier::L section.length_modifier = printf_core::LengthModifier::L; - printf_core::WriteBuffer wb(s, (n > 0 ? n - 1 : 0)); - printf_core::Writer writer(&wb); + printf_core::WriteBuffer::value> + wb(s, (n > 0 ? n - 1 : 0)); + printf_core::Writer writer(wb); int result = 0; if (section.has_conv) diff --git a/system/lib/llvm-libc/src/stdlib/strtod.cpp b/system/lib/llvm-libc/src/stdlib/strtod.cpp index 2c6819163aa46..deb2390c7fcde 100644 --- a/system/lib/llvm-libc/src/stdlib/strtod.cpp +++ b/system/lib/llvm-libc/src/stdlib/strtod.cpp @@ -8,9 +8,9 @@ #include "src/stdlib/strtod.h" #include "src/__support/common.h" +#include "src/__support/libc_errno.h" #include "src/__support/macros/config.h" #include "src/__support/str_to_float.h" -#include "src/errno/libc_errno.h" namespace LIBC_NAMESPACE_DECL { diff --git a/system/lib/llvm-libc/src/stdlib/strtod_l.cpp b/system/lib/llvm-libc/src/stdlib/strtod_l.cpp index 247314398315b..ad333b32d2406 100644 --- a/system/lib/llvm-libc/src/stdlib/strtod_l.cpp +++ b/system/lib/llvm-libc/src/stdlib/strtod_l.cpp @@ -8,9 +8,9 @@ #include "src/stdlib/strtod_l.h" #include "src/__support/common.h" +#include "src/__support/libc_errno.h" #include "src/__support/macros/config.h" #include "src/__support/str_to_float.h" -#include "src/errno/libc_errno.h" namespace LIBC_NAMESPACE_DECL { diff --git a/system/lib/llvm-libc/src/stdlib/strtof.cpp b/system/lib/llvm-libc/src/stdlib/strtof.cpp index 351bf64ad4f70..fc52dc85ffc50 100644 --- a/system/lib/llvm-libc/src/stdlib/strtof.cpp +++ b/system/lib/llvm-libc/src/stdlib/strtof.cpp @@ -8,9 +8,9 @@ #include "src/stdlib/strtof.h" #include "src/__support/common.h" +#include "src/__support/libc_errno.h" #include "src/__support/macros/config.h" #include "src/__support/str_to_float.h" -#include "src/errno/libc_errno.h" namespace LIBC_NAMESPACE_DECL { diff --git a/system/lib/llvm-libc/src/stdlib/strtof_l.cpp b/system/lib/llvm-libc/src/stdlib/strtof_l.cpp index d54efa66e0846..c6e03ff51fa2f 100644 --- a/system/lib/llvm-libc/src/stdlib/strtof_l.cpp +++ b/system/lib/llvm-libc/src/stdlib/strtof_l.cpp @@ -8,9 +8,9 @@ #include "src/stdlib/strtof_l.h" #include "src/__support/common.h" +#include "src/__support/libc_errno.h" #include "src/__support/macros/config.h" #include "src/__support/str_to_float.h" -#include "src/errno/libc_errno.h" namespace LIBC_NAMESPACE_DECL { diff --git a/system/lib/llvm-libc/src/stdlib/strtol.cpp b/system/lib/llvm-libc/src/stdlib/strtol.cpp index 77f8712d7c136..42db36b2052b4 100644 --- a/system/lib/llvm-libc/src/stdlib/strtol.cpp +++ b/system/lib/llvm-libc/src/stdlib/strtol.cpp @@ -8,9 +8,9 @@ #include "src/stdlib/strtol.h" #include "src/__support/common.h" +#include "src/__support/libc_errno.h" #include "src/__support/macros/config.h" #include "src/__support/str_to_integer.h" -#include "src/errno/libc_errno.h" namespace LIBC_NAMESPACE_DECL { diff --git a/system/lib/llvm-libc/src/stdlib/strtol_l.cpp b/system/lib/llvm-libc/src/stdlib/strtol_l.cpp index f94aff1a0d7b2..497a4403eff4b 100644 --- a/system/lib/llvm-libc/src/stdlib/strtol_l.cpp +++ b/system/lib/llvm-libc/src/stdlib/strtol_l.cpp @@ -8,9 +8,9 @@ #include "src/stdlib/strtol_l.h" #include "src/__support/common.h" +#include "src/__support/libc_errno.h" #include "src/__support/macros/config.h" #include "src/__support/str_to_integer.h" -#include "src/errno/libc_errno.h" namespace LIBC_NAMESPACE_DECL { diff --git a/system/lib/llvm-libc/src/stdlib/strtold.cpp b/system/lib/llvm-libc/src/stdlib/strtold.cpp index 88d29c9f36278..44046c2c6f613 100644 --- a/system/lib/llvm-libc/src/stdlib/strtold.cpp +++ b/system/lib/llvm-libc/src/stdlib/strtold.cpp @@ -8,9 +8,9 @@ #include "src/stdlib/strtold.h" #include "src/__support/common.h" +#include "src/__support/libc_errno.h" #include "src/__support/macros/config.h" #include "src/__support/str_to_float.h" -#include "src/errno/libc_errno.h" namespace LIBC_NAMESPACE_DECL { diff --git a/system/lib/llvm-libc/src/stdlib/strtold_l.cpp b/system/lib/llvm-libc/src/stdlib/strtold_l.cpp index d0c57f50246b5..c3af30a1b9ecc 100644 --- a/system/lib/llvm-libc/src/stdlib/strtold_l.cpp +++ b/system/lib/llvm-libc/src/stdlib/strtold_l.cpp @@ -8,9 +8,9 @@ #include "src/stdlib/strtold_l.h" #include "src/__support/common.h" +#include "src/__support/libc_errno.h" #include "src/__support/macros/config.h" #include "src/__support/str_to_float.h" -#include "src/errno/libc_errno.h" namespace LIBC_NAMESPACE_DECL { diff --git a/system/lib/llvm-libc/src/stdlib/strtoll.cpp b/system/lib/llvm-libc/src/stdlib/strtoll.cpp index 8d1b3efdcf87d..c1dca13112e0f 100644 --- a/system/lib/llvm-libc/src/stdlib/strtoll.cpp +++ b/system/lib/llvm-libc/src/stdlib/strtoll.cpp @@ -8,9 +8,9 @@ #include "src/stdlib/strtoll.h" #include "src/__support/common.h" +#include "src/__support/libc_errno.h" #include "src/__support/macros/config.h" #include "src/__support/str_to_integer.h" -#include "src/errno/libc_errno.h" namespace LIBC_NAMESPACE_DECL { diff --git a/system/lib/llvm-libc/src/stdlib/strtoll_l.cpp b/system/lib/llvm-libc/src/stdlib/strtoll_l.cpp index e82971d59c48d..6f30d7794c5ca 100644 --- a/system/lib/llvm-libc/src/stdlib/strtoll_l.cpp +++ b/system/lib/llvm-libc/src/stdlib/strtoll_l.cpp @@ -8,9 +8,9 @@ #include "src/stdlib/strtoll_l.h" #include "src/__support/common.h" +#include "src/__support/libc_errno.h" #include "src/__support/macros/config.h" #include "src/__support/str_to_integer.h" -#include "src/errno/libc_errno.h" namespace LIBC_NAMESPACE_DECL { diff --git a/system/lib/llvm-libc/src/stdlib/strtoul.cpp b/system/lib/llvm-libc/src/stdlib/strtoul.cpp index 1d832318c4489..d26ca5e5a10a1 100644 --- a/system/lib/llvm-libc/src/stdlib/strtoul.cpp +++ b/system/lib/llvm-libc/src/stdlib/strtoul.cpp @@ -8,9 +8,9 @@ #include "src/stdlib/strtoul.h" #include "src/__support/common.h" +#include "src/__support/libc_errno.h" #include "src/__support/macros/config.h" #include "src/__support/str_to_integer.h" -#include "src/errno/libc_errno.h" namespace LIBC_NAMESPACE_DECL { diff --git a/system/lib/llvm-libc/src/stdlib/strtoul_l.cpp b/system/lib/llvm-libc/src/stdlib/strtoul_l.cpp index 74fce00a0ac3c..9a875ddee9029 100644 --- a/system/lib/llvm-libc/src/stdlib/strtoul_l.cpp +++ b/system/lib/llvm-libc/src/stdlib/strtoul_l.cpp @@ -8,9 +8,9 @@ #include "src/stdlib/strtoul_l.h" #include "src/__support/common.h" +#include "src/__support/libc_errno.h" #include "src/__support/macros/config.h" #include "src/__support/str_to_integer.h" -#include "src/errno/libc_errno.h" namespace LIBC_NAMESPACE_DECL { diff --git a/system/lib/llvm-libc/src/stdlib/strtoull.cpp b/system/lib/llvm-libc/src/stdlib/strtoull.cpp index dba22611cfb09..8f929f577311e 100644 --- a/system/lib/llvm-libc/src/stdlib/strtoull.cpp +++ b/system/lib/llvm-libc/src/stdlib/strtoull.cpp @@ -8,9 +8,9 @@ #include "src/stdlib/strtoull.h" #include "src/__support/common.h" +#include "src/__support/libc_errno.h" #include "src/__support/macros/config.h" #include "src/__support/str_to_integer.h" -#include "src/errno/libc_errno.h" namespace LIBC_NAMESPACE_DECL { diff --git a/system/lib/llvm-libc/src/stdlib/strtoull_l.cpp b/system/lib/llvm-libc/src/stdlib/strtoull_l.cpp index 2ea8a43a40ef2..9eb056b0e59b4 100644 --- a/system/lib/llvm-libc/src/stdlib/strtoull_l.cpp +++ b/system/lib/llvm-libc/src/stdlib/strtoull_l.cpp @@ -8,9 +8,9 @@ #include "src/stdlib/strtoull_l.h" #include "src/__support/common.h" +#include "src/__support/libc_errno.h" #include "src/__support/macros/config.h" #include "src/__support/str_to_integer.h" -#include "src/errno/libc_errno.h" namespace LIBC_NAMESPACE_DECL { diff --git a/system/lib/llvm-libc/src/string/allocating_string_utils.h b/system/lib/llvm-libc/src/string/allocating_string_utils.h index b3a8663c2f477..1dece510e7969 100644 --- a/system/lib/llvm-libc/src/string/allocating_string_utils.h +++ b/system/lib/llvm-libc/src/string/allocating_string_utils.h @@ -11,7 +11,7 @@ #include "src/__support/CPP/new.h" #include "src/__support/CPP/optional.h" -#include "src/__support/macros/config.h" +#include "src/__support/macros/config.h" // LIBC_NAMESPACE_DECL #include "src/string/memory_utils/inline_memcpy.h" #include "src/string/string_utils.h" diff --git a/system/lib/llvm-libc/src/string/memccpy.cpp b/system/lib/llvm-libc/src/string/memccpy.cpp index ae90cf9370d49..d5654fc5e46ab 100644 --- a/system/lib/llvm-libc/src/string/memccpy.cpp +++ b/system/lib/llvm-libc/src/string/memccpy.cpp @@ -10,6 +10,7 @@ #include "src/__support/common.h" #include "src/__support/macros/config.h" +#include "src/__support/macros/null_check.h" #include // For size_t. namespace LIBC_NAMESPACE_DECL { @@ -17,6 +18,10 @@ namespace LIBC_NAMESPACE_DECL { LLVM_LIBC_FUNCTION(void *, memccpy, (void *__restrict dest, const void *__restrict src, int c, size_t count)) { + if (count) { + LIBC_CRASH_ON_NULLPTR(dest); + LIBC_CRASH_ON_NULLPTR(src); + } unsigned char end = static_cast(c); const unsigned char *uc_src = static_cast(src); unsigned char *uc_dest = static_cast(dest); diff --git a/system/lib/llvm-libc/src/string/memchr.cpp b/system/lib/llvm-libc/src/string/memchr.cpp index ba52f14afa9d6..ccdc262837cd4 100644 --- a/system/lib/llvm-libc/src/string/memchr.cpp +++ b/system/lib/llvm-libc/src/string/memchr.cpp @@ -8,6 +8,7 @@ #include "src/string/memchr.h" #include "src/__support/macros/config.h" +#include "src/__support/macros/null_check.h" #include "src/string/string_utils.h" #include "src/__support/common.h" @@ -17,6 +18,8 @@ namespace LIBC_NAMESPACE_DECL { // TODO: Look at performance benefits of comparing words. LLVM_LIBC_FUNCTION(void *, memchr, (const void *src, int c, size_t n)) { + if (n) + LIBC_CRASH_ON_NULLPTR(src); return internal::find_first_character( reinterpret_cast(src), static_cast(c), n); diff --git a/system/lib/llvm-libc/src/string/memcmp.cpp b/system/lib/llvm-libc/src/string/memcmp.cpp index 68996fb787236..d2f67f0478ef9 100644 --- a/system/lib/llvm-libc/src/string/memcmp.cpp +++ b/system/lib/llvm-libc/src/string/memcmp.cpp @@ -8,6 +8,7 @@ #include "src/string/memcmp.h" #include "src/__support/macros/config.h" +#include "src/__support/macros/null_check.h" #include "src/string/memory_utils/inline_memcmp.h" #include // size_t @@ -16,6 +17,10 @@ namespace LIBC_NAMESPACE_DECL { LLVM_LIBC_FUNCTION(int, memcmp, (const void *lhs, const void *rhs, size_t count)) { + if (count) { + LIBC_CRASH_ON_NULLPTR(lhs); + LIBC_CRASH_ON_NULLPTR(rhs); + } return inline_memcmp(lhs, rhs, count); } diff --git a/system/lib/llvm-libc/src/string/memcpy.cpp b/system/lib/llvm-libc/src/string/memcpy.cpp index 0eb7f2c170e08..4d4ff4dd38726 100644 --- a/system/lib/llvm-libc/src/string/memcpy.cpp +++ b/system/lib/llvm-libc/src/string/memcpy.cpp @@ -9,6 +9,7 @@ #include "src/string/memcpy.h" #include "src/__support/common.h" #include "src/__support/macros/config.h" +#include "src/__support/macros/null_check.h" #include "src/string/memory_utils/inline_memcpy.h" namespace LIBC_NAMESPACE_DECL { @@ -16,6 +17,10 @@ namespace LIBC_NAMESPACE_DECL { LLVM_LIBC_FUNCTION(void *, memcpy, (void *__restrict dst, const void *__restrict src, size_t size)) { + if (size) { + LIBC_CRASH_ON_NULLPTR(dst); + LIBC_CRASH_ON_NULLPTR(src); + } inline_memcpy(dst, src, size); return dst; } diff --git a/system/lib/llvm-libc/src/string/memmove.cpp b/system/lib/llvm-libc/src/string/memmove.cpp index 26a8c4187f3d3..04ed51b84f8f1 100644 --- a/system/lib/llvm-libc/src/string/memmove.cpp +++ b/system/lib/llvm-libc/src/string/memmove.cpp @@ -8,6 +8,7 @@ #include "src/string/memmove.h" #include "src/__support/macros/config.h" +#include "src/__support/macros/null_check.h" #include "src/string/memory_utils/inline_memcpy.h" #include "src/string/memory_utils/inline_memmove.h" #include // size_t @@ -16,6 +17,10 @@ namespace LIBC_NAMESPACE_DECL { LLVM_LIBC_FUNCTION(void *, memmove, (void *dst, const void *src, size_t count)) { + if (count) { + LIBC_CRASH_ON_NULLPTR(dst); + LIBC_CRASH_ON_NULLPTR(src); + } // Memmove may handle some small sizes as efficiently as inline_memcpy. // For these sizes we may not do is_disjoint check. // This both avoids additional code for the most frequent smaller sizes diff --git a/system/lib/llvm-libc/src/string/memory_utils/aarch64/inline_bcmp.h b/system/lib/llvm-libc/src/string/memory_utils/aarch64/inline_bcmp.h index 93196415f2662..66d24378095b9 100644 --- a/system/lib/llvm-libc/src/string/memory_utils/aarch64/inline_bcmp.h +++ b/system/lib/llvm-libc/src/string/memory_utils/aarch64/inline_bcmp.h @@ -9,7 +9,7 @@ #define LIBC_SRC_STRING_MEMORY_UTILS_AARCH64_INLINE_BCMP_H #include "src/__support/macros/attributes.h" // LIBC_INLINE -#include "src/__support/macros/config.h" +#include "src/__support/macros/config.h" // LIBC_NAMESPACE_DECL #include "src/__support/macros/optimization.h" // LIBC_UNLIKELY #include "src/string/memory_utils/op_aarch64.h" #include "src/string/memory_utils/op_generic.h" @@ -19,9 +19,43 @@ namespace LIBC_NAMESPACE_DECL { -[[maybe_unused]] LIBC_INLINE BcmpReturnType inline_bcmp_aarch64(CPtr p1, - CPtr p2, - size_t count) { +[[maybe_unused]] LIBC_INLINE BcmpReturnType +inline_bcmp_aarch64_no_fp(CPtr p1, CPtr p2, size_t count) { + if (LIBC_LIKELY(count < 16)) { + switch (count) { + case 0: + return BcmpReturnType::zero(); + case 1: + return generic::Bcmp::block(p1, p2); + case 2: + return generic::Bcmp::block(p1, p2); + case 3: + return generic::Bcmp::head_tail(p1, p2, count); + case 4: + return generic::Bcmp::block(p1, p2); + case 5: + case 6: + case 7: + return generic::Bcmp::head_tail(p1, p2, count); + case 8: + return generic::Bcmp::block(p1, p2); + case 9: + case 10: + case 11: + case 12: + case 13: + case 14: + case 15: + return generic::Bcmp::head_tail(p1, p2, count); + } + } + + return generic::Bcmp::loop_and_tail_align_above(256, p1, p2, count); +} + +#ifdef __ARM_NEON +[[maybe_unused]] LIBC_INLINE BcmpReturnType +inline_bcmp_aarch64_with_fp(CPtr p1, CPtr p2, size_t count) { if (LIBC_LIKELY(count <= 32)) { if (LIBC_UNLIKELY(count >= 16)) { return aarch64::Bcmp<16>::head_tail(p1, p2, count); @@ -65,6 +99,16 @@ namespace LIBC_NAMESPACE_DECL { } return aarch64::Bcmp<32>::loop_and_tail(p1, p2, count); } +#endif + +[[gnu::flatten]] LIBC_INLINE BcmpReturnType +inline_bcmp_aarch64_dispatch(CPtr p1, CPtr p2, size_t count) { +#if defined(__ARM_NEON) + return inline_bcmp_aarch64_with_fp(p1, p2, count); +#else + return inline_bcmp_aarch64_no_fp(p1, p2, count); +#endif +} } // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/string/memory_utils/aarch64/inline_memcmp.h b/system/lib/llvm-libc/src/string/memory_utils/aarch64/inline_memcmp.h index 35ca077dab526..380ebb410efb5 100644 --- a/system/lib/llvm-libc/src/string/memory_utils/aarch64/inline_memcmp.h +++ b/system/lib/llvm-libc/src/string/memory_utils/aarch64/inline_memcmp.h @@ -17,17 +17,40 @@ namespace LIBC_NAMESPACE_DECL { [[maybe_unused]] LIBC_INLINE MemcmpReturnType -inline_memcmp_generic_gt16(CPtr p1, CPtr p2, size_t count) { - if (LIBC_UNLIKELY(count >= 384)) { - if (auto value = generic::Memcmp::block(p1, p2)) - return value; - align_to_next_boundary<16, Arg::P1>(p1, p2, count); - } - return generic::Memcmp::loop_and_tail(p1, p2, count); +inline_memcmp_aarch64_no_fp(CPtr p1, CPtr p2, size_t count) { + if (count == 0) + return MemcmpReturnType::zero(); + if (count == 1) + return generic::Memcmp::block(p1, p2); + if (count == 2) + return generic::Memcmp::block(p1, p2); + if (count == 3) + return generic::MemcmpSequence::block(p1, p2); + if (count <= 8) + return generic::Memcmp::head_tail(p1, p2, count); + if (count <= 16) + return generic::Memcmp::head_tail(p1, p2, count); + + return generic::Memcmp::loop_and_tail_align_above(384, p1, p2, + count); } +#if defined(__ARM_NEON) [[maybe_unused]] LIBC_INLINE MemcmpReturnType -inline_memcmp_aarch64_neon_gt16(CPtr p1, CPtr p2, size_t count) { +inline_memcmp_aarch64_with_fp(CPtr p1, CPtr p2, size_t count) { + if (count == 0) + return MemcmpReturnType::zero(); + if (count == 1) + return generic::Memcmp::block(p1, p2); + if (count == 2) + return generic::Memcmp::block(p1, p2); + if (count == 3) + return generic::MemcmpSequence::block(p1, p2); + if (count <= 8) + return generic::Memcmp::head_tail(p1, p2, count); + if (count <= 16) + return generic::Memcmp::head_tail(p1, p2, count); + if (LIBC_UNLIKELY(count >= 128)) { // [128, ∞] if (auto value = generic::Memcmp::block(p1, p2)) return value; @@ -46,25 +69,15 @@ inline_memcmp_aarch64_neon_gt16(CPtr p1, CPtr p2, size_t count) { return generic::Memcmp::loop_and_tail(p1 + 32, p2 + 32, count - 32); } +#endif -LIBC_INLINE MemcmpReturnType inline_memcmp_aarch64(CPtr p1, CPtr p2, - size_t count) { - if (count == 0) - return MemcmpReturnType::zero(); - if (count == 1) - return generic::Memcmp::block(p1, p2); - if (count == 2) - return generic::Memcmp::block(p1, p2); - if (count == 3) - return generic::MemcmpSequence::block(p1, p2); - if (count <= 8) - return generic::Memcmp::head_tail(p1, p2, count); - if (count <= 16) - return generic::Memcmp::head_tail(p1, p2, count); - if constexpr (aarch64::kNeon) - return inline_memcmp_aarch64_neon_gt16(p1, p2, count); - else - return inline_memcmp_generic_gt16(p1, p2, count); +[[gnu::flatten]] LIBC_INLINE MemcmpReturnType +inline_memcmp_aarch64_dispatch(CPtr p1, CPtr p2, size_t count) { +#if defined(__ARM_NEON) + return inline_memcmp_aarch64_with_fp(p1, p2, count); +#else + return inline_memcmp_aarch64_no_fp(p1, p2, count); +#endif } } // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/string/memory_utils/aarch64/inline_memmove.h b/system/lib/llvm-libc/src/string/memory_utils/aarch64/inline_memmove.h index 2b238031af49d..d8d276966fd27 100644 --- a/system/lib/llvm-libc/src/string/memory_utils/aarch64/inline_memmove.h +++ b/system/lib/llvm-libc/src/string/memory_utils/aarch64/inline_memmove.h @@ -8,8 +8,7 @@ #ifndef LIBC_SRC_STRING_MEMORY_UTILS_AARCH64_INLINE_MEMMOVE_H #define LIBC_SRC_STRING_MEMORY_UTILS_AARCH64_INLINE_MEMMOVE_H -#include "src/__support/macros/attributes.h" // LIBC_INLINE -#include "src/string/memory_utils/op_aarch64.h" // aarch64::kNeon +#include "src/__support/macros/attributes.h" // LIBC_INLINE #include "src/string/memory_utils/op_builtin.h" #include "src/string/memory_utils/op_generic.h" #include "src/string/memory_utils/utils.h" @@ -19,7 +18,6 @@ namespace LIBC_NAMESPACE_DECL { LIBC_INLINE void inline_memmove_aarch64(Ptr dst, CPtr src, size_t count) { - static_assert(aarch64::kNeon, "aarch64 supports vector types"); using uint128_t = generic_v128; using uint256_t = generic_v256; using uint512_t = generic_v512; diff --git a/system/lib/llvm-libc/src/string/memory_utils/aarch64/inline_memset.h b/system/lib/llvm-libc/src/string/memory_utils/aarch64/inline_memset.h index a2c05534aa330..1b4b871792c69 100644 --- a/system/lib/llvm-libc/src/string/memory_utils/aarch64/inline_memset.h +++ b/system/lib/llvm-libc/src/string/memory_utils/aarch64/inline_memset.h @@ -9,7 +9,7 @@ #define LIBC_SRC_STRING_MEMORY_UTILS_AARCH64_INLINE_MEMSET_H #include "src/__support/macros/attributes.h" // LIBC_INLINE -#include "src/__support/macros/config.h" +#include "src/__support/macros/config.h" // LIBC_NAMESPACE_DECL #include "src/string/memory_utils/op_aarch64.h" #include "src/string/memory_utils/op_generic.h" #include "src/string/memory_utils/utils.h" // Ptr, CPtr @@ -18,12 +18,12 @@ namespace LIBC_NAMESPACE_DECL { +using uint128_t = generic_v128; +using uint256_t = generic_v256; +using uint512_t = generic_v512; + [[maybe_unused]] LIBC_INLINE static void -inline_memset_aarch64(Ptr dst, uint8_t value, size_t count) { - static_assert(aarch64::kNeon, "aarch64 supports vector types"); - using uint128_t = generic_v128; - using uint256_t = generic_v256; - using uint512_t = generic_v512; +inline_memset_aarch64_no_fp(Ptr dst, uint8_t value, size_t count) { if (count == 0) return; if (count <= 3) { @@ -46,15 +46,57 @@ inline_memset_aarch64(Ptr dst, uint8_t value, size_t count) { generic::Memset::tail(dst, value, count); return; } + + generic::Memset::block(dst, value); + align_to_next_boundary<16>(dst, count); + return generic::Memset::loop_and_tail(dst, value, count); +} + +#if defined(__ARM_NEON) +[[maybe_unused]] LIBC_INLINE static void +inline_memset_aarch64_with_fp(Ptr dst, uint8_t value, size_t count) { + if (count == 0) + return; + if (count <= 3) { + generic::Memset::block(dst, value); + if (count > 1) + generic::Memset::tail(dst, value, count); + return; + } + if (count <= 8) + return generic::Memset::head_tail(dst, value, count); + if (count <= 16) + return generic::Memset::head_tail(dst, value, count); + if (count <= 32) + return generic::Memset::head_tail(dst, value, count); + if (count <= (32 + 64)) { + generic::Memset::block(dst, value); + if (count <= 64) + return generic::Memset::tail(dst, value, count); + generic::Memset::block(dst + 32, value); + generic::Memset::tail(dst, value, count); + return; + } + if (count >= 448 && value == 0 && aarch64::neon::hasZva()) { generic::Memset::block(dst, 0); align_to_next_boundary<64>(dst, count); return aarch64::neon::BzeroCacheLine::loop_and_tail(dst, 0, count); - } else { - generic::Memset::block(dst, value); - align_to_next_boundary<16>(dst, count); - return generic::Memset::loop_and_tail(dst, value, count); } + + generic::Memset::block(dst, value); + align_to_next_boundary<16>(dst, count); + return generic::Memset::loop_and_tail(dst, value, count); +} +#endif + +[[gnu::flatten]] [[maybe_unused]] LIBC_INLINE static void +inline_memset_aarch64_dispatch(Ptr dst, uint8_t value, size_t count) { +#if defined(__ARM_NEON) + return inline_memset_aarch64_with_fp(dst, value, count); +#else + return inline_memset_aarch64_no_fp(dst, value, count); +#endif } } // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/string/memory_utils/generic/builtin.h b/system/lib/llvm-libc/src/string/memory_utils/generic/builtin.h index 5670a4e610565..2b9ecc0c046eb 100644 --- a/system/lib/llvm-libc/src/string/memory_utils/generic/builtin.h +++ b/system/lib/llvm-libc/src/string/memory_utils/generic/builtin.h @@ -10,7 +10,7 @@ #define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_GENERIC_BUILTIN_H #include "src/__support/macros/attributes.h" // LIBC_INLINE -#include "src/__support/macros/config.h" +#include "src/__support/macros/config.h" // LIBC_NAMESPACE_DECL #include "src/string/memory_utils/utils.h" // Ptr, CPtr #include // size_t diff --git a/system/lib/llvm-libc/src/string/memory_utils/generic/byte_per_byte.h b/system/lib/llvm-libc/src/string/memory_utils/generic/byte_per_byte.h index 2aecf0126cc43..862aeecab7f55 100644 --- a/system/lib/llvm-libc/src/string/memory_utils/generic/byte_per_byte.h +++ b/system/lib/llvm-libc/src/string/memory_utils/generic/byte_per_byte.h @@ -38,7 +38,8 @@ inline_memmove_byte_per_byte(Ptr dst, CPtr src, size_t count) { dst[offset] = src[offset]; } else { LIBC_LOOP_NOUNROLL - for (ptrdiff_t offset = count - 1; offset >= 0; --offset) + for (ptrdiff_t offset = static_cast(count - 1); offset >= 0; + --offset) dst[offset] = src[offset]; } } diff --git a/system/lib/llvm-libc/src/string/memory_utils/inline_bcmp.h b/system/lib/llvm-libc/src/string/memory_utils/inline_bcmp.h index 126da4e004f7f..955d764aade2b 100644 --- a/system/lib/llvm-libc/src/string/memory_utils/inline_bcmp.h +++ b/system/lib/llvm-libc/src/string/memory_utils/inline_bcmp.h @@ -10,7 +10,8 @@ #define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_INLINE_BCMP_H #include "src/__support/common.h" -#include "src/__support/macros/config.h" +#include "src/__support/macros/attributes.h" // LIBC_INLINE +#include "src/__support/macros/config.h" // LIBC_NAMESPACE_DECL #include "src/__support/macros/properties/architectures.h" // LIBC_TARGET_ARCH_IS_ #include // size_t @@ -20,15 +21,13 @@ #define LIBC_SRC_STRING_MEMORY_UTILS_BCMP inline_bcmp_x86 #elif defined(LIBC_TARGET_ARCH_IS_AARCH64) #include "src/string/memory_utils/aarch64/inline_bcmp.h" -#define LIBC_SRC_STRING_MEMORY_UTILS_BCMP inline_bcmp_aarch64 +#define LIBC_SRC_STRING_MEMORY_UTILS_BCMP inline_bcmp_aarch64_dispatch #elif defined(LIBC_TARGET_ARCH_IS_ANY_RISCV) #include "src/string/memory_utils/riscv/inline_bcmp.h" #define LIBC_SRC_STRING_MEMORY_UTILS_BCMP inline_bcmp_riscv -#elif defined(LIBC_TARGET_ARCH_IS_ARM) || defined(LIBC_TARGET_ARCH_IS_GPU) || defined(LIBC_TARGET_ARCH_IS_WASM) +#else #include "src/string/memory_utils/generic/byte_per_byte.h" #define LIBC_SRC_STRING_MEMORY_UTILS_BCMP inline_bcmp_byte_per_byte -#else -#error "Unsupported architecture" #endif namespace LIBC_NAMESPACE_DECL { diff --git a/system/lib/llvm-libc/src/string/memory_utils/inline_bzero.h b/system/lib/llvm-libc/src/string/memory_utils/inline_bzero.h index 4a92e8b57a68e..a131b68725de7 100644 --- a/system/lib/llvm-libc/src/string/memory_utils/inline_bzero.h +++ b/system/lib/llvm-libc/src/string/memory_utils/inline_bzero.h @@ -10,7 +10,8 @@ #define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_INLINE_BZERO_H #include "src/__support/common.h" -#include "src/__support/macros/config.h" +#include "src/__support/macros/attributes.h" // LIBC_INLINE +#include "src/__support/macros/config.h" // LIBC_NAMESPACE_DECL #include "src/string/memory_utils/inline_memset.h" #include // size_t diff --git a/system/lib/llvm-libc/src/string/memory_utils/inline_memcmp.h b/system/lib/llvm-libc/src/string/memory_utils/inline_memcmp.h index e9c949f42bc3b..85a614b2fb95e 100644 --- a/system/lib/llvm-libc/src/string/memory_utils/inline_memcmp.h +++ b/system/lib/llvm-libc/src/string/memory_utils/inline_memcmp.h @@ -20,15 +20,13 @@ #define LIBC_SRC_STRING_MEMORY_UTILS_MEMCMP inline_memcmp_x86 #elif defined(LIBC_TARGET_ARCH_IS_AARCH64) #include "src/string/memory_utils/aarch64/inline_memcmp.h" -#define LIBC_SRC_STRING_MEMORY_UTILS_MEMCMP inline_memcmp_aarch64 +#define LIBC_SRC_STRING_MEMORY_UTILS_MEMCMP inline_memcmp_aarch64_dispatch #elif defined(LIBC_TARGET_ARCH_IS_ANY_RISCV) #include "src/string/memory_utils/riscv/inline_memcmp.h" #define LIBC_SRC_STRING_MEMORY_UTILS_MEMCMP inline_memcmp_riscv -#elif defined(LIBC_TARGET_ARCH_IS_ARM) || defined(LIBC_TARGET_ARCH_IS_GPU) || defined(LIBC_TARGET_ARCH_IS_WASM) +#else #include "src/string/memory_utils/generic/byte_per_byte.h" #define LIBC_SRC_STRING_MEMORY_UTILS_MEMCMP inline_memcmp_byte_per_byte -#else -#error "Unsupported architecture" #endif namespace LIBC_NAMESPACE_DECL { diff --git a/system/lib/llvm-libc/src/string/memory_utils/inline_memcpy.h b/system/lib/llvm-libc/src/string/memory_utils/inline_memcpy.h index 8ba0e8d464af5..f98e55321a9b5 100644 --- a/system/lib/llvm-libc/src/string/memory_utils/inline_memcpy.h +++ b/system/lib/llvm-libc/src/string/memory_utils/inline_memcpy.h @@ -28,14 +28,12 @@ #elif defined(LIBC_TARGET_ARCH_IS_ANY_RISCV) #include "src/string/memory_utils/riscv/inline_memcpy.h" #define LIBC_SRC_STRING_MEMORY_UTILS_MEMCPY inline_memcpy_riscv -#elif defined(LIBC_TARGET_ARCH_IS_ARM) -#include "src/string/memory_utils/generic/byte_per_byte.h" -#define LIBC_SRC_STRING_MEMORY_UTILS_MEMCPY inline_memcpy_byte_per_byte -#elif defined(LIBC_TARGET_ARCH_IS_GPU) || defined(LIBC_TARGET_ARCH_IS_WASM) +#elif defined(LIBC_TARGET_ARCH_IS_GPU) #include "src/string/memory_utils/generic/builtin.h" #define LIBC_SRC_STRING_MEMORY_UTILS_MEMCPY inline_memcpy_builtin #else -#error "Unsupported architecture" +#include "src/string/memory_utils/generic/byte_per_byte.h" +#define LIBC_SRC_STRING_MEMORY_UTILS_MEMCPY inline_memcpy_byte_per_byte #endif namespace LIBC_NAMESPACE_DECL { diff --git a/system/lib/llvm-libc/src/string/memory_utils/inline_memmem.h b/system/lib/llvm-libc/src/string/memory_utils/inline_memmem.h index 15e3d633985d6..1e9649c6e65e3 100644 --- a/system/lib/llvm-libc/src/string/memory_utils/inline_memmem.h +++ b/system/lib/llvm-libc/src/string/memory_utils/inline_memmem.h @@ -9,8 +9,8 @@ #ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_INLINE_MEMMEM_H #define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_INLINE_MEMMEM_H -#include "src/__support/macros/attributes.h" -#include "src/__support/macros/config.h" +#include "src/__support/macros/attributes.h" // LIBC_INLINE +#include "src/__support/macros/config.h" // LIBC_NAMESPACE_DECL #include diff --git a/system/lib/llvm-libc/src/string/memory_utils/inline_memmove.h b/system/lib/llvm-libc/src/string/memory_utils/inline_memmove.h index 4c988e1155eab..71a28c32e4c56 100644 --- a/system/lib/llvm-libc/src/string/memory_utils/inline_memmove.h +++ b/system/lib/llvm-libc/src/string/memory_utils/inline_memmove.h @@ -9,8 +9,9 @@ #ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_INLINE_MEMMOVE_H #define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_INLINE_MEMMOVE_H -#include "src/__support/macros/config.h" -#include // size_t, ptrdiff_t +#include "src/__support/macros/attributes.h" // LIBC_INLINE +#include "src/__support/macros/config.h" // LIBC_NAMESPACE_DECL +#include // size_t, ptrdiff_t #if defined(LIBC_TARGET_ARCH_IS_X86) #include "src/string/memory_utils/x86_64/inline_memmove.h" @@ -28,19 +29,17 @@ #define LIBC_SRC_STRING_MEMORY_UTILS_MEMMOVE_SMALL_SIZE \ inline_memmove_no_small_size #define LIBC_SRC_STRING_MEMORY_UTILS_MEMMOVE_FOLLOW_UP inline_memmove_riscv -#elif defined(LIBC_TARGET_ARCH_IS_ARM) -#include "src/string/memory_utils/generic/byte_per_byte.h" -#define LIBC_SRC_STRING_MEMORY_UTILS_MEMMOVE_SMALL_SIZE \ - inline_memmove_no_small_size -#define LIBC_SRC_STRING_MEMORY_UTILS_MEMMOVE_FOLLOW_UP \ - inline_memmove_byte_per_byte -#elif defined(LIBC_TARGET_ARCH_IS_GPU) || defined(LIBC_TARGET_ARCH_IS_WASM) +#elif defined(LIBC_TARGET_ARCH_IS_GPU) #include "src/string/memory_utils/generic/builtin.h" #define LIBC_SRC_STRING_MEMORY_UTILS_MEMMOVE_SMALL_SIZE \ inline_memmove_no_small_size #define LIBC_SRC_STRING_MEMORY_UTILS_MEMMOVE_FOLLOW_UP inline_memmove_builtin #else -#error "Unsupported architecture" +#include "src/string/memory_utils/generic/byte_per_byte.h" +#define LIBC_SRC_STRING_MEMORY_UTILS_MEMMOVE_SMALL_SIZE \ + inline_memmove_no_small_size +#define LIBC_SRC_STRING_MEMORY_UTILS_MEMMOVE_FOLLOW_UP \ + inline_memmove_byte_per_byte #endif namespace LIBC_NAMESPACE_DECL { diff --git a/system/lib/llvm-libc/src/string/memory_utils/inline_memset.h b/system/lib/llvm-libc/src/string/memory_utils/inline_memset.h index d16fa8247c32d..fd9c29ea4410a 100644 --- a/system/lib/llvm-libc/src/string/memory_utils/inline_memset.h +++ b/system/lib/llvm-libc/src/string/memory_utils/inline_memset.h @@ -20,18 +20,16 @@ #define LIBC_SRC_STRING_MEMORY_UTILS_MEMSET inline_memset_x86 #elif defined(LIBC_TARGET_ARCH_IS_AARCH64) #include "src/string/memory_utils/aarch64/inline_memset.h" -#define LIBC_SRC_STRING_MEMORY_UTILS_MEMSET inline_memset_aarch64 +#define LIBC_SRC_STRING_MEMORY_UTILS_MEMSET inline_memset_aarch64_dispatch #elif defined(LIBC_TARGET_ARCH_IS_ANY_RISCV) #include "src/string/memory_utils/riscv/inline_memset.h" #define LIBC_SRC_STRING_MEMORY_UTILS_MEMSET inline_memset_riscv -#elif defined(LIBC_TARGET_ARCH_IS_ARM) -#include "src/string/memory_utils/generic/byte_per_byte.h" -#define LIBC_SRC_STRING_MEMORY_UTILS_MEMSET inline_memset_byte_per_byte -#elif defined(LIBC_TARGET_ARCH_IS_GPU) || defined(LIBC_TARGET_ARCH_IS_WASM) +#elif defined(LIBC_TARGET_ARCH_IS_GPU) #include "src/string/memory_utils/generic/builtin.h" #define LIBC_SRC_STRING_MEMORY_UTILS_MEMSET inline_memset_builtin #else -#error "Unsupported architecture" +#include "src/string/memory_utils/generic/byte_per_byte.h" +#define LIBC_SRC_STRING_MEMORY_UTILS_MEMSET inline_memset_byte_per_byte #endif namespace LIBC_NAMESPACE_DECL { diff --git a/system/lib/llvm-libc/src/string/memory_utils/inline_strcmp.h b/system/lib/llvm-libc/src/string/memory_utils/inline_strcmp.h index 281d5b14c6cba..6758e79ae9ca3 100644 --- a/system/lib/llvm-libc/src/string/memory_utils/inline_strcmp.h +++ b/system/lib/llvm-libc/src/string/memory_utils/inline_strcmp.h @@ -9,7 +9,8 @@ #ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_INLINE_STRCMP_H #define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_INLINE_STRCMP_H -#include "src/__support/macros/config.h" +#include "src/__support/macros/attributes.h" // LIBC_INLINE +#include "src/__support/macros/config.h" // LIBC_NAMESPACE_DECL #include namespace LIBC_NAMESPACE_DECL { diff --git a/system/lib/llvm-libc/src/string/memory_utils/inline_strstr.h b/system/lib/llvm-libc/src/string/memory_utils/inline_strstr.h index 9c99e920b4b57..5495cc4803ec9 100644 --- a/system/lib/llvm-libc/src/string/memory_utils/inline_strstr.h +++ b/system/lib/llvm-libc/src/string/memory_utils/inline_strstr.h @@ -9,7 +9,8 @@ #ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_INLINE_STRSTR_H #define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_INLINE_STRSTR_H -#include "src/__support/macros/config.h" +#include "src/__support/macros/attributes.h" // LIBC_INLINE +#include "src/__support/macros/config.h" // LIBC_NAMESPACE_DECL #include "src/string/memory_utils/inline_memmem.h" #include "src/string/string_utils.h" #include diff --git a/system/lib/llvm-libc/src/string/memory_utils/op_aarch64.h b/system/lib/llvm-libc/src/string/memory_utils/op_aarch64.h index 1090ea2617f09..e552601fbb708 100644 --- a/system/lib/llvm-libc/src/string/memory_utils/op_aarch64.h +++ b/system/lib/llvm-libc/src/string/memory_utils/op_aarch64.h @@ -13,7 +13,8 @@ #ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_OP_AARCH64_H #define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_OP_AARCH64_H -#include "src/__support/macros/config.h" +#include "src/__support/macros/attributes.h" // LIBC_INLINE +#include "src/__support/macros/config.h" // LIBC_NAMESPACE_DECL #include "src/__support/macros/properties/architectures.h" #if defined(LIBC_TARGET_ARCH_IS_AARCH64) @@ -24,7 +25,6 @@ #ifdef __ARM_NEON #include -#endif //__ARM_NEON namespace LIBC_NAMESPACE_DECL { namespace aarch64 { @@ -175,6 +175,8 @@ template struct Bcmp { } // namespace aarch64 } // namespace LIBC_NAMESPACE_DECL +#endif //__ARM_NEON + namespace LIBC_NAMESPACE_DECL { namespace generic { @@ -224,6 +226,8 @@ LIBC_INLINE MemcmpReturnType cmp(CPtr p1, CPtr p2, size_t offset) { return MemcmpReturnType::zero(); } +#if defined(__ARM_NEON) + /////////////////////////////////////////////////////////////////////////////// // Specializations for uint8x16_t template <> struct is_vector : cpp::true_type {}; @@ -268,6 +272,9 @@ LIBC_INLINE MemcmpReturnType cmp(CPtr p1, CPtr p2, } return MemcmpReturnType::zero(); } + +#endif // __ARM_NEON + } // namespace generic } // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/string/memory_utils/op_builtin.h b/system/lib/llvm-libc/src/string/memory_utils/op_builtin.h index d7c1b1f870115..27b621d97f01a 100644 --- a/system/lib/llvm-libc/src/string/memory_utils/op_builtin.h +++ b/system/lib/llvm-libc/src/string/memory_utils/op_builtin.h @@ -16,7 +16,8 @@ #define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_OP_BUILTIN_H #include "src/__support/CPP/type_traits.h" -#include "src/__support/macros/config.h" +#include "src/__support/macros/attributes.h" // LIBC_INLINE +#include "src/__support/macros/config.h" // LIBC_NAMESPACE_DECL #include "src/string/memory_utils/utils.h" namespace LIBC_NAMESPACE_DECL { diff --git a/system/lib/llvm-libc/src/string/memory_utils/op_generic.h b/system/lib/llvm-libc/src/string/memory_utils/op_generic.h index da20a84dede4d..9349cfdd1d020 100644 --- a/system/lib/llvm-libc/src/string/memory_utils/op_generic.h +++ b/system/lib/llvm-libc/src/string/memory_utils/op_generic.h @@ -27,7 +27,8 @@ #include "src/__support/CPP/type_traits.h" #include "src/__support/common.h" #include "src/__support/endian_internal.h" -#include "src/__support/macros/config.h" +#include "src/__support/macros/attributes.h" // LIBC_INLINE +#include "src/__support/macros/config.h" // LIBC_NAMESPACE_DECL #include "src/__support/macros/optimization.h" #include "src/__support/macros/properties/types.h" // LIBC_TYPES_HAS_INT64 #include "src/string/memory_utils/op_builtin.h" diff --git a/system/lib/llvm-libc/src/string/memory_utils/op_riscv.h b/system/lib/llvm-libc/src/string/memory_utils/op_riscv.h index 2d211de0facfc..4292d7a627a54 100644 --- a/system/lib/llvm-libc/src/string/memory_utils/op_riscv.h +++ b/system/lib/llvm-libc/src/string/memory_utils/op_riscv.h @@ -12,7 +12,8 @@ #ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_OP_RISCV_H #define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_OP_RISCV_H -#include "src/__support/macros/config.h" +#include "src/__support/macros/attributes.h" // LIBC_INLINE +#include "src/__support/macros/config.h" // LIBC_NAMESPACE_DECL #include "src/__support/macros/properties/architectures.h" #if defined(LIBC_TARGET_ARCH_IS_ANY_RISCV) diff --git a/system/lib/llvm-libc/src/string/memory_utils/op_x86.h b/system/lib/llvm-libc/src/string/memory_utils/op_x86.h index 309610e4ad630..8bd84120c4ffa 100644 --- a/system/lib/llvm-libc/src/string/memory_utils/op_x86.h +++ b/system/lib/llvm-libc/src/string/memory_utils/op_x86.h @@ -12,7 +12,8 @@ #ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_OP_X86_H #define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_OP_X86_H -#include "src/__support/macros/config.h" +#include "src/__support/macros/attributes.h" // LIBC_INLINE +#include "src/__support/macros/config.h" // LIBC_NAMESPACE_DECL #include "src/__support/macros/properties/architectures.h" #if defined(LIBC_TARGET_ARCH_IS_X86) diff --git a/system/lib/llvm-libc/src/string/memory_utils/riscv/inline_bcmp.h b/system/lib/llvm-libc/src/string/memory_utils/riscv/inline_bcmp.h index 4bdde27cd16be..666ad6382412e 100644 --- a/system/lib/llvm-libc/src/string/memory_utils/riscv/inline_bcmp.h +++ b/system/lib/llvm-libc/src/string/memory_utils/riscv/inline_bcmp.h @@ -9,7 +9,7 @@ #define LIBC_SRC_STRING_MEMORY_UTILS_RISCV_INLINE_BCMP_H #include "src/__support/macros/attributes.h" // LIBC_INLINE -#include "src/__support/macros/config.h" +#include "src/__support/macros/config.h" // LIBC_NAMESPACE_DECL #include "src/__support/macros/properties/architectures.h" // LIBC_TARGET_ARCH_IS_RISCV64 #include "src/string/memory_utils/generic/aligned_access.h" #include "src/string/memory_utils/utils.h" // Ptr, CPtr diff --git a/system/lib/llvm-libc/src/string/memory_utils/riscv/inline_memcmp.h b/system/lib/llvm-libc/src/string/memory_utils/riscv/inline_memcmp.h index ca834952d331e..c49e8d0a4ff2f 100644 --- a/system/lib/llvm-libc/src/string/memory_utils/riscv/inline_memcmp.h +++ b/system/lib/llvm-libc/src/string/memory_utils/riscv/inline_memcmp.h @@ -9,7 +9,7 @@ #define LIBC_SRC_STRING_MEMORY_UTILS_RISCV_INLINE_MEMCMP_H #include "src/__support/macros/attributes.h" // LIBC_INLINE -#include "src/__support/macros/config.h" +#include "src/__support/macros/config.h" // LIBC_NAMESPACE_DECL #include "src/__support/macros/properties/architectures.h" // LIBC_TARGET_ARCH_IS_RISCV64 #include "src/string/memory_utils/generic/aligned_access.h" #include "src/string/memory_utils/utils.h" // Ptr, CPtr diff --git a/system/lib/llvm-libc/src/string/memory_utils/riscv/inline_memcpy.h b/system/lib/llvm-libc/src/string/memory_utils/riscv/inline_memcpy.h index 8eb87e0f38ce5..e907ae4c079e0 100644 --- a/system/lib/llvm-libc/src/string/memory_utils/riscv/inline_memcpy.h +++ b/system/lib/llvm-libc/src/string/memory_utils/riscv/inline_memcpy.h @@ -9,7 +9,7 @@ #define LIBC_SRC_STRING_MEMORY_UTILS_RISCV_INLINE_MEMCPY_H #include "src/__support/macros/attributes.h" // LIBC_INLINE -#include "src/__support/macros/config.h" +#include "src/__support/macros/config.h" // LIBC_NAMESPACE_DECL #include "src/__support/macros/properties/architectures.h" // LIBC_TARGET_ARCH_IS_RISCV64 #include "src/string/memory_utils/generic/aligned_access.h" #include "src/string/memory_utils/utils.h" // Ptr, CPtr diff --git a/system/lib/llvm-libc/src/string/memory_utils/riscv/inline_memmove.h b/system/lib/llvm-libc/src/string/memory_utils/riscv/inline_memmove.h index 28de4c240260d..01a9fa5c4bbdc 100644 --- a/system/lib/llvm-libc/src/string/memory_utils/riscv/inline_memmove.h +++ b/system/lib/llvm-libc/src/string/memory_utils/riscv/inline_memmove.h @@ -9,7 +9,7 @@ #define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_RISCV_INLINE_MEMMOVE_H #include "src/__support/macros/attributes.h" // LIBC_INLINE -#include "src/__support/macros/config.h" +#include "src/__support/macros/config.h" // LIBC_NAMESPACE_DECL #include "src/__support/macros/properties/architectures.h" // LIBC_TARGET_ARCH_IS_RISCV64 #include "src/string/memory_utils/generic/byte_per_byte.h" #include "src/string/memory_utils/utils.h" // Ptr, CPtr diff --git a/system/lib/llvm-libc/src/string/memory_utils/riscv/inline_memset.h b/system/lib/llvm-libc/src/string/memory_utils/riscv/inline_memset.h index d6ab523f16414..09a7a765e4cb6 100644 --- a/system/lib/llvm-libc/src/string/memory_utils/riscv/inline_memset.h +++ b/system/lib/llvm-libc/src/string/memory_utils/riscv/inline_memset.h @@ -9,7 +9,7 @@ #define LIBC_SRC_STRING_MEMORY_UTILS_RISCV_INLINE_MEMSET_H #include "src/__support/macros/attributes.h" // LIBC_INLINE -#include "src/__support/macros/config.h" +#include "src/__support/macros/config.h" // LIBC_NAMESPACE_DECL #include "src/__support/macros/properties/architectures.h" // LIBC_TARGET_ARCH_IS_RISCV64 #include "src/string/memory_utils/generic/aligned_access.h" #include "src/string/memory_utils/utils.h" // Ptr, CPtr diff --git a/system/lib/llvm-libc/src/string/memory_utils/utils.h b/system/lib/llvm-libc/src/string/memory_utils/utils.h index cae65bddd92f6..bdf0b8652188b 100644 --- a/system/lib/llvm-libc/src/string/memory_utils/utils.h +++ b/system/lib/llvm-libc/src/string/memory_utils/utils.h @@ -14,7 +14,7 @@ #include "src/__support/CPP/type_traits.h" #include "src/__support/endian_internal.h" #include "src/__support/macros/attributes.h" // LIBC_INLINE -#include "src/__support/macros/config.h" +#include "src/__support/macros/config.h" // LIBC_NAMESPACE_DECL #include "src/__support/macros/properties/architectures.h" #include // size_t @@ -263,7 +263,7 @@ LIBC_INLINE void store_aligned(ValueType value, Ptr dst) { static_assert(sizeof(ValueType) >= (sizeof(T) + ... + sizeof(TS))); constexpr size_t SHIFT = sizeof(T) * 8; if constexpr (Endian::IS_LITTLE) { - store(assume_aligned(dst), value & ~T(0)); + store(assume_aligned(dst), T(value & T(~0))); if constexpr (sizeof...(TS) > 0) store_aligned(value >> SHIFT, dst + sizeof(T)); } else if constexpr (Endian::IS_BIG) { @@ -297,7 +297,7 @@ LIBC_INLINE void adjust(ptrdiff_t offset, T1 *__restrict &p1, T2 *__restrict &p2, size_t &count) { p1 += offset; p2 += offset; - count -= offset; + count -= static_cast(offset); } // Advances p1 and p2 so p1 gets aligned to the next SIZE bytes boundary @@ -306,7 +306,8 @@ LIBC_INLINE void adjust(ptrdiff_t offset, T1 *__restrict &p1, template void align_p1_to_next_boundary(T1 *__restrict &p1, T2 *__restrict &p2, size_t &count) { - adjust(distance_to_next_aligned(p1), p1, p2, count); + adjust(static_cast(distance_to_next_aligned(p1)), p1, p2, + count); p1 = assume_aligned(p1); } diff --git a/system/lib/llvm-libc/src/string/memory_utils/x86_64/inline_bcmp.h b/system/lib/llvm-libc/src/string/memory_utils/x86_64/inline_bcmp.h index cc54c4140ee6e..8be391b428910 100644 --- a/system/lib/llvm-libc/src/string/memory_utils/x86_64/inline_bcmp.h +++ b/system/lib/llvm-libc/src/string/memory_utils/x86_64/inline_bcmp.h @@ -9,7 +9,7 @@ #define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_X86_64_INLINE_BCMP_H #include "src/__support/macros/attributes.h" // LIBC_INLINE -#include "src/__support/macros/config.h" +#include "src/__support/macros/config.h" // LIBC_NAMESPACE_DECL #include "src/string/memory_utils/op_generic.h" #include "src/string/memory_utils/op_x86.h" #include "src/string/memory_utils/utils.h" // Ptr, CPtr diff --git a/system/lib/llvm-libc/src/string/memory_utils/x86_64/inline_memset.h b/system/lib/llvm-libc/src/string/memory_utils/x86_64/inline_memset.h index 9f8e584d2bbb4..35719a89fa600 100644 --- a/system/lib/llvm-libc/src/string/memory_utils/x86_64/inline_memset.h +++ b/system/lib/llvm-libc/src/string/memory_utils/x86_64/inline_memset.h @@ -8,8 +8,8 @@ #ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_X86_64_INLINE_MEMSET_H #define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_X86_64_INLINE_MEMSET_H -#include "src/__support/macros/attributes.h" // LIBC_INLINE -#include "src/__support/macros/config.h" +#include "src/__support/macros/attributes.h" // LIBC_INLINE, LIBC_INLINE_VAR +#include "src/__support/macros/config.h" // LIBC_NAMESPACE_DECL #include "src/string/memory_utils/op_generic.h" #include "src/string/memory_utils/op_x86.h" #include "src/string/memory_utils/utils.h" // Ptr, CPtr diff --git a/system/lib/llvm-libc/src/string/mempcpy.cpp b/system/lib/llvm-libc/src/string/mempcpy.cpp index 09392ceb966d6..b6a9721960ce3 100644 --- a/system/lib/llvm-libc/src/string/mempcpy.cpp +++ b/system/lib/llvm-libc/src/string/mempcpy.cpp @@ -8,6 +8,7 @@ #include "src/string/mempcpy.h" #include "src/__support/macros/config.h" +#include "src/__support/macros/null_check.h" #include "src/string/memory_utils/inline_memcpy.h" #include "src/__support/common.h" @@ -18,6 +19,10 @@ namespace LIBC_NAMESPACE_DECL { LLVM_LIBC_FUNCTION(void *, mempcpy, (void *__restrict dst, const void *__restrict src, size_t count)) { + if (count) { + LIBC_CRASH_ON_NULLPTR(dst); + LIBC_CRASH_ON_NULLPTR(src); + } inline_memcpy(dst, src, count); return reinterpret_cast(dst) + count; } diff --git a/system/lib/llvm-libc/src/string/memrchr.cpp b/system/lib/llvm-libc/src/string/memrchr.cpp index d665e225bbb7d..d5c843c733b39 100644 --- a/system/lib/llvm-libc/src/string/memrchr.cpp +++ b/system/lib/llvm-libc/src/string/memrchr.cpp @@ -9,11 +9,16 @@ #include "src/string/memrchr.h" #include "src/__support/common.h" #include "src/__support/macros/config.h" +#include "src/__support/macros/null_check.h" #include namespace LIBC_NAMESPACE_DECL { LLVM_LIBC_FUNCTION(void *, memrchr, (const void *src, int c, size_t n)) { + + if (n) + LIBC_CRASH_ON_NULLPTR(src); + const unsigned char *str = reinterpret_cast(src); const unsigned char ch = static_cast(c); for (; n != 0; --n) { diff --git a/system/lib/llvm-libc/src/string/memset.cpp b/system/lib/llvm-libc/src/string/memset.cpp index c2868afa91031..a0b96b3cf88bb 100644 --- a/system/lib/llvm-libc/src/string/memset.cpp +++ b/system/lib/llvm-libc/src/string/memset.cpp @@ -9,11 +9,15 @@ #include "src/string/memset.h" #include "src/__support/common.h" #include "src/__support/macros/config.h" +#include "src/__support/macros/null_check.h" #include "src/string/memory_utils/inline_memset.h" namespace LIBC_NAMESPACE_DECL { LLVM_LIBC_FUNCTION(void *, memset, (void *dst, int value, size_t count)) { + if (count) + LIBC_CRASH_ON_NULLPTR(dst); + inline_memset(dst, static_cast(value), count); return dst; } diff --git a/system/lib/llvm-libc/src/string/stpcpy.cpp b/system/lib/llvm-libc/src/string/stpcpy.cpp index 979edd72c1f1d..48c0db950ace0 100644 --- a/system/lib/llvm-libc/src/string/stpcpy.cpp +++ b/system/lib/llvm-libc/src/string/stpcpy.cpp @@ -8,7 +8,6 @@ #include "src/string/stpcpy.h" #include "src/__support/macros/config.h" -#include "src/string/mempcpy.h" #include "src/string/string_utils.h" #include "src/__support/common.h" @@ -18,8 +17,8 @@ namespace LIBC_NAMESPACE_DECL { LLVM_LIBC_FUNCTION(char *, stpcpy, (char *__restrict dest, const char *__restrict src)) { size_t size = internal::string_length(src) + 1; - char *result = - reinterpret_cast(LIBC_NAMESPACE::mempcpy(dest, src, size)); + __builtin_memcpy(dest, src, size); + char *result = dest + size; if (result != nullptr) return result - 1; diff --git a/system/lib/llvm-libc/src/string/stpncpy.cpp b/system/lib/llvm-libc/src/string/stpncpy.cpp index d2a6e04749820..47bf4c6e86fba 100644 --- a/system/lib/llvm-libc/src/string/stpncpy.cpp +++ b/system/lib/llvm-libc/src/string/stpncpy.cpp @@ -8,6 +8,7 @@ #include "src/string/stpncpy.h" #include "src/__support/macros/config.h" +#include "src/__support/macros/null_check.h" #include "src/string/memory_utils/inline_bzero.h" #include "src/__support/common.h" @@ -17,6 +18,10 @@ namespace LIBC_NAMESPACE_DECL { LLVM_LIBC_FUNCTION(char *, stpncpy, (char *__restrict dest, const char *__restrict src, size_t n)) { + if (n) { + LIBC_CRASH_ON_NULLPTR(dest); + LIBC_CRASH_ON_NULLPTR(src); + } size_t i; // Copy up until \0 is found. for (i = 0; i < n && src[i] != '\0'; ++i) diff --git a/system/lib/llvm-libc/src/string/strcasestr.cpp b/system/lib/llvm-libc/src/string/strcasestr.cpp index 1da1e3f025079..de8e4bec7fe0b 100644 --- a/system/lib/llvm-libc/src/string/strcasestr.cpp +++ b/system/lib/llvm-libc/src/string/strcasestr.cpp @@ -11,6 +11,7 @@ #include "src/__support/common.h" #include "src/__support/ctype_utils.h" #include "src/__support/macros/config.h" +#include "src/__support/macros/null_check.h" #include "src/string/memory_utils/inline_strstr.h" namespace LIBC_NAMESPACE_DECL { @@ -23,6 +24,9 @@ LLVM_LIBC_FUNCTION(char *, strcasestr, return LIBC_NAMESPACE::internal::tolower(a) - LIBC_NAMESPACE::internal::tolower(b); }; + + LIBC_CRASH_ON_NULLPTR(haystack); + LIBC_CRASH_ON_NULLPTR(needle); return inline_strstr(haystack, needle, case_cmp); } diff --git a/system/lib/llvm-libc/src/string/strcat.cpp b/system/lib/llvm-libc/src/string/strcat.cpp index 0eb189ce204f0..7dce6d15a65c1 100644 --- a/system/lib/llvm-libc/src/string/strcat.cpp +++ b/system/lib/llvm-libc/src/string/strcat.cpp @@ -8,7 +8,7 @@ #include "src/string/strcat.h" #include "src/__support/macros/config.h" -#include "src/string/strcpy.h" +#include "src/__support/macros/null_check.h" #include "src/string/string_utils.h" #include "src/__support/common.h" @@ -17,10 +17,14 @@ namespace LIBC_NAMESPACE_DECL { LLVM_LIBC_FUNCTION(char *, strcat, (char *__restrict dest, const char *__restrict src)) { + LIBC_CRASH_ON_NULLPTR(dest); + LIBC_CRASH_ON_NULLPTR(src); size_t dest_length = internal::string_length(dest); - size_t src_length = internal::string_length(src); - LIBC_NAMESPACE::strcpy(dest + dest_length, src); - dest[dest_length + src_length] = '\0'; + size_t i; + for (i = 0; src[i] != '\0'; ++i) + dest[dest_length + i] = src[i]; + + dest[dest_length + i] = '\0'; return dest; } diff --git a/system/lib/llvm-libc/src/string/strcoll.cpp b/system/lib/llvm-libc/src/string/strcoll.cpp index eeb2c79e38074..aa08f7194c9e5 100644 --- a/system/lib/llvm-libc/src/string/strcoll.cpp +++ b/system/lib/llvm-libc/src/string/strcoll.cpp @@ -10,11 +10,14 @@ #include "src/__support/common.h" #include "src/__support/macros/config.h" +#include "src/__support/macros/null_check.h" namespace LIBC_NAMESPACE_DECL { // TODO: Add support for locales. LLVM_LIBC_FUNCTION(int, strcoll, (const char *left, const char *right)) { + LIBC_CRASH_ON_NULLPTR(left); + LIBC_CRASH_ON_NULLPTR(right); for (; *left && *left == *right; ++left, ++right) ; return static_cast(*left) - static_cast(*right); diff --git a/system/lib/llvm-libc/src/string/strcoll_l.cpp b/system/lib/llvm-libc/src/string/strcoll_l.cpp index f664a3c7c03f3..e820efa564a3d 100644 --- a/system/lib/llvm-libc/src/string/strcoll_l.cpp +++ b/system/lib/llvm-libc/src/string/strcoll_l.cpp @@ -10,12 +10,15 @@ #include "src/__support/common.h" #include "src/__support/macros/config.h" +#include "src/__support/macros/null_check.h" namespace LIBC_NAMESPACE_DECL { // TODO: Add support for locales. LLVM_LIBC_FUNCTION(int, strcoll_l, (const char *left, const char *right, locale_t)) { + LIBC_CRASH_ON_NULLPTR(left); + LIBC_CRASH_ON_NULLPTR(right); for (; *left && *left == *right; ++left, ++right) ; return static_cast(*left) - static_cast(*right); diff --git a/system/lib/llvm-libc/src/string/strcpy.cpp b/system/lib/llvm-libc/src/string/strcpy.cpp index 60b73ab3aa823..2013593db24a6 100644 --- a/system/lib/llvm-libc/src/string/strcpy.cpp +++ b/system/lib/llvm-libc/src/string/strcpy.cpp @@ -8,6 +8,7 @@ #include "src/string/strcpy.h" #include "src/__support/macros/config.h" +#include "src/__support/macros/null_check.h" #include "src/string/memory_utils/inline_memcpy.h" #include "src/string/string_utils.h" @@ -17,6 +18,7 @@ namespace LIBC_NAMESPACE_DECL { LLVM_LIBC_FUNCTION(char *, strcpy, (char *__restrict dest, const char *__restrict src)) { + LIBC_CRASH_ON_NULLPTR(dest); size_t size = internal::string_length(src) + 1; inline_memcpy(dest, src, size); return dest; diff --git a/system/lib/llvm-libc/src/string/strdup.cpp b/system/lib/llvm-libc/src/string/strdup.cpp index 4cf4173a27bf3..dab0ab4288c9e 100644 --- a/system/lib/llvm-libc/src/string/strdup.cpp +++ b/system/lib/llvm-libc/src/string/strdup.cpp @@ -8,8 +8,8 @@ #include "src/string/strdup.h" #include "hdr/stdlib_macros.h" +#include "src/__support/libc_errno.h" #include "src/__support/macros/config.h" -#include "src/errno/libc_errno.h" #include "src/string/allocating_string_utils.h" #include "src/string/memory_utils/inline_memcpy.h" diff --git a/system/lib/llvm-libc/src/string/string_utils.h b/system/lib/llvm-libc/src/string/string_utils.h index 583d35014d398..4f56263fce8ec 100644 --- a/system/lib/llvm-libc/src/string/string_utils.h +++ b/system/lib/llvm-libc/src/string/string_utils.h @@ -14,19 +14,19 @@ #ifndef LLVM_LIBC_SRC_STRING_STRING_UTILS_H #define LLVM_LIBC_SRC_STRING_STRING_UTILS_H +#include "hdr/limits_macros.h" #include "hdr/types/size_t.h" #include "src/__support/CPP/bitset.h" #include "src/__support/CPP/type_traits.h" // cpp::is_same_v #include "src/__support/macros/config.h" #include "src/__support/macros/optimization.h" // LIBC_UNLIKELY -#include "src/string/memory_utils/inline_bzero.h" -#include "src/string/memory_utils/inline_memcpy.h" namespace LIBC_NAMESPACE_DECL { namespace internal { template LIBC_INLINE constexpr Word repeat_byte(Word byte) { - constexpr size_t BITS_IN_BYTE = 8; + static_assert(CHAR_BIT == 8, "repeat_byte assumes a byte is 8 bits."); + constexpr size_t BITS_IN_BYTE = CHAR_BIT; constexpr size_t BYTE_MASK = 0xff; Word result = 0; byte = byte & BYTE_MASK; @@ -66,7 +66,7 @@ LIBC_INLINE size_t string_length_wide_read(const char *src) { for (; reinterpret_cast(char_ptr) % sizeof(Word) != 0; ++char_ptr) { if (*char_ptr == '\0') - return char_ptr - src; + return static_cast(char_ptr - src); } // Step 2: read blocks for (const Word *block_ptr = reinterpret_cast(char_ptr); @@ -77,7 +77,7 @@ LIBC_INLINE size_t string_length_wide_read(const char *src) { for (; *char_ptr != '\0'; ++char_ptr) { ; } - return char_ptr - src; + return static_cast(char_ptr - src); } // Returns the length of a string, denoted by the first occurrence @@ -90,12 +90,11 @@ template LIBC_INLINE size_t string_length(const T *src) { // string a block at a time. if constexpr (cpp::is_same_v) return string_length_wide_read(src); -#else +#endif size_t length; for (length = 0; *src; ++src, ++length) ; return length; -#endif } template @@ -169,7 +168,7 @@ LIBC_INLINE size_t complementary_span(const char *src, const char *segment) { for (; *src && !bitset.test(*reinterpret_cast(src)); ++src) ; - return src - initial; + return static_cast(src - initial); } // Given the similarities between strtok and strtok_r, we can implement both @@ -189,12 +188,13 @@ LIBC_INLINE char *string_token(char *__restrict src, if (LIBC_UNLIKELY(src == nullptr && ((src = *saveptr) == nullptr))) return nullptr; + static_assert(CHAR_BIT == 8, "bitset of 256 assumes char is 8 bits"); cpp::bitset<256> delimiter_set; for (; *delimiter_string != '\0'; ++delimiter_string) - delimiter_set.set(*delimiter_string); + delimiter_set.set(static_cast(*delimiter_string)); if constexpr (SkipDelim) - for (; *src != '\0' && delimiter_set.test(*src); ++src) + for (; *src != '\0' && delimiter_set.test(static_cast(*src)); ++src) ; if (*src == '\0') { *saveptr = src; @@ -202,7 +202,7 @@ LIBC_INLINE char *string_token(char *__restrict src, } char *token = src; for (; *src != '\0'; ++src) { - if (delimiter_set.test(*src)) { + if (delimiter_set.test(static_cast(*src))) { *src = '\0'; ++src; break; @@ -218,7 +218,7 @@ LIBC_INLINE size_t strlcpy(char *__restrict dst, const char *__restrict src, if (!size) return len; size_t n = len < size - 1 ? len : size - 1; - inline_memcpy(dst, src, n); + __builtin_memcpy(dst, src, n); dst[n] = '\0'; return len; } diff --git a/system/lib/llvm-libc/src/string/strlen.cpp b/system/lib/llvm-libc/src/string/strlen.cpp index ff7ab14dd3134..234edb81d4c8c 100644 --- a/system/lib/llvm-libc/src/string/strlen.cpp +++ b/system/lib/llvm-libc/src/string/strlen.cpp @@ -8,6 +8,7 @@ #include "src/string/strlen.h" #include "src/__support/macros/config.h" +#include "src/__support/macros/null_check.h" #include "src/string/string_utils.h" #include "src/__support/common.h" @@ -17,6 +18,7 @@ namespace LIBC_NAMESPACE_DECL { // TODO: investigate the performance of this function. // There might be potential for compiler optimization. LLVM_LIBC_FUNCTION(size_t, strlen, (const char *src)) { + LIBC_CRASH_ON_NULLPTR(src); return internal::string_length(src); } diff --git a/system/lib/llvm-libc/src/string/strncat.cpp b/system/lib/llvm-libc/src/string/strncat.cpp index 221881f93c47a..6d8bb69607486 100644 --- a/system/lib/llvm-libc/src/string/strncat.cpp +++ b/system/lib/llvm-libc/src/string/strncat.cpp @@ -8,8 +8,8 @@ #include "src/string/strncat.h" #include "src/__support/macros/config.h" +#include "src/__support/macros/null_check.h" #include "src/string/string_utils.h" -#include "src/string/strncpy.h" #include "src/__support/common.h" @@ -18,11 +18,16 @@ namespace LIBC_NAMESPACE_DECL { LLVM_LIBC_FUNCTION(char *, strncat, (char *__restrict dest, const char *__restrict src, size_t count)) { - size_t src_length = internal::string_length(src); - size_t copy_amount = src_length > count ? count : src_length; + if (count) { + LIBC_CRASH_ON_NULLPTR(dest); + LIBC_CRASH_ON_NULLPTR(src); + } size_t dest_length = internal::string_length(dest); - LIBC_NAMESPACE::strncpy(dest + dest_length, src, copy_amount); - dest[dest_length + copy_amount] = '\0'; + size_t i; + for (i = 0; i < count && src[i] != '\0'; ++i) + dest[dest_length + i] = src[i]; + + dest[dest_length + i] = '\0'; return dest; } diff --git a/system/lib/llvm-libc/src/string/strncmp.cpp b/system/lib/llvm-libc/src/string/strncmp.cpp index 16d4601fe845b..f21fd769f3943 100644 --- a/system/lib/llvm-libc/src/string/strncmp.cpp +++ b/system/lib/llvm-libc/src/string/strncmp.cpp @@ -10,6 +10,7 @@ #include "src/__support/common.h" #include "src/__support/macros/config.h" +#include "src/__support/macros/null_check.h" #include "src/string/memory_utils/inline_strcmp.h" #include @@ -18,6 +19,10 @@ namespace LIBC_NAMESPACE_DECL { LLVM_LIBC_FUNCTION(int, strncmp, (const char *left, const char *right, size_t n)) { + if (n) { + LIBC_CRASH_ON_NULLPTR(left); + LIBC_CRASH_ON_NULLPTR(right); + } auto comp = [](char l, char r) -> int { return l - r; }; return inline_strncmp(left, right, n, comp); } diff --git a/system/lib/llvm-libc/src/string/strncpy.cpp b/system/lib/llvm-libc/src/string/strncpy.cpp index 4976ad94708c7..e271009502f2a 100644 --- a/system/lib/llvm-libc/src/string/strncpy.cpp +++ b/system/lib/llvm-libc/src/string/strncpy.cpp @@ -10,6 +10,7 @@ #include "src/__support/common.h" #include "src/__support/macros/config.h" +#include "src/__support/macros/null_check.h" #include // For size_t. namespace LIBC_NAMESPACE_DECL { @@ -17,6 +18,10 @@ namespace LIBC_NAMESPACE_DECL { LLVM_LIBC_FUNCTION(char *, strncpy, (char *__restrict dest, const char *__restrict src, size_t n)) { + if (n) { + LIBC_CRASH_ON_NULLPTR(dest); + LIBC_CRASH_ON_NULLPTR(src); + } size_t i = 0; // Copy up until \0 is found. for (; i < n && src[i] != '\0'; ++i) diff --git a/system/lib/llvm-libc/src/string/strsep.cpp b/system/lib/llvm-libc/src/string/strsep.cpp index 4c275122de52f..41874b6af2263 100644 --- a/system/lib/llvm-libc/src/string/strsep.cpp +++ b/system/lib/llvm-libc/src/string/strsep.cpp @@ -9,14 +9,19 @@ #include "src/string/strsep.h" #include "src/__support/macros/config.h" +#include "src/__support/macros/null_check.h" #include "src/string/string_utils.h" +#include "src/__support/common.h" + namespace LIBC_NAMESPACE_DECL { LLVM_LIBC_FUNCTION(char *, strsep, (char **__restrict stringp, const char *__restrict delim)) { + LIBC_CRASH_ON_NULLPTR(stringp); if (!*stringp) return nullptr; + LIBC_CRASH_ON_NULLPTR(delim); return internal::string_token(*stringp, delim, stringp); } diff --git a/system/lib/llvm-libc/src/string/strspn.cpp b/system/lib/llvm-libc/src/string/strspn.cpp index 66bb39903ab38..b205bedf80cab 100644 --- a/system/lib/llvm-libc/src/string/strspn.cpp +++ b/system/lib/llvm-libc/src/string/strspn.cpp @@ -11,11 +11,14 @@ #include "src/__support/CPP/bitset.h" #include "src/__support/common.h" #include "src/__support/macros/config.h" +#include "src/__support/macros/null_check.h" #include namespace LIBC_NAMESPACE_DECL { LLVM_LIBC_FUNCTION(size_t, strspn, (const char *src, const char *segment)) { + LIBC_CRASH_ON_NULLPTR(src); + LIBC_CRASH_ON_NULLPTR(segment); const char *initial = src; cpp::bitset<256> bitset; diff --git a/system/lib/llvm-libc/src/string/strstr.cpp b/system/lib/llvm-libc/src/string/strstr.cpp index 5132f06ef53f3..44797ef670b9f 100644 --- a/system/lib/llvm-libc/src/string/strstr.cpp +++ b/system/lib/llvm-libc/src/string/strstr.cpp @@ -10,6 +10,7 @@ #include "src/__support/common.h" #include "src/__support/macros/config.h" +#include "src/__support/macros/null_check.h" #include "src/string/memory_utils/inline_strstr.h" namespace LIBC_NAMESPACE_DECL { @@ -18,6 +19,8 @@ namespace LIBC_NAMESPACE_DECL { // improved upon using well known string matching algorithms. LLVM_LIBC_FUNCTION(char *, strstr, (const char *haystack, const char *needle)) { auto comp = [](char l, char r) -> int { return l - r; }; + LIBC_CRASH_ON_NULLPTR(haystack); + LIBC_CRASH_ON_NULLPTR(needle); return inline_strstr(haystack, needle, comp); } diff --git a/system/lib/llvm-libc/src/strings/ffs.cpp b/system/lib/llvm-libc/src/strings/ffs.cpp new file mode 100644 index 0000000000000..5e1efafeab56a --- /dev/null +++ b/system/lib/llvm-libc/src/strings/ffs.cpp @@ -0,0 +1,20 @@ +//===-- Implementation of ffs ---------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/strings/ffs.h" +#include "src/__support/common.h" +#include "src/__support/macros/config.h" +#include "src/__support/math_extras.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(int, ffs, (int i)) { + return first_trailing_one(static_cast(i)); +} + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/strings/ffs.h b/system/lib/llvm-libc/src/strings/ffs.h new file mode 100644 index 0000000000000..bf43c43caedb2 --- /dev/null +++ b/system/lib/llvm-libc/src/strings/ffs.h @@ -0,0 +1,20 @@ +//===-- Implementation header for ffs ---------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_STRINGS_FFS_H +#define LLVM_LIBC_SRC_STRINGS_FFS_H + +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +int ffs(int i); + +} // namespace LIBC_NAMESPACE_DECL + +#endif // LLVM_LIBC_SRC_STRINGS_FFS_H diff --git a/system/lib/llvm-libc/src/strings/ffsl.cpp b/system/lib/llvm-libc/src/strings/ffsl.cpp new file mode 100644 index 0000000000000..b1b25598e405b --- /dev/null +++ b/system/lib/llvm-libc/src/strings/ffsl.cpp @@ -0,0 +1,20 @@ +//===-- Implementation of ffsl --------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/strings/ffsl.h" +#include "src/__support/common.h" +#include "src/__support/macros/config.h" +#include "src/__support/math_extras.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(int, ffsl, (long i)) { + return first_trailing_one(static_cast(i)); +} + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/strings/ffsl.h b/system/lib/llvm-libc/src/strings/ffsl.h new file mode 100644 index 0000000000000..1feca010b2ebb --- /dev/null +++ b/system/lib/llvm-libc/src/strings/ffsl.h @@ -0,0 +1,20 @@ +//===-- Implementation header for ffsl --------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_STRINGS_FFSL_H +#define LLVM_LIBC_SRC_STRINGS_FFSL_H + +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +int ffsl(long i); + +} // namespace LIBC_NAMESPACE_DECL + +#endif // LLVM_LIBC_SRC_STRINGS_FFSL_H diff --git a/system/lib/llvm-libc/src/strings/ffsll.cpp b/system/lib/llvm-libc/src/strings/ffsll.cpp new file mode 100644 index 0000000000000..be16e81a54711 --- /dev/null +++ b/system/lib/llvm-libc/src/strings/ffsll.cpp @@ -0,0 +1,20 @@ +//===-- Implementation of ffsll -------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/strings/ffsll.h" +#include "src/__support/common.h" +#include "src/__support/macros/config.h" +#include "src/__support/math_extras.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(int, ffsll, (long long i)) { + return first_trailing_one(static_cast(i)); +} + +} // namespace LIBC_NAMESPACE_DECL diff --git a/system/lib/llvm-libc/src/strings/ffsll.h b/system/lib/llvm-libc/src/strings/ffsll.h new file mode 100644 index 0000000000000..f059b8ab89b4e --- /dev/null +++ b/system/lib/llvm-libc/src/strings/ffsll.h @@ -0,0 +1,20 @@ +//===-- Implementation header for ffsll -------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_STRINGS_FFSLL_H +#define LLVM_LIBC_SRC_STRINGS_FFSLL_H + +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +int ffsll(long long i); + +} // namespace LIBC_NAMESPACE_DECL + +#endif // LLVM_LIBC_SRC_STRINGS_FFSLL_H diff --git a/system/lib/llvm-libc/src/strings/rindex.cpp b/system/lib/llvm-libc/src/strings/rindex.cpp index 1242e0faf85fc..2540222ffb0eb 100644 --- a/system/lib/llvm-libc/src/strings/rindex.cpp +++ b/system/lib/llvm-libc/src/strings/rindex.cpp @@ -10,11 +10,13 @@ #include "src/__support/common.h" #include "src/__support/macros/config.h" +#include "src/__support/macros/null_check.h" #include "src/string/string_utils.h" namespace LIBC_NAMESPACE_DECL { LLVM_LIBC_FUNCTION(char *, rindex, (const char *src, int c)) { + LIBC_CRASH_ON_NULLPTR(src); return internal::strrchr_implementation(src, c); } diff --git a/system/lib/update_libcxx.py b/system/lib/update_libcxx.py index c71832417c65c..3dacb46eae950 100755 --- a/system/lib/update_libcxx.py +++ b/system/lib/update_libcxx.py @@ -25,14 +25,6 @@ 'include/llvm-libc-types', 'shared', 'config', - 'src/__support', - 'src/string', - 'src/strings', - 'src/errno', - 'src/math', - 'src/stdlib', - 'src/inttypes', - 'src/stdio/printf_core', ] def clean_dir(dirname): diff --git a/system/lib/update_llvm_libc.py b/system/lib/update_llvm_libc.py new file mode 100644 index 0000000000000..d6ba9c0e03517 --- /dev/null +++ b/system/lib/update_llvm_libc.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +# Copyright 2019 The Emscripten Authors. All rights reserved. +# Emscripten is available under two separate licenses, the MIT license and the +# University of Illinois/NCSA Open Source License. Both these licenses can be +# found in the LICENSE file. + +import os +import sys +import shutil +import glob + +script_dir = os.path.abspath(os.path.dirname(__file__)) +emscripten_root = os.path.dirname(os.path.dirname(script_dir)) +default_llvm_dir = os.path.join(os.path.dirname(emscripten_root), 'llvm-project') + +preserve_files = ('readme.txt', '__assertion_handler', '__config_site') +# ryu_long_double_constants.h from libc is unused (and very large) +excludes = ('CMakeLists.txt', 'ryu_long_double_constants.h') + +libc_copy_dirs = [ + 'hdr', + 'include/llvm-libc-macros', + 'include/llvm-libc-types', + 'shared', + 'config', + 'src/__support', + 'src/string', + 'src/strings', + 'src/errno', + 'src/math', + 'src/stdlib', + 'src/inttypes', + 'src/stdio/printf_core', +] + +libc_exclusion_patterns = [ + 'src/math/generic/*f16*', # float16 is unsupported in Emscripten. + 'src/strings/str*casecmp_l*' # locale_t is unsupported in Overlay Mode. +] + +def clean_dir(dirname): + if not os.path.exists(dirname): + return + for f in os.listdir(dirname): + if f in preserve_files: + continue + full = os.path.join(dirname, f) + if os.path.isdir(full): + shutil.rmtree(full) + else: + os.remove(full) + + +def copy_tree(upstream_dir, local_dir): + if not os.path.exists(local_dir): + os.makedirs(local_dir) + for f in os.listdir(upstream_dir): + full = os.path.join(upstream_dir, f) + if os.path.isdir(full): + shutil.copytree(full, os.path.join(local_dir, f)) + elif f not in excludes: + shutil.copy2(full, os.path.join(local_dir, f)) + for root, dirs, files in os.walk(local_dir): + for f in files: + if f in excludes: + full = os.path.join(root, f) + os.remove(full) + +def main(): + if len(sys.argv) > 1: + llvm_dir = os.path.abspath(sys.argv[1]) + else: + llvm_dir = default_llvm_dir + libcxx_dir = os.path.join(llvm_dir, 'libcxx') + + libc_upstream_dir = os.path.join(llvm_dir, 'libc') + assert os.path.exists(libc_upstream_dir) + libc_local_dir = os.path.join(script_dir, 'llvm-libc') + + for dirname in libc_copy_dirs: + local_dir = os.path.join(libc_local_dir, dirname) + clean_dir(local_dir) + + for dirname in libc_copy_dirs: + upstream_dir = os.path.join(libc_upstream_dir, dirname) + local_dir = os.path.join(libc_local_dir, dirname) + copy_tree(upstream_dir, local_dir) + + # Certain llvm-libc files that are incompatible in Emscripten + for excludsion_pattern in libc_exclusion_patterns: + files_to_exclude = glob.glob(os.path.join(libc_local_dir, excludsion_pattern)) + for file in files_to_exclude: + os.remove(file) + +if __name__ == '__main__': + main() diff --git a/test/code_size/audio_worklet_wasm.json b/test/code_size/audio_worklet_wasm.json index 5aad516bd2316..753ef4971f2aa 100644 --- a/test/code_size/audio_worklet_wasm.json +++ b/test/code_size/audio_worklet_wasm.json @@ -3,8 +3,8 @@ "a.html.gz": 364, "a.js": 3853, "a.js.gz": 2050, - "a.wasm": 1294, - "a.wasm.gz": 864, - "total": 5666, - "total_gz": 3278 + "a.wasm": 1287, + "a.wasm.gz": 859, + "total": 5659, + "total_gz": 3273 } diff --git a/test/code_size/embind_hello_wasm.json b/test/code_size/embind_hello_wasm.json index a9884cc49d60f..f0ed7f9061df1 100644 --- a/test/code_size/embind_hello_wasm.json +++ b/test/code_size/embind_hello_wasm.json @@ -1,10 +1,10 @@ { "a.html": 552, "a.html.gz": 380, - "a.js": 7266, - "a.js.gz": 3321, - "a.wasm": 7300, - "a.wasm.gz": 3348, - "total": 15118, - "total_gz": 7049 + "a.js": 7680, + "a.js.gz": 3363, + "a.wasm": 7290, + "a.wasm.gz": 3343, + "total": 15522, + "total_gz": 7086 } diff --git a/test/code_size/embind_val_wasm.json b/test/code_size/embind_val_wasm.json index dd1c00e7322f8..c728c849e521a 100644 --- a/test/code_size/embind_val_wasm.json +++ b/test/code_size/embind_val_wasm.json @@ -1,10 +1,10 @@ { "a.html": 552, "a.html.gz": 380, - "a.js": 5367, - "a.js.gz": 2540, - "a.wasm": 9101, - "a.wasm.gz": 4698, - "total": 15020, - "total_gz": 7618 + "a.js": 5685, + "a.js.gz": 2538, + "a.wasm": 9083, + "a.wasm.gz": 4682, + "total": 15320, + "total_gz": 7600 } diff --git a/test/code_size/hello_wasm_worker_wasm.json b/test/code_size/hello_wasm_worker_wasm.json index 4b31168c56d25..05d3767404b39 100644 --- a/test/code_size/hello_wasm_worker_wasm.json +++ b/test/code_size/hello_wasm_worker_wasm.json @@ -3,8 +3,8 @@ "a.html.gz": 364, "a.js": 830, "a.js.gz": 530, - "a.wasm": 1891, - "a.wasm.gz": 1082, - "total": 3240, - "total_gz": 1976 + "a.wasm": 1881, + "a.wasm.gz": 1069, + "total": 3230, + "total_gz": 1963 } diff --git a/test/code_size/random_printf_wasm.json b/test/code_size/random_printf_wasm.json index 66b36767c9c4f..f1d7d9c5eeeb8 100644 --- a/test/code_size/random_printf_wasm.json +++ b/test/code_size/random_printf_wasm.json @@ -1,6 +1,6 @@ { - "a.html": 12515, - "a.html.gz": 6858, - "total": 12515, - "total_gz": 6858 + "a.html": 12519, + "a.html.gz": 6849, + "total": 12519, + "total_gz": 6849 } diff --git a/test/code_size/random_printf_wasm2js.json b/test/code_size/random_printf_wasm2js.json index d1adadf3f7fe2..b20d638983b5f 100644 --- a/test/code_size/random_printf_wasm2js.json +++ b/test/code_size/random_printf_wasm2js.json @@ -1,6 +1,6 @@ { - "a.html": 17233, - "a.html.gz": 7547, - "total": 17233, - "total_gz": 7547 + "a.html": 17199, + "a.html.gz": 7525, + "total": 17199, + "total_gz": 7525 } diff --git a/test/other/codesize/test_codesize_cxx_ctors1.size b/test/other/codesize/test_codesize_cxx_ctors1.size index 3308094165994..a479a3cfb63da 100644 --- a/test/other/codesize/test_codesize_cxx_ctors1.size +++ b/test/other/codesize/test_codesize_cxx_ctors1.size @@ -1 +1 @@ -129753 +129745 diff --git a/test/other/codesize/test_codesize_cxx_ctors2.size b/test/other/codesize/test_codesize_cxx_ctors2.size index b2ff9e0759f6f..3b8091cb3a236 100644 --- a/test/other/codesize/test_codesize_cxx_ctors2.size +++ b/test/other/codesize/test_codesize_cxx_ctors2.size @@ -1 +1 @@ -129138 +129130 diff --git a/test/other/codesize/test_codesize_cxx_except.size b/test/other/codesize/test_codesize_cxx_except.size index 9badc0f7f5176..e849b405ae737 100644 --- a/test/other/codesize/test_codesize_cxx_except.size +++ b/test/other/codesize/test_codesize_cxx_except.size @@ -1 +1 @@ -171423 +171415 diff --git a/test/other/codesize/test_codesize_cxx_except_wasm.size b/test/other/codesize/test_codesize_cxx_except_wasm.size index 2c22f318394f9..78bcec982cede 100644 --- a/test/other/codesize/test_codesize_cxx_except_wasm.size +++ b/test/other/codesize/test_codesize_cxx_except_wasm.size @@ -1 +1 @@ -144869 +144861 diff --git a/test/other/codesize/test_codesize_cxx_except_wasm_legacy.size b/test/other/codesize/test_codesize_cxx_except_wasm_legacy.size index b151408672a7b..7d3848e6c64e7 100644 --- a/test/other/codesize/test_codesize_cxx_except_wasm_legacy.size +++ b/test/other/codesize/test_codesize_cxx_except_wasm_legacy.size @@ -1 +1 @@ -142459 +142451 diff --git a/test/other/codesize/test_codesize_cxx_lto.size b/test/other/codesize/test_codesize_cxx_lto.size index 5e47e2b576a23..c0bb84b7c4577 100644 --- a/test/other/codesize/test_codesize_cxx_lto.size +++ b/test/other/codesize/test_codesize_cxx_lto.size @@ -1 +1 @@ -121983 +121990 diff --git a/test/other/codesize/test_codesize_cxx_mangle.size b/test/other/codesize/test_codesize_cxx_mangle.size index 49938bda6e698..18b5cfac0dc89 100644 --- a/test/other/codesize/test_codesize_cxx_mangle.size +++ b/test/other/codesize/test_codesize_cxx_mangle.size @@ -1 +1 @@ -235487 +235492 diff --git a/test/other/codesize/test_codesize_cxx_noexcept.size b/test/other/codesize/test_codesize_cxx_noexcept.size index af3b299d9add0..80a7b4a09bf8d 100644 --- a/test/other/codesize/test_codesize_cxx_noexcept.size +++ b/test/other/codesize/test_codesize_cxx_noexcept.size @@ -1 +1 @@ -132126 +132118 diff --git a/test/other/codesize/test_codesize_cxx_wasmfs.size b/test/other/codesize/test_codesize_cxx_wasmfs.size index c45c0566d0efc..b3f62fb43ee6d 100644 --- a/test/other/codesize/test_codesize_cxx_wasmfs.size +++ b/test/other/codesize/test_codesize_cxx_wasmfs.size @@ -1 +1 @@ -169996 +169988 diff --git a/test/other/codesize/test_codesize_hello_O0.size b/test/other/codesize/test_codesize_hello_O0.size index 15056771f58d7..4ed4cbb4a695f 100644 --- a/test/other/codesize/test_codesize_hello_O0.size +++ b/test/other/codesize/test_codesize_hello_O0.size @@ -1 +1 @@ -15167 +15170 diff --git a/test/other/codesize/test_codesize_hello_dylink.size b/test/other/codesize/test_codesize_hello_dylink.size index afd43f8827d5b..912b7ba8cf298 100644 --- a/test/other/codesize/test_codesize_hello_dylink.size +++ b/test/other/codesize/test_codesize_hello_dylink.size @@ -1 +1 @@ -18547 +18550 diff --git a/test/other/test_unoptimized_code_size.wasm.size b/test/other/test_unoptimized_code_size.wasm.size index 15056771f58d7..4ed4cbb4a695f 100644 --- a/test/other/test_unoptimized_code_size.wasm.size +++ b/test/other/test_unoptimized_code_size.wasm.size @@ -1 +1 @@ -15167 +15170 diff --git a/test/other/test_unoptimized_code_size_no_asserts.wasm.size b/test/other/test_unoptimized_code_size_no_asserts.wasm.size index 0a888765b3d17..927075cad6eed 100644 --- a/test/other/test_unoptimized_code_size_no_asserts.wasm.size +++ b/test/other/test_unoptimized_code_size_no_asserts.wasm.size @@ -1 +1 @@ -12248 +12251 diff --git a/test/other/test_unoptimized_code_size_strict.wasm.size b/test/other/test_unoptimized_code_size_strict.wasm.size index 15056771f58d7..4ed4cbb4a695f 100644 --- a/test/other/test_unoptimized_code_size_strict.wasm.size +++ b/test/other/test_unoptimized_code_size_strict.wasm.size @@ -1 +1 @@ -15167 +15170