1 #pragma once
2 #ifndef FP16_PSIMD_H
3 #define FP16_PSIMD_H
4
5 #if defined(__cplusplus) && (__cplusplus >= 201103L)
6 #include <cstdint>
7 #elif !defined(__OPENCL_VERSION__)
8 #include <stdint.h>
9 #endif
10
11 #include <psimd.h>
12
13
fp16_ieee_to_fp32_psimd(psimd_u16 half)14 PSIMD_INTRINSIC psimd_f32 fp16_ieee_to_fp32_psimd(psimd_u16 half) {
15 const psimd_u32 word = (psimd_u32) psimd_interleave_lo_u16(psimd_zero_u16(), half);
16
17 const psimd_u32 sign = word & psimd_splat_u32(UINT32_C(0x80000000));
18 const psimd_u32 shr3_nonsign = (word + word) >> psimd_splat_u32(4);
19
20 const psimd_u32 exp_offset = psimd_splat_u32(UINT32_C(0x70000000));
21 #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) || defined(__GNUC__) && !defined(__STRICT_ANSI__)
22 const psimd_f32 exp_scale = psimd_splat_f32(0x1.0p-112f);
23 #else
24 const psimd_f32 exp_scale = psimd_splat_f32(fp32_from_bits(UINT32_C(0x7800000)));
25 #endif
26 const psimd_f32 norm_nonsign = psimd_mul_f32((psimd_f32) (shr3_nonsign + exp_offset), exp_scale);
27
28 const psimd_u16 magic_mask = psimd_splat_u16(UINT16_C(0x3E80));
29 const psimd_f32 magic_bias = psimd_splat_f32(0.25f);
30 const psimd_f32 denorm_nonsign = psimd_sub_f32((psimd_f32) psimd_interleave_lo_u16(half + half, magic_mask), magic_bias);
31
32 const psimd_s32 denorm_cutoff = psimd_splat_s32(INT32_C(0x00800000));
33 const psimd_s32 denorm_mask = (psimd_s32) shr3_nonsign < denorm_cutoff;
34 return (psimd_f32) (sign | (psimd_s32) psimd_blend_f32(denorm_mask, denorm_nonsign, norm_nonsign));
35 }
36
fp16_ieee_to_fp32x2_psimd(psimd_u16 half)37 PSIMD_INTRINSIC psimd_f32x2 fp16_ieee_to_fp32x2_psimd(psimd_u16 half) {
38 const psimd_u32 word_lo = (psimd_u32) psimd_interleave_lo_u16(psimd_zero_u16(), half);
39 const psimd_u32 word_hi = (psimd_u32) psimd_interleave_hi_u16(psimd_zero_u16(), half);
40
41 const psimd_u32 sign_mask = psimd_splat_u32(UINT32_C(0x80000000));
42 const psimd_u32 sign_lo = word_lo & sign_mask;
43 const psimd_u32 sign_hi = word_hi & sign_mask;
44 const psimd_u32 shr3_nonsign_lo = (word_lo + word_lo) >> psimd_splat_u32(4);
45 const psimd_u32 shr3_nonsign_hi = (word_hi + word_hi) >> psimd_splat_u32(4);
46
47 const psimd_u32 exp_offset = psimd_splat_u32(UINT32_C(0x70000000));
48 #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) || defined(__GNUC__) && !defined(__STRICT_ANSI__)
49 const psimd_f32 exp_scale = psimd_splat_f32(0x1.0p-112f);
50 #else
51 const psimd_f32 exp_scale = psimd_splat_f32(fp32_from_bits(UINT32_C(0x7800000)));
52 #endif
53 const psimd_f32 norm_nonsign_lo = psimd_mul_f32((psimd_f32) (shr3_nonsign_lo + exp_offset), exp_scale);
54 const psimd_f32 norm_nonsign_hi = psimd_mul_f32((psimd_f32) (shr3_nonsign_hi + exp_offset), exp_scale);
55
56 const psimd_u16 magic_mask = psimd_splat_u16(UINT16_C(0x3E80));
57 const psimd_u16 shl1_half = half + half;
58 const psimd_f32 magic_bias = psimd_splat_f32(0.25f);
59 const psimd_f32 denorm_nonsign_lo = psimd_sub_f32((psimd_f32) psimd_interleave_lo_u16(shl1_half, magic_mask), magic_bias);
60 const psimd_f32 denorm_nonsign_hi = psimd_sub_f32((psimd_f32) psimd_interleave_hi_u16(shl1_half, magic_mask), magic_bias);
61
62 const psimd_s32 denorm_cutoff = psimd_splat_s32(INT32_C(0x00800000));
63 const psimd_s32 denorm_mask_lo = (psimd_s32) shr3_nonsign_lo < denorm_cutoff;
64 const psimd_s32 denorm_mask_hi = (psimd_s32) shr3_nonsign_hi < denorm_cutoff;
65
66 psimd_f32x2 result;
67 result.lo = (psimd_f32) (sign_lo | (psimd_s32) psimd_blend_f32(denorm_mask_lo, denorm_nonsign_lo, norm_nonsign_lo));
68 result.hi = (psimd_f32) (sign_hi | (psimd_s32) psimd_blend_f32(denorm_mask_hi, denorm_nonsign_hi, norm_nonsign_hi));
69 return result;
70 }
71
fp16_alt_to_fp32_psimd(psimd_u16 half)72 PSIMD_INTRINSIC psimd_f32 fp16_alt_to_fp32_psimd(psimd_u16 half) {
73 const psimd_u32 word = (psimd_u32) psimd_interleave_lo_u16(psimd_zero_u16(), half);
74
75 const psimd_u32 sign = word & psimd_splat_u32(INT32_C(0x80000000));
76 const psimd_u32 shr3_nonsign = (word + word) >> psimd_splat_u32(4);
77
78 #if 0
79 const psimd_s32 exp112_offset = psimd_splat_s32(INT32_C(0x38000000));
80 const psimd_s32 nonsign_bits = (psimd_s32) shr3_nonsign + exp112_offset;
81 const psimd_s32 exp1_offset = psimd_splat_s32(INT32_C(0x00800000));
82 const psimd_f32 two_nonsign = (psimd_f32) (nonsign_bits + exp1_offset);
83 const psimd_s32 exp113_offset = exp112_offset | exp1_offset;
84 return (psimd_f32) (sign | (psimd_s32) psimd_sub_f32(two_nonsign, (psimd_f32) psimd_max_s32(nonsign_bits, exp113_offset)));
85 #else
86 const psimd_u32 exp_offset = psimd_splat_u32(UINT32_C(0x38000000));
87 const psimd_f32 nonsign = (psimd_f32) (shr3_nonsign + exp_offset);
88 #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) || defined(__GNUC__) && !defined(__STRICT_ANSI__)
89 const psimd_f32 denorm_bias = psimd_splat_f32(0x1.0p-14f);
90 #else
91 const psimd_f32 denorm_bias = psimd_splat_f32(fp32_from_bits(UINT32_C(0x38800000)));
92 #endif
93 return (psimd_f32) (sign | (psimd_s32) psimd_sub_f32(psimd_add_f32(nonsign, nonsign), psimd_max_f32(nonsign, denorm_bias)));
94 #endif
95 }
96
fp16_alt_to_fp32x2_psimd(psimd_u16 half)97 PSIMD_INTRINSIC psimd_f32x2 fp16_alt_to_fp32x2_psimd(psimd_u16 half) {
98 const psimd_u32 word_lo = (psimd_u32) psimd_interleave_lo_u16(psimd_zero_u16(), half);
99 const psimd_u32 word_hi = (psimd_u32) psimd_interleave_hi_u16(psimd_zero_u16(), half);
100
101 const psimd_u32 sign_mask = psimd_splat_u32(UINT32_C(0x80000000));
102 const psimd_u32 sign_lo = word_lo & sign_mask;
103 const psimd_u32 sign_hi = word_hi & sign_mask;
104 const psimd_u32 shr3_nonsign_lo = (word_lo + word_lo) >> psimd_splat_u32(4);
105 const psimd_u32 shr3_nonsign_hi = (word_hi + word_hi) >> psimd_splat_u32(4);
106
107 #if 1
108 const psimd_s32 exp112_offset = psimd_splat_s32(INT32_C(0x38000000));
109 const psimd_s32 nonsign_bits_lo = (psimd_s32) shr3_nonsign_lo + exp112_offset;
110 const psimd_s32 nonsign_bits_hi = (psimd_s32) shr3_nonsign_hi + exp112_offset;
111 const psimd_s32 exp1_offset = psimd_splat_s32(INT32_C(0x00800000));
112 const psimd_f32 two_nonsign_lo = (psimd_f32) (nonsign_bits_lo + exp1_offset);
113 const psimd_f32 two_nonsign_hi = (psimd_f32) (nonsign_bits_hi + exp1_offset);
114 const psimd_s32 exp113_offset = exp1_offset | exp112_offset;
115 psimd_f32x2 result;
116 result.lo = (psimd_f32) (sign_lo | (psimd_s32) psimd_sub_f32(two_nonsign_lo, (psimd_f32) psimd_max_s32(nonsign_bits_lo, exp113_offset)));
117 result.hi = (psimd_f32) (sign_hi | (psimd_s32) psimd_sub_f32(two_nonsign_hi, (psimd_f32) psimd_max_s32(nonsign_bits_hi, exp113_offset)));
118 return result;
119 #else
120 const psimd_u32 exp_offset = psimd_splat_u32(UINT32_C(0x38000000));
121 const psimd_f32 nonsign_lo = (psimd_f32) (shr3_nonsign_lo + exp_offset);
122 const psimd_f32 nonsign_hi = (psimd_f32) (shr3_nonsign_hi + exp_offset);
123 const psimd_f32 denorm_bias = psimd_splat_f32(0x1.0p-14f);
124 psimd_f32x2 result;
125 result.lo = (psimd_f32) (sign_lo | (psimd_s32) psimd_sub_f32(psimd_add_f32(nonsign_lo, nonsign_lo), psimd_max_f32(nonsign_lo, denorm_bias)));
126 result.hi = (psimd_f32) (sign_hi | (psimd_s32) psimd_sub_f32(psimd_add_f32(nonsign_hi, nonsign_hi), psimd_max_f32(nonsign_hi, denorm_bias)));
127 return result;
128 #endif
129 }
130
131 #endif /* FP16_PSIMD_H */
132