fftw_mkapiplan: 87| 285|{ 88| 285| apiplan *p = 0; 89| 285| plan *pln; 90| 285| unsigned flags_used_for_planning; 91| 285| planner *plnr; 92| 285| static const unsigned int pats[] = {FFTW_ESTIMATE, FFTW_MEASURE, ------------------ | | 502| 285|#define FFTW_ESTIMATE (1U << 6) ------------------ static const unsigned int pats[] = {FFTW_ESTIMATE, FFTW_MEASURE, ------------------ | | 495| 285|#define FFTW_MEASURE (0U) ------------------ 93| 285| FFTW_PATIENT, FFTW_EXHAUSTIVE}; ------------------ | | 501| 285|#define FFTW_PATIENT (1U << 5) /* IMPATIENT is default */ ------------------ FFTW_PATIENT, FFTW_EXHAUSTIVE}; ------------------ | | 499| 285|#define FFTW_EXHAUSTIVE (1U << 3) /* NO_EXHAUSTIVE is default */ ------------------ 94| 285| int pat, pat_max; 95| 285| double pcost = 0; 96| | 97| 285| if (before_planner_hook) ------------------ | Branch (97:10): [True: 0, False: 285] ------------------ 98| 0| before_planner_hook(); 99| | 100| 285| plnr = X(the_planner)(); ------------------ | | 78| 285|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 285|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 101| | 102| 285| if (flags & FFTW_WISDOM_ONLY) { ------------------ | | 503| 285|#define FFTW_WISDOM_ONLY (1U << 21) ------------------ | Branch (102:10): [True: 0, False: 285] ------------------ 103| | /* Special mode that returns a plan only if wisdom is present, 104| | and returns 0 otherwise. This is now documented in the manual, 105| | as a way to detect whether wisdom is available for a problem. */ 106| 0| flags_used_for_planning = flags; 107| 0| pln = mkplan0(plnr, flags, prb, 0, WISDOM_ONLY); 108| 285| } else { 109| 285| pat_max = flags & FFTW_ESTIMATE ? 0 : ------------------ | | 502| 285|#define FFTW_ESTIMATE (1U << 6) ------------------ | Branch (109:14): [True: 285, False: 0] ------------------ 110| 285| (flags & FFTW_EXHAUSTIVE ? 3 : ------------------ | | 499| 0|#define FFTW_EXHAUSTIVE (1U << 3) /* NO_EXHAUSTIVE is default */ ------------------ | Branch (110:10): [True: 0, False: 0] ------------------ 111| 0| (flags & FFTW_PATIENT ? 2 : 1)); ------------------ | | 501| 0|#define FFTW_PATIENT (1U << 5) /* IMPATIENT is default */ ------------------ | Branch (111:4): [True: 0, False: 0] ------------------ 112| 285| pat = plnr->timelimit >= 0 ? 0 : pat_max; ------------------ | Branch (112:10): [True: 0, False: 285] ------------------ 113| | 114| 285| flags &= ~(FFTW_ESTIMATE | FFTW_MEASURE | ------------------ | | 502| 285|#define FFTW_ESTIMATE (1U << 6) ------------------ flags &= ~(FFTW_ESTIMATE | FFTW_MEASURE | ------------------ | | 495| 285|#define FFTW_MEASURE (0U) ------------------ 115| 285| FFTW_PATIENT | FFTW_EXHAUSTIVE); ------------------ | | 501| 285|#define FFTW_PATIENT (1U << 5) /* IMPATIENT is default */ ------------------ FFTW_PATIENT | FFTW_EXHAUSTIVE); ------------------ | | 499| 285|#define FFTW_EXHAUSTIVE (1U << 3) /* NO_EXHAUSTIVE is default */ ------------------ 116| | 117| 285| plnr->start_time = X(get_crude_time)(); ------------------ | | 78| 285|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 285|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 118| | 119| | /* plan at incrementally increasing patience until we run 120| | out of time */ 121| 570| for (pln = 0, flags_used_for_planning = 0; pat <= pat_max; ++pat) { ------------------ | Branch (121:47): [True: 285, False: 285] ------------------ 122| 285| plan *pln1; 123| 285| unsigned tmpflags = flags | pats[pat]; 124| 285| pln1 = mkplan(plnr, tmpflags, prb, 0u); 125| | 126| 285| if (!pln1) { ------------------ | Branch (126:13): [True: 0, False: 285] ------------------ 127| | /* don't bother continuing if planner failed or timed out */ 128| 0| A(!pln || plnr->timed_out); 129| 0| break; 130| 0| } 131| | 132| 285| X(plan_destroy_internal)(pln); ------------------ | | 78| 285|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 285|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 133| 285| pln = pln1; 134| 285| flags_used_for_planning = tmpflags; 135| 285| pcost = pln->pcost; 136| 285| } 137| 285| } 138| | 139| 285| if (pln) { ------------------ | Branch (139:10): [True: 285, False: 0] ------------------ 140| | /* build apiplan */ 141| 285| p = (apiplan *) MALLOC(sizeof(apiplan), PLANS); ------------------ | | 306| 285|#define MALLOC(n, what) X(malloc_plain)(n) | | ------------------ | | | | 78| 285|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 285|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 142| 285| p->prb = prb; 143| 285| p->refcount = 1u; 144| 285| p->sign = sign; /* cache for execute_dft */ 145| | 146| | /* re-create plan from wisdom, adding blessing */ 147| 285| p->pln = mkplan(plnr, flags_used_for_planning, prb, BLESSING); 148| | 149| | /* record pcost from most recent measurement for use in X(cost) */ 150| 285| p->pln->pcost = pcost; 151| | 152| 285| if (sizeof(trigreal) > sizeof(R)) { ------------------ | Branch (152:8): [Folded, False: 285] ------------------ 153| | /* this is probably faster, and we have enough trigreal 154| | bits to maintain accuracy */ 155| 0| X(plan_awake)(p->pln, AWAKE_SQRTN_TABLE); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 156| 285| } else { 157| | /* more accurate */ 158| 285| X(plan_awake)(p->pln, AWAKE_SINCOS); ------------------ | | 78| 285|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 285|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 159| 285| } 160| | 161| | /* we don't use pln for p->pln, above, since by re-creating the 162| | plan we might use more patient wisdom from a timed-out mkplan */ 163| 285| X(plan_destroy_internal)(pln); ------------------ | | 78| 285|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 285|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 164| 285| } else 165| 0| X(problem_destroy)(prb); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 166| | 167| | /* discard all information not necessary to reconstruct the plan */ 168| 285| plnr->adt->forget(plnr, FORGET_ACCURSED); 169| | 170| |#ifdef FFTW_RANDOM_ESTIMATOR 171| | X(random_estimate_seed)++; /* subsequent "random" plans are distinct */ 172| |#endif 173| | 174| 285| if (after_planner_hook) ------------------ | Branch (174:10): [True: 0, False: 285] ------------------ 175| 0| after_planner_hook(); 176| | 177| 285| return p; 178| 285|} fftw_destroy_plan: 195| 285|{ 196| 285| if (p) { ------------------ | Branch (196:10): [True: 285, False: 0] ------------------ 197| 285| if (before_planner_hook) ------------------ | Branch (197:15): [True: 0, False: 285] ------------------ 198| 0| before_planner_hook(); 199| | 200| 285| if (p->refcount-- == 1u) { ------------------ | Branch (200:15): [True: 285, False: 0] ------------------ 201| 285| X(plan_awake)(p->pln, SLEEPY); ------------------ | | 78| 285|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 285|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 202| 285| X(plan_destroy_internal)(p->pln); ------------------ | | 78| 285|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 285|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 203| 285| X(problem_destroy)(p->prb); ------------------ | | 78| 285|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 285|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 204| 285| X(ifree)(p); ------------------ | | 78| 285|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 285|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 205| 285| } 206| | 207| 285| if (after_planner_hook) ------------------ | Branch (207:15): [True: 0, False: 285] ------------------ 208| 0| after_planner_hook(); 209| 285| } 210| 285|} apiplan.c:mkplan0: 34| 570|{ 35| | /* map API flags into FFTW flags */ 36| 570| X(mapflags)(plnr, flags); ------------------ | | 78| 570|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 570|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 37| | 38| 570| plnr->flags.hash_info = hash_info; 39| 570| plnr->wisdom_state = wisdom_state; 40| | 41| | /* create plan */ 42| 570| return plnr->adt->mkplan(plnr, prb); 43| 570|} apiplan.c:mkplan: 53| 570|{ 54| 570| plan *pln; 55| | 56| 570| pln = mkplan0(plnr, flags, prb, hash_info, WISDOM_NORMAL); 57| | 58| 570| if (plnr->wisdom_state == WISDOM_NORMAL && !pln) { ------------------ | Branch (58:10): [True: 570, False: 0] | Branch (58:49): [True: 0, False: 570] ------------------ 59| | /* maybe the planner failed because of inconsistent wisdom; 60| | plan again ignoring infeasible wisdom */ 61| 0| pln = mkplan0(plnr, force_estimator(flags), prb, 62| 0| hash_info, WISDOM_IGNORE_INFEASIBLE); 63| 0| } 64| | 65| 570| if (plnr->wisdom_state == WISDOM_IS_BOGUS) { ------------------ | Branch (65:10): [True: 0, False: 570] ------------------ 66| | /* if the planner detected a wisdom inconsistency, 67| | forget all wisdom and plan again */ 68| 0| plnr->adt->forget(plnr, FORGET_EVERYTHING); 69| | 70| 0| A(!pln); 71| 0| pln = mkplan0(plnr, flags, prb, hash_info, WISDOM_NORMAL); 72| | 73| 0| if (plnr->wisdom_state == WISDOM_IS_BOGUS) { ------------------ | Branch (73:8): [True: 0, False: 0] ------------------ 74| | /* if it still fails, plan without wisdom */ 75| 0| plnr->adt->forget(plnr, FORGET_EVERYTHING); 76| | 77| 0| A(!pln); 78| 0| pln = mkplan0(plnr, force_estimator(flags), 79| 0| prb, hash_info, WISDOM_IGNORE_ALL); 80| 0| } 81| 0| } 82| | 83| 570| return pln; 84| 570|} fftw_configure_planner: 27| 1|{ 28| 1| X(dft_conf_standard)(plnr); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 29| 1| X(rdft_conf_standard)(plnr); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 30| 1| X(reodft_conf_standard)(plnr); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 31| 1|} fftw_execute: 24| 285|{ 25| 285| plan *pln = p->pln; 26| 285| pln->adt->solve(pln, p->prb); 27| 285|} fftw_mapflags: 78| 570|{ 79| 570| unsigned l, u, t; 80| | 81| | /* map of api flags -> api flags, to implement consistency rules 82| | and combination flags */ 83| 570| const flagop self_flagmap[] = { 84| | /* in some cases (notably for halfcomplex->real transforms), 85| | DESTROY_INPUT is the default, so we need to support 86| | an inverse flag to disable it. 87| | 88| | (PRESERVE, DESTROY) -> (PRESERVE, DESTROY) 89| | (0, 0) (1, 0) 90| | (0, 1) (0, 1) 91| | (1, 0) (1, 0) 92| | (1, 1) (1, 0) 93| | */ 94| 570| IMPLIES(YES(FFTW_PRESERVE_INPUT), NO(FFTW_DESTROY_INPUT)), ------------------ | | 41| 570|#define IMPLIES(predicate, consequence) { predicate, consequence } ------------------ 95| 570| IMPLIES(NO(FFTW_DESTROY_INPUT), YES(FFTW_PRESERVE_INPUT)), ------------------ | | 41| 570|#define IMPLIES(predicate, consequence) { predicate, consequence } ------------------ 96| | 97| 570| IMPLIES(YES(FFTW_EXHAUSTIVE), YES(FFTW_PATIENT)), ------------------ | | 41| 570|#define IMPLIES(predicate, consequence) { predicate, consequence } ------------------ 98| | 99| 570| IMPLIES(YES(FFTW_ESTIMATE), NO(FFTW_PATIENT)), ------------------ | | 41| 570|#define IMPLIES(predicate, consequence) { predicate, consequence } ------------------ 100| 570| IMPLIES(YES(FFTW_ESTIMATE), ------------------ | | 41| 570|#define IMPLIES(predicate, consequence) { predicate, consequence } ------------------ 101| 570| YES(FFTW_ESTIMATE_PATIENT 102| 570| | FFTW_NO_INDIRECT_OP 103| 570| | FFTW_ALLOW_PRUNING)), 104| | 105| 570| IMPLIES(NO(FFTW_EXHAUSTIVE), ------------------ | | 41| 570|#define IMPLIES(predicate, consequence) { predicate, consequence } ------------------ 106| 570| YES(FFTW_NO_SLOW)), 107| | 108| | /* a canonical set of fftw2-like impatience flags */ 109| 570| IMPLIES(NO(FFTW_PATIENT), ------------------ | | 41| 570|#define IMPLIES(predicate, consequence) { predicate, consequence } ------------------ 110| 570| YES(FFTW_NO_VRECURSE 111| 570| | FFTW_NO_RANK_SPLITS 112| 570| | FFTW_NO_VRANK_SPLITS 113| 570| | FFTW_NO_NONTHREADED 114| 570| | FFTW_NO_DFT_R2HC 115| 570| | FFTW_NO_FIXED_RADIX_LARGE_N 116| 570| | FFTW_BELIEVE_PCOST)) 117| 570| }; 118| | 119| | /* map of (processed) api flags to internal problem/planner flags */ 120| 570| const flagop l_flagmap[] = { 121| 570| EQV(FFTW_PRESERVE_INPUT, NO_DESTROY_INPUT), ------------------ | | 42| 570|#define EQV(a, b) IMPLIES(YES(a), YES(b)), IMPLIES(NO(a), NO(b)) | | ------------------ | | | | 41| 570|#define IMPLIES(predicate, consequence) { predicate, consequence } | | ------------------ | | #define EQV(a, b) IMPLIES(YES(a), YES(b)), IMPLIES(NO(a), NO(b)) | | ------------------ | | | | 41| 570|#define IMPLIES(predicate, consequence) { predicate, consequence } | | ------------------ ------------------ 122| 570| EQV(FFTW_NO_SIMD, NO_SIMD), ------------------ | | 42| 570|#define EQV(a, b) IMPLIES(YES(a), YES(b)), IMPLIES(NO(a), NO(b)) | | ------------------ | | | | 41| 570|#define IMPLIES(predicate, consequence) { predicate, consequence } | | ------------------ | | #define EQV(a, b) IMPLIES(YES(a), YES(b)), IMPLIES(NO(a), NO(b)) | | ------------------ | | | | 41| 570|#define IMPLIES(predicate, consequence) { predicate, consequence } | | ------------------ ------------------ 123| 570| EQV(FFTW_CONSERVE_MEMORY, CONSERVE_MEMORY), ------------------ | | 42| 570|#define EQV(a, b) IMPLIES(YES(a), YES(b)), IMPLIES(NO(a), NO(b)) | | ------------------ | | | | 41| 570|#define IMPLIES(predicate, consequence) { predicate, consequence } | | ------------------ | | #define EQV(a, b) IMPLIES(YES(a), YES(b)), IMPLIES(NO(a), NO(b)) | | ------------------ | | | | 41| 570|#define IMPLIES(predicate, consequence) { predicate, consequence } | | ------------------ ------------------ 124| 570| EQV(FFTW_NO_BUFFERING, NO_BUFFERING), ------------------ | | 42| 570|#define EQV(a, b) IMPLIES(YES(a), YES(b)), IMPLIES(NO(a), NO(b)) | | ------------------ | | | | 41| 570|#define IMPLIES(predicate, consequence) { predicate, consequence } | | ------------------ | | #define EQV(a, b) IMPLIES(YES(a), YES(b)), IMPLIES(NO(a), NO(b)) | | ------------------ | | | | 41| 570|#define IMPLIES(predicate, consequence) { predicate, consequence } | | ------------------ ------------------ 125| 570| NEQV(FFTW_ALLOW_LARGE_GENERIC, NO_LARGE_GENERIC) ------------------ | | 43| 570|#define NEQV(a, b) IMPLIES(YES(a), NO(b)), IMPLIES(NO(a), YES(b)) | | ------------------ | | | | 41| 570|#define IMPLIES(predicate, consequence) { predicate, consequence } | | ------------------ | | #define NEQV(a, b) IMPLIES(YES(a), NO(b)), IMPLIES(NO(a), YES(b)) | | ------------------ | | | | 41| 570|#define IMPLIES(predicate, consequence) { predicate, consequence } | | ------------------ ------------------ 126| 570| }; 127| | 128| 570| const flagop u_flagmap[] = { 129| 570| IMPLIES(YES(FFTW_EXHAUSTIVE), NO(0xFFFFFFFF)), ------------------ | | 41| 570|#define IMPLIES(predicate, consequence) { predicate, consequence } ------------------ 130| 570| IMPLIES(NO(FFTW_EXHAUSTIVE), YES(NO_UGLY)), ------------------ | | 41| 570|#define IMPLIES(predicate, consequence) { predicate, consequence } ------------------ 131| | 132| | /* the following are undocumented, "beyond-guru" flags that 133| | require some understanding of FFTW internals */ 134| 570| EQV(FFTW_ESTIMATE_PATIENT, ESTIMATE), ------------------ | | 42| 570|#define EQV(a, b) IMPLIES(YES(a), YES(b)), IMPLIES(NO(a), NO(b)) | | ------------------ | | | | 41| 570|#define IMPLIES(predicate, consequence) { predicate, consequence } | | ------------------ | | #define EQV(a, b) IMPLIES(YES(a), YES(b)), IMPLIES(NO(a), NO(b)) | | ------------------ | | | | 41| 570|#define IMPLIES(predicate, consequence) { predicate, consequence } | | ------------------ ------------------ 135| 570| EQV(FFTW_ALLOW_PRUNING, ALLOW_PRUNING), ------------------ | | 42| 570|#define EQV(a, b) IMPLIES(YES(a), YES(b)), IMPLIES(NO(a), NO(b)) | | ------------------ | | | | 41| 570|#define IMPLIES(predicate, consequence) { predicate, consequence } | | ------------------ | | #define EQV(a, b) IMPLIES(YES(a), YES(b)), IMPLIES(NO(a), NO(b)) | | ------------------ | | | | 41| 570|#define IMPLIES(predicate, consequence) { predicate, consequence } | | ------------------ ------------------ 136| 570| EQV(FFTW_BELIEVE_PCOST, BELIEVE_PCOST), ------------------ | | 42| 570|#define EQV(a, b) IMPLIES(YES(a), YES(b)), IMPLIES(NO(a), NO(b)) | | ------------------ | | | | 41| 570|#define IMPLIES(predicate, consequence) { predicate, consequence } | | ------------------ | | #define EQV(a, b) IMPLIES(YES(a), YES(b)), IMPLIES(NO(a), NO(b)) | | ------------------ | | | | 41| 570|#define IMPLIES(predicate, consequence) { predicate, consequence } | | ------------------ ------------------ 137| 570| EQV(FFTW_NO_DFT_R2HC, NO_DFT_R2HC), ------------------ | | 42| 570|#define EQV(a, b) IMPLIES(YES(a), YES(b)), IMPLIES(NO(a), NO(b)) | | ------------------ | | | | 41| 570|#define IMPLIES(predicate, consequence) { predicate, consequence } | | ------------------ | | #define EQV(a, b) IMPLIES(YES(a), YES(b)), IMPLIES(NO(a), NO(b)) | | ------------------ | | | | 41| 570|#define IMPLIES(predicate, consequence) { predicate, consequence } | | ------------------ ------------------ 138| 570| EQV(FFTW_NO_NONTHREADED, NO_NONTHREADED), ------------------ | | 42| 570|#define EQV(a, b) IMPLIES(YES(a), YES(b)), IMPLIES(NO(a), NO(b)) | | ------------------ | | | | 41| 570|#define IMPLIES(predicate, consequence) { predicate, consequence } | | ------------------ | | #define EQV(a, b) IMPLIES(YES(a), YES(b)), IMPLIES(NO(a), NO(b)) | | ------------------ | | | | 41| 570|#define IMPLIES(predicate, consequence) { predicate, consequence } | | ------------------ ------------------ 139| 570| EQV(FFTW_NO_INDIRECT_OP, NO_INDIRECT_OP), ------------------ | | 42| 570|#define EQV(a, b) IMPLIES(YES(a), YES(b)), IMPLIES(NO(a), NO(b)) | | ------------------ | | | | 41| 570|#define IMPLIES(predicate, consequence) { predicate, consequence } | | ------------------ | | #define EQV(a, b) IMPLIES(YES(a), YES(b)), IMPLIES(NO(a), NO(b)) | | ------------------ | | | | 41| 570|#define IMPLIES(predicate, consequence) { predicate, consequence } | | ------------------ ------------------ 140| 570| EQV(FFTW_NO_RANK_SPLITS, NO_RANK_SPLITS), ------------------ | | 42| 570|#define EQV(a, b) IMPLIES(YES(a), YES(b)), IMPLIES(NO(a), NO(b)) | | ------------------ | | | | 41| 570|#define IMPLIES(predicate, consequence) { predicate, consequence } | | ------------------ | | #define EQV(a, b) IMPLIES(YES(a), YES(b)), IMPLIES(NO(a), NO(b)) | | ------------------ | | | | 41| 570|#define IMPLIES(predicate, consequence) { predicate, consequence } | | ------------------ ------------------ 141| 570| EQV(FFTW_NO_VRANK_SPLITS, NO_VRANK_SPLITS), ------------------ | | 42| 570|#define EQV(a, b) IMPLIES(YES(a), YES(b)), IMPLIES(NO(a), NO(b)) | | ------------------ | | | | 41| 570|#define IMPLIES(predicate, consequence) { predicate, consequence } | | ------------------ | | #define EQV(a, b) IMPLIES(YES(a), YES(b)), IMPLIES(NO(a), NO(b)) | | ------------------ | | | | 41| 570|#define IMPLIES(predicate, consequence) { predicate, consequence } | | ------------------ ------------------ 142| 570| EQV(FFTW_NO_VRECURSE, NO_VRECURSE), ------------------ | | 42| 570|#define EQV(a, b) IMPLIES(YES(a), YES(b)), IMPLIES(NO(a), NO(b)) | | ------------------ | | | | 41| 570|#define IMPLIES(predicate, consequence) { predicate, consequence } | | ------------------ | | #define EQV(a, b) IMPLIES(YES(a), YES(b)), IMPLIES(NO(a), NO(b)) | | ------------------ | | | | 41| 570|#define IMPLIES(predicate, consequence) { predicate, consequence } | | ------------------ ------------------ 143| 570| EQV(FFTW_NO_SLOW, NO_SLOW), ------------------ | | 42| 570|#define EQV(a, b) IMPLIES(YES(a), YES(b)), IMPLIES(NO(a), NO(b)) | | ------------------ | | | | 41| 570|#define IMPLIES(predicate, consequence) { predicate, consequence } | | ------------------ | | #define EQV(a, b) IMPLIES(YES(a), YES(b)), IMPLIES(NO(a), NO(b)) | | ------------------ | | | | 41| 570|#define IMPLIES(predicate, consequence) { predicate, consequence } | | ------------------ ------------------ 144| 570| EQV(FFTW_NO_FIXED_RADIX_LARGE_N, NO_FIXED_RADIX_LARGE_N) ------------------ | | 42| 570|#define EQV(a, b) IMPLIES(YES(a), YES(b)), IMPLIES(NO(a), NO(b)) | | ------------------ | | | | 41| 570|#define IMPLIES(predicate, consequence) { predicate, consequence } | | ------------------ | | #define EQV(a, b) IMPLIES(YES(a), YES(b)), IMPLIES(NO(a), NO(b)) | | ------------------ | | | | 41| 570|#define IMPLIES(predicate, consequence) { predicate, consequence } | | ------------------ ------------------ 145| 570| }; 146| | 147| 570| map_flags(&flags, &flags, self_flagmap, NELEM(self_flagmap)); ------------------ | | 90| 570|#define NELEM(array) ((sizeof(array) / sizeof((array)[0]))) ------------------ 148| | 149| 570| l = u = 0; 150| 570| map_flags(&flags, &l, l_flagmap, NELEM(l_flagmap)); ------------------ | | 90| 570|#define NELEM(array) ((sizeof(array) / sizeof((array)[0]))) ------------------ 151| 570| map_flags(&flags, &u, u_flagmap, NELEM(u_flagmap)); ------------------ | | 90| 570|#define NELEM(array) ((sizeof(array) / sizeof((array)[0]))) ------------------ 152| | 153| | /* enforce l <= u */ 154| 570| PLNR_L(plnr) = l; ------------------ | | 674| 570|#define PLNR_L(plnr) ((plnr)->flags.l) ------------------ 155| 570| PLNR_U(plnr) = u | l; ------------------ | | 675| 570|#define PLNR_U(plnr) ((plnr)->flags.u) ------------------ 156| | 157| | /* assert that the conversion didn't lose bits */ 158| 570| A(PLNR_L(plnr) == l); 159| 570| A(PLNR_U(plnr) == (u | l)); 160| | 161| | /* compute flags representation of the timelimit */ 162| 570| t = timelimit_to_flags(plnr->timelimit); 163| | 164| 570| PLNR_TIMELIMIT_IMPATIENCE(plnr) = t; ------------------ | | 676| 570|#define PLNR_TIMELIMIT_IMPATIENCE(plnr) ((plnr)->flags.timelimit_impatience) ------------------ 165| 570| A(PLNR_TIMELIMIT_IMPATIENCE(plnr) == t); 166| 570|} mapflags.c:map_flags: 47| 1.71k|{ 48| 1.71k| size_t i; 49| 25.0k| for (i = 0; i < nmap; ++i) ------------------ | Branch (49:18): [True: 23.3k, False: 1.71k] ------------------ 50| 23.3k| if (FLAGP(*iflags, flagmap[i].flag)) ------------------ | | 36| 23.3k|#define FLAGP(f, msk)(((f) & (msk).x) ^ (msk).xm) | | ------------------ | | | Branch (36:22): [True: 12.5k, False: 10.8k] | | ------------------ ------------------ 51| 12.5k| *oflags = OP(*oflags, flagmap[i].op); ------------------ | | 37| 12.5k|#define OP(f, msk)(((f) | (msk).x) ^ (msk).xm) ------------------ 52| 1.71k|} mapflags.c:timelimit_to_flags: 59| 570|{ 60| 570| const double tmax = 365 * 24 * 3600; 61| 570| const double tstep = 1.05; 62| 570| const int nsteps = (1 << BITS_FOR_TIMELIMIT); ------------------ | | 634| 570|# define BITS_FOR_TIMELIMIT 9 ------------------ 63| 570| int x; 64| | 65| 570| if (timelimit < 0 || timelimit >= tmax) ------------------ | Branch (65:10): [True: 570, False: 0] | Branch (65:27): [True: 0, False: 0] ------------------ 66| 570| return 0; 67| 0| if (timelimit <= 1.0e-10) ------------------ | Branch (67:10): [True: 0, False: 0] ------------------ 68| 0| return nsteps - 1; 69| | 70| 0| x = (int) (0.5 + (log(tmax / timelimit) / log(tstep))); 71| | 72| 0| if (x < 0) x = 0; ------------------ | Branch (72:10): [True: 0, False: 0] ------------------ 73| 0| if (x >= nsteps) x = nsteps - 1; ------------------ | Branch (73:10): [True: 0, False: 0] ------------------ 74| 0| return x; 75| 0|} fftw_mktensor_rowmajor: 26| 285|{ 27| 285| tensor *x = X(mktensor)(rnk); ------------------ | | 78| 285|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 285|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 28| | 29| 285| if (FINITE_RNK(rnk) && rnk > 0) { ------------------ | | 442| 570|#define FINITE_RNK(rnk) ((rnk) != RNK_MINFTY) | | ------------------ | | | | 441| 285|#define RNK_MINFTY INT_MAX | | ------------------ | | | Branch (442:25): [True: 285, False: 0] | | ------------------ ------------------ | Branch (29:29): [True: 285, False: 0] ------------------ 30| 285| int i; 31| | 32| 285| A(n && niphys && nophys); 33| 285| x->dims[rnk - 1].is = is; 34| 285| x->dims[rnk - 1].os = os; 35| 285| x->dims[rnk - 1].n = n[rnk - 1]; 36| 285| for (i = rnk - 1; i > 0; --i) { ------------------ | Branch (36:29): [True: 0, False: 285] ------------------ 37| 0| x->dims[i - 1].is = x->dims[i].is * niphys[i]; 38| 0| x->dims[i - 1].os = x->dims[i].os * nophys[i]; 39| 0| x->dims[i - 1].n = n[i - 1]; 40| 0| } 41| 285| } 42| 285| return x; 43| 285|} fftw_many_kosherp: 59| 285|{ 60| 285| return (howmany >= 0) && rowmajor_kosherp(rnk, n); ------------------ | Branch (60:13): [True: 285, False: 0] | Branch (60:31): [True: 285, False: 0] ------------------ 61| 285|} mktensor-rowmajor.c:rowmajor_kosherp: 46| 285|{ 47| 285| int i; 48| | 49| 285| if (!FINITE_RNK(rnk)) return 0; ------------------ | | 442| 285|#define FINITE_RNK(rnk) ((rnk) != RNK_MINFTY) | | ------------------ | | | | 441| 285|#define RNK_MINFTY INT_MAX | | ------------------ ------------------ | Branch (49:10): [True: 0, False: 285] ------------------ 50| 285| if (rnk < 0) return 0; ------------------ | Branch (50:10): [True: 0, False: 285] ------------------ 51| | 52| 570| for (i = 0; i < rnk; ++i) ------------------ | Branch (52:18): [True: 285, False: 285] ------------------ 53| 285| if (n[i] <= 0) return 0; ------------------ | Branch (53:8): [True: 0, False: 285] ------------------ 54| | 55| 285| return 1; 56| 285|} fftw_plan_dft_1d: 25| 285|{ 26| 285| return X(plan_dft)(1, &n, in, out, sign, flags); ------------------ | | 78| 285|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 285|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 27| 285|} fftw_plan_dft: 25| 285|{ 26| 285| return X(plan_many_dft)(rank, n, 1, ------------------ | | 78| 285|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 285|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 27| 285| in, 0, 1, 1, 28| 285| out, 0, 1, 1, 29| 285| sign, flags); 30| 285|} fftw_plan_many_dft: 32| 285|{ 33| 285| R *ri, *ii, *ro, *io; 34| | 35| 285| if (!X(many_kosherp)(rank, n, howmany)) return 0; ------------------ | | 78| 285|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 285|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ | Branch (35:10): [True: 0, False: 285] ------------------ 36| | 37| 285| EXTRACT_REIM(sign, in, &ri, &ii); ------------------ | | 75| 285|#define EXTRACT_REIM(sign, c, r, i) X(extract_reim)(sign, (c)[0], r, i) | | ------------------ | | | | 78| 285|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 285|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 38| 285| EXTRACT_REIM(sign, out, &ro, &io); ------------------ | | 75| 285|#define EXTRACT_REIM(sign, c, r, i) X(extract_reim)(sign, (c)[0], r, i) | | ------------------ | | | | 78| 285|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 285|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 39| | 40| 285| return 41| 285| X(mkapiplan)(sign, flags, ------------------ | | 78| 285|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 285|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 42| 285| X(mkproblem_dft_d)( ------------------ | | 78| 285|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 285|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 43| 285| X(mktensor_rowmajor)(rank, n, ------------------ | | 78| 285|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 285|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 44| 285| N0(inembed), N0(onembed), ------------------ | | 24| 285|#define N0(nembed)((nembed) ? (nembed) : n) | | ------------------ | | | Branch (24:20): [True: 0, False: 285] | | ------------------ ------------------ N0(inembed), N0(onembed), ------------------ | | 24| 285|#define N0(nembed)((nembed) ? (nembed) : n) | | ------------------ | | | Branch (24:20): [True: 0, False: 285] | | ------------------ ------------------ 45| 285| 2 * istride, 2 * ostride), 46| 285| X(mktensor_1d)(howmany, 2 * idist, 2 * odist), ------------------ | | 78| 285|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 285|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 47| 285| TAINT_UNALIGNED(ri, flags), ------------------ | | 77| 285|#define TAINT_UNALIGNED(p, flg) TAINT(p, ((flg) & FFTW_UNALIGNED) != 0) | | ------------------ | | | | 1036| 285|#define TAINT(p, s) (p) | | ------------------ ------------------ 48| 285| TAINT_UNALIGNED(ii, flags), ------------------ | | 77| 285|#define TAINT_UNALIGNED(p, flg) TAINT(p, ((flg) & FFTW_UNALIGNED) != 0) | | ------------------ | | | | 1036| 285|#define TAINT(p, s) (p) | | ------------------ ------------------ 49| 285| TAINT_UNALIGNED(ro, flags), ------------------ | | 77| 285|#define TAINT_UNALIGNED(p, flg) TAINT(p, ((flg) & FFTW_UNALIGNED) != 0) | | ------------------ | | | | 1036| 285|#define TAINT(p, s) (p) | | ------------------ ------------------ 50| 285| TAINT_UNALIGNED(io, flags))); ------------------ | | 77| 285|#define TAINT_UNALIGNED(p, flg) TAINT(p, ((flg) & FFTW_UNALIGNED) != 0) | | ------------------ | | | | 1036| 285|#define TAINT(p, s) (p) | | ------------------ ------------------ 51| 285|} fftw_the_planner: 27| 285|{ 28| 285| if (!plnr) { ------------------ | Branch (28:10): [True: 1, False: 284] ------------------ 29| 1| plnr = X(mkplanner)(); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 30| 1| X(configure_planner)(plnr); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 31| 1| } 32| | 33| 285| return plnr; 34| 285|} fftw_dft_bluestein_register: 248| 1|{ 249| 1| REGISTER_SOLVER(p, mksolver()); ------------------ | | 95| 1|#define REGISTER_SOLVER(p, s) X(solver_register)(p, s) | | ------------------ | | | | 78| 1|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 250| 1|} bluestein.c:mksolver: 241| 1|{ 242| 1| static const solver_adt sadt = { PROBLEM_DFT, mkplan, 0 }; 243| 1| S *slv = MKSOLVER(S, &sadt); ------------------ | | 604| 1|#define MKSOLVER(type, adt) (type *)X(mksolver)(sizeof(type), adt) | | ------------------ | | | | 78| 1|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 244| 1| return &(slv->super); 245| 1|} bluestein.c:mkplan: 188| 1.14k|{ 189| 1.14k| const problem_dft *p = (const problem_dft *) p_; 190| 1.14k| P *pln; 191| 1.14k| INT n, nb; 192| 1.14k| plan *cldf = 0; 193| 1.14k| R *buf = (R *) 0; 194| | 195| 1.14k| static const plan_adt padt = { 196| 1.14k| X(dft_solve), awake, print, destroy ------------------ | | 78| 1.14k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1.14k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 197| 1.14k| }; 198| | 199| 1.14k| if (!applicable(ego, p_, plnr)) ------------------ | Branch (199:10): [True: 1.03k, False: 109] ------------------ 200| 1.03k| return (plan *) 0; 201| | 202| 109| n = p->sz->dims[0].n; 203| 109| nb = choose_transform_size(2 * n - 1); 204| 109| buf = (R *) MALLOC(2 * nb * sizeof(R), BUFFERS); ------------------ | | 306| 109|#define MALLOC(n, what) X(malloc_plain)(n) | | ------------------ | | | | 78| 109|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 109|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 205| | 206| 109| cldf = X(mkplan_f_d)(plnr, ------------------ | | 78| 109|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 109|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 207| 109| X(mkproblem_dft_d)(X(mktensor_1d)(nb, 2, 2), ------------------ | | 78| 109|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 109|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ X(mkproblem_dft_d)(X(mktensor_1d)(nb, 2, 2), ------------------ | | 78| 109|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 109|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 208| 109| X(mktensor_1d)(1, 0, 0), ------------------ | | 78| 109|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 109|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 209| 109| buf, buf+1, 210| 109| buf, buf+1), 211| 109| NO_SLOW, 0, 0); 212| 109| if (!cldf) goto nada; ------------------ | Branch (212:10): [True: 0, False: 109] ------------------ 213| | 214| 109| X(ifree)(buf); ------------------ | | 78| 109|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 109|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 215| | 216| 109| pln = MKPLAN_DFT(P, &padt, apply); ------------------ | | 60| 109| (type *)X(mkplan_dft)(sizeof(type), adt, apply) | | ------------------ | | | | 78| 109|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 109|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 217| | 218| 109| pln->n = n; 219| 109| pln->nb = nb; 220| 109| pln->w = 0; 221| 109| pln->W = 0; 222| 109| pln->cldf = cldf; 223| 109| pln->is = p->sz->dims[0].is; 224| 109| pln->os = p->sz->dims[0].os; 225| | 226| 109| X(ops_add)(&cldf->ops, &cldf->ops, &pln->super.super.ops); ------------------ | | 78| 109|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 109|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 227| 109| pln->super.super.ops.add += 4 * n + 2 * nb; 228| 109| pln->super.super.ops.mul += 8 * n + 4 * nb; 229| 109| pln->super.super.ops.other += 6 * (n + nb); 230| | 231| 109| return &(pln->super.super); 232| | 233| 0| nada: 234| 0| X(ifree0)(buf); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 235| 0| X(plan_destroy_internal)(cldf); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 236| 0| return (plan *)0; 237| 109|} bluestein.c:awake: 131| 48|{ 132| 48| P *ego = (P *) ego_; 133| | 134| 48| X(plan_awake)(ego->cldf, wakefulness); ------------------ | | 78| 48|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 48|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 135| | 136| 48| switch (wakefulness) { 137| 24| case SLEEPY: ------------------ | Branch (137:3): [True: 24, False: 24] ------------------ 138| 24| X(ifree0)(ego->w); ego->w = 0; ------------------ | | 78| 24|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 24|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 139| 24| X(ifree0)(ego->W); ego->W = 0; ------------------ | | 78| 24|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 24|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 140| 24| break; 141| 24| default: ------------------ | Branch (141:3): [True: 24, False: 24] ------------------ 142| 24| A(!ego->w); 143| 24| mktwiddle(wakefulness, ego); 144| 24| break; 145| 48| } 146| 48|} bluestein.c:mktwiddle: 53| 24|{ 54| 24| INT i; 55| 24| INT n = p->n, nb = p->nb; 56| 24| R *w, *W; 57| 24| E nbf = (E)nb; 58| | 59| 24| p->w = w = (R *) MALLOC(2 * n * sizeof(R), TWIDDLES); ------------------ | | 306| 24|#define MALLOC(n, what) X(malloc_plain)(n) | | ------------------ | | | | 78| 24|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 24|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 60| 24| p->W = W = (R *) MALLOC(2 * nb * sizeof(R), TWIDDLES); ------------------ | | 306| 24|#define MALLOC(n, what) X(malloc_plain)(n) | | ------------------ | | | | 78| 24|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 24|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 61| | 62| 24| bluestein_sequence(wakefulness, n, w); 63| | 64| 7.73k| for (i = 0; i < nb; ++i) ------------------ | Branch (64:18): [True: 7.71k, False: 24] ------------------ 65| 7.71k| W[2*i] = W[2*i+1] = K(0.0); ------------------ | | 1054| 7.71k|# define K(x) ((E) x) ------------------ 66| | 67| 24| W[0] = w[0] / nbf; 68| 24| W[1] = w[1] / nbf; 69| | 70| 3.77k| for (i = 1; i < n; ++i) { ------------------ | Branch (70:18): [True: 3.75k, False: 24] ------------------ 71| 3.75k| W[2*i] = W[2*(nb-i)] = w[2*i] / nbf; 72| 3.75k| W[2*i+1] = W[2*(nb-i)+1] = w[2*i+1] / nbf; 73| 3.75k| } 74| | 75| 24| { 76| 24| plan_dft *cldf = (plan_dft *)p->cldf; 77| | /* cldf must be awake */ 78| 24| cldf->apply(p->cldf, W, W+1, W, W+1); 79| 24| } 80| 24|} bluestein.c:bluestein_sequence: 38| 24|{ 39| 24| INT k, ksq, n2 = 2 * n; 40| 24| triggen *t = X(mktriggen)(wakefulness, n2); ------------------ | | 78| 24|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 24|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 41| | 42| 24| ksq = 0; 43| 3.80k| for (k = 0; k < n; ++k) { ------------------ | Branch (43:18): [True: 3.77k, False: 24] ------------------ 44| 3.77k| t->cexp(t, ksq, w+2*k); 45| | /* careful with overflow */ 46| 5.65k| ksq += 2*k + 1; while (ksq > n2) ksq -= n2; ------------------ | Branch (46:34): [True: 1.87k, False: 3.77k] ------------------ 47| 3.77k| } 48| | 49| 24| X(triggen_destroy)(t); ------------------ | | 78| 24|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 24|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 50| 24|} bluestein.c:destroy: 168| 109|{ 169| 109| P *ego = (P *) ego_; 170| 109| X(plan_destroy_internal)(ego->cldf); ------------------ | | 78| 109|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 109|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 171| 109|} bluestein.c:applicable: 150| 1.14k|{ 151| 1.14k| const problem_dft *p = (const problem_dft *) p_; 152| 1.14k| UNUSED(ego); ------------------ | | 88| 1.14k|#define UNUSED(x) (void)x ------------------ 153| 1.14k| return (1 ------------------ | Branch (153:14): [True: 1.14k, Folded] ------------------ 154| 1.14k| && p->sz->rnk == 1 ------------------ | Branch (154:10): [True: 810, False: 335] ------------------ 155| 810| && p->vecsz->rnk == 0 ------------------ | Branch (155:10): [True: 289, False: 521] ------------------ 156| | /* FIXME: allow other sizes */ 157| 289| && X(is_prime)(p->sz->dims[0].n) ------------------ | | 78| 289|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 289|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ | Branch (157:10): [True: 132, False: 157] ------------------ 158| | 159| | /* FIXME: avoid infinite recursion of bluestein with itself. 160| | This works because all factors in child problems are 2, 3, 5 */ 161| 132| && p->sz->dims[0].n > 16 ------------------ | Branch (161:10): [True: 132, False: 0] ------------------ 162| | 163| 132| && CIMPLIES(NO_SLOWP(plnr), p->sz->dims[0].n > BLUESTEIN_MAX_SLOW) ------------------ | | 99| 132|#define CIMPLIES(ante, post) (!(ante) || (post)) | | ------------------ | | | Branch (99:31): [True: 0, False: 132] | | | Branch (99:42): [True: 109, False: 23] | | ------------------ ------------------ 164| 1.14k| ); 165| 1.14k|} bluestein.c:choose_transform_size: 181| 109|{ 182| 795| while (!X(factors_into_small_primes)(minsz)) ------------------ | | 78| 795|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 795|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ | Branch (182:13): [True: 686, False: 109] ------------------ 183| 686| ++minsz; 184| 109| return minsz; 185| 109|} bluestein.c:apply: 83| 27|{ 84| 27| const P *ego = (const P *) ego_; 85| 27| INT i, n = ego->n, nb = ego->nb, is = ego->is, os = ego->os; 86| 27| R *w = ego->w, *W = ego->W; 87| 27| R *b = (R *) MALLOC(2 * nb * sizeof(R), BUFFERS); ------------------ | | 306| 27|#define MALLOC(n, what) X(malloc_plain)(n) | | ------------------ | | | | 78| 27|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 27|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 88| | 89| | /* multiply input by conjugate bluestein sequence */ 90| 4.13k| for (i = 0; i < n; ++i) { ------------------ | Branch (90:18): [True: 4.10k, False: 27] ------------------ 91| 4.10k| E xr = ri[i*is], xi = ii[i*is]; 92| 4.10k| E wr = w[2*i], wi = w[2*i+1]; 93| 4.10k| b[2*i] = xr * wr + xi * wi; 94| 4.10k| b[2*i+1] = xi * wr - xr * wi; 95| 4.10k| } 96| | 97| 4.29k| for (; i < nb; ++i) b[2*i] = b[2*i+1] = K(0.0); ------------------ | | 1054| 4.26k|# define K(x) ((E) x) ------------------ | Branch (97:13): [True: 4.26k, False: 27] ------------------ 98| | 99| | /* convolution: FFT */ 100| 27| { 101| 27| plan_dft *cldf = (plan_dft *)ego->cldf; 102| 27| cldf->apply(ego->cldf, b, b+1, b, b+1); 103| 27| } 104| | 105| | /* convolution: pointwise multiplication */ 106| 8.39k| for (i = 0; i < nb; ++i) { ------------------ | Branch (106:18): [True: 8.36k, False: 27] ------------------ 107| 8.36k| E xr = b[2*i], xi = b[2*i+1]; 108| 8.36k| E wr = W[2*i], wi = W[2*i+1]; 109| 8.36k| b[2*i] = xi * wr + xr * wi; 110| 8.36k| b[2*i+1] = xr * wr - xi * wi; 111| 8.36k| } 112| | 113| | /* convolution: IFFT by FFT with real/imag input/output swapped */ 114| 27| { 115| 27| plan_dft *cldf = (plan_dft *)ego->cldf; 116| 27| cldf->apply(ego->cldf, b, b+1, b, b+1); 117| 27| } 118| | 119| | /* multiply output by conjugate bluestein sequence */ 120| 4.13k| for (i = 0; i < n; ++i) { ------------------ | Branch (120:18): [True: 4.10k, False: 27] ------------------ 121| 4.10k| E xi = b[2*i], xr = b[2*i+1]; 122| 4.10k| E wr = w[2*i], wi = w[2*i+1]; 123| 4.10k| ro[i*os] = xr * wr + xi * wi; 124| 4.10k| io[i*os] = xi * wr - xr * wi; 125| 4.10k| } 126| | 127| 27| X(ifree)(b); ------------------ | | 78| 27|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 27|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 128| 27|} fftw_dft_buffered_register: 280| 1|{ 281| 1| size_t i; 282| 3| for (i = 0; i < NELEM(maxnbufs); ++i) ------------------ | | 90| 3|#define NELEM(array) ((sizeof(array) / sizeof((array)[0]))) ------------------ | Branch (282:18): [True: 2, False: 1] ------------------ 283| 2| REGISTER_SOLVER(p, mksolver(i)); ------------------ | | 95| 2|#define REGISTER_SOLVER(p, s) X(solver_register)(p, s) | | ------------------ | | | | 78| 2|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 2|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 284| 1|} buffered.c:mksolver: 272| 4|{ 273| 4| static const solver_adt sadt = { PROBLEM_DFT, mkplan, 0 }; 274| 4| S *slv = MKSOLVER(S, &sadt); ------------------ | | 604| 4|#define MKSOLVER(type, adt) (type *)X(mksolver)(sizeof(type), adt) | | ------------------ | | | | 78| 4|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 4|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 275| 4| slv->maxnbuf_ndx = maxnbuf_ndx; 276| 4| return &(slv->super); 277| 4|} buffered.c:mkplan: 164| 2.27k|{ 165| 2.27k| P *pln; 166| 2.27k| const S *ego = (const S *)ego_; 167| 2.27k| plan *cld = (plan *) 0; 168| 2.27k| plan *cldcpy = (plan *) 0; 169| 2.27k| plan *cldrest = (plan *) 0; 170| 2.27k| const problem_dft *p = (const problem_dft *) p_; 171| 2.27k| R *bufs = (R *) 0; 172| 2.27k| INT nbuf = 0, bufdist, n, vl; 173| 2.27k| INT ivs, ovs, roffset, ioffset; 174| | 175| 2.27k| static const plan_adt padt = { 176| 2.27k| X(dft_solve), awake, print, destroy ------------------ | | 78| 2.27k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 2.27k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 177| 2.27k| }; 178| | 179| 2.27k| if (!applicable(ego, p_, plnr)) ------------------ | Branch (179:10): [True: 2.07k, False: 204] ------------------ 180| 2.07k| goto nada; 181| | 182| 204| n = X(tensor_sz)(p->sz); ------------------ | | 78| 204|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 204|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 183| | 184| 204| X(tensor_tornk1)(p->vecsz, &vl, &ivs, &ovs); ------------------ | | 78| 204|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 204|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 185| | 186| 204| nbuf = X(nbuf)(n, vl, maxnbufs[ego->maxnbuf_ndx]); ------------------ | | 78| 204|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 204|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 187| 204| bufdist = X(bufdist)(n, vl); ------------------ | | 78| 204|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 204|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 188| 204| A(nbuf > 0); 189| | 190| | /* attempt to keep real and imaginary part in the same order, 191| | so as to allow optimizations in the the copy plan */ 192| 204| roffset = (p->ri - p->ii > 0) ? (INT)1 : (INT)0; ------------------ | Branch (192:16): [True: 0, False: 204] ------------------ 193| 204| ioffset = 1 - roffset; 194| | 195| | /* initial allocation for the purpose of planning */ 196| 204| bufs = (R *) MALLOC(sizeof(R) * nbuf * bufdist * 2, BUFFERS); ------------------ | | 306| 204|#define MALLOC(n, what) X(malloc_plain)(n) | | ------------------ | | | | 78| 204|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 204|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 197| | 198| | /* allow destruction of input if problem is in place */ 199| 204| cld = X(mkplan_f_d)(plnr, ------------------ | | 78| 204|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 204|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 200| 204| X(mkproblem_dft_d)( ------------------ | | 78| 204|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 204|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 201| 204| X(mktensor_1d)(n, p->sz->dims[0].is, 2), ------------------ | | 78| 204|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 204|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 202| 204| X(mktensor_1d)(nbuf, ivs, bufdist * 2), ------------------ | | 78| 204|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 204|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 203| 204| TAINT(p->ri, ivs * nbuf), ------------------ | | 1036| 204|#define TAINT(p, s) (p) ------------------ 204| 204| TAINT(p->ii, ivs * nbuf), ------------------ | | 1036| 204|#define TAINT(p, s) (p) ------------------ 205| 204| bufs + roffset, 206| 204| bufs + ioffset), 207| 204| 0, 0, (p->ri == p->ro) ? NO_DESTROY_INPUT : 0); ------------------ | Branch (207:11): [True: 204, False: 0] ------------------ 208| 204| if (!cld) ------------------ | Branch (208:10): [True: 0, False: 204] ------------------ 209| 0| goto nada; 210| | 211| | /* copying back from the buffer is a rank-0 transform: */ 212| 204| cldcpy = X(mkplan_d)(plnr, ------------------ | | 78| 204|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 204|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 213| 204| X(mkproblem_dft_d)( ------------------ | | 78| 204|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 204|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 214| 204| X(mktensor_0d)(), ------------------ | | 78| 204|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 204|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 215| 204| X(mktensor_2d)(nbuf, bufdist * 2, ovs, ------------------ | | 78| 204|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 204|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 216| 204| n, 2, p->sz->dims[0].os), 217| 204| bufs + roffset, 218| 204| bufs + ioffset, 219| 204| TAINT(p->ro, ovs * nbuf), ------------------ | | 1036| 204|#define TAINT(p, s) (p) ------------------ 220| 204| TAINT(p->io, ovs * nbuf))); ------------------ | | 1036| 204|#define TAINT(p, s) (p) ------------------ 221| 204| if (!cldcpy) ------------------ | Branch (221:10): [True: 0, False: 204] ------------------ 222| 0| goto nada; 223| | 224| | /* deallocate buffers, let apply() allocate them for real */ 225| 204| X(ifree)(bufs); ------------------ | | 78| 204|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 204|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 226| 204| bufs = 0; 227| | 228| | /* plan the leftover transforms (cldrest): */ 229| 204| { 230| 204| INT id = ivs * (nbuf * (vl / nbuf)); 231| 204| INT od = ovs * (nbuf * (vl / nbuf)); 232| 204| cldrest = X(mkplan_d)(plnr, ------------------ | | 78| 204|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 204|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 233| 204| X(mkproblem_dft_d)( ------------------ | | 78| 204|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 204|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 234| 204| X(tensor_copy)(p->sz), ------------------ | | 78| 204|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 204|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 235| 204| X(mktensor_1d)(vl % nbuf, ivs, ovs), ------------------ | | 78| 204|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 204|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 236| 204| p->ri+id, p->ii+id, p->ro+od, p->io+od)); 237| 204| } 238| 204| if (!cldrest) ------------------ | Branch (238:10): [True: 0, False: 204] ------------------ 239| 0| goto nada; 240| | 241| 204| pln = MKPLAN_DFT(P, &padt, apply); ------------------ | | 60| 204| (type *)X(mkplan_dft)(sizeof(type), adt, apply) | | ------------------ | | | | 78| 204|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 204|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 242| 204| pln->cld = cld; 243| 204| pln->cldcpy = cldcpy; 244| 204| pln->cldrest = cldrest; 245| 204| pln->n = n; 246| 204| pln->vl = vl; 247| 204| pln->ivs_by_nbuf = ivs * nbuf; 248| 204| pln->ovs_by_nbuf = ovs * nbuf; 249| 204| pln->roffset = roffset; 250| 204| pln->ioffset = ioffset; 251| | 252| 204| pln->nbuf = nbuf; 253| 204| pln->bufdist = bufdist; 254| | 255| 204| { 256| 204| opcnt t; 257| 204| X(ops_add)(&cld->ops, &cldcpy->ops, &t); ------------------ | | 78| 204|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 204|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 258| 204| X(ops_madd)(vl / nbuf, &t, &cldrest->ops, &pln->super.super.ops); ------------------ | | 78| 204|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 204|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 259| 204| } 260| | 261| 204| return &(pln->super.super); 262| | 263| 2.07k| nada: 264| 2.07k| X(ifree0)(bufs); ------------------ | | 78| 2.07k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 2.07k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 265| 2.07k| X(plan_destroy_internal)(cldrest); ------------------ | | 78| 2.07k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 2.07k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 266| 2.07k| X(plan_destroy_internal)(cldcpy); ------------------ | | 78| 2.07k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 2.07k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 267| 2.07k| X(plan_destroy_internal)(cld); ------------------ | | 78| 2.07k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 2.07k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 268| 2.07k| return (plan *) 0; 269| 204|} buffered.c:awake: 73| 54|{ 74| 54| P *ego = (P *) ego_; 75| | 76| 54| X(plan_awake)(ego->cld, wakefulness); ------------------ | | 78| 54|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 54|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 77| 54| X(plan_awake)(ego->cldcpy, wakefulness); ------------------ | | 78| 54|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 54|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 78| 54| X(plan_awake)(ego->cldrest, wakefulness); ------------------ | | 78| 54|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 54|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 79| 54|} buffered.c:destroy: 82| 204|{ 83| 204| P *ego = (P *) ego_; 84| 204| X(plan_destroy_internal)(ego->cldrest); ------------------ | | 78| 204|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 204|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 85| 204| X(plan_destroy_internal)(ego->cldcpy); ------------------ | | 78| 204|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 204|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 86| 204| X(plan_destroy_internal)(ego->cld); ------------------ | | 78| 204|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 204|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 87| 204|} buffered.c:applicable: 151| 2.27k|{ 152| 2.27k| if (NO_BUFFERINGP(plnr)) return 0; ------------------ | | 699| 2.27k|#define NO_BUFFERINGP(plnr) (PLNR_L(plnr) & NO_BUFFERING) | | ------------------ | | | | 674| 2.27k|#define PLNR_L(plnr) ((plnr)->flags.l) | | ------------------ | | | Branch (699:29): [True: 0, False: 2.27k] | | ------------------ ------------------ 153| 2.27k| if (!applicable0(ego, p_, plnr)) return 0; ------------------ | Branch (153:10): [True: 2.07k, False: 204] ------------------ 154| | 155| 204| if (NO_UGLYP(plnr)) { ------------------ | | 689| 204|#define NO_UGLYP(plnr) (PLNR_L(plnr) & NO_UGLY) | | ------------------ | | | | 674| 204|#define PLNR_L(plnr) ((plnr)->flags.l) | | ------------------ | | | Branch (689:24): [True: 204, False: 0] | | ------------------ ------------------ 156| 204| const problem_dft *p = (const problem_dft *) p_; 157| 204| if (p->ri != p->ro) return 0; ------------------ | Branch (157:8): [True: 0, False: 204] ------------------ 158| 204| if (X(toobig)(p->sz->dims[0].n)) return 0; ------------------ | | 78| 204|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 204|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ | Branch (158:8): [True: 0, False: 204] ------------------ 159| 204| } 160| 204| return 1; 161| 204|} buffered.c:applicable0: 99| 2.27k|{ 100| 2.27k| const problem_dft *p = (const problem_dft *) p_; 101| 2.27k| const iodim *d = p->sz->dims; 102| | 103| 2.27k| if (1 ------------------ | Branch (103:10): [True: 2.27k, Folded] ------------------ 104| 2.27k| && p->vecsz->rnk <= 1 ------------------ | Branch (104:6): [True: 1.23k, False: 1.03k] ------------------ 105| 1.23k| && p->sz->rnk == 1 ------------------ | Branch (105:6): [True: 1.20k, False: 36] ------------------ 106| 2.27k| ) { 107| 1.20k| INT vl, ivs, ovs; 108| 1.20k| X(tensor_tornk1)(p->vecsz, &vl, &ivs, &ovs); ------------------ | | 78| 1.20k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1.20k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 109| | 110| 1.20k| if (X(toobig)(p->sz->dims[0].n) && CONSERVE_MEMORYP(plnr)) ------------------ | | 78| 1.20k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1.20k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ if (X(toobig)(p->sz->dims[0].n) && CONSERVE_MEMORYP(plnr)) ------------------ | | 697| 0|#define CONSERVE_MEMORYP(plnr) (PLNR_L(plnr) & CONSERVE_MEMORY) | | ------------------ | | | | 674| 0|#define PLNR_L(plnr) ((plnr)->flags.l) | | ------------------ | | | Branch (697:32): [True: 0, False: 0] | | ------------------ ------------------ | Branch (110:8): [True: 0, False: 1.20k] ------------------ 111| 0| return 0; 112| | 113| | /* if this solver is redundant, in the sense that a solver 114| | of lower index generates the same plan, then prune this 115| | solver */ 116| 1.20k| if (X(nbuf_redundant)(d[0].n, vl, ------------------ | | 78| 1.20k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1.20k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ | Branch (116:8): [True: 441, False: 762] ------------------ 117| 1.20k| ego->maxnbuf_ndx, 118| 1.20k| maxnbufs, NELEM(maxnbufs))) ------------------ | | 90| 1.20k|#define NELEM(array) ((sizeof(array) / sizeof((array)[0]))) ------------------ 119| 441| return 0; 120| | 121| | /* 122| | In principle, the buffered transforms might be useful 123| | when working out of place. However, in order to 124| | prevent infinite loops in the planner, we require 125| | that the output stride of the buffered transforms be 126| | greater than 2. 127| | */ 128| 762| if (p->ri != p->ro) ------------------ | Branch (128:8): [True: 450, False: 312] ------------------ 129| 450| return (d[0].os > 2); 130| | 131| | /* 132| | * If the problem is in place, the input/output strides must 133| | * be the same or the whole thing must fit in the buffer. 134| | */ 135| 312| if (X(tensor_inplace_strides2)(p->sz, p->vecsz)) ------------------ | | 78| 312|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 312|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ | Branch (135:8): [True: 0, False: 312] ------------------ 136| 0| return 1; 137| | 138| 312| if (/* fits into buffer: */ 139| 312| ((p->vecsz->rnk == 0) ------------------ | Branch (139:10): [True: 0, False: 312] ------------------ 140| 312| || 141| 312| (X(nbuf)(d[0].n, p->vecsz->dims[0].n, ------------------ | | 78| 312|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 312|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ | Branch (141:3): [True: 204, False: 108] ------------------ 142| 312| maxnbufs[ego->maxnbuf_ndx]) 143| 312| == p->vecsz->dims[0].n))) 144| 204| return 1; 145| 312| } 146| | 147| 1.18k| return 0; 148| 2.27k|} buffered.c:apply: 42| 69|{ 43| 69| const P *ego = (const P *) ego_; 44| 69| INT nbuf = ego->nbuf; 45| 69| R *bufs = (R *)MALLOC(sizeof(R) * nbuf * ego->bufdist * 2, BUFFERS); ------------------ | | 306| 69|#define MALLOC(n, what) X(malloc_plain)(n) | | ------------------ | | | | 78| 69|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 69|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 46| | 47| 69| plan_dft *cld = (plan_dft *) ego->cld; 48| 69| plan_dft *cldcpy = (plan_dft *) ego->cldcpy; 49| 69| plan_dft *cldrest; 50| 69| INT i, vl = ego->vl; 51| 69| INT ivs_by_nbuf = ego->ivs_by_nbuf, ovs_by_nbuf = ego->ovs_by_nbuf; 52| 69| INT roffset = ego->roffset, ioffset = ego->ioffset; 53| | 54| 138| for (i = nbuf; i <= vl; i += nbuf) { ------------------ | Branch (54:21): [True: 69, False: 69] ------------------ 55| | /* transform to bufs: */ 56| 69| cld->apply((plan *) cld, ri, ii, bufs + roffset, bufs + ioffset); 57| 69| ri += ivs_by_nbuf; ii += ivs_by_nbuf; 58| | 59| | /* copy back */ 60| 69| cldcpy->apply((plan *) cldcpy, bufs+roffset, bufs+ioffset, ro, io); 61| 69| ro += ovs_by_nbuf; io += ovs_by_nbuf; 62| 69| } 63| | 64| 69| X(ifree)(bufs); ------------------ | | 78| 69|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 69|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 65| | 66| | /* Do the remaining transforms, if any: */ 67| 69| cldrest = (plan_dft *) ego->cldrest; 68| 69| cldrest->apply((plan *) cldrest, ri, ii, ro, io); 69| 69|} fftw_dft_conf_standard: 41| 1|{ 42| 1| X(solvtab_exec)(s, p); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 43| 1| X(solvtab_exec)(X(solvtab_dft_standard), p); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ X(solvtab_exec)(X(solvtab_dft_standard), p); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 44| |#if HAVE_SSE2 45| | if (X(have_simd_sse2)()) 46| | X(solvtab_exec)(X(solvtab_dft_sse2), p); 47| |#endif 48| |#if HAVE_AVX 49| | if (X(have_simd_avx)()) 50| | X(solvtab_exec)(X(solvtab_dft_avx), p); 51| |#endif 52| |#if HAVE_AVX_128_FMA 53| | if (X(have_simd_avx_128_fma)()) 54| | X(solvtab_exec)(X(solvtab_dft_avx_128_fma), p); 55| |#endif 56| |#if HAVE_AVX2 57| | if (X(have_simd_avx2)()) 58| | X(solvtab_exec)(X(solvtab_dft_avx2), p); 59| | if (X(have_simd_avx2_128)()) 60| | X(solvtab_exec)(X(solvtab_dft_avx2_128), p); 61| |#endif 62| |#if HAVE_AVX512 63| | if (X(have_simd_avx512)()) 64| | X(solvtab_exec)(X(solvtab_dft_avx512), p); 65| |#endif 66| |#if HAVE_KCVI 67| | if (X(have_simd_kcvi)()) 68| | X(solvtab_exec)(X(solvtab_dft_kcvi), p); 69| |#endif 70| |#if HAVE_ALTIVEC 71| | if (X(have_simd_altivec)()) 72| | X(solvtab_exec)(X(solvtab_dft_altivec), p); 73| |#endif 74| |#if HAVE_VSX 75| | if (X(have_simd_vsx)()) 76| | X(solvtab_exec)(X(solvtab_dft_vsx), p); 77| |#endif 78| |#if HAVE_NEON 79| | if (X(have_simd_neon)()) 80| | X(solvtab_exec)(X(solvtab_dft_neon), p); 81| |#endif 82| |#if HAVE_SVE 83| | if (X(have_simd_sve)(128)) 84| | X(solvtab_exec)(X(solvtab_dft_sve128), p); 85| | if (X(have_simd_sve)(256)) 86| | X(solvtab_exec)(X(solvtab_dft_sve256), p); 87| | if (X(have_simd_sve)(512)) 88| | X(solvtab_exec)(X(solvtab_dft_sve512), p); 89| | if (X(have_simd_sve)(1024)) 90| | X(solvtab_exec)(X(solvtab_dft_sve1024), p); 91| | if (X(have_simd_sve)(2048)) 92| | X(solvtab_exec)(X(solvtab_dft_sve2048), p); 93| |#endif 94| |#if HAVE_LSX 95| | if (X(have_simd_lsx)()) 96| | X(solvtab_exec)(X(solvtab_dft_lsx), p); 97| |#endif 98| |#if HAVE_LASX 99| | if (X(have_simd_lasx)()) 100| | X(solvtab_exec)(X(solvtab_dft_lasx), p); 101| |#endif 102| |#if HAVE_GENERIC_SIMD128 103| | X(solvtab_exec)(X(solvtab_dft_generic_simd128), p); 104| |#endif 105| |#if HAVE_GENERIC_SIMD256 106| | X(solvtab_exec)(X(solvtab_dft_generic_simd256), p); 107| |#endif 108| 1|} fftw_ct_applicable: 102| 136k|{ 103| 136k| const problem_dft *p; 104| | 105| 136k| if (!applicable0(ego, p_, plnr)) ------------------ | Branch (105:10): [True: 129k, False: 6.94k] ------------------ 106| 129k| return 0; 107| | 108| 6.94k| p = (const problem_dft *) p_; 109| | 110| 6.94k| return (0 ------------------ | Branch (110:14): [Folded, False: 6.94k] ------------------ 111| 6.94k| || ego->dec == DECDIF+TRANSPOSE ------------------ | | 48| 6.94k|# define DECDIF 0 ------------------ || ego->dec == DECDIF+TRANSPOSE ------------------ | | 50| 13.8k|# define TRANSPOSE 2 ------------------ | Branch (111:10): [True: 1.12k, False: 5.81k] ------------------ 112| 5.81k| || p->vecsz->rnk == 0 ------------------ | Branch (112:10): [True: 2.24k, False: 3.56k] ------------------ 113| 3.56k| || !NO_VRECURSEP(plnr) ------------------ | | 686| 10.5k|#define NO_VRECURSEP(plnr) (PLNR_L(plnr) & NO_VRECURSE) | | ------------------ | | | | 674| 3.56k|#define PLNR_L(plnr) ((plnr)->flags.l) | | ------------------ ------------------ | Branch (113:10): [True: 36, False: 3.53k] ------------------ 114| 3.53k| || (ego->force_vrecursionp && ego->force_vrecursionp(ego, p)) ------------------ | Branch (114:11): [True: 0, False: 3.53k] | Branch (114:37): [True: 0, False: 0] ------------------ 115| 6.94k| ); 116| 136k|} fftw_mksolver_ct: 237| 93|{ 238| 93| static const solver_adt sadt = { PROBLEM_DFT, mkplan, 0 }; 239| 93| ct_solver *slv = (ct_solver *)X(mksolver)(size, &sadt); ------------------ | | 78| 93|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 93|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 240| 93| slv->r = r; 241| 93| slv->dec = dec; 242| 93| slv->mkcldw = mkcldw; 243| 93| slv->force_vrecursionp = force_vrecursionp; 244| 93| return slv; 245| 93|} fftw_mkplan_dftw: 248| 1.20k|{ 249| 1.20k| plan_dftw *ego; 250| | 251| 1.20k| ego = (plan_dftw *) X(mkplan)(size, adt); ------------------ | | 78| 1.20k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1.20k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 252| 1.20k| ego->apply = apply; 253| | 254| 1.20k| return &(ego->super); 255| 1.20k|} ct.c:applicable0: 83| 136k|{ 84| 136k| const problem_dft *p = (const problem_dft *) p_; 85| 136k| INT r; 86| | 87| 136k| return (1 ------------------ | Branch (87:14): [True: 136k, Folded] ------------------ 88| 136k| && p->sz->rnk == 1 ------------------ | Branch (88:10): [True: 105k, False: 31.1k] ------------------ 89| 105k| && p->vecsz->rnk <= 1 ------------------ | Branch (89:10): [True: 86.9k, False: 18.6k] ------------------ 90| | 91| | /* DIF destroys the input and we don't like it */ 92| 86.9k| && (ego->dec == DECDIT || ------------------ | | 49| 173k|# define DECDIT 1 ------------------ | Branch (92:11): [True: 78.8k, False: 8.09k] ------------------ 93| 8.09k| p->ri == p->ro || ------------------ | Branch (93:4): [True: 1.99k, False: 6.10k] ------------------ 94| 6.10k| !NO_DESTROY_INPUTP(plnr)) ------------------ | | 695| 6.10k|#define NO_DESTROY_INPUTP(plnr) (PLNR_L(plnr) & NO_DESTROY_INPUT) | | ------------------ | | | | 674| 6.10k|#define PLNR_L(plnr) ((plnr)->flags.l) | | ------------------ ------------------ | Branch (94:4): [True: 1.96k, False: 4.14k] ------------------ 95| | 96| 82.7k| && ((r = X(choose_radix)(ego->r, p->sz->dims[0].n)) > 1) ------------------ | | 78| 82.7k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 82.7k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ | Branch (96:10): [True: 8.90k, False: 73.8k] ------------------ 97| 8.90k| && p->sz->dims[0].n > r); ------------------ | Branch (97:10): [True: 6.94k, False: 1.96k] ------------------ 98| 136k|} ct.c:mkplan: 120| 136k|{ 121| 136k| const ct_solver *ego = (const ct_solver *) ego_; 122| 136k| const problem_dft *p; 123| 136k| P *pln = 0; 124| 136k| plan *cld = 0, *cldw = 0; 125| 136k| INT n, r, m, v, ivs, ovs; 126| 136k| iodim *d; 127| | 128| 136k| static const plan_adt padt = { 129| 136k| X(dft_solve), awake, print, destroy ------------------ | | 78| 136k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 136k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 130| 136k| }; 131| | 132| 136k| if ((NO_NONTHREADEDP(plnr)) || !X(ct_applicable)(ego, p_, plnr)) ------------------ | | 693| 136k| ((PLNR_L(plnr) & NO_NONTHREADED) && (plnr)->nthr > 1) | | ------------------ | | | | 674| 136k|#define PLNR_L(plnr) ((plnr)->flags.l) | | ------------------ | | | Branch (693:4): [True: 136k, False: 0] | | | Branch (693:39): [True: 0, False: 136k] | | ------------------ ------------------ if ((NO_NONTHREADEDP(plnr)) || !X(ct_applicable)(ego, p_, plnr)) ------------------ | | 78| 136k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 136k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ | Branch (132:37): [True: 133k, False: 3.40k] ------------------ 133| 133k| return (plan *) 0; 134| | 135| 3.40k| p = (const problem_dft *) p_; 136| 3.40k| d = p->sz->dims; 137| 3.40k| n = d[0].n; 138| 3.40k| r = X(choose_radix)(ego->r, n); ------------------ | | 78| 3.40k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 3.40k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 139| 3.40k| m = n / r; 140| | 141| 3.40k| X(tensor_tornk1)(p->vecsz, &v, &ivs, &ovs); ------------------ | | 78| 3.40k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 3.40k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 142| | 143| 3.40k| switch (ego->dec) { 144| 2.25k| case DECDIT: ------------------ | | 49| 2.25k|# define DECDIT 1 ------------------ | Branch (144:3): [True: 2.25k, False: 1.15k] ------------------ 145| 2.25k| { 146| 2.25k| cldw = ego->mkcldw(ego, 147| 2.25k| r, m * d[0].os, m * d[0].os, 148| 2.25k| m, d[0].os, 149| 2.25k| v, ovs, ovs, 150| 2.25k| 0, m, 151| 2.25k| p->ro, p->io, plnr); 152| 2.25k| if (!cldw) goto nada; ------------------ | Branch (152:12): [True: 1.07k, False: 1.18k] ------------------ 153| | 154| 1.18k| cld = X(mkplan_d)(plnr, ------------------ | | 78| 1.18k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1.18k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 155| 1.18k| X(mkproblem_dft_d)( ------------------ | | 78| 1.18k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1.18k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 156| 1.18k| X(mktensor_1d)(m, r * d[0].is, d[0].os), ------------------ | | 78| 1.18k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1.18k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 157| 1.18k| X(mktensor_2d)(r, d[0].is, m * d[0].os, ------------------ | | 78| 1.18k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1.18k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 158| 1.18k| v, ivs, ovs), 159| 1.18k| p->ri, p->ii, p->ro, p->io) 160| 1.18k| ); 161| 1.18k| if (!cld) goto nada; ------------------ | Branch (161:12): [True: 3, False: 1.18k] ------------------ 162| | 163| 1.18k| pln = MKPLAN_DFT(P, &padt, apply_dit); ------------------ | | 60| 1.18k| (type *)X(mkplan_dft)(sizeof(type), adt, apply) | | ------------------ | | | | 78| 1.18k|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 1.18k|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 164| 1.18k| break; 165| 1.18k| } 166| 28| case DECDIF: ------------------ | | 48| 28|# define DECDIF 0 ------------------ | Branch (166:3): [True: 28, False: 3.38k] ------------------ 167| 1.15k| case DECDIF+TRANSPOSE: ------------------ | | 48| 1.15k|# define DECDIF 0 ------------------ case DECDIF+TRANSPOSE: ------------------ | | 50| 1.15k|# define TRANSPOSE 2 ------------------ | Branch (167:3): [True: 1.12k, False: 2.28k] ------------------ 168| 1.15k| { 169| 1.15k| INT cors, covs; /* cldw ors, ovs */ 170| 1.15k| if (ego->dec == DECDIF+TRANSPOSE) { ------------------ | | 48| 1.15k|# define DECDIF 0 ------------------ if (ego->dec == DECDIF+TRANSPOSE) { ------------------ | | 50| 1.15k|# define TRANSPOSE 2 ------------------ | Branch (170:12): [True: 1.12k, False: 28] ------------------ 171| 1.12k| cors = ivs; 172| 1.12k| covs = m * d[0].is; 173| | /* ensure that we generate well-formed dftw subproblems */ 174| | /* FIXME: too conservative */ 175| 1.12k| if (!(1 ------------------ | Branch (175:12): [True: 1.12k, Folded] ------------------ 176| 1.12k| && r == v ------------------ | Branch (176:8): [True: 64, False: 1.06k] ------------------ 177| 64| && d[0].is == r * cors)) ------------------ | Branch (177:8): [True: 64, False: 0] ------------------ 178| 1.06k| goto nada; 179| | 180| | /* FIXME: allow in-place only for now, like in 181| | fftw-3.[01] */ 182| 64| if (!(1 ------------------ | Branch (182:12): [True: 64, Folded] ------------------ 183| 64| && p->ri == p->ro ------------------ | Branch (183:8): [True: 21, False: 43] ------------------ 184| 21| && d[0].is == r * d[0].os ------------------ | Branch (184:8): [True: 20, False: 1] ------------------ 185| 20| && cors == d[0].os ------------------ | Branch (185:8): [True: 20, False: 0] ------------------ 186| 20| && covs == ovs ------------------ | Branch (186:8): [True: 20, False: 0] ------------------ 187| 64| )) 188| 44| goto nada; 189| 64| } else { 190| 28| cors = m * d[0].is; 191| 28| covs = ivs; 192| 28| } 193| | 194| 48| cldw = ego->mkcldw(ego, 195| 48| r, m * d[0].is, cors, 196| 48| m, d[0].is, 197| 48| v, ivs, covs, 198| 48| 0, m, 199| 48| p->ri, p->ii, plnr); 200| 48| if (!cldw) goto nada; ------------------ | Branch (200:12): [True: 28, False: 20] ------------------ 201| | 202| 20| cld = X(mkplan_d)(plnr, ------------------ | | 78| 20|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 20|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 203| 20| X(mkproblem_dft_d)( ------------------ | | 78| 20|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 20|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 204| 20| X(mktensor_1d)(m, d[0].is, r * d[0].os), ------------------ | | 78| 20|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 20|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 205| 20| X(mktensor_2d)(r, cors, d[0].os, ------------------ | | 78| 20|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 20|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 206| 20| v, covs, ovs), 207| 20| p->ri, p->ii, p->ro, p->io) 208| 20| ); 209| 20| if (!cld) goto nada; ------------------ | Branch (209:12): [True: 0, False: 20] ------------------ 210| | 211| 20| pln = MKPLAN_DFT(P, &padt, apply_dif); ------------------ | | 60| 20| (type *)X(mkplan_dft)(sizeof(type), adt, apply) | | ------------------ | | | | 78| 20|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 20|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 212| 20| break; 213| 20| } 214| | 215| 0| default: A(0); ------------------ | Branch (215:3): [True: 0, False: 3.40k] ------------------ 216| | 217| 3.40k| } 218| | 219| 1.20k| pln->cld = cld; 220| 1.20k| pln->cldw = cldw; 221| 1.20k| pln->r = r; 222| 1.20k| X(ops_add)(&cld->ops, &cldw->ops, &pln->super.super.ops); ------------------ | | 78| 1.20k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1.20k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 223| | 224| | /* inherit could_prune_now_p attribute from cldw */ 225| 1.20k| pln->super.super.could_prune_now_p = cldw->could_prune_now_p; 226| 1.20k| return &(pln->super.super); 227| | 228| 2.20k| nada: 229| 2.20k| X(plan_destroy_internal)(cldw); ------------------ | | 78| 2.20k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 2.20k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 230| 2.20k| X(plan_destroy_internal)(cld); ------------------ | | 78| 2.20k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 2.20k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 231| 2.20k| return (plan *) 0; 232| 3.40k|} ct.c:awake: 61| 748|{ 62| 748| P *ego = (P *) ego_; 63| 748| X(plan_awake)(ego->cld, wakefulness); ------------------ | | 78| 748|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 748|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 64| 748| X(plan_awake)(ego->cldw, wakefulness); ------------------ | | 78| 748|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 748|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 65| 748|} ct.c:destroy: 68| 1.20k|{ 69| 1.20k| P *ego = (P *) ego_; 70| 1.20k| X(plan_destroy_internal)(ego->cldw); ------------------ | | 78| 1.20k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1.20k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 71| 1.20k| X(plan_destroy_internal)(ego->cld); ------------------ | | 78| 1.20k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1.20k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 72| 1.20k|} ct.c:apply_dit: 35| 1.41k|{ 36| 1.41k| const P *ego = (const P *) ego_; 37| 1.41k| plan_dft *cld; 38| 1.41k| plan_dftw *cldw; 39| | 40| 1.41k| cld = (plan_dft *) ego->cld; 41| 1.41k| cld->apply(ego->cld, ri, ii, ro, io); 42| | 43| 1.41k| cldw = (plan_dftw *) ego->cldw; 44| 1.41k| cldw->apply(ego->cldw, ro, io); 45| 1.41k|} ct.c:apply_dif: 48| 11|{ 49| 11| const P *ego = (const P *) ego_; 50| 11| plan_dft *cld; 51| 11| plan_dftw *cldw; 52| | 53| 11| cldw = (plan_dftw *) ego->cldw; 54| 11| cldw->apply(ego->cldw, ri, ii); 55| | 56| 11| cld = (plan_dft *) ego->cld; 57| 11| cld->apply(ego->cld, ri, ii, ro, io); 58| 11|} fftw_regsolver_ct_directw: 329| 25|{ 330| 25| regone(plnr, codelet, desc, dec, /* bufferedp */ 0); 331| 25| regone(plnr, codelet, desc, dec, /* bufferedp */ 1); 332| 25|} dftw-direct.c:regone: 311| 50|{ 312| 50| S *slv = (S *)X(mksolver_ct)(sizeof(S), desc->radix, dec, mkcldw, 0); ------------------ | | 78| 50|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 50|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 313| 50| slv->k = codelet; 314| 50| slv->desc = desc; 315| 50| slv->bufferedp = bufferedp; 316| 50| REGISTER_SOLVER(plnr, &(slv->super.super)); ------------------ | | 95| 50|#define REGISTER_SOLVER(p, s) X(solver_register)(p, s) | | ------------------ | | | | 78| 50|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 50|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 317| 50| if (X(mksolver_ct_hook)) { ------------------ | | 78| 50|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 50|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | | | | Branch (63:30): [True: 0, False: 50] | | | | ------------------ | | ------------------ ------------------ 318| 0| slv = (S *)X(mksolver_ct_hook)(sizeof(S), desc->radix, ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 319| 0| dec, mkcldw, 0); 320| 0| slv->k = codelet; 321| 0| slv->desc = desc; 322| 0| slv->bufferedp = bufferedp; 323| 0| REGISTER_SOLVER(plnr, &(slv->super.super)); ------------------ | | 95| 0|#define REGISTER_SOLVER(p, s) X(solver_register)(p, s) | | ------------------ | | | | 78| 0|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 324| 0| } 325| 50|} dftw-direct.c:mkcldw: 260| 1.84k|{ 261| 1.84k| const S *ego = (const S *) ego_; 262| 1.84k| P *pln; 263| 1.84k| const ct_desc *e = ego->desc; 264| 1.84k| INT extra_iter; 265| | 266| 1.84k| static const plan_adt padt = { 267| 1.84k| 0, awake, print, destroy 268| 1.84k| }; 269| | 270| 1.84k| A(mstart >= 0 && mstart + mcount <= m); 271| 1.84k| if (!applicable(ego, ------------------ | Branch (271:10): [True: 679, False: 1.16k] ------------------ 272| 1.84k| r, irs, ors, m, ms, v, ivs, ovs, mstart, mstart + mcount, 273| 1.84k| rio, iio, plnr, &extra_iter)) 274| 679| return (plan *)0; 275| | 276| 1.16k| if (ego->bufferedp) { ------------------ | Branch (276:10): [True: 0, False: 1.16k] ------------------ 277| 0| pln = MKPLAN_DFTW(P, &padt, apply_buf); ------------------ | | 42| 0| (type *)X(mkplan_dftw)(sizeof(type), adt, apply) | | ------------------ | | | | 78| 0|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 278| 1.16k| } else { 279| 1.16k| pln = MKPLAN_DFTW(P, &padt, extra_iter ? apply_extra_iter : apply); ------------------ | | 42| 2.32k| (type *)X(mkplan_dftw)(sizeof(type), adt, apply) | | ------------------ | | | | 78| 1.16k|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 1.16k|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ | | | Branch (42:45): [True: 0, False: 1.16k] | | ------------------ ------------------ 280| 1.16k| } 281| | 282| 1.16k| pln->k = ego->k; 283| 1.16k| pln->rs = X(mkstride)(r, irs); ------------------ | | 78| 1.16k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1.16k|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | | | | | 827| 1.16k|#define fftw_mkstride(n, stride) stride | | | | ------------------ | | ------------------ ------------------ 284| 1.16k| pln->td = 0; 285| 1.16k| pln->r = r; 286| 1.16k| pln->m = m; 287| 1.16k| pln->ms = ms; 288| 1.16k| pln->v = v; 289| 1.16k| pln->vs = ivs; 290| 1.16k| pln->mb = mstart; 291| 1.16k| pln->me = mstart + mcount; 292| 1.16k| pln->slv = ego; 293| 1.16k| pln->brs = X(mkstride)(r, 2 * compute_batchsize(r)); ------------------ | | 78| 1.16k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1.16k|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | | | | | 827| 1.16k|#define fftw_mkstride(n, stride) stride | | | | ------------------ | | ------------------ ------------------ 294| 1.16k| pln->extra_iter = extra_iter; 295| | 296| 1.16k| X(ops_zero)(&pln->super.super.ops); ------------------ | | 78| 1.16k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1.16k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 297| 1.16k| X(ops_madd2)(v * (mcount/e->genus->vl), &e->ops, &pln->super.super.ops); ------------------ | | 78| 1.16k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1.16k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 298| | 299| 1.16k| if (ego->bufferedp) { ------------------ | Branch (299:10): [True: 0, False: 1.16k] ------------------ 300| | /* 8 load/stores * N * V */ 301| 0| pln->super.super.ops.other += 8 * r * mcount * v; 302| 0| } 303| | 304| 1.16k| pln->super.super.could_prune_now_p = 305| 1.16k| (!ego->bufferedp && r >= 5 && r < 64 && m >= r); ------------------ | Branch (305:5): [True: 1.16k, False: 0] | Branch (305:24): [True: 800, False: 362] | Branch (305:34): [True: 797, False: 3] | Branch (305:44): [True: 539, False: 258] ------------------ 306| 1.16k| return &(pln->super.super); 307| 1.84k|} dftw-direct.c:awake: 126| 724|{ 127| 724| P *ego = (P *) ego_; 128| | 129| 724| X(twiddle_awake)(wakefulness, &ego->td, ego->slv->desc->tw, ------------------ | | 78| 724|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 724|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 130| 724| ego->r * ego->m, ego->r, ego->m + ego->extra_iter); 131| 724|} dftw-direct.c:destroy: 134| 1.16k|{ 135| 1.16k| P *ego = (P *) ego_; 136| 1.16k| X(stride_destroy)(ego->brs); ------------------ | | 78| 1.16k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1.16k|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | | | | | 830| 1.16k|#define fftw_stride_destroy(p) ((void) p) | | | | ------------------ | | ------------------ ------------------ 137| 1.16k| X(stride_destroy)(ego->rs); ------------------ | | 78| 1.16k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1.16k|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | | | | | 830| 1.16k|#define fftw_stride_destroy(p) ((void) p) | | | | ------------------ | | ------------------ ------------------ 138| 1.16k|} dftw-direct.c:applicable: 229| 1.84k|{ 230| 1.84k| if (ego->bufferedp) { ------------------ | Branch (230:10): [True: 635, False: 1.20k] ------------------ 231| 635| *extra_iter = 0; 232| 635| if (!applicable0_buf(ego, ------------------ | Branch (232:8): [True: 0, False: 635] ------------------ 233| 635| r, irs, ors, m, ms, v, ivs, ovs, mb, me, 234| 635| rio, iio, plnr)) 235| 0| return 0; 236| 1.20k| } else { 237| 1.20k| if (!applicable0(ego, ------------------ | Branch (237:8): [True: 0, False: 1.20k] ------------------ 238| 1.20k| r, irs, ors, m, ms, v, ivs, ovs, mb, me, 239| 1.20k| rio, iio, plnr, extra_iter)) 240| 0| return 0; 241| 1.20k| } 242| | 243| 1.84k| if (NO_UGLYP(plnr) && X(ct_uglyp)((ego->bufferedp? (INT)512 : (INT)16), ------------------ | | 689| 3.68k|#define NO_UGLYP(plnr) (PLNR_L(plnr) & NO_UGLY) | | ------------------ | | | | 674| 1.84k|#define PLNR_L(plnr) ((plnr)->flags.l) | | ------------------ | | | Branch (689:24): [True: 1.84k, False: 0] | | ------------------ ------------------ if (NO_UGLYP(plnr) && X(ct_uglyp)((ego->bufferedp? (INT)512 : (INT)16), ------------------ | | 78| 1.84k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1.84k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ | Branch (243:28): [True: 679, False: 1.16k] | Branch (243:41): [True: 635, False: 1.20k] ------------------ 244| 1.84k| v, m * r, r)) 245| 679| return 0; 246| | 247| 1.16k| if (m * r > 262144 && NO_FIXED_RADIX_LARGE_NP(plnr)) ------------------ | | 691| 0| (PLNR_L(plnr) & NO_FIXED_RADIX_LARGE_N) | | ------------------ | | | | 674| 0|#define PLNR_L(plnr) ((plnr)->flags.l) | | ------------------ | | | Branch (691:3): [True: 0, False: 0] | | ------------------ ------------------ | Branch (247:10): [True: 0, False: 1.16k] ------------------ 248| 0| return 0; 249| | 250| 1.16k| return 1; 251| 1.16k|} dftw-direct.c:applicable0_buf: 201| 635|{ 202| 635| const ct_desc *e = ego->desc; 203| 635| INT batchsz; 204| 635| UNUSED(v); UNUSED(ms); UNUSED(rio); UNUSED(iio); ------------------ | | 88| 635|#define UNUSED(x) (void)x ------------------ UNUSED(v); UNUSED(ms); UNUSED(rio); UNUSED(iio); ------------------ | | 88| 635|#define UNUSED(x) (void)x ------------------ UNUSED(v); UNUSED(ms); UNUSED(rio); UNUSED(iio); ------------------ | | 88| 635|#define UNUSED(x) (void)x ------------------ UNUSED(v); UNUSED(ms); UNUSED(rio); UNUSED(iio); ------------------ | | 88| 635|#define UNUSED(x) (void)x ------------------ 205| | 206| 635| return ( 207| 635| 1 ------------------ | Branch (207:4): [True: 635, Folded] ------------------ 208| 635| && r == e->radix ------------------ | Branch (208:7): [True: 635, False: 0] ------------------ 209| 635| && irs == ors /* in-place along R */ ------------------ | Branch (209:7): [True: 635, False: 0] ------------------ 210| 635| && ivs == ovs /* in-place along V */ ------------------ | Branch (210:7): [True: 635, False: 0] ------------------ 211| | 212| | /* check for alignment/vector length restrictions, both for 213| | batchsize and for the remainder */ 214| 635| && (batchsz = compute_batchsize(r), 1) ------------------ | Branch (214:7): [True: 635, False: 0] ------------------ 215| 635| && (e->genus->okp(e, 0, ((const R *)0) + 1, 2 * batchsz, 0, ------------------ | Branch (215:7): [True: 635, False: 0] ------------------ 216| 635| m, mb, mb + batchsz, 2, plnr)) 217| 635| && (e->genus->okp(e, 0, ((const R *)0) + 1, 2 * batchsz, 0, ------------------ | Branch (217:7): [True: 635, False: 0] ------------------ 218| 635| m, mb, me, 2, plnr)) 219| 635| ); 220| 635|} dftw-direct.c:applicable0: 162| 1.20k|{ 163| 1.20k| const ct_desc *e = ego->desc; 164| 1.20k| UNUSED(v); ------------------ | | 88| 1.20k|#define UNUSED(x) (void)x ------------------ 165| | 166| 1.20k| return ( 167| 1.20k| 1 ------------------ | Branch (167:4): [True: 1.20k, Folded] ------------------ 168| 1.20k| && r == e->radix ------------------ | Branch (168:7): [True: 1.20k, False: 0] ------------------ 169| 1.20k| && irs == ors /* in-place along R */ ------------------ | Branch (169:7): [True: 1.20k, False: 0] ------------------ 170| 1.20k| && ivs == ovs /* in-place along V */ ------------------ | Branch (170:7): [True: 1.20k, False: 0] ------------------ 171| | 172| | /* check for alignment/vector length restrictions */ 173| 1.20k| && ((*extra_iter = 0, ------------------ | Branch (173:8): [True: 1.20k, False: 0] ------------------ 174| 1.20k| e->genus->okp(e, rio, iio, irs, ivs, m, mb, me, ms, plnr)) 175| 0| || 176| 0| (*extra_iter = 1, ------------------ | Branch (176:8): [True: 0, False: 0] ------------------ 177| 0| (1 ------------------ | Branch (177:10): [True: 0, Folded] ------------------ 178| | /* FIXME: require full array, otherwise some threads 179| | may be extra_iter and other threads won't be. 180| | Generating the proper twiddle factors is a pain in 181| | this case */ 182| 0| && mb == 0 && me == m ------------------ | Branch (182:6): [True: 0, False: 0] | Branch (182:17): [True: 0, False: 0] ------------------ 183| 0| && e->genus->okp(e, rio, iio, irs, ivs, ------------------ | Branch (183:6): [True: 0, False: 0] ------------------ 184| 0| m, mb, me - 1, ms, plnr) 185| 0| && e->genus->okp(e, rio, iio, irs, ivs, ------------------ | Branch (185:6): [True: 0, False: 0] ------------------ 186| 0| m, me - 1, me + 1, ms, plnr)))) 187| | 188| 1.20k| && (e->genus->okp(e, rio + ivs, iio + ivs, irs, ivs, ------------------ | Branch (188:7): [True: 1.20k, False: 0] ------------------ 189| 1.20k| m, mb, me - *extra_iter, ms, plnr)) 190| | 191| 1.20k| ); 192| 1.20k|} dftw-direct.c:apply: 47| 1.40k|{ 48| 1.40k| const P *ego = (const P *) ego_; 49| 1.40k| INT i; 50| 1.40k| ASSERT_ALIGNED_DOUBLE; 51| 2.81k| for (i = 0; i < ego->v; ++i, rio += ego->vs, iio += ego->vs) { ------------------ | Branch (51:18): [True: 1.40k, False: 1.40k] ------------------ 52| 1.40k| INT mb = ego->mb, ms = ego->ms; 53| 1.40k| ego->k(rio + mb*ms, iio + mb*ms, ego->td->W, 54| 1.40k| ego->rs, mb, ego->me, ms); 55| 1.40k| } 56| 1.40k|} dftw-direct.c:compute_batchsize: 93| 1.79k|{ 94| | /* round up to multiple of 4 */ 95| 1.79k| radix += 3; 96| 1.79k| radix &= -4; 97| | 98| 1.79k| return (radix + 2); 99| 1.79k|} fftw_regsolver_ct_directwsq: 160| 6|{ 161| 6| regone(plnr, codelet, desc, dec+TRANSPOSE); ------------------ | | 50| 6|# define TRANSPOSE 2 ------------------ 162| 6|} dftw-directsq.c:regone: 144| 6|{ 145| 6| S *slv = (S *)X(mksolver_ct)(sizeof(S), desc->radix, dec, mkcldw, 0); ------------------ | | 78| 6|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 6|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 146| 6| slv->k = codelet; 147| 6| slv->desc = desc; 148| 6| REGISTER_SOLVER(plnr, &(slv->super.super)); ------------------ | | 95| 6|#define REGISTER_SOLVER(p, s) X(solver_register)(p, s) | | ------------------ | | | | 78| 6|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 6|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 149| 6| if (X(mksolver_ct_hook)) { ------------------ | | 78| 6|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 6|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | | | | Branch (63:30): [True: 0, False: 6] | | | | ------------------ | | ------------------ ------------------ 150| 0| slv = (S *)X(mksolver_ct_hook)(sizeof(S), desc->radix, dec, ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 151| 0| mkcldw, 0); 152| 0| slv->k = codelet; 153| 0| slv->desc = desc; 154| 0| REGISTER_SOLVER(plnr, &(slv->super.super)); ------------------ | | 95| 0|#define REGISTER_SOLVER(p, s) X(solver_register)(p, s) | | ------------------ | | | | 78| 0|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 155| 0| } 156| 6|} dftw-directsq.c:mkcldw: 107| 20|{ 108| 20| const S *ego = (const S *) ego_; 109| 20| P *pln; 110| 20| const ct_desc *e = ego->desc; 111| | 112| 20| static const plan_adt padt = { 113| 20| 0, awake, print, destroy 114| 20| }; 115| | 116| 20| A(mstart >= 0 && mstart + mcount <= m); 117| 20| if (!applicable(ego, ------------------ | Branch (117:10): [True: 0, False: 20] ------------------ 118| 20| r, irs, ors, m, ms, v, ivs, ovs, mstart, mstart + mcount, 119| 20| rio, iio, plnr)) 120| 0| return (plan *)0; 121| | 122| 20| pln = MKPLAN_DFTW(P, &padt, apply); ------------------ | | 42| 20| (type *)X(mkplan_dftw)(sizeof(type), adt, apply) | | ------------------ | | | | 78| 20|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 20|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 123| | 124| 20| pln->k = ego->k; 125| 20| pln->rs = X(mkstride)(r, irs); ------------------ | | 78| 20|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 20|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | | | | | 827| 20|#define fftw_mkstride(n, stride) stride | | | | ------------------ | | ------------------ ------------------ 126| 20| pln->vs = X(mkstride)(v, ivs); ------------------ | | 78| 20|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 20|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | | | | | 827| 20|#define fftw_mkstride(n, stride) stride | | | | ------------------ | | ------------------ ------------------ 127| 20| pln->td = 0; 128| 20| pln->r = r; 129| 20| pln->m = m; 130| 20| pln->ms = ms; 131| 20| pln->v = v; 132| 20| pln->mb = mstart; 133| 20| pln->me = mstart + mcount; 134| 20| pln->slv = ego; 135| | 136| 20| X(ops_zero)(&pln->super.super.ops); ------------------ | | 78| 20|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 20|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 137| 20| X(ops_madd2)(mcount/e->genus->vl, &e->ops, &pln->super.super.ops); ------------------ | | 78| 20|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 20|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 138| | 139| 20| return &(pln->super.super); 140| 20|} dftw-directsq.c:awake: 50| 6|{ 51| 6| P *ego = (P *) ego_; 52| | 53| 6| X(twiddle_awake)(wakefulness, &ego->td, ego->slv->desc->tw, ------------------ | | 78| 6|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 6|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 54| 6| ego->r * ego->m, ego->r, ego->m); 55| 6|} dftw-directsq.c:destroy: 58| 20|{ 59| 20| P *ego = (P *) ego_; 60| 20| X(stride_destroy)(ego->rs); ------------------ | | 78| 20|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 20|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | | | | | 830| 20|#define fftw_stride_destroy(p) ((void) p) | | | | ------------------ | | ------------------ ------------------ 61| 20| X(stride_destroy)(ego->vs); ------------------ | | 78| 20|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 20|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | | | | | 830| 20|#define fftw_stride_destroy(p) ((void) p) | | | | ------------------ | | ------------------ ------------------ 62| 20|} dftw-directsq.c:applicable: 81| 20|{ 82| 20| const ct_desc *e = ego->desc; 83| 20| UNUSED(v); ------------------ | | 88| 20|#define UNUSED(x) (void)x ------------------ 84| | 85| 20| return ( 86| 20| 1 ------------------ | Branch (86:4): [True: 20, Folded] ------------------ 87| 20| && r == e->radix ------------------ | Branch (87:7): [True: 20, False: 0] ------------------ 88| | 89| | /* transpose r, v */ 90| 20| && r == v ------------------ | Branch (90:7): [True: 20, False: 0] ------------------ 91| 20| && irs == ovs ------------------ | Branch (91:7): [True: 20, False: 0] ------------------ 92| 20| && ivs == ors ------------------ | Branch (92:7): [True: 20, False: 0] ------------------ 93| | 94| | /* check for alignment/vector length restrictions */ 95| 20| && e->genus->okp(e, rio, iio, irs, ivs, m, mb, me, ms, plnr) ------------------ | Branch (95:7): [True: 20, False: 0] ------------------ 96| | 97| 20| ); 98| 20|} dftw-directsq.c:apply: 42| 11|{ 43| 11| const P *ego = (const P *) ego_; 44| 11| INT mb = ego->mb, ms = ego->ms; 45| 11| ego->k(rio + mb*ms, iio + mb*ms, ego->td->W, ego->rs, ego->vs, 46| 11| mb, ego->me, ms); 47| 11|} fftw_ct_generic_register: 201| 1|{ 202| 1| regsolver(p, 0, DECDIT); ------------------ | | 49| 1|# define DECDIT 1 ------------------ 203| 1| regsolver(p, 0, DECDIF); ------------------ | | 48| 1|# define DECDIF 0 ------------------ 204| 1|} dftw-generic.c:regsolver: 191| 2|{ 192| 2| S *slv = (S *)X(mksolver_ct)(sizeof(S), r, dec, mkcldw, 0); ------------------ | | 78| 2|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 2|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 193| 2| REGISTER_SOLVER(plnr, &(slv->super)); ------------------ | | 95| 2|#define REGISTER_SOLVER(p, s) X(solver_register)(p, s) | | ------------------ | | | | 78| 2|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 2|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 194| 2| if (X(mksolver_ct_hook)) { ------------------ | | 78| 2|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 2|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | | | | Branch (63:30): [True: 0, False: 2] | | | | ------------------ | | ------------------ ------------------ 195| 0| slv = (S *)X(mksolver_ct_hook)(sizeof(S), r, dec, mkcldw, 0); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 196| 0| REGISTER_SOLVER(plnr, &(slv->super)); ------------------ | | 95| 0|#define REGISTER_SOLVER(p, s) X(solver_register)(p, s) | | ------------------ | | | | 78| 0|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 197| 0| } 198| 2|} dftw-generic.c:mkcldw: 140| 203|{ 141| 203| const S *ego = (const S *)ego_; 142| 203| P *pln; 143| 203| plan *cld = 0; 144| 203| INT dm = ms * mstart; 145| | 146| 203| static const plan_adt padt = { 147| 203| 0, awake, print, destroy 148| 203| }; 149| | 150| 203| A(mstart >= 0 && mstart + mcount <= m); 151| 203| if (!applicable(irs, ors, ivs, ovs, plnr)) ------------------ | Branch (151:10): [True: 182, False: 21] ------------------ 152| 182| return (plan *)0; 153| | 154| 21| cld = X(mkplan_d)(plnr, ------------------ | | 78| 21|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 21|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 155| 21| X(mkproblem_dft_d)( ------------------ | | 78| 21|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 21|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 156| 21| X(mktensor_1d)(r, irs, irs), ------------------ | | 78| 21|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 21|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 157| 21| X(mktensor_2d)(mcount, ms, ms, v, ivs, ivs), ------------------ | | 78| 21|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 21|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 158| 21| rio + dm, iio + dm, rio + dm, iio + dm) 159| 21| ); 160| 21| if (!cld) goto nada; ------------------ | Branch (160:10): [True: 0, False: 21] ------------------ 161| | 162| 21| pln = MKPLAN_DFTW(P, &padt, ego->dec == DECDIT ? apply_dit : apply_dif); ------------------ | | 42| 42| (type *)X(mkplan_dftw)(sizeof(type), adt, apply) | | ------------------ | | | | 78| 21|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 21|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ | | | Branch (42:45): [True: 21, False: 0] | | ------------------ ------------------ 163| 21| pln->slv = ego; 164| 21| pln->cld = cld; 165| 21| pln->r = r; 166| 21| pln->rs = irs; 167| 21| pln->m = m; 168| 21| pln->ms = ms; 169| 21| pln->v = v; 170| 21| pln->vs = ivs; 171| 21| pln->mb = mstart; 172| 21| pln->me = mstart + mcount; 173| 21| pln->dec = ego->dec; 174| 21| pln->td = 0; 175| | 176| 21| { 177| 21| double n0 = (r - 1) * (mcount - 1) * v; 178| 21| pln->super.super.ops = cld->ops; 179| 21| pln->super.super.ops.mul += 8 * n0; 180| 21| pln->super.super.ops.add += 4 * n0; 181| 21| pln->super.super.ops.other += 8 * n0; 182| 21| } 183| 21| return &(pln->super.super); 184| | 185| 0| nada: 186| 0| X(plan_destroy_internal)(cld); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 187| 0| return (plan *) 0; 188| 21|} dftw-generic.c:awake: 113| 18|{ 114| 18| P *ego = (P *) ego_; 115| 18| X(plan_awake)(ego->cld, wakefulness); ------------------ | | 78| 18|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 18|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 116| 18| mktwiddle(ego, wakefulness); 117| 18|} dftw-generic.c:mktwiddle: 42| 18|{ 43| 18| static const tw_instr tw[] = { { TW_FULL, 0, 0 }, { TW_NEXT, 1, 0 } }; 44| | 45| | /* note that R and M are swapped, to allow for sequential 46| | access both to data and twiddles */ 47| 18| X(twiddle_awake)(wakefulness, &ego->td, tw, ------------------ | | 78| 18|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 18|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 48| 18| ego->r * ego->m, ego->m, ego->r); 49| 18|} dftw-generic.c:destroy: 120| 21|{ 121| 21| P *ego = (P *) ego_; 122| 21| X(plan_destroy_internal)(ego->cld); ------------------ | | 78| 21|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 21|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 123| 21|} dftw-generic.c:applicable: 80| 203|{ 81| 203| return (1 ------------------ | Branch (81:14): [True: 203, Folded] ------------------ 82| 203| && irs == ors ------------------ | Branch (82:10): [True: 203, False: 0] ------------------ 83| 203| && ivs == ovs ------------------ | Branch (83:10): [True: 203, False: 0] ------------------ 84| 203| && !NO_SLOWP(plnr) ------------------ | | 688| 203|#define NO_SLOWP(plnr) (PLNR_L(plnr) & NO_SLOW) | | ------------------ | | | | 674| 203|#define PLNR_L(plnr) ((plnr)->flags.l) | | ------------------ ------------------ | Branch (84:10): [True: 21, False: 182] ------------------ 85| 203| ); 86| 203|} dftw-generic.c:apply_dit: 89| 11|{ 90| 11| const P *ego = (const P *) ego_; 91| 11| plan_dft *cld; 92| 11| INT dm = ego->ms * ego->mb; 93| | 94| 11| bytwiddle(ego, rio, iio); 95| | 96| 11| cld = (plan_dft *) ego->cld; 97| 11| cld->apply(ego->cld, rio + dm, iio + dm, rio + dm, iio + dm); 98| 11|} dftw-generic.c:bytwiddle: 52| 11|{ 53| 11| INT iv, ir, im; 54| 11| INT r = ego->r, rs = ego->rs; 55| 11| INT m = ego->m, mb = ego->mb, me = ego->me, ms = ego->ms; 56| 11| INT v = ego->v, vs = ego->vs; 57| 11| const R *W = ego->td->W; 58| | 59| 11| mb += (mb == 0); /* skip m=0 iteration */ 60| 22| for (iv = 0; iv < v; ++iv) { ------------------ | Branch (60:19): [True: 11, False: 11] ------------------ 61| 127| for (ir = 1; ir < r; ++ir) { ------------------ | Branch (61:17): [True: 116, False: 11] ------------------ 62| 1.54k| for (im = mb; im < me; ++im) { ------------------ | Branch (62:23): [True: 1.43k, False: 116] ------------------ 63| 1.43k| R *pr = rio + ms * im + rs * ir; 64| 1.43k| R *pi = iio + ms * im + rs * ir; 65| 1.43k| E xr = *pr; 66| 1.43k| E xi = *pi; 67| 1.43k| E wr = W[2 * im + (2 * (m-1)) * ir - 2]; 68| 1.43k| E wi = W[2 * im + (2 * (m-1)) * ir - 1]; 69| 1.43k| *pr = xr * wr + xi * wi; 70| 1.43k| *pi = xi * wr - xr * wi; 71| 1.43k| } 72| 116| } 73| 11| rio += vs; 74| 11| iio += vs; 75| 11| } 76| 11|} fftw_ct_genericbuf_register: 223| 1|{ 224| 1| static const INT radices[] = { -1, -2, -4, -8, -16, -32, -64 }; 225| 1| static const INT batchsizes[] = { 4, 8, 16, 32, 64 }; 226| 1| unsigned i, j; 227| | 228| 8| for (i = 0; i < sizeof(radices) / sizeof(radices[0]); ++i) ------------------ | Branch (228:18): [True: 7, False: 1] ------------------ 229| 42| for (j = 0; j < sizeof(batchsizes) / sizeof(batchsizes[0]); ++j) ------------------ | Branch (229:16): [True: 35, False: 7] ------------------ 230| 35| regsolver(p, radices[i], batchsizes[j]); 231| 1|} dftw-genericbuf.c:regsolver: 209| 35|{ 210| 35| S *slv = (S *)X(mksolver_ct)(sizeof(S), r, DECDIT, mkcldw, 0); ------------------ | | 78| 35|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 35|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ S *slv = (S *)X(mksolver_ct)(sizeof(S), r, DECDIT, mkcldw, 0); ------------------ | | 49| 35|# define DECDIT 1 ------------------ 211| 35| slv->batchsz = batchsz; 212| 35| REGISTER_SOLVER(plnr, &(slv->super.super)); ------------------ | | 95| 35|#define REGISTER_SOLVER(p, s) X(solver_register)(p, s) | | ------------------ | | | | 78| 35|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 35|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 213| | 214| 35| if (X(mksolver_ct_hook)) { ------------------ | | 78| 35|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 35|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | | | | Branch (63:30): [True: 0, False: 35] | | | | ------------------ | | ------------------ ------------------ 215| 0| slv = (S *)X(mksolver_ct_hook)(sizeof(S), r, DECDIT, mkcldw, 0); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ slv = (S *)X(mksolver_ct_hook)(sizeof(S), r, DECDIT, mkcldw, 0); ------------------ | | 49| 0|# define DECDIT 1 ------------------ 216| 0| slv->batchsz = batchsz; 217| 0| REGISTER_SOLVER(plnr, &(slv->super.super)); ------------------ | | 95| 0|#define REGISTER_SOLVER(p, s) X(solver_register)(p, s) | | ------------------ | | | | 78| 0|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 218| 0| } 219| | 220| 35|} dftw-genericbuf.c:mkcldw: 154| 240|{ 155| 240| const S *ego = (const S *)ego_; 156| 240| P *pln; 157| 240| plan *cld = 0; 158| 240| R *buf; 159| | 160| 240| static const plan_adt padt = { 161| 240| 0, awake, print, destroy 162| 240| }; 163| | 164| 240| UNUSED(ivs); UNUSED(ovs); UNUSED(rio); UNUSED(iio); ------------------ | | 88| 240|#define UNUSED(x) (void)x ------------------ UNUSED(ivs); UNUSED(ovs); UNUSED(rio); UNUSED(iio); ------------------ | | 88| 240|#define UNUSED(x) (void)x ------------------ UNUSED(ivs); UNUSED(ovs); UNUSED(rio); UNUSED(iio); ------------------ | | 88| 240|#define UNUSED(x) (void)x ------------------ UNUSED(ivs); UNUSED(ovs); UNUSED(rio); UNUSED(iio); ------------------ | | 88| 240|#define UNUSED(x) (void)x ------------------ 165| | 166| 240| A(mstart >= 0 && mstart + mcount <= m); 167| 240| if (!applicable(ego, r, irs, ors, m, v, mcount, plnr)) ------------------ | Branch (167:10): [True: 240, False: 0] ------------------ 168| 240| return (plan *)0; 169| | 170| 0| buf = (R *) MALLOC(sizeof(R) * 2 * BATCHDIST(r) * ego->batchsz, BUFFERS); ------------------ | | 306| 0|#define MALLOC(n, what) X(malloc_plain)(n) | | ------------------ | | | | 78| 0|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 171| 0| cld = X(mkplan_d)(plnr, ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 172| 0| X(mkproblem_dft_d)( ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 173| 0| X(mktensor_1d)(r, 2, 2), ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 174| 0| X(mktensor_1d)(ego->batchsz, ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 175| 0| 2 * BATCHDIST(r), ------------------ | | 43| 0|#define BATCHDIST(r) ((r) + 16) ------------------ 176| 0| 2 * BATCHDIST(r)), ------------------ | | 43| 0|#define BATCHDIST(r) ((r) + 16) ------------------ 177| 0| buf, buf + 1, buf, buf + 1 178| 0| ) 179| 0| ); 180| 0| X(ifree)(buf); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 181| 0| if (!cld) goto nada; ------------------ | Branch (181:10): [True: 0, False: 0] ------------------ 182| | 183| 0| pln = MKPLAN_DFTW(P, &padt, apply); ------------------ | | 42| 0| (type *)X(mkplan_dftw)(sizeof(type), adt, apply) | | ------------------ | | | | 78| 0|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 184| 0| pln->slv = ego; 185| 0| pln->cld = cld; 186| 0| pln->r = r; 187| 0| pln->m = m; 188| 0| pln->ms = ms; 189| 0| pln->rs = irs; 190| 0| pln->batchsz = ego->batchsz; 191| 0| pln->mb = mstart; 192| 0| pln->me = mstart + mcount; 193| | 194| 0| { 195| 0| double n0 = (r - 1) * (mcount - 1); 196| 0| pln->super.super.ops = cld->ops; 197| 0| pln->super.super.ops.mul += 8 * n0; 198| 0| pln->super.super.ops.add += 4 * n0; 199| 0| pln->super.super.ops.other += 8 * n0; 200| 0| } 201| 0| return &(pln->super.super); 202| | 203| 0| nada: 204| 0| X(plan_destroy_internal)(cld); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 205| 0| return (plan *) 0; 206| 0|} dftw-genericbuf.c:applicable: 80| 240|{ 81| 240| if (!applicable0(ego, r, irs, ors, m, v, mcount)) ------------------ | Branch (81:10): [True: 240, False: 0] ------------------ 82| 240| return 0; 83| 0| if (NO_UGLYP(plnr) && m * r < 65536) ------------------ | | 689| 0|#define NO_UGLYP(plnr) (PLNR_L(plnr) & NO_UGLY) | | ------------------ | | | | 674| 0|#define PLNR_L(plnr) ((plnr)->flags.l) | | ------------------ | | | Branch (689:24): [True: 0, False: 0] | | ------------------ ------------------ | Branch (83:28): [True: 0, False: 0] ------------------ 84| 0| return 0; 85| | 86| 0| return 1; 87| 0|} dftw-genericbuf.c:applicable0: 64| 240|{ 65| 240| return (1 ------------------ | Branch (65:14): [True: 240, Folded] ------------------ 66| 240| && v == 1 ------------------ | Branch (66:10): [True: 210, False: 30] ------------------ 67| 210| && irs == ors ------------------ | Branch (67:10): [True: 210, False: 0] ------------------ 68| 210| && mcount >= ego->batchsz ------------------ | Branch (68:10): [True: 68, False: 142] ------------------ 69| 68| && mcount % ego->batchsz == 0 ------------------ | Branch (69:10): [True: 0, False: 68] ------------------ 70| 0| && r >= 64 ------------------ | Branch (70:10): [True: 0, False: 0] ------------------ 71| 0| && m >= r ------------------ | Branch (71:10): [True: 0, False: 0] ------------------ 72| 240| ); 73| 240|} fftw_mksolver_dft_direct: 286| 19|{ 287| 19| return mksolver(k, desc, 0); 288| 19|} fftw_mksolver_dft_directbuf: 291| 19|{ 292| 19| return mksolver(k, desc, 1); 293| 19|} direct.c:mksolver: 276| 38|{ 277| 38| static const solver_adt sadt = { PROBLEM_DFT, mkplan, 0 }; 278| 38| S *slv = MKSOLVER(S, &sadt); ------------------ | | 604| 38|#define MKSOLVER(type, adt) (type *)X(mksolver)(sizeof(type), adt) | | ------------------ | | | | 78| 38|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 38|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 279| 38| slv->k = k; 280| 38| slv->desc = desc; 281| 38| slv->bufferedp = bufferedp; 282| 38| return &(slv->super); 283| 38|} direct.c:mkplan: 230| 56.5k|{ 231| 56.5k| const S *ego = (const S *) ego_; 232| 56.5k| P *pln; 233| 56.5k| const problem_dft *p; 234| 56.5k| iodim *d; 235| 56.5k| const kdft_desc *e = ego->desc; 236| | 237| 56.5k| static const plan_adt padt = { 238| 56.5k| X(dft_solve), X(null_awake), print, destroy ------------------ | | 78| 56.5k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 56.5k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ X(dft_solve), X(null_awake), print, destroy ------------------ | | 78| 56.5k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 56.5k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 239| 56.5k| }; 240| | 241| 56.5k| UNUSED(plnr); ------------------ | | 88| 56.5k|#define UNUSED(x) (void)x ------------------ 242| | 243| 56.5k| if (ego->bufferedp) { ------------------ | Branch (243:10): [True: 28.0k, False: 28.4k] ------------------ 244| 28.0k| if (!applicable_buf(ego_, p_, plnr)) ------------------ | Branch (244:8): [True: 27.5k, False: 576] ------------------ 245| 27.5k| return (plan *)0; 246| 576| pln = MKPLAN_DFT(P, &padt, apply_buf); ------------------ | | 60| 576| (type *)X(mkplan_dft)(sizeof(type), adt, apply) | | ------------------ | | | | 78| 576|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 576|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 247| 28.4k| } else { 248| 28.4k| int extra_iterp = 0; 249| 28.4k| if (!applicable(ego_, p_, plnr, &extra_iterp)) ------------------ | Branch (249:8): [True: 27.4k, False: 962] ------------------ 250| 27.4k| return (plan *)0; 251| 962| pln = MKPLAN_DFT(P, &padt, extra_iterp ? apply_extra_iter : apply); ------------------ | | 60| 1.92k| (type *)X(mkplan_dft)(sizeof(type), adt, apply) | | ------------------ | | | | 78| 962|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 962|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ | | | Branch (60:44): [True: 0, False: 962] | | ------------------ ------------------ 252| 962| } 253| | 254| 1.53k| p = (const problem_dft *) p_; 255| 1.53k| d = p->sz->dims; 256| 1.53k| pln->k = ego->k; 257| 1.53k| pln->n = d[0].n; 258| 1.53k| pln->is = X(mkstride)(pln->n, d[0].is); ------------------ | | 78| 1.53k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1.53k|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | | | | | 827| 1.53k|#define fftw_mkstride(n, stride) stride | | | | ------------------ | | ------------------ ------------------ 259| 1.53k| pln->os = X(mkstride)(pln->n, d[0].os); ------------------ | | 78| 1.53k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1.53k|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | | | | | 827| 1.53k|#define fftw_mkstride(n, stride) stride | | | | ------------------ | | ------------------ ------------------ 260| 1.53k| pln->bufstride = X(mkstride)(pln->n, 2 * compute_batchsize(pln->n)); ------------------ | | 78| 1.53k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1.53k|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | | | | | 827| 1.53k|#define fftw_mkstride(n, stride) stride | | | | ------------------ | | ------------------ ------------------ 261| | 262| 1.53k| X(tensor_tornk1)(p->vecsz, &pln->vl, &pln->ivs, &pln->ovs); ------------------ | | 78| 1.53k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1.53k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 263| 1.53k| pln->slv = ego; 264| | 265| 1.53k| X(ops_zero)(&pln->super.super.ops); ------------------ | | 78| 1.53k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1.53k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 266| 1.53k| X(ops_madd2)(pln->vl / e->genus->vl, &e->ops, &pln->super.super.ops); ------------------ | | 78| 1.53k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1.53k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 267| | 268| 1.53k| if (ego->bufferedp) ------------------ | Branch (268:10): [True: 576, False: 962] ------------------ 269| 576| pln->super.super.ops.other += 4 * pln->n * pln->vl; 270| | 271| 1.53k| pln->super.super.could_prune_now_p = !ego->bufferedp; 272| 1.53k| return &(pln->super.super); 273| 56.5k|} direct.c:destroy: 117| 1.53k|{ 118| 1.53k| P *ego = (P *) ego_; 119| 1.53k| X(stride_destroy)(ego->is); ------------------ | | 78| 1.53k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1.53k|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | | | | | 830| 1.53k|#define fftw_stride_destroy(p) ((void) p) | | | | ------------------ | | ------------------ ------------------ 120| 1.53k| X(stride_destroy)(ego->os); ------------------ | | 78| 1.53k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1.53k|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | | | | | 830| 1.53k|#define fftw_stride_destroy(p) ((void) p) | | | | ------------------ | | ------------------ ------------------ 121| 1.53k| X(stride_destroy)(ego->bufstride); ------------------ | | 78| 1.53k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1.53k|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | | | | | 830| 1.53k|#define fftw_stride_destroy(p) ((void) p) | | | | ------------------ | | ------------------ ------------------ 122| 1.53k|} direct.c:applicable_buf: 139| 28.0k|{ 140| 28.0k| const S *ego = (const S *) ego_; 141| 28.0k| const problem_dft *p = (const problem_dft *) p_; 142| 28.0k| const kdft_desc *d = ego->desc; 143| 28.0k| INT vl; 144| 28.0k| INT ivs, ovs; 145| 28.0k| INT batchsz; 146| | 147| 28.0k| return ( 148| 28.0k| 1 ------------------ | Branch (148:4): [True: 28.0k, Folded] ------------------ 149| 28.0k| && p->sz->rnk == 1 ------------------ | Branch (149:7): [True: 21.7k, False: 6.36k] ------------------ 150| 21.7k| && p->vecsz->rnk == 1 ------------------ | Branch (150:7): [True: 12.7k, False: 8.94k] ------------------ 151| 12.7k| && p->sz->dims[0].n == d->sz ------------------ | Branch (151:7): [True: 695, False: 12.0k] ------------------ 152| | 153| | /* check strides etc */ 154| 695| && X(tensor_tornk1)(p->vecsz, &vl, &ivs, &ovs) ------------------ | | 78| 695|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 695|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ | Branch (154:7): [True: 695, False: 0] ------------------ 155| | 156| | /* UGLY if IS <= IVS */ 157| 695| && !(NO_UGLYP(plnr) && ------------------ | | 689| 1.39k|#define NO_UGLYP(plnr) (PLNR_L(plnr) & NO_UGLY) | | ------------------ | | | | 674| 695|#define PLNR_L(plnr) ((plnr)->flags.l) | | ------------------ | | | Branch (689:24): [True: 695, False: 0] | | ------------------ ------------------ 158| 695| X(iabs)(p->sz->dims[0].is) <= X(iabs)(ivs)) ------------------ | | 78| 695|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 695|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ X(iabs)(p->sz->dims[0].is) <= X(iabs)(ivs)) ------------------ | | 78| 695|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 695|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ | Branch (158:9): [True: 63, False: 632] ------------------ 159| | 160| 632| && (batchsz = compute_batchsize(d->sz), 1) ------------------ | Branch (160:7): [True: 632, False: 0] ------------------ 161| 632| && (d->genus->okp(d, 0, ((const R *)0) + 1, p->ro, p->io, ------------------ | Branch (161:7): [True: 632, False: 0] ------------------ 162| 632| 2 * batchsz, p->sz->dims[0].os, 163| 632| batchsz, 2, ovs, plnr)) 164| 632| && (d->genus->okp(d, 0, ((const R *)0) + 1, p->ro, p->io, ------------------ | Branch (164:7): [True: 632, False: 0] ------------------ 165| 632| 2 * batchsz, p->sz->dims[0].os, 166| 632| vl % batchsz, 2, ovs, plnr)) 167| | 168| | 169| 632| && (0 ------------------ | Branch (169:8): [Folded, False: 632] ------------------ 170| | /* can operate out-of-place */ 171| 632| || p->ri != p->ro ------------------ | Branch (171:11): [True: 411, False: 221] ------------------ 172| | 173| | /* can operate in-place as long as strides are the same */ 174| 221| || X(tensor_inplace_strides2)(p->sz, p->vecsz) ------------------ | | 78| 221|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 221|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ | Branch (174:11): [True: 28, False: 193] ------------------ 175| | 176| | /* can do it if the problem fits in the buffer, no matter 177| | what the strides are */ 178| 193| || vl <= batchsz ------------------ | Branch (178:11): [True: 137, False: 56] ------------------ 179| 632| ) 180| 28.0k| ); 181| 28.0k|} direct.c:apply_buf: 73| 43|{ 74| 43| const P *ego = (const P *) ego_; 75| 43| R *buf; 76| 43| INT vl = ego->vl, n = ego->n, batchsz = compute_batchsize(n); 77| 43| INT i; 78| 43| size_t bufsz = n * batchsz * 2 * sizeof(R); 79| | 80| 43| BUF_ALLOC(R *, buf, bufsz); ------------------ | | 212| 43|#define BUF_ALLOC(T, p, n) \ | | 213| 43|{ \ | | 214| 43| if (n < MAX_STACK_ALLOC) { \ | | ------------------ | | | | 210| 86|#define MAX_STACK_ALLOC ((size_t)64 * 1024) | | ------------------ | | | Branch (214:10): [True: 43, False: 0] | | ------------------ | | 215| 43| STACK_MALLOC(T, p, n); \ | | ------------------ | | | | 196| 43|# define STACK_MALLOC(T, p, n) p = (T)alloca(n) | | ------------------ | | 216| 43| } else { \ | | 217| 0| p = (T)MALLOC(n, BUFFERS); \ | | ------------------ | | | | 306| 0|#define MALLOC(n, what) X(malloc_plain)(n) | | | | ------------------ | | | | | | 78| 0|# define X(name) CONCAT(fftw_, name) | | | | | | ------------------ | | | | | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | | | | | ------------------ | | | | ------------------ | | ------------------ | | 218| 0| } \ | | 219| 43|} ------------------ 81| | 82| 43| for (i = 0; i < vl - batchsz; i += batchsz) { ------------------ | Branch (82:18): [True: 0, False: 43] ------------------ 83| 0| dobatch(ego, ri, ii, ro, io, buf, batchsz); 84| 0| ri += batchsz * ego->ivs; ii += batchsz * ego->ivs; 85| 0| ro += batchsz * ego->ovs; io += batchsz * ego->ovs; 86| 0| } 87| 43| dobatch(ego, ri, ii, ro, io, buf, vl - i); 88| | 89| 43| BUF_FREE(buf, bufsz); ------------------ | | 221| 43|#define BUF_FREE(p, n) \ | | 222| 43|{ \ | | 223| 43| if (n < MAX_STACK_ALLOC) { \ | | ------------------ | | | | 210| 86|#define MAX_STACK_ALLOC ((size_t)64 * 1024) | | ------------------ | | | Branch (223:10): [True: 43, False: 0] | | ------------------ | | 224| 43| STACK_FREE(p); \ | | 225| 43| } else { \ | | 226| 0| X(ifree)(p); \ | | ------------------ | | | | 78| 0|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ | | 227| 0| } \ | | 228| 43|} ------------------ 90| 43|} direct.c:dobatch: 44| 43|{ 45| 43| X(cpy2d_pair_ci)(ri, ii, buf, buf+1, ------------------ | | 78| 43|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 43|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 46| 43| ego->n, WS(ego->is, 1), WS(ego->bufstride, 1), ------------------ | | 825| 43|#define WS(stride, i) (stride * i) ------------------ ego->n, WS(ego->is, 1), WS(ego->bufstride, 1), ------------------ | | 825| 43|#define WS(stride, i) (stride * i) ------------------ 47| 43| batchsz, ego->ivs, 2); 48| | 49| 43| if (IABS(WS(ego->os, 1)) < IABS(ego->ovs)) { ------------------ | | 381| 43|#define IABS(x) (((x) < 0) ? (0 - (x)) : (x)) | | ------------------ | | | Branch (381:18): [True: 0, False: 43] | | ------------------ ------------------ if (IABS(WS(ego->os, 1)) < IABS(ego->ovs)) { ------------------ | | 381| 43|#define IABS(x) (((x) < 0) ? (0 - (x)) : (x)) | | ------------------ | | | Branch (381:18): [True: 0, False: 43] | | ------------------ ------------------ | Branch (49:10): [True: 43, False: 0] ------------------ 50| | /* transform directly to output */ 51| 43| ego->k(buf, buf+1, ro, io, 52| 43| ego->bufstride, ego->os, batchsz, 2, ego->ovs); 53| 43| } else { 54| | /* transform to buffer and copy back */ 55| 0| ego->k(buf, buf+1, buf, buf+1, 56| 0| ego->bufstride, ego->bufstride, batchsz, 2, 2); 57| 0| X(cpy2d_pair_co)(buf, buf+1, ro, io, ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 58| 0| ego->n, WS(ego->bufstride, 1), WS(ego->os, 1), ------------------ | | 825| 0|#define WS(stride, i) (stride * i) ------------------ ego->n, WS(ego->bufstride, 1), WS(ego->os, 1), ------------------ | | 825| 0|#define WS(stride, i) (stride * i) ------------------ 59| 0| batchsz, 2, ego->ovs); 60| 0| } 61| 43|} direct.c:applicable: 185| 28.4k|{ 186| 28.4k| const S *ego = (const S *) ego_; 187| 28.4k| const problem_dft *p = (const problem_dft *) p_; 188| 28.4k| const kdft_desc *d = ego->desc; 189| 28.4k| INT vl; 190| 28.4k| INT ivs, ovs; 191| | 192| 28.4k| return ( 193| 28.4k| 1 ------------------ | Branch (193:4): [True: 28.4k, Folded] ------------------ 194| 28.4k| && p->sz->rnk == 1 ------------------ | Branch (194:7): [True: 22.0k, False: 6.36k] ------------------ 195| 22.0k| && p->vecsz->rnk <= 1 ------------------ | Branch (195:7): [True: 18.2k, False: 3.81k] ------------------ 196| 18.2k| && p->sz->dims[0].n == d->sz ------------------ | Branch (196:7): [True: 1.08k, False: 17.1k] ------------------ 197| | 198| | /* check strides etc */ 199| 1.08k| && X(tensor_tornk1)(p->vecsz, &vl, &ivs, &ovs) ------------------ | | 78| 1.08k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1.08k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ | Branch (199:7): [True: 1.08k, False: 0] ------------------ 200| | 201| 1.08k| && ((*extra_iterp = 0, ------------------ | Branch (201:8): [True: 1.08k, False: 0] ------------------ 202| 1.08k| (d->genus->okp(d, p->ri, p->ii, p->ro, p->io, 203| 1.08k| p->sz->dims[0].is, p->sz->dims[0].os, 204| 1.08k| vl, ivs, ovs, plnr))) 205| 0| || 206| 0| (*extra_iterp = 1, ------------------ | Branch (206:8): [True: 0, False: 0] ------------------ 207| 0| ((d->genus->okp(d, p->ri, p->ii, p->ro, p->io, ------------------ | Branch (207:10): [True: 0, False: 0] ------------------ 208| 0| p->sz->dims[0].is, p->sz->dims[0].os, 209| 0| vl - 1, ivs, ovs, plnr)) 210| 0| && 211| 0| (d->genus->okp(d, p->ri, p->ii, p->ro, p->io, ------------------ | Branch (211:3): [True: 0, False: 0] ------------------ 212| 0| p->sz->dims[0].is, p->sz->dims[0].os, 213| 0| 2, 0, 0, plnr))))) 214| | 215| 1.08k| && (0 ------------------ | Branch (215:8): [Folded, False: 1.08k] ------------------ 216| | /* can operate out-of-place */ 217| 1.08k| || p->ri != p->ro ------------------ | Branch (217:11): [True: 875, False: 212] ------------------ 218| | 219| | /* can always compute one transform */ 220| 212| || vl == 1 ------------------ | Branch (220:11): [True: 0, False: 212] ------------------ 221| | 222| | /* can operate in-place as long as strides are the same */ 223| 212| || X(tensor_inplace_strides2)(p->sz, p->vecsz) ------------------ | | 78| 212|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 212|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ | Branch (223:11): [True: 87, False: 125] ------------------ 224| 1.08k| ) 225| 28.4k| ); 226| 28.4k|} direct.c:apply: 93| 1.31k|{ 94| 1.31k| const P *ego = (const P *) ego_; 95| 1.31k| ASSERT_ALIGNED_DOUBLE; 96| 1.31k| ego->k(ri, ii, ro, io, ego->is, ego->os, ego->vl, ego->ivs, ego->ovs); 97| 1.31k|} direct.c:compute_batchsize: 64| 2.21k|{ 65| | /* round up to multiple of 4 */ 66| 2.21k| n += 3; 67| 2.21k| n &= -4; 68| | 69| 2.21k| return (n + 2); 70| 2.21k|} fftw_dft_generic_register: 167| 1|{ 168| 1| REGISTER_SOLVER(p, mksolver()); ------------------ | | 95| 1|#define REGISTER_SOLVER(p, s) X(solver_register)(p, s) | | ------------------ | | | | 78| 1|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 169| 1|} generic.c:mksolver: 160| 3|{ 161| 3| static const solver_adt sadt = { PROBLEM_DFT, mkplan, 0 }; 162| 3| S *slv = MKSOLVER(S, &sadt); ------------------ | | 604| 3|#define MKSOLVER(type, adt) (type *)X(mksolver)(sizeof(type), adt) | | ------------------ | | | | 78| 3|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 3|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 163| 3| return &(slv->super); 164| 3|} generic.c:mkplan: 129| 1.21k|{ 130| 1.21k| const problem_dft *p; 131| 1.21k| P *pln; 132| 1.21k| INT n; 133| | 134| 1.21k| static const plan_adt padt = { 135| 1.21k| X(dft_solve), awake, print, X(plan_null_destroy) ------------------ | | 78| 1.21k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1.21k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ X(dft_solve), awake, print, X(plan_null_destroy) ------------------ | | 78| 1.21k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1.21k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 136| 1.21k| }; 137| | 138| 1.21k| if (!applicable(ego, p_, plnr)) ------------------ | Branch (138:10): [True: 1.02k, False: 186] ------------------ 139| 1.02k| return (plan *)0; 140| | 141| 186| pln = MKPLAN_DFT(P, &padt, apply); ------------------ | | 60| 186| (type *)X(mkplan_dft)(sizeof(type), adt, apply) | | ------------------ | | | | 78| 186|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 186|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 142| | 143| 186| p = (const problem_dft *) p_; 144| 186| pln->n = n = p->sz->dims[0].n; 145| 186| pln->is = p->sz->dims[0].is; 146| 186| pln->os = p->sz->dims[0].os; 147| 186| pln->td = 0; 148| | 149| 186| pln->super.super.ops.add = (n-1) * 5; 150| 186| pln->super.super.ops.mul = 0; 151| 186| pln->super.super.ops.fma = (n-1) * (n-1) ; 152| |#if 0 /* these are nice pipelined sequential loads and should cost nothing */ 153| | pln->super.super.ops.other = (n-1)*(4 + 1 + 2 * (n-1)); /* approximate */ 154| |#endif 155| | 156| 186| return &(pln->super.super); 157| 1.21k|} generic.c:awake: 94| 136|{ 95| 136| P *ego = (P *) ego_; 96| 136| static const tw_instr half_tw[] = { 97| 136| { TW_HALF, 1, 0 }, 98| 136| { TW_NEXT, 1, 0 } 99| 136| }; 100| | 101| 136| X(twiddle_awake)(wakefulness, &ego->td, half_tw, ego->n, ego->n, ------------------ | | 78| 136|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 136|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 102| 136| (ego->n - 1) / 2); 103| 136|} generic.c:applicable: 114| 1.21k|{ 115| 1.21k| const problem_dft *p = (const problem_dft *) p_; 116| 1.21k| UNUSED(ego); ------------------ | | 88| 1.21k|#define UNUSED(x) (void)x ------------------ 117| | 118| 1.21k| return (1 ------------------ | Branch (118:14): [True: 1.21k, Folded] ------------------ 119| 1.21k| && p->sz->rnk == 1 ------------------ | Branch (119:10): [True: 875, False: 335] ------------------ 120| 875| && p->vecsz->rnk == 0 ------------------ | Branch (120:10): [True: 354, False: 521] ------------------ 121| 354| && (p->sz->dims[0].n % 2) == 1 ------------------ | Branch (121:10): [True: 267, False: 87] ------------------ 122| 267| && CIMPLIES(NO_LARGE_GENERICP(plnr), p->sz->dims[0].n < GENERIC_MIN_BAD) ------------------ | | 99| 1.47k|#define CIMPLIES(ante, post) (!(ante) || (post)) | | ------------------ | | | Branch (99:31): [True: 0, False: 267] | | | Branch (99:42): [True: 240, False: 27] | | ------------------ ------------------ 123| 240| && CIMPLIES(NO_SLOWP(plnr), p->sz->dims[0].n > GENERIC_MAX_SLOW) ------------------ | | 99| 1.45k|#define CIMPLIES(ante, post) (!(ante) || (post)) | | ------------------ | | | Branch (99:31): [True: 4, False: 236] | | | Branch (99:42): [True: 236, False: 0] | | ------------------ ------------------ 124| 240| && X(is_prime)(p->sz->dims[0].n) ------------------ | | 78| 240|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 240|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ | Branch (124:10): [True: 186, False: 54] ------------------ 125| 1.21k| ); 126| 1.21k|} generic.c:apply: 72| 277|{ 73| 277| const P *ego = (const P *) ego_; 74| 277| INT i; 75| 277| INT n = ego->n, is = ego->is, os = ego->os; 76| 277| const R *W = ego->td->W; 77| 277| E *buf; 78| 277| size_t bufsz = n * 2 * sizeof(E); 79| | 80| 277| BUF_ALLOC(E *, buf, bufsz); ------------------ | | 212| 277|#define BUF_ALLOC(T, p, n) \ | | 213| 277|{ \ | | 214| 277| if (n < MAX_STACK_ALLOC) { \ | | ------------------ | | | | 210| 554|#define MAX_STACK_ALLOC ((size_t)64 * 1024) | | ------------------ | | | Branch (214:10): [True: 277, False: 0] | | ------------------ | | 215| 277| STACK_MALLOC(T, p, n); \ | | ------------------ | | | | 196| 277|# define STACK_MALLOC(T, p, n) p = (T)alloca(n) | | ------------------ | | 216| 277| } else { \ | | 217| 0| p = (T)MALLOC(n, BUFFERS); \ | | ------------------ | | | | 306| 0|#define MALLOC(n, what) X(malloc_plain)(n) | | | | ------------------ | | | | | | 78| 0|# define X(name) CONCAT(fftw_, name) | | | | | | ------------------ | | | | | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | | | | | ------------------ | | | | ------------------ | | ------------------ | | 218| 0| } \ | | 219| 277|} ------------------ 81| 277| hartley(n, ri, ii, is, buf, ro, io); 82| | 83| 4.18k| for (i = 1; i + i < n; ++i) { ------------------ | Branch (83:18): [True: 3.91k, False: 277] ------------------ 84| 3.91k| cdot(n, buf, W, 85| 3.91k| ro + i * os, io + i * os, 86| 3.91k| ro + (n - i) * os, io + (n - i) * os); 87| 3.91k| W += n - 1; 88| 3.91k| } 89| | 90| 277| BUF_FREE(buf, bufsz); ------------------ | | 221| 277|#define BUF_FREE(p, n) \ | | 222| 277|{ \ | | 223| 277| if (n < MAX_STACK_ALLOC) { \ | | ------------------ | | | | 210| 554|#define MAX_STACK_ALLOC ((size_t)64 * 1024) | | ------------------ | | | Branch (223:10): [True: 277, False: 0] | | ------------------ | | 224| 277| STACK_FREE(p); \ | | 225| 277| } else { \ | | 226| 0| X(ifree)(p); \ | | ------------------ | | | | 78| 0|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ | | 227| 0| } \ | | 228| 277|} ------------------ 91| 277|} generic.c:hartley: 56| 277|{ 57| 277| INT i; 58| 277| E sr, si; 59| 277| o[0] = sr = xr[0]; o[1] = si = xi[0]; o += 2; 60| 4.18k| for (i = 1; i + i < n; ++i) { ------------------ | Branch (60:18): [True: 3.91k, False: 277] ------------------ 61| 3.91k| sr += (o[0] = xr[i * xs] + xr[(n - i) * xs]); 62| 3.91k| si += (o[1] = xi[i * xs] + xi[(n - i) * xs]); 63| 3.91k| o[2] = xr[i * xs] - xr[(n - i) * xs]; 64| 3.91k| o[3] = xi[i * xs] - xi[(n - i) * xs]; 65| 3.91k| o += 4; 66| 3.91k| } 67| 277| *pr = sr; 68| 277| *pi = si; 69| 277|} generic.c:cdot: 36| 3.91k|{ 37| 3.91k| INT i; 38| | 39| 3.91k| E rr = x[0], ri = 0, ir = x[1], ii = 0; 40| 3.91k| x += 2; 41| 78.9k| for (i = 1; i + i < n; ++i) { ------------------ | Branch (41:18): [True: 75.0k, False: 3.91k] ------------------ 42| 75.0k| rr += x[0] * w[0]; 43| 75.0k| ir += x[1] * w[0]; 44| 75.0k| ri += x[2] * w[1]; 45| 75.0k| ii += x[3] * w[1]; 46| 75.0k| x += 4; w += 2; 47| 75.0k| } 48| 3.91k| *or0 = rr + ii; 49| 3.91k| *oi0 = ir - ri; 50| 3.91k| *or1 = rr - ii; 51| 3.91k| *oi1 = ir + ri; 52| 3.91k|} fftw_dft_indirect_transpose_register: 232| 1|{ 233| 1| REGISTER_SOLVER(p, mksolver()); ------------------ | | 95| 1|#define REGISTER_SOLVER(p, s) X(solver_register)(p, s) | | ------------------ | | | | 78| 1|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 234| 1|} indirect-transpose.c:mksolver: 225| 1|{ 226| 1| static const solver_adt sadt = { PROBLEM_DFT, mkplan, 0 }; 227| 1| S *slv = MKSOLVER(S, &sadt); ------------------ | | 604| 1|#define MKSOLVER(type, adt) (type *)X(mksolver)(sizeof(type), adt) | | ------------------ | | | | 78| 1|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 228| 1| return slv; 229| 1|} indirect-transpose.c:mkplan: 148| 1.11k|{ 149| 1.11k| const problem_dft *p = (const problem_dft *) p_; 150| 1.11k| P *pln; 151| 1.11k| plan *cld = 0, *cldtrans = 0, *cldrest = 0; 152| 1.11k| int pdim0, pdim1; 153| 1.11k| tensor *ts, *tv; 154| 1.11k| INT vl, ivs, ovs; 155| 1.11k| R *rit, *iit, *rot, *iot; 156| | 157| 1.11k| static const plan_adt padt = { 158| 1.11k| X(dft_solve), awake, print, destroy ------------------ | | 78| 1.11k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1.11k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 159| 1.11k| }; 160| | 161| 1.11k| if (!applicable(ego_, p_, plnr, &pdim0, &pdim1)) ------------------ | Branch (161:10): [True: 1.10k, False: 12] ------------------ 162| 1.10k| return (plan *) 0; 163| | 164| 12| vl = p->vecsz->dims[pdim0].n / p->sz->dims[pdim1].n; 165| 12| A(vl >= 1); 166| 12| ivs = p->sz->dims[pdim1].n * p->vecsz->dims[pdim0].is; 167| 12| ovs = p->sz->dims[pdim1].n * p->vecsz->dims[pdim0].os; 168| 12| rit = TAINT(p->ri, vl == 1 ? 0 : ivs); ------------------ | | 1036| 12|#define TAINT(p, s) (p) ------------------ 169| 12| iit = TAINT(p->ii, vl == 1 ? 0 : ivs); ------------------ | | 1036| 12|#define TAINT(p, s) (p) ------------------ 170| 12| rot = TAINT(p->ro, vl == 1 ? 0 : ovs); ------------------ | | 1036| 12|#define TAINT(p, s) (p) ------------------ 171| 12| iot = TAINT(p->io, vl == 1 ? 0 : ovs); ------------------ | | 1036| 12|#define TAINT(p, s) (p) ------------------ 172| | 173| 12| ts = X(tensor_copy_inplace)(p->sz, INPLACE_IS); ------------------ | | 78| 12|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 12|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 174| 12| ts->dims[pdim1].os = p->vecsz->dims[pdim0].is; 175| 12| tv = X(tensor_copy_inplace)(p->vecsz, INPLACE_IS); ------------------ | | 78| 12|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 12|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 176| 12| tv->dims[pdim0].os = p->sz->dims[pdim1].is; 177| 12| tv->dims[pdim0].n = p->sz->dims[pdim1].n; 178| 12| cldtrans = X(mkplan_d)(plnr, ------------------ | | 78| 12|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 12|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 179| 12| X(mkproblem_dft_d)(X(mktensor_0d)(), ------------------ | | 78| 12|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 12|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ X(mkproblem_dft_d)(X(mktensor_0d)(), ------------------ | | 78| 12|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 12|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 180| 12| X(tensor_append)(tv, ts), ------------------ | | 78| 12|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 12|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 181| 12| rit, iit, 182| 12| rot, iot)); 183| 12| X(tensor_destroy2)(ts, tv); ------------------ | | 78| 12|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 12|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 184| 12| if (!cldtrans) goto nada; ------------------ | Branch (184:10): [True: 0, False: 12] ------------------ 185| | 186| 12| ts = X(tensor_copy)(p->sz); ------------------ | | 78| 12|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 12|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 187| 12| ts->dims[pdim1].is = p->vecsz->dims[pdim0].is; 188| 12| tv = X(tensor_copy)(p->vecsz); ------------------ | | 78| 12|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 12|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 189| 12| tv->dims[pdim0].is = p->sz->dims[pdim1].is; 190| 12| tv->dims[pdim0].n = p->sz->dims[pdim1].n; 191| 12| cld = X(mkplan_d)(plnr, X(mkproblem_dft_d)(ts, tv, ------------------ | | 78| 12|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 12|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ cld = X(mkplan_d)(plnr, X(mkproblem_dft_d)(ts, tv, ------------------ | | 78| 12|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 12|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 192| 12| rot, iot, 193| 12| rot, iot)); 194| 12| if (!cld) goto nada; ------------------ | Branch (194:10): [True: 0, False: 12] ------------------ 195| | 196| 12| tv = X(tensor_copy)(p->vecsz); ------------------ | | 78| 12|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 12|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 197| 12| tv->dims[pdim0].n -= vl * p->sz->dims[pdim1].n; 198| 12| cldrest = X(mkplan_d)(plnr, X(mkproblem_dft_d)(X(tensor_copy)(p->sz), tv, ------------------ | | 78| 12|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 12|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ cldrest = X(mkplan_d)(plnr, X(mkproblem_dft_d)(X(tensor_copy)(p->sz), tv, ------------------ | | 78| 12|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 12|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ cldrest = X(mkplan_d)(plnr, X(mkproblem_dft_d)(X(tensor_copy)(p->sz), tv, ------------------ | | 78| 12|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 12|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 199| 12| p->ri + ivs * vl, 200| 12| p->ii + ivs * vl, 201| 12| p->ro + ovs * vl, 202| 12| p->io + ovs * vl)); 203| 12| if (!cldrest) goto nada; ------------------ | Branch (203:10): [True: 0, False: 12] ------------------ 204| | 205| 12| pln = MKPLAN_DFT(P, &padt, apply_op); ------------------ | | 60| 12| (type *)X(mkplan_dft)(sizeof(type), adt, apply) | | ------------------ | | | | 78| 12|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 12|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 206| 12| pln->cldtrans = cldtrans; 207| 12| pln->cld = cld; 208| 12| pln->cldrest = cldrest; 209| 12| pln->vl = vl; 210| 12| pln->ivs = ivs; 211| 12| pln->ovs = ovs; 212| 12| X(ops_cpy)(&cldrest->ops, &pln->super.super.ops); ------------------ | | 78| 12|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 12|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 213| 12| X(ops_madd2)(vl, &cld->ops, &pln->super.super.ops); ------------------ | | 78| 12|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 12|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 214| 12| X(ops_madd2)(vl, &cldtrans->ops, &pln->super.super.ops); ------------------ | | 78| 12|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 12|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 215| 12| return &(pln->super.super); 216| | 217| 0| nada: 218| 0| X(plan_destroy_internal)(cldrest); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 219| 0| X(plan_destroy_internal)(cld); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 220| 0| X(plan_destroy_internal)(cldtrans); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 221| 0| return (plan *)0; 222| 12|} indirect-transpose.c:destroy: 62| 12|{ 63| 12| P *ego = (P *) ego_; 64| 12| X(plan_destroy_internal)(ego->cldrest); ------------------ | | 78| 12|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 12|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 65| 12| X(plan_destroy_internal)(ego->cld); ------------------ | | 78| 12|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 12|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 66| 12| X(plan_destroy_internal)(ego->cldtrans); ------------------ | | 78| 12|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 12|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 67| 12|} indirect-transpose.c:applicable: 125| 1.11k|{ 126| 1.11k| if (!applicable0(ego_, p_, plnr, pdim0, pdim1)) return 0; ------------------ | Branch (126:10): [True: 1.10k, False: 12] ------------------ 127| 12| { 128| 12| const problem_dft *p = (const problem_dft *) p_; 129| 12| INT u = p->ri == p->ii + 1 || p->ii == p->ri + 1 ? (INT)2 : (INT)1; ------------------ | Branch (129:12): [True: 0, False: 12] | Branch (129:34): [True: 12, False: 0] ------------------ 130| | 131| | /* UGLY if does not result in contiguous transforms or 132| | transforms of contiguous vectors (since the latter at 133| | least have efficient transpositions) */ 134| 12| if (NO_UGLYP(plnr) ------------------ | | 689| 24|#define NO_UGLYP(plnr) (PLNR_L(plnr) & NO_UGLY) | | ------------------ | | | | 674| 12|#define PLNR_L(plnr) ((plnr)->flags.l) | | ------------------ | | | Branch (689:24): [True: 12, False: 0] | | ------------------ ------------------ 135| 12| && p->vecsz->dims[*pdim0].is != u ------------------ | Branch (135:11): [True: 0, False: 12] ------------------ 136| 0| && !(p->vecsz->rnk == 2 ------------------ | Branch (136:13): [True: 0, False: 0] ------------------ 137| 0| && p->vecsz->dims[1-*pdim0].is == u ------------------ | Branch (137:9): [True: 0, False: 0] ------------------ 138| 0| && p->vecsz->dims[*pdim0].is ------------------ | Branch (138:9): [True: 0, False: 0] ------------------ 139| 0| == u * p->vecsz->dims[1-*pdim0].n)) 140| 0| return 0; 141| | 142| 12| if (NO_INDIRECT_OP_P(plnr) && p->ri != p->ro) return 0; ------------------ | | 682| 24|#define NO_INDIRECT_OP_P(plnr) (PLNR_L(plnr) & NO_INDIRECT_OP) | | ------------------ | | | | 674| 12|#define PLNR_L(plnr) ((plnr)->flags.l) | | ------------------ | | | Branch (682:32): [True: 12, False: 0] | | ------------------ ------------------ | Branch (142:34): [True: 0, False: 12] ------------------ 143| 12| } 144| 12| return 1; 145| 12|} indirect-transpose.c:applicable0: 104| 1.11k|{ 105| 1.11k| const problem_dft *p = (const problem_dft *) p_; 106| 1.11k| UNUSED(ego_); UNUSED(plnr); ------------------ | | 88| 1.11k|#define UNUSED(x) (void)x ------------------ UNUSED(ego_); UNUSED(plnr); ------------------ | | 88| 1.11k|#define UNUSED(x) (void)x ------------------ 107| | 108| 1.11k| return (1 ------------------ | Branch (108:14): [True: 1.11k, Folded] ------------------ 109| 1.11k| && FINITE_RNK(p->vecsz->rnk) && FINITE_RNK(p->sz->rnk) ------------------ | | 442| 2.23k|#define FINITE_RNK(rnk) ((rnk) != RNK_MINFTY) | | ------------------ | | | | 441| 1.11k|#define RNK_MINFTY INT_MAX | | ------------------ | | | Branch (442:25): [True: 950, False: 166] | | ------------------ ------------------ && FINITE_RNK(p->vecsz->rnk) && FINITE_RNK(p->sz->rnk) ------------------ | | 442| 2.06k|#define FINITE_RNK(rnk) ((rnk) != RNK_MINFTY) | | ------------------ | | | | 441| 950|#define RNK_MINFTY INT_MAX | | ------------------ | | | Branch (442:25): [True: 950, False: 0] | | ------------------ ------------------ 110| | 111| | /* FIXME: can/should we relax this constraint? */ 112| 950| && X(tensor_inplace_strides2)(p->vecsz, p->sz) ------------------ | | 78| 950|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 950|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ | Branch (112:10): [True: 163, False: 787] ------------------ 113| | 114| 163| && pickdim(p->vecsz, p->sz, pdim0, pdim1) ------------------ | Branch (114:10): [True: 12, False: 151] ------------------ 115| | 116| | /* output should not *already* include the transpose 117| | (in which case we duplicate the regular indirect.c) */ 118| 12| && (p->sz->dims[*pdim1].os != p->vecsz->dims[*pdim0].is) ------------------ | Branch (118:10): [True: 12, False: 0] ------------------ 119| 1.11k| ); 120| 1.11k|} indirect-transpose.c:pickdim: 85| 163|{ 86| 163| int dim0, dim1; 87| 163| *pdim0 = *pdim1 = -1; 88| 224| for (dim0 = 0; dim0 < vs->rnk; ++dim0) ------------------ | Branch (88:21): [True: 61, False: 163] ------------------ 89| 105| for (dim1 = 0; dim1 < s->rnk; ++dim1) ------------------ | Branch (89:26): [True: 44, False: 61] ------------------ 90| 44| if (vs->dims[dim0].n * X(iabs)(vs->dims[dim0].is) <= X(iabs)(s->dims[dim1].is) ------------------ | | 78| 44|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 44|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ if (vs->dims[dim0].n * X(iabs)(vs->dims[dim0].is) <= X(iabs)(s->dims[dim1].is) ------------------ | | 78| 44|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 44|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ | Branch (90:13): [True: 22, False: 22] ------------------ 91| 22| && vs->dims[dim0].n >= s->dims[dim1].n ------------------ | Branch (91:9): [True: 12, False: 10] ------------------ 92| 12| && (*pdim0 == -1 ------------------ | Branch (92:10): [True: 12, False: 0] ------------------ 93| 0| || (X(iabs)(vs->dims[dim0].is) <= X(iabs)(vs->dims[*pdim0].is) ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ || (X(iabs)(vs->dims[dim0].is) <= X(iabs)(vs->dims[*pdim0].is) ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ | Branch (93:14): [True: 0, False: 0] ------------------ 94| 12| && X(iabs)(s->dims[dim1].is) >= X(iabs)(s->dims[*pdim1].is)))) { ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ && X(iabs)(s->dims[dim1].is) >= X(iabs)(s->dims[*pdim1].is)))) { ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ | Branch (94:10): [True: 0, False: 0] ------------------ 95| 12| *pdim0 = dim0; 96| 12| *pdim1 = dim1; 97| 12| } 98| 163| return (*pdim0 != -1 && *pdim1 != -1); ------------------ | Branch (98:14): [True: 12, False: 151] | Branch (98:30): [True: 12, False: 0] ------------------ 99| 163|} fftw_dft_indirect_register: 232| 1|{ 233| 1| unsigned i; 234| 1| static const ndrct_adt *const adts[] = { 235| 1| &adt_before, &adt_after 236| 1| }; 237| | 238| 3| for (i = 0; i < sizeof(adts) / sizeof(adts[0]); ++i) ------------------ | Branch (238:18): [True: 2, False: 1] ------------------ 239| 2| REGISTER_SOLVER(p, mksolver(adts[i])); ------------------ | | 95| 2|#define REGISTER_SOLVER(p, s) X(solver_register)(p, s) | | ------------------ | | | | 78| 2|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 2|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 240| 1|} indirect.c:mkcld_before: 70| 15|{ 71| 15| return X(mkproblem_dft_d)(X(tensor_copy_inplace)(p->sz, INPLACE_OS), ------------------ | | 78| 15|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 15|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ return X(mkproblem_dft_d)(X(tensor_copy_inplace)(p->sz, INPLACE_OS), ------------------ | | 78| 15|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 15|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 72| 15| X(tensor_copy_inplace)(p->vecsz, INPLACE_OS), ------------------ | | 78| 15|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 15|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 73| 15| p->ro, p->io, p->ro, p->io); 74| 15|} indirect.c:mkcld_after: 99| 24|{ 100| 24| return X(mkproblem_dft_d)(X(tensor_copy_inplace)(p->sz, INPLACE_IS), ------------------ | | 78| 24|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 24|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ return X(mkproblem_dft_d)(X(tensor_copy_inplace)(p->sz, INPLACE_IS), ------------------ | | 78| 24|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 24|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 101| 24| X(tensor_copy_inplace)(p->vecsz, INPLACE_IS), ------------------ | | 78| 24|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 24|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 102| 24| p->ri, p->ii, p->ri, p->ii); 103| 24|} indirect.c:mksolver: 224| 4|{ 225| 4| static const solver_adt sadt = { PROBLEM_DFT, mkplan, 0 }; 226| 4| S *slv = MKSOLVER(S, &sadt); ------------------ | | 604| 4|#define MKSOLVER(type, adt) (type *)X(mksolver)(sizeof(type), adt) | | ------------------ | | | | 78| 4|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 4|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 227| 4| slv->adt = adt; 228| 4| return &(slv->super); 229| 4|} indirect.c:mkplan: 185| 2.90k|{ 186| 2.90k| const problem_dft *p = (const problem_dft *) p_; 187| 2.90k| const S *ego = (const S *) ego_; 188| 2.90k| P *pln; 189| 2.90k| plan *cld = 0, *cldcpy = 0; 190| | 191| 2.90k| static const plan_adt padt = { 192| 2.90k| X(dft_solve), awake, print, destroy ------------------ | | 78| 2.90k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 2.90k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 193| 2.90k| }; 194| | 195| 2.90k| if (!applicable(ego_, p_, plnr)) ------------------ | Branch (195:10): [True: 2.72k, False: 173] ------------------ 196| 2.72k| return (plan *) 0; 197| | 198| 173| cldcpy = 199| 173| X(mkplan_d)(plnr, ------------------ | | 78| 173|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 173|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 200| 173| X(mkproblem_dft_d)(X(mktensor_0d)(), ------------------ | | 78| 173|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 173|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ X(mkproblem_dft_d)(X(mktensor_0d)(), ------------------ | | 78| 173|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 173|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 201| 173| X(tensor_append)(p->vecsz, p->sz), ------------------ | | 78| 173|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 173|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 202| 173| p->ri, p->ii, p->ro, p->io)); 203| | 204| 173| if (!cldcpy) goto nada; ------------------ | Branch (204:10): [True: 134, False: 39] ------------------ 205| | 206| 39| cld = X(mkplan_f_d)(plnr, ego->adt->mkcld(p), NO_BUFFERING, 0, 0); ------------------ | | 78| 39|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 39|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 207| 39| if (!cld) goto nada; ------------------ | Branch (207:10): [True: 0, False: 39] ------------------ 208| | 209| 39| pln = MKPLAN_DFT(P, &padt, ego->adt->apply); ------------------ | | 60| 39| (type *)X(mkplan_dft)(sizeof(type), adt, apply) | | ------------------ | | | | 78| 39|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 39|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 210| 39| pln->cld = cld; 211| 39| pln->cldcpy = cldcpy; 212| 39| pln->slv = ego; 213| 39| X(ops_add)(&cld->ops, &cldcpy->ops, &pln->super.super.ops); ------------------ | | 78| 39|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 39|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 214| | 215| 39| return &(pln->super.super); 216| | 217| 134| nada: 218| 134| X(plan_destroy_internal)(cld); ------------------ | | 78| 134|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 134|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 219| 134| X(plan_destroy_internal)(cldcpy); ------------------ | | 78| 134|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 134|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 220| 134| return (plan *)0; 221| 39|} indirect.c:destroy: 112| 39|{ 113| 39| P *ego = (P *) ego_; 114| 39| X(plan_destroy_internal)(ego->cld); ------------------ | | 78| 39|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 39|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 115| 39| X(plan_destroy_internal)(ego->cldcpy); ------------------ | | 78| 39|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 39|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 116| 39|} indirect.c:applicable: 175| 2.23k|{ 176| 2.23k| if (!applicable0(ego_, p_, plnr)) return 0; ------------------ | Branch (176:10): [True: 1.76k, False: 470] ------------------ 177| 470| { 178| 470| const problem_dft *p = (const problem_dft *) p_; 179| 470| if (NO_INDIRECT_OP_P(plnr) && p->ri != p->ro) return 0; ------------------ | | 682| 940|#define NO_INDIRECT_OP_P(plnr) (PLNR_L(plnr) & NO_INDIRECT_OP) | | ------------------ | | | | 674| 470|#define PLNR_L(plnr) ((plnr)->flags.l) | | ------------------ | | | Branch (682:32): [True: 470, False: 0] | | ------------------ ------------------ | Branch (179:34): [True: 297, False: 173] ------------------ 180| 470| } 181| 173| return 1; 182| 470|} indirect.c:applicable0: 134| 2.23k|{ 135| 2.23k| const S *ego = (const S *) ego_; 136| 2.23k| const problem_dft *p = (const problem_dft *) p_; 137| 2.23k| return (1 ------------------ | Branch (137:14): [True: 2.23k, Folded] ------------------ 138| 2.23k| && FINITE_RNK(p->vecsz->rnk) ------------------ | | 442| 4.46k|#define FINITE_RNK(rnk) ((rnk) != RNK_MINFTY) | | ------------------ | | | | 441| 2.23k|#define RNK_MINFTY INT_MAX | | ------------------ | | | Branch (442:25): [True: 1.90k, False: 332] | | ------------------ ------------------ 139| | 140| | /* problem must be a nontrivial transform, not just a copy */ 141| 1.90k| && p->sz->rnk > 0 ------------------ | Branch (141:10): [True: 1.23k, False: 670] ------------------ 142| | 143| 1.23k| && (0 ------------------ | Branch (143:11): [Folded, False: 1.23k] ------------------ 144| | 145| | /* problem must be in-place & require some 146| | rearrangement of the data; to prevent 147| | infinite loops with indirect-transpose, we 148| | further require that at least some transform 149| | strides must decrease */ 150| 1.23k| || (p->ri == p->ro ------------------ | Branch (150:8): [True: 390, False: 840] ------------------ 151| 390| && !X(tensor_inplace_strides2)(p->sz, p->vecsz) ------------------ | | 78| 390|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 390|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ | Branch (151:11): [True: 346, False: 44] ------------------ 152| 346| && X(tensor_strides_decrease)( ------------------ | | 78| 346|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 346|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ | Branch (152:11): [True: 173, False: 173] ------------------ 153| 346| p->sz, p->vecsz, 154| 346| ego->adt->apply == apply_after ? ------------------ | Branch (154:6): [True: 173, False: 173] ------------------ 155| 173| INPLACE_IS : INPLACE_OS)) 156| | 157| | /* or problem must be out of place, transforming 158| | from stride 1/2 to bigger stride, for apply_after */ 159| 1.05k| || (p->ri != p->ro && ego->adt->apply == apply_after ------------------ | Branch (159:8): [True: 840, False: 217] | Branch (159:26): [True: 420, False: 420] ------------------ 160| 420| && !NO_DESTROY_INPUTP(plnr) ------------------ | | 695| 1.47k|#define NO_DESTROY_INPUTP(plnr) (PLNR_L(plnr) & NO_DESTROY_INPUT) | | ------------------ | | | | 674| 420|#define PLNR_L(plnr) ((plnr)->flags.l) | | ------------------ ------------------ | Branch (160:11): [True: 64, False: 356] ------------------ 161| 64| && X(tensor_min_istride)(p->sz) <= 2 ------------------ | | 78| 64|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 64|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ | Branch (161:11): [True: 0, False: 64] ------------------ 162| 0| && X(tensor_min_ostride)(p->sz) > 2) ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ | Branch (162:11): [True: 0, False: 0] ------------------ 163| | 164| | /* or problem must be out of place, transforming 165| | to stride 1/2 from bigger stride, for apply_before */ 166| 1.05k| || (p->ri != p->ro && ego->adt->apply == apply_before ------------------ | Branch (166:8): [True: 840, False: 217] | Branch (166:26): [True: 420, False: 420] ------------------ 167| 420| && X(tensor_min_ostride)(p->sz) <= 2 ------------------ | | 78| 420|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 420|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ | Branch (167:11): [True: 420, False: 0] ------------------ 168| 420| && X(tensor_min_istride)(p->sz) > 2) ------------------ | | 78| 420|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 420|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ | Branch (168:11): [True: 297, False: 123] ------------------ 169| 1.23k| ) 170| 2.23k| ); 171| 2.23k|} fftw_kdft_difsq_register: 25| 6|{ 26| 6| X(regsolver_ct_directwsq)(p, k, desc, DECDIF); ------------------ | | 78| 6|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 6|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ X(regsolver_ct_directwsq)(p, k, desc, DECDIF); ------------------ | | 48| 6|# define DECDIF 0 ------------------ 27| 6|} fftw_kdft_dit_register: 25| 25|{ 26| 25| X(regsolver_ct_directw)(p, codelet, desc, DECDIT); ------------------ | | 78| 25|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 25|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ X(regsolver_ct_directw)(p, codelet, desc, DECDIT); ------------------ | | 49| 25|# define DECDIT 1 ------------------ 27| 25|} fftw_kdft_register: 25| 19|{ 26| 19| REGISTER_SOLVER(p, X(mksolver_dft_direct)(codelet, desc)); ------------------ | | 95| 19|#define REGISTER_SOLVER(p, s) X(solver_register)(p, s) | | ------------------ | | | | 78| 19|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 19|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 27| 19| REGISTER_SOLVER(p, X(mksolver_dft_directbuf)(codelet, desc)); ------------------ | | 95| 19|#define REGISTER_SOLVER(p, s) X(solver_register)(p, s) | | ------------------ | | | | 78| 19|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 19|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 28| 19|} fftw_dft_nop_register: 84| 1|{ 85| 1| REGISTER_SOLVER(p, mksolver()); ------------------ | | 95| 1|#define REGISTER_SOLVER(p, s) X(solver_register)(p, s) | | ------------------ | | | | 78| 1|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 86| 1|} nop.c:mksolver: 78| 2|{ 79| 2| static const solver_adt sadt = { PROBLEM_DFT, mkplan, 0 }; 80| 2| return MKSOLVER(solver, &sadt); ------------------ | | 604| 2|#define MKSOLVER(type, adt) (type *)X(mksolver)(sizeof(type), adt) | | ------------------ | | | | 78| 2|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 2|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 81| 2|} nop.c:mkplan: 61| 1.49k|{ 62| 1.49k| static const plan_adt padt = { 63| 1.49k| X(dft_solve), X(null_awake), print, X(plan_null_destroy) ------------------ | | 78| 1.49k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1.49k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ X(dft_solve), X(null_awake), print, X(plan_null_destroy) ------------------ | | 78| 1.49k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1.49k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ X(dft_solve), X(null_awake), print, X(plan_null_destroy) ------------------ | | 78| 1.49k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1.49k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 64| 1.49k| }; 65| 1.49k| plan_dft *pln; 66| | 67| 1.49k| UNUSED(plnr); ------------------ | | 88| 1.49k|#define UNUSED(x) (void)x ------------------ 68| | 69| 1.49k| if (!applicable(ego, p)) ------------------ | Branch (69:10): [True: 1.28k, False: 209] ------------------ 70| 1.28k| return (plan *) 0; 71| 209| pln = MKPLAN_DFT(plan_dft, &padt, apply); ------------------ | | 60| 209| (type *)X(mkplan_dft)(sizeof(type), adt, apply) | | ------------------ | | | | 78| 209|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 209|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 72| 209| X(ops_zero)(&pln->super.ops); ------------------ | | 78| 209|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 209|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 73| | 74| 209| return &(pln->super); 75| 1.49k|} nop.c:applicable: 36| 1.15k|{ 37| 1.15k| const problem_dft *p = (const problem_dft *) p_; 38| | 39| 1.15k| UNUSED(ego_); ------------------ | | 88| 1.15k|#define UNUSED(x) (void)x ------------------ 40| | 41| 1.15k| return 0 ------------------ | Branch (41:13): [Folded, False: 1.15k] ------------------ 42| | /* case 1 : -infty vector rank */ 43| 1.15k| || (!FINITE_RNK(p->vecsz->rnk)) ------------------ | | 442| 1.15k|#define FINITE_RNK(rnk) ((rnk) != RNK_MINFTY) | | ------------------ | | | | 441| 1.15k|#define RNK_MINFTY INT_MAX | | ------------------ ------------------ | Branch (43:7): [True: 209, False: 950] ------------------ 44| | 45| | /* case 2 : rank-0 in-place dft */ 46| 950| || (1 ------------------ | Branch (46:8): [True: 950, Folded] ------------------ 47| 950| && p->sz->rnk == 0 ------------------ | Branch (47:11): [True: 335, False: 615] ------------------ 48| 335| && FINITE_RNK(p->vecsz->rnk) ------------------ | | 442| 1.28k|#define FINITE_RNK(rnk) ((rnk) != RNK_MINFTY) | | ------------------ | | | | 441| 335|#define RNK_MINFTY INT_MAX | | ------------------ | | | Branch (442:25): [True: 335, False: 0] | | ------------------ ------------------ 49| 335| && p->ro == p->ri ------------------ | Branch (49:11): [True: 173, False: 162] ------------------ 50| 173| && X(tensor_inplace_strides)(p->vecsz) ------------------ | | 78| 173|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 173|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ | Branch (50:11): [True: 0, False: 173] ------------------ 51| 950| ); 52| 1.15k|} nop.c:apply: 27| 69|{ 28| 69| UNUSED(ego_); ------------------ | | 88| 69|#define UNUSED(x) (void)x ------------------ 29| 69| UNUSED(ri); ------------------ | | 88| 69|#define UNUSED(x) (void)x ------------------ 30| 69| UNUSED(ii); ------------------ | | 88| 69|#define UNUSED(x) (void)x ------------------ 31| 69| UNUSED(ro); ------------------ | | 88| 69|#define UNUSED(x) (void)x ------------------ 32| 69| UNUSED(io); ------------------ | | 88| 69|#define UNUSED(x) (void)x ------------------ 33| 69|} fftw_mkplan_dft: 25| 4.22k|{ 26| 4.22k| plan_dft *ego; 27| | 28| 4.22k| ego = (plan_dft *) X(mkplan)(size, adt); ------------------ | | 78| 4.22k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 4.22k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 29| 4.22k| ego->apply = apply; 30| | 31| 4.22k| return &(ego->super); 32| 4.22k|} fftw_mkproblem_dft: 79| 3.12k|{ 80| 3.12k| problem_dft *ego; 81| | 82| | /* enforce pointer equality if untainted pointers are equal */ 83| 3.12k| if (UNTAINT(ri) == UNTAINT(ro)) ------------------ | | 1037| 3.12k|#define UNTAINT(p) (p) ------------------ if (UNTAINT(ri) == UNTAINT(ro)) ------------------ | | 1037| 3.12k|#define UNTAINT(p) (p) ------------------ | Branch (83:10): [True: 812, False: 2.31k] ------------------ 84| 812| ri = ro = JOIN_TAINT(ri, ro); ------------------ | | 1039| 812|#define JOIN_TAINT(p1, p2) p1 ------------------ 85| 3.12k| if (UNTAINT(ii) == UNTAINT(io)) ------------------ | | 1037| 3.12k|#define UNTAINT(p) (p) ------------------ if (UNTAINT(ii) == UNTAINT(io)) ------------------ | | 1037| 3.12k|#define UNTAINT(p) (p) ------------------ | Branch (85:10): [True: 812, False: 2.31k] ------------------ 86| 812| ii = io = JOIN_TAINT(ii, io); ------------------ | | 1039| 812|#define JOIN_TAINT(p1, p2) p1 ------------------ 87| | 88| | /* more correctness conditions: */ 89| 3.12k| A(TAINTOF(ri) == TAINTOF(ii)); 90| 3.12k| A(TAINTOF(ro) == TAINTOF(io)); 91| | 92| 3.12k| A(X(tensor_kosherp)(sz)); 93| 3.12k| A(X(tensor_kosherp)(vecsz)); 94| | 95| 3.12k| if (ri == ro || ii == io) { ------------------ | Branch (95:10): [True: 812, False: 2.31k] | Branch (95:22): [True: 0, False: 2.31k] ------------------ 96| | /* If either real or imag pointers are in place, both must be. */ 97| 812| if (ri != ro || ii != io || !X(tensor_inplace_locations)(sz, vecsz)) ------------------ | | 78| 812|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 812|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ | Branch (97:8): [True: 0, False: 812] | Branch (97:20): [True: 0, False: 812] | Branch (97:32): [True: 0, False: 812] ------------------ 98| 0| return X(mkproblem_unsolvable)(); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 99| 812| } 100| | 101| 3.12k| ego = (problem_dft *)X(mkproblem)(sizeof(problem_dft), &padt); ------------------ | | 78| 3.12k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 3.12k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 102| | 103| 3.12k| ego->sz = X(tensor_compress)(sz); ------------------ | | 78| 3.12k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 3.12k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 104| 3.12k| ego->vecsz = X(tensor_compress_contiguous)(vecsz); ------------------ | | 78| 3.12k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 3.12k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 105| 3.12k| ego->ri = ri; 106| 3.12k| ego->ii = ii; 107| 3.12k| ego->ro = ro; 108| 3.12k| ego->io = io; 109| | 110| 3.12k| A(FINITE_RNK(ego->sz->rnk)); 111| 3.12k| return &(ego->super); 112| 3.12k|} fftw_mkproblem_dft_d: 117| 3.12k|{ 118| 3.12k| problem *p = X(mkproblem_dft)(sz, vecsz, ri, ii, ro, io); ------------------ | | 78| 3.12k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 3.12k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 119| 3.12k| X(tensor_destroy2)(vecsz, sz); ------------------ | | 78| 3.12k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 3.12k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 120| 3.12k| return p; 121| 3.12k|} problem.c:hash: 33| 3.82k|{ 34| 3.82k| const problem_dft *p = (const problem_dft *) p_; 35| 3.82k| X(md5puts)(m, "dft"); ------------------ | | 78| 3.82k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 3.82k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 36| 3.82k| X(md5int)(m, p->ri == p->ro); ------------------ | | 78| 3.82k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 3.82k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 37| 3.82k| X(md5INT)(m, p->ii - p->ri); ------------------ | | 78| 3.82k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 3.82k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 38| 3.82k| X(md5INT)(m, p->io - p->ro); ------------------ | | 78| 3.82k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 3.82k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 39| 3.82k| X(md5int)(m, X(ialignment_of)(p->ri)); ------------------ | | 78| 3.82k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 3.82k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ X(md5int)(m, X(ialignment_of)(p->ri)); ------------------ | | 78| 3.82k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 3.82k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 40| 3.82k| X(md5int)(m, X(ialignment_of)(p->ii)); ------------------ | | 78| 3.82k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 3.82k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ X(md5int)(m, X(ialignment_of)(p->ii)); ------------------ | | 78| 3.82k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 3.82k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 41| 3.82k| X(md5int)(m, X(ialignment_of)(p->ro)); ------------------ | | 78| 3.82k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 3.82k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ X(md5int)(m, X(ialignment_of)(p->ro)); ------------------ | | 78| 3.82k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 3.82k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 42| 3.82k| X(md5int)(m, X(ialignment_of)(p->io)); ------------------ | | 78| 3.82k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 3.82k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ X(md5int)(m, X(ialignment_of)(p->io)); ------------------ | | 78| 3.82k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 3.82k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 43| 3.82k| X(tensor_md5)(m, p->sz); ------------------ | | 78| 3.82k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 3.82k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 44| 3.82k| X(tensor_md5)(m, p->vecsz); ------------------ | | 78| 3.82k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 3.82k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 45| 3.82k|} problem.c:destroy: 26| 3.54k|{ 27| 3.54k| problem_dft *ego = (problem_dft *) ego_; 28| 3.54k| X(tensor_destroy2)(ego->vecsz, ego->sz); ------------------ | | 78| 3.54k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 3.54k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 29| 3.54k| X(ifree)(ego_); ------------------ | | 78| 3.54k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 3.54k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 30| 3.54k|} fftw_dft_rader_register: 325| 1|{ 326| 1| REGISTER_SOLVER(p, mksolver()); ------------------ | | 95| 1|#define REGISTER_SOLVER(p, s) X(solver_register)(p, s) | | ------------------ | | | | 78| 1|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 327| 1|} rader.c:mksolver: 318| 1|{ 319| 1| static const solver_adt sadt = { PROBLEM_DFT, mkplan, 0 }; 320| 1| S *slv = MKSOLVER(S, &sadt); ------------------ | | 604| 1|#define MKSOLVER(type, adt) (type *)X(mksolver)(sizeof(type), adt) | | ------------------ | | | | 78| 1|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 321| 1| return &(slv->super); 322| 1|} rader.c:mkplan: 292| 1.18k|{ 293| 1.18k| const problem_dft *p = (const problem_dft *) p_; 294| 1.18k| P *pln; 295| 1.18k| INT n; 296| 1.18k| INT is, os; 297| | 298| 1.18k| static const plan_adt padt = { 299| 1.18k| X(dft_solve), awake, print, destroy ------------------ | | 78| 1.18k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1.18k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 300| 1.18k| }; 301| | 302| 1.18k| if (!applicable(ego, p_, plnr)) ------------------ | Branch (302:10): [True: 1.08k, False: 96] ------------------ 303| 1.08k| return (plan *) 0; 304| | 305| 96| n = p->sz->dims[0].n; 306| 96| is = p->sz->dims[0].is; 307| 96| os = p->sz->dims[0].os; 308| | 309| 96| pln = MKPLAN_DFT(P, &padt, apply); ------------------ | | 60| 96| (type *)X(mkplan_dft)(sizeof(type), adt, apply) | | ------------------ | | | | 78| 96|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 96|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 310| 96| if (!mkP(pln, n, is, os, p->ro, p->io, plnr)) { ------------------ | Branch (310:10): [True: 0, False: 96] ------------------ 311| 0| X(ifree)(pln); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 312| 0| return (plan *) 0; 313| 0| } 314| 96| return &(pln->super.super); 315| 96|} rader.c:awake: 170| 90|{ 171| 90| P *ego = (P *) ego_; 172| | 173| 90| X(plan_awake)(ego->cld1, wakefulness); ------------------ | | 78| 90|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 90|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 174| 90| X(plan_awake)(ego->cld2, wakefulness); ------------------ | | 78| 90|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 90|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 175| 90| X(plan_awake)(ego->cld_omega, wakefulness); ------------------ | | 78| 90|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 90|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 176| | 177| 90| switch (wakefulness) { 178| 45| case SLEEPY: ------------------ | Branch (178:3): [True: 45, False: 45] ------------------ 179| 45| free_omega(ego->omega); 180| 45| ego->omega = 0; 181| 45| break; 182| 45| default: ------------------ | Branch (182:3): [True: 45, False: 45] ------------------ 183| 45| ego->g = X(find_generator)(ego->n); ------------------ | | 78| 45|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 45|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 184| 45| ego->ginv = X(power_mod)(ego->g, ego->n - 2, ego->n); ------------------ | | 78| 45|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 45|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 185| 45| A(MULMOD(ego->g, ego->ginv, ego->n) == 1); 186| | 187| 45| ego->omega = mkomega(wakefulness, 188| 45| ego->cld_omega, ego->n, ego->ginv); 189| 45| break; 190| 90| } 191| 90|} rader.c:free_omega: 77| 45|{ 78| 45| X(rader_tl_delete)(omega, &omegas); ------------------ | | 78| 45|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 45|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 79| 45|} rader.c:mkomega: 46| 45|{ 47| 45| plan_dft *p = (plan_dft *) p_; 48| 45| R *omega; 49| 45| INT i, gpower; 50| 45| trigreal scale; 51| 45| triggen *t; 52| | 53| 45| if ((omega = X(rader_tl_find)(n, n, ginv, omegas))) ------------------ | | 78| 45|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 45|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ | Branch (53:10): [True: 0, False: 45] ------------------ 54| 0| return omega; 55| | 56| 45| omega = (R *)MALLOC(sizeof(R) * (n - 1) * 2, TWIDDLES); ------------------ | | 306| 45|#define MALLOC(n, what) X(malloc_plain)(n) | | ------------------ | | | | 78| 45|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 45|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 57| | 58| 45| scale = n - 1.0; /* normalization for convolution */ 59| | 60| 45| t = X(mktriggen)(wakefulness, n); ------------------ | | 78| 45|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 45|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 61| 3.73k| for (i = 0, gpower = 1; i < n-1; ++i, gpower = MULMOD(gpower, ginv, n)) { ------------------ | | 925| 3.69k| (((x) <= 92681 - (y)) ? ((x) * (y)) % (p) : X(safe_mulmod)(x, y, p)) | | ------------------ | | | | 78| 0|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ | | | Branch (925:5): [True: 3.69k, False: 0] | | ------------------ ------------------ | Branch (61:30): [True: 3.69k, False: 45] ------------------ 62| 3.69k| trigreal w[2]; 63| 3.69k| t->cexpl(t, gpower, w); 64| 3.69k| omega[2*i] = w[0] / scale; 65| 3.69k| omega[2*i+1] = FFT_SIGN * w[1] / scale; ------------------ | | 92| 3.69k|#define FFT_SIGN (-1) /* sign convention for forward transforms */ ------------------ 66| 3.69k| } 67| 45| X(triggen_destroy)(t); ------------------ | | 78| 45|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 45|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 68| 45| A(gpower == 1); 69| | 70| 45| p->apply(p_, omega, omega + 1, omega, omega + 1); 71| | 72| 45| X(rader_tl_insert)(n, n, ginv, omega, &omegas); ------------------ | | 78| 45|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 45|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 73| 45| return omega; 74| 45|} rader.c:destroy: 194| 96|{ 195| 96| P *ego = (P *) ego_; 196| 96| X(plan_destroy_internal)(ego->cld_omega); ------------------ | | 78| 96|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 96|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 197| 96| X(plan_destroy_internal)(ego->cld2); ------------------ | | 78| 96|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 96|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 198| 96| X(plan_destroy_internal)(ego->cld1); ------------------ | | 78| 96|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 96|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 199| 96|} rader.c:applicable: 215| 1.18k|{ 216| 1.18k| const problem_dft *p = (const problem_dft *) p_; 217| 1.18k| UNUSED(ego_); ------------------ | | 88| 1.18k|#define UNUSED(x) (void)x ------------------ 218| 1.18k| return (1 ------------------ | Branch (218:14): [True: 1.18k, Folded] ------------------ 219| 1.18k| && p->sz->rnk == 1 ------------------ | Branch (219:10): [True: 850, False: 335] ------------------ 220| 850| && p->vecsz->rnk == 0 ------------------ | Branch (220:10): [True: 329, False: 521] ------------------ 221| 329| && CIMPLIES(NO_SLOWP(plnr), p->sz->dims[0].n > RADER_MAX_SLOW) ------------------ | | 99| 1.51k|#define CIMPLIES(ante, post) (!(ante) || (post)) | | ------------------ | | | Branch (99:31): [True: 8, False: 321] | | | Branch (99:42): [True: 229, False: 92] | | ------------------ ------------------ 222| 237| && X(is_prime)(p->sz->dims[0].n) ------------------ | | 78| 237|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 237|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ | Branch (222:10): [True: 138, False: 99] ------------------ 223| | 224| | /* proclaim the solver SLOW if p-1 is not easily factorizable. 225| | Bluestein should take care of this case. */ 226| 138| && CIMPLIES(NO_SLOWP(plnr), X(factors_into_small_primes)(p->sz->dims[0].n - 1)) ------------------ | | 99| 138|#define CIMPLIES(ante, post) (!(ante) || (post)) | | ------------------ | | | Branch (99:31): [True: 0, False: 138] | | | Branch (99:42): [True: 96, False: 42] | | ------------------ ------------------ 227| 1.18k| ); 228| 1.18k|} rader.c:apply: 96| 83|{ 97| 83| const P *ego = (const P *) ego_; 98| 83| INT is, os; 99| 83| INT k, gpower, g, r; 100| 83| R *buf; 101| 83| R r0 = ri[0], i0 = ii[0]; 102| | 103| 83| r = ego->n; is = ego->is; os = ego->os; g = ego->g; 104| 83| buf = (R *) MALLOC(sizeof(R) * (r - 1) * 2, BUFFERS); ------------------ | | 306| 83|#define MALLOC(n, what) X(malloc_plain)(n) | | ------------------ | | | | 78| 83|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 83|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 105| | 106| | /* First, permute the input, storing in buf: */ 107| 5.55k| for (gpower = 1, k = 0; k < r - 1; ++k, gpower = MULMOD(gpower, g, r)) { ------------------ | | 925| 5.47k| (((x) <= 92681 - (y)) ? ((x) * (y)) % (p) : X(safe_mulmod)(x, y, p)) | | ------------------ | | | | 78| 0|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ | | | Branch (925:5): [True: 5.47k, False: 0] | | ------------------ ------------------ | Branch (107:30): [True: 5.47k, False: 83] ------------------ 108| 5.47k| R rA, iA; 109| 5.47k| rA = ri[gpower * is]; 110| 5.47k| iA = ii[gpower * is]; 111| 5.47k| buf[2*k] = rA; buf[2*k + 1] = iA; 112| 5.47k| } 113| 83| /* gpower == g^(r-1) mod r == 1 */; 114| | 115| | 116| | /* compute DFT of buf, storing in output (except DC): */ 117| 83| { 118| 83| plan_dft *cld = (plan_dft *) ego->cld1; 119| 83| cld->apply(ego->cld1, buf, buf+1, ro+os, io+os); 120| 83| } 121| | 122| | /* set output DC component: */ 123| 83| { 124| 83| ro[0] = r0 + ro[os]; 125| 83| io[0] = i0 + io[os]; 126| 83| } 127| | 128| | /* now, multiply by omega: */ 129| 83| { 130| 83| const R *omega = ego->omega; 131| 5.55k| for (k = 0; k < r - 1; ++k) { ------------------ | Branch (131:16): [True: 5.47k, False: 83] ------------------ 132| 5.47k| E rB, iB, rW, iW; 133| 5.47k| rW = omega[2*k]; 134| 5.47k| iW = omega[2*k+1]; 135| 5.47k| rB = ro[(k+1)*os]; 136| 5.47k| iB = io[(k+1)*os]; 137| 5.47k| ro[(k+1)*os] = rW * rB - iW * iB; 138| 5.47k| io[(k+1)*os] = -(rW * iB + iW * rB); 139| 5.47k| } 140| 83| } 141| | 142| | /* this will add input[0] to all of the outputs after the ifft */ 143| 83| ro[os] += r0; 144| 83| io[os] -= i0; 145| | 146| | /* inverse FFT: */ 147| 83| { 148| 83| plan_dft *cld = (plan_dft *) ego->cld2; 149| 83| cld->apply(ego->cld2, ro+os, io+os, buf, buf+1); 150| 83| } 151| | 152| | /* finally, do inverse permutation to unshuffle the output: */ 153| 83| { 154| 83| INT ginv = ego->ginv; 155| 83| gpower = 1; 156| 5.55k| for (k = 0; k < r - 1; ++k, gpower = MULMOD(gpower, ginv, r)) { ------------------ | | 925| 5.47k| (((x) <= 92681 - (y)) ? ((x) * (y)) % (p) : X(safe_mulmod)(x, y, p)) | | ------------------ | | | | 78| 0|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ | | | Branch (925:5): [True: 5.47k, False: 0] | | ------------------ ------------------ | Branch (156:16): [True: 5.47k, False: 83] ------------------ 157| 5.47k| ro[gpower * os] = buf[2*k]; 158| 5.47k| io[gpower * os] = -buf[2*k+1]; 159| 5.47k| } 160| 83| A(gpower == 1); 161| 83| } 162| | 163| | 164| 83| X(ifree)(buf); ------------------ | | 78| 83|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 83|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 165| 83|} rader.c:mkP: 232| 96|{ 233| 96| plan *cld1 = (plan *) 0; 234| 96| plan *cld2 = (plan *) 0; 235| 96| plan *cld_omega = (plan *) 0; 236| 96| R *buf = (R *) 0; 237| | 238| | /* initial allocation for the purpose of planning */ 239| 96| buf = (R *) MALLOC(sizeof(R) * (n - 1) * 2, BUFFERS); ------------------ | | 306| 96|#define MALLOC(n, what) X(malloc_plain)(n) | | ------------------ | | | | 78| 96|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 96|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 240| | 241| 96| cld1 = X(mkplan_f_d)(plnr, ------------------ | | 78| 96|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 96|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 242| 96| X(mkproblem_dft_d)(X(mktensor_1d)(n - 1, 2, os), ------------------ | | 78| 96|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 96|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ X(mkproblem_dft_d)(X(mktensor_1d)(n - 1, 2, os), ------------------ | | 78| 96|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 96|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 243| 96| X(mktensor_1d)(1, 0, 0), ------------------ | | 78| 96|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 96|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 244| 96| buf, buf + 1, ro + os, io + os), 245| 96| NO_SLOW, 0, 0); 246| 96| if (!cld1) goto nada; ------------------ | Branch (246:10): [True: 0, False: 96] ------------------ 247| | 248| 96| cld2 = X(mkplan_f_d)(plnr, ------------------ | | 78| 96|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 96|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 249| 96| X(mkproblem_dft_d)(X(mktensor_1d)(n - 1, os, 2), ------------------ | | 78| 96|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 96|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ X(mkproblem_dft_d)(X(mktensor_1d)(n - 1, os, 2), ------------------ | | 78| 96|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 96|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 250| 96| X(mktensor_1d)(1, 0, 0), ------------------ | | 78| 96|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 96|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 251| 96| ro + os, io + os, buf, buf + 1), 252| 96| NO_SLOW, 0, 0); 253| | 254| 96| if (!cld2) goto nada; ------------------ | Branch (254:10): [True: 0, False: 96] ------------------ 255| | 256| | /* plan for omega array */ 257| 96| cld_omega = X(mkplan_f_d)(plnr, ------------------ | | 78| 96|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 96|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 258| 96| X(mkproblem_dft_d)(X(mktensor_1d)(n - 1, 2, 2), ------------------ | | 78| 96|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 96|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ X(mkproblem_dft_d)(X(mktensor_1d)(n - 1, 2, 2), ------------------ | | 78| 96|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 96|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 259| 96| X(mktensor_1d)(1, 0, 0), ------------------ | | 78| 96|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 96|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 260| 96| buf, buf + 1, buf, buf + 1), 261| 96| NO_SLOW, ESTIMATE, 0); 262| 96| if (!cld_omega) goto nada; ------------------ | Branch (262:10): [True: 0, False: 96] ------------------ 263| | 264| | /* deallocate buffers; let awake() or apply() allocate them for real */ 265| 96| X(ifree)(buf); ------------------ | | 78| 96|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 96|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 266| 96| buf = 0; 267| | 268| 96| pln->cld1 = cld1; 269| 96| pln->cld2 = cld2; 270| 96| pln->cld_omega = cld_omega; 271| 96| pln->omega = 0; 272| 96| pln->n = n; 273| 96| pln->is = is; 274| 96| pln->os = os; 275| | 276| 96| X(ops_add)(&cld1->ops, &cld2->ops, &pln->super.super.ops); ------------------ | | 78| 96|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 96|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 277| 96| pln->super.super.ops.other += (n - 1) * (4 * 2 + 6) + 6; 278| 96| pln->super.super.ops.add += (n - 1) * 2 + 4; 279| 96| pln->super.super.ops.mul += (n - 1) * 4; 280| | 281| 96| return 1; 282| | 283| 0| nada: 284| 0| X(ifree0)(buf); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 285| 0| X(plan_destroy_internal)(cld_omega); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 286| 0| X(plan_destroy_internal)(cld2); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 287| 0| X(plan_destroy_internal)(cld1); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 288| 0| return 0; 289| 96|} fftw_dft_rank_geq2_register: 185| 1|{ 186| 1| static const int buddies[] = { 1, 0, -2 }; 187| 1| size_t i; 188| | 189| 4| for (i = 0; i < NELEM(buddies); ++i) ------------------ | | 90| 4|#define NELEM(array) ((sizeof(array) / sizeof((array)[0]))) ------------------ | Branch (189:18): [True: 3, False: 1] ------------------ 190| 3| REGISTER_SOLVER(p, mksolver(buddies[i], buddies, NELEM(buddies))); ------------------ | | 95| 3|#define REGISTER_SOLVER(p, s) X(solver_register)(p, s) | | ------------------ | | | | 78| 3|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 3|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 191| | 192| | /* FIXME: 193| | 194| | Should we try more buddies? 195| | 196| | Another possible variant is to swap cld1 and cld2 (or rather, 197| | to swap their problems; they are not interchangeable because 198| | cld2 must be in-place). In past versions of FFTW, however, I 199| | seem to recall that such rearrangements have made little or no 200| | difference. 201| | */ 202| 1|} rank-geq2.c:mksolver: 175| 6|{ 176| 6| static const solver_adt sadt = { PROBLEM_DFT, mkplan, 0 }; 177| 6| S *slv = MKSOLVER(S, &sadt); ------------------ | | 604| 6|#define MKSOLVER(type, adt) (type *)X(mksolver)(sizeof(type), adt) | | ------------------ | | | | 78| 6|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 6|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 178| 6| slv->spltrnk = spltrnk; 179| 6| slv->buddies = buddies; 180| 6| slv->nbuddies = nbuddies; 181| 6| return &(slv->super); 182| 6|} rank-geq2.c:mkplan: 122| 4.35k|{ 123| 4.35k| const S *ego = (const S *) ego_; 124| 4.35k| const problem_dft *p; 125| 4.35k| P *pln; 126| 4.35k| plan *cld1 = 0, *cld2 = 0; 127| 4.35k| tensor *sz1, *sz2, *vecszi, *sz2i; 128| 4.35k| int spltrnk; 129| | 130| 4.35k| static const plan_adt padt = { 131| 4.35k| X(dft_solve), awake, print, destroy ------------------ | | 78| 4.35k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 4.35k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 132| 4.35k| }; 133| | 134| 4.35k| if (!applicable(ego_, p_, plnr, &spltrnk)) ------------------ | Branch (134:10): [True: 4.35k, False: 0] ------------------ 135| 4.35k| return (plan *) 0; 136| | 137| 0| p = (const problem_dft *) p_; 138| 0| X(tensor_split)(p->sz, &sz1, spltrnk, &sz2); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 139| 0| vecszi = X(tensor_copy_inplace)(p->vecsz, INPLACE_OS); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 140| 0| sz2i = X(tensor_copy_inplace)(sz2, INPLACE_OS); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 141| | 142| 0| cld1 = X(mkplan_d)(plnr, ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 143| 0| X(mkproblem_dft_d)(X(tensor_copy)(sz2), ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ X(mkproblem_dft_d)(X(tensor_copy)(sz2), ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 144| 0| X(tensor_append)(p->vecsz, sz1), ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 145| 0| p->ri, p->ii, p->ro, p->io)); 146| 0| if (!cld1) goto nada; ------------------ | Branch (146:10): [True: 0, False: 0] ------------------ 147| | 148| 0| cld2 = X(mkplan_d)(plnr, ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 149| 0| X(mkproblem_dft_d)( ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 150| 0| X(tensor_copy_inplace)(sz1, INPLACE_OS), ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 151| 0| X(tensor_append)(vecszi, sz2i), ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 152| 0| p->ro, p->io, p->ro, p->io)); 153| 0| if (!cld2) goto nada; ------------------ | Branch (153:10): [True: 0, False: 0] ------------------ 154| | 155| 0| pln = MKPLAN_DFT(P, &padt, apply); ------------------ | | 60| 0| (type *)X(mkplan_dft)(sizeof(type), adt, apply) | | ------------------ | | | | 78| 0|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 156| | 157| 0| pln->cld1 = cld1; 158| 0| pln->cld2 = cld2; 159| | 160| 0| pln->solver = ego; 161| 0| X(ops_add)(&cld1->ops, &cld2->ops, &pln->super.super.ops); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 162| | 163| 0| X(tensor_destroy4)(sz1, sz2, vecszi, sz2i); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 164| | 165| 0| return &(pln->super.super); 166| | 167| 0| nada: 168| 0| X(plan_destroy_internal)(cld2); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 169| 0| X(plan_destroy_internal)(cld1); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 170| 0| X(tensor_destroy4)(sz1, sz2, vecszi, sz2i); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 171| 0| return (plan *) 0; 172| 0|} rank-geq2.c:applicable: 102| 4.35k|{ 103| 4.35k| const S *ego = (const S *)ego_; 104| 4.35k| const problem_dft *p = (const problem_dft *) p_; 105| | 106| 4.35k| if (!applicable0(ego_, p_, rp)) return 0; ------------------ | Branch (106:10): [True: 4.35k, False: 0] ------------------ 107| | 108| 0| if (NO_RANK_SPLITSP(plnr) && (ego->spltrnk != ego->buddies[0])) return 0; ------------------ | | 684| 0|#define NO_RANK_SPLITSP(plnr) (PLNR_L(plnr) & NO_RANK_SPLITS) | | ------------------ | | | | 674| 0|#define PLNR_L(plnr) ((plnr)->flags.l) | | ------------------ | | | Branch (684:31): [True: 0, False: 0] | | ------------------ ------------------ | Branch (108:35): [True: 0, False: 0] ------------------ 109| | 110| | /* Heuristic: if the vector stride is greater than the transform 111| | sz, don't use (prefer to do the vector loop first with a 112| | vrank-geq1 plan). */ 113| 0| if (NO_UGLYP(plnr)) ------------------ | | 689| 0|#define NO_UGLYP(plnr) (PLNR_L(plnr) & NO_UGLY) | | ------------------ | | | | 674| 0|#define PLNR_L(plnr) ((plnr)->flags.l) | | ------------------ | | | Branch (689:24): [True: 0, False: 0] | | ------------------ ------------------ 114| 0| if (p->vecsz->rnk > 0 && ------------------ | Branch (114:8): [True: 0, False: 0] ------------------ 115| 0| X(tensor_min_stride)(p->vecsz) > X(tensor_max_index)(p->sz)) ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ X(tensor_min_stride)(p->vecsz) > X(tensor_max_index)(p->sz)) ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ | Branch (115:8): [True: 0, False: 0] ------------------ 116| 0| return 0; 117| | 118| 0| return 1; 119| 0|} rank-geq2.c:applicable0: 89| 4.35k|{ 90| 4.35k| const problem_dft *p = (const problem_dft *) p_; 91| 4.35k| const S *ego = (const S *)ego_; 92| 4.35k| return (1 ------------------ | Branch (92:14): [True: 4.35k, Folded] ------------------ 93| 4.35k| && FINITE_RNK(p->sz->rnk) && FINITE_RNK(p->vecsz->rnk) ------------------ | | 442| 8.70k|#define FINITE_RNK(rnk) ((rnk) != RNK_MINFTY) | | ------------------ | | | | 441| 4.35k|#define RNK_MINFTY INT_MAX | | ------------------ | | | Branch (442:25): [True: 4.35k, False: 0] | | ------------------ ------------------ && FINITE_RNK(p->sz->rnk) && FINITE_RNK(p->vecsz->rnk) ------------------ | | 442| 8.70k|#define FINITE_RNK(rnk) ((rnk) != RNK_MINFTY) | | ------------------ | | | | 441| 4.35k|#define RNK_MINFTY INT_MAX | | ------------------ | | | Branch (442:25): [True: 3.85k, False: 498] | | ------------------ ------------------ 94| 3.85k| && p->sz->rnk >= 2 ------------------ | Branch (94:10): [True: 0, False: 3.85k] ------------------ 95| 0| && picksplit(ego, p->sz, rp) ------------------ | Branch (95:10): [True: 0, False: 0] ------------------ 96| 4.35k| ); 97| 4.35k|} fftw_codelet_n1_10: 359| 1|void X(codelet_n1_10) (planner *p) { X(kdft_register) (p, n1_10, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 360| 1|} n1_10.c:n1_10: 206| 19|{ 207| 19| DK(KP250000000, +0.250000000000000000000000000000000000000000000); ------------------ | | 1056| 19|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 19|# define K(x) ((E) x) | | ------------------ ------------------ 208| 19| DK(KP559016994, +0.559016994374947424102293417182819058860154590); ------------------ | | 1056| 19|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 19|# define K(x) ((E) x) | | ------------------ ------------------ 209| 19| DK(KP587785252, +0.587785252292473129168705954639072768597652438); ------------------ | | 1056| 19|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 19|# define K(x) ((E) x) | | ------------------ ------------------ 210| 19| DK(KP951056516, +0.951056516295153572116439333379382143405698634); ------------------ | | 1056| 19|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 19|# define K(x) ((E) x) | | ------------------ ------------------ 211| 19| { 212| 19| INT i; 213| 177| for (i = v; i > 0; i = i - 1, ri = ri + ivs, ii = ii + ivs, ro = ro + ovs, io = io + ovs, MAKE_VOLATILE_STRIDE(40, is), MAKE_VOLATILE_STRIDE(40, os)) { ------------------ | | 851| 158| (nptr <= ESTIMATED_AVAILABLE_INDEX_REGISTERS ? \ | | ------------------ | | | | 849| 316|#define ESTIMATED_AVAILABLE_INDEX_REGISTERS 16 | | ------------------ | | | Branch (851:7): [Folded, False: 158] | | ------------------ | | 852| 158| 0 : \ | | 853| 158| ((x) = (x) ^ X(an_INT_guaranteed_to_be_zero))) | | ------------------ | | | | 78| 158|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 158|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ for (i = v; i > 0; i = i - 1, ri = ri + ivs, ii = ii + ivs, ro = ro + ovs, io = io + ovs, MAKE_VOLATILE_STRIDE(40, is), MAKE_VOLATILE_STRIDE(40, os)) { ------------------ | | 851| 158| (nptr <= ESTIMATED_AVAILABLE_INDEX_REGISTERS ? \ | | ------------------ | | | | 849| 316|#define ESTIMATED_AVAILABLE_INDEX_REGISTERS 16 | | ------------------ | | | Branch (851:7): [Folded, False: 158] | | ------------------ | | 852| 158| 0 : \ | | 853| 158| ((x) = (x) ^ X(an_INT_guaranteed_to_be_zero))) | | ------------------ | | | | 78| 158|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 158|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ | Branch (213:16): [True: 158, False: 19] ------------------ 214| 158| E T3, Tj, TQ, T1e, TU, TV, T1c, T1b, Tm, Tp, Tq, Ta, Th, Ti, TA; 215| 158| E TH, T17, T14, T1f, T1g, T1h, TL, TM, TR; 216| 158| { 217| 158| E T1, T2, TO, TP; 218| 158| T1 = ri[0]; 219| 158| T2 = ri[WS(is, 5)]; ------------------ | | 825| 158|#define WS(stride, i) (stride * i) ------------------ 220| 158| T3 = T1 - T2; 221| 158| Tj = T1 + T2; 222| 158| TO = ii[0]; 223| 158| TP = ii[WS(is, 5)]; ------------------ | | 825| 158|#define WS(stride, i) (stride * i) ------------------ 224| 158| TQ = TO - TP; 225| 158| T1e = TO + TP; 226| 158| } 227| 158| { 228| 158| E T6, Tk, Tg, To, T9, Tl, Td, Tn; 229| 158| { 230| 158| E T4, T5, Te, Tf; 231| 158| T4 = ri[WS(is, 2)]; ------------------ | | 825| 158|#define WS(stride, i) (stride * i) ------------------ 232| 158| T5 = ri[WS(is, 7)]; ------------------ | | 825| 158|#define WS(stride, i) (stride * i) ------------------ 233| 158| T6 = T4 - T5; 234| 158| Tk = T4 + T5; 235| 158| Te = ri[WS(is, 6)]; ------------------ | | 825| 158|#define WS(stride, i) (stride * i) ------------------ 236| 158| Tf = ri[WS(is, 1)]; ------------------ | | 825| 158|#define WS(stride, i) (stride * i) ------------------ 237| 158| Tg = Te - Tf; 238| 158| To = Te + Tf; 239| 158| } 240| 158| { 241| 158| E T7, T8, Tb, Tc; 242| 158| T7 = ri[WS(is, 8)]; ------------------ | | 825| 158|#define WS(stride, i) (stride * i) ------------------ 243| 158| T8 = ri[WS(is, 3)]; ------------------ | | 825| 158|#define WS(stride, i) (stride * i) ------------------ 244| 158| T9 = T7 - T8; 245| 158| Tl = T7 + T8; 246| 158| Tb = ri[WS(is, 4)]; ------------------ | | 825| 158|#define WS(stride, i) (stride * i) ------------------ 247| 158| Tc = ri[WS(is, 9)]; ------------------ | | 825| 158|#define WS(stride, i) (stride * i) ------------------ 248| 158| Td = Tb - Tc; 249| 158| Tn = Tb + Tc; 250| 158| } 251| 158| TU = T6 - T9; 252| 158| TV = Td - Tg; 253| 158| T1c = Tk - Tl; 254| 158| T1b = Tn - To; 255| 158| Tm = Tk + Tl; 256| 158| Tp = Tn + To; 257| 158| Tq = Tm + Tp; 258| 158| Ta = T6 + T9; 259| 158| Th = Td + Tg; 260| 158| Ti = Ta + Th; 261| 158| } 262| 158| { 263| 158| E Tw, T15, TG, T13, Tz, T16, TD, T12; 264| 158| { 265| 158| E Tu, Tv, TE, TF; 266| 158| Tu = ii[WS(is, 2)]; ------------------ | | 825| 158|#define WS(stride, i) (stride * i) ------------------ 267| 158| Tv = ii[WS(is, 7)]; ------------------ | | 825| 158|#define WS(stride, i) (stride * i) ------------------ 268| 158| Tw = Tu - Tv; 269| 158| T15 = Tu + Tv; 270| 158| TE = ii[WS(is, 6)]; ------------------ | | 825| 158|#define WS(stride, i) (stride * i) ------------------ 271| 158| TF = ii[WS(is, 1)]; ------------------ | | 825| 158|#define WS(stride, i) (stride * i) ------------------ 272| 158| TG = TE - TF; 273| 158| T13 = TE + TF; 274| 158| } 275| 158| { 276| 158| E Tx, Ty, TB, TC; 277| 158| Tx = ii[WS(is, 8)]; ------------------ | | 825| 158|#define WS(stride, i) (stride * i) ------------------ 278| 158| Ty = ii[WS(is, 3)]; ------------------ | | 825| 158|#define WS(stride, i) (stride * i) ------------------ 279| 158| Tz = Tx - Ty; 280| 158| T16 = Tx + Ty; 281| 158| TB = ii[WS(is, 4)]; ------------------ | | 825| 158|#define WS(stride, i) (stride * i) ------------------ 282| 158| TC = ii[WS(is, 9)]; ------------------ | | 825| 158|#define WS(stride, i) (stride * i) ------------------ 283| 158| TD = TB - TC; 284| 158| T12 = TB + TC; 285| 158| } 286| 158| TA = Tw - Tz; 287| 158| TH = TD - TG; 288| 158| T17 = T15 - T16; 289| 158| T14 = T12 - T13; 290| 158| T1f = T15 + T16; 291| 158| T1g = T12 + T13; 292| 158| T1h = T1f + T1g; 293| 158| TL = Tw + Tz; 294| 158| TM = TD + TG; 295| 158| TR = TL + TM; 296| 158| } 297| 158| ro[WS(os, 5)] = T3 + Ti; ------------------ | | 825| 158|#define WS(stride, i) (stride * i) ------------------ 298| 158| io[WS(os, 5)] = TQ + TR; ------------------ | | 825| 158|#define WS(stride, i) (stride * i) ------------------ 299| 158| ro[0] = Tj + Tq; 300| 158| io[0] = T1e + T1h; 301| 158| { 302| 158| E TI, TK, Tt, TJ, Tr, Ts; 303| 158| TI = FMA(KP951056516, TA, KP587785252 * TH); ------------------ | | 1131| 158|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 304| 158| TK = FNMS(KP587785252, TA, KP951056516 * TH); ------------------ | | 1134| 158|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 305| 158| Tr = KP559016994 * (Ta - Th); 306| 158| Ts = FNMS(KP250000000, Ti, T3); ------------------ | | 1134| 158|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 307| 158| Tt = Tr + Ts; 308| 158| TJ = Ts - Tr; 309| 158| ro[WS(os, 9)] = Tt - TI; ------------------ | | 825| 158|#define WS(stride, i) (stride * i) ------------------ 310| 158| ro[WS(os, 3)] = TJ + TK; ------------------ | | 825| 158|#define WS(stride, i) (stride * i) ------------------ 311| 158| ro[WS(os, 1)] = Tt + TI; ------------------ | | 825| 158|#define WS(stride, i) (stride * i) ------------------ 312| 158| ro[WS(os, 7)] = TJ - TK; ------------------ | | 825| 158|#define WS(stride, i) (stride * i) ------------------ 313| 158| } 314| 158| { 315| 158| E TW, TY, TT, TX, TN, TS; 316| 158| TW = FMA(KP951056516, TU, KP587785252 * TV); ------------------ | | 1131| 158|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 317| 158| TY = FNMS(KP587785252, TU, KP951056516 * TV); ------------------ | | 1134| 158|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 318| 158| TN = KP559016994 * (TL - TM); 319| 158| TS = FNMS(KP250000000, TR, TQ); ------------------ | | 1134| 158|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 320| 158| TT = TN + TS; 321| 158| TX = TS - TN; 322| 158| io[WS(os, 1)] = TT - TW; ------------------ | | 825| 158|#define WS(stride, i) (stride * i) ------------------ 323| 158| io[WS(os, 7)] = TY + TX; ------------------ | | 825| 158|#define WS(stride, i) (stride * i) ------------------ 324| 158| io[WS(os, 9)] = TW + TT; ------------------ | | 825| 158|#define WS(stride, i) (stride * i) ------------------ 325| 158| io[WS(os, 3)] = TX - TY; ------------------ | | 825| 158|#define WS(stride, i) (stride * i) ------------------ 326| 158| } 327| 158| { 328| 158| E T18, T1a, T11, T19, TZ, T10; 329| 158| T18 = FNMS(KP587785252, T17, KP951056516 * T14); ------------------ | | 1134| 158|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 330| 158| T1a = FMA(KP951056516, T17, KP587785252 * T14); ------------------ | | 1131| 158|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 331| 158| TZ = FNMS(KP250000000, Tq, Tj); ------------------ | | 1134| 158|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 332| 158| T10 = KP559016994 * (Tm - Tp); 333| 158| T11 = TZ - T10; 334| 158| T19 = T10 + TZ; 335| 158| ro[WS(os, 2)] = T11 - T18; ------------------ | | 825| 158|#define WS(stride, i) (stride * i) ------------------ 336| 158| ro[WS(os, 6)] = T19 + T1a; ------------------ | | 825| 158|#define WS(stride, i) (stride * i) ------------------ 337| 158| ro[WS(os, 8)] = T11 + T18; ------------------ | | 825| 158|#define WS(stride, i) (stride * i) ------------------ 338| 158| ro[WS(os, 4)] = T19 - T1a; ------------------ | | 825| 158|#define WS(stride, i) (stride * i) ------------------ 339| 158| } 340| 158| { 341| 158| E T1d, T1l, T1k, T1m, T1i, T1j; 342| 158| T1d = FNMS(KP587785252, T1c, KP951056516 * T1b); ------------------ | | 1134| 158|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 343| 158| T1l = FMA(KP951056516, T1c, KP587785252 * T1b); ------------------ | | 1131| 158|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 344| 158| T1i = FNMS(KP250000000, T1h, T1e); ------------------ | | 1134| 158|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 345| 158| T1j = KP559016994 * (T1f - T1g); 346| 158| T1k = T1i - T1j; 347| 158| T1m = T1j + T1i; 348| 158| io[WS(os, 2)] = T1d + T1k; ------------------ | | 825| 158|#define WS(stride, i) (stride * i) ------------------ 349| 158| io[WS(os, 6)] = T1m - T1l; ------------------ | | 825| 158|#define WS(stride, i) (stride * i) ------------------ 350| 158| io[WS(os, 8)] = T1k - T1d; ------------------ | | 825| 158|#define WS(stride, i) (stride * i) ------------------ 351| 158| io[WS(os, 4)] = T1l + T1m; ------------------ | | 825| 158|#define WS(stride, i) (stride * i) ------------------ 352| 158| } 353| 158| } 354| 19| } 355| 19|} fftw_codelet_n1_11: 423| 1|void X(codelet_n1_11) (planner *p) { X(kdft_register) (p, n1_11, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 424| 1|} n1_11.c:n1_11: 287| 52|{ 288| 52| DK(KP654860733, +0.654860733945285064056925072466293553183791199); ------------------ | | 1056| 52|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 52|# define K(x) ((E) x) | | ------------------ ------------------ 289| 52| DK(KP142314838, +0.142314838273285140443792668616369668791051361); ------------------ | | 1056| 52|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 52|# define K(x) ((E) x) | | ------------------ ------------------ 290| 52| DK(KP959492973, +0.959492973614497389890368057066327699062454848); ------------------ | | 1056| 52|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 52|# define K(x) ((E) x) | | ------------------ ------------------ 291| 52| DK(KP415415013, +0.415415013001886425529274149229623203524004910); ------------------ | | 1056| 52|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 52|# define K(x) ((E) x) | | ------------------ ------------------ 292| 52| DK(KP841253532, +0.841253532831181168861811648919367717513292498); ------------------ | | 1056| 52|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 52|# define K(x) ((E) x) | | ------------------ ------------------ 293| 52| DK(KP989821441, +0.989821441880932732376092037776718787376519372); ------------------ | | 1056| 52|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 52|# define K(x) ((E) x) | | ------------------ ------------------ 294| 52| DK(KP909631995, +0.909631995354518371411715383079028460060241051); ------------------ | | 1056| 52|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 52|# define K(x) ((E) x) | | ------------------ ------------------ 295| 52| DK(KP281732556, +0.281732556841429697711417915346616899035777899); ------------------ | | 1056| 52|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 52|# define K(x) ((E) x) | | ------------------ ------------------ 296| 52| DK(KP540640817, +0.540640817455597582107635954318691695431770608); ------------------ | | 1056| 52|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 52|# define K(x) ((E) x) | | ------------------ ------------------ 297| 52| DK(KP755749574, +0.755749574354258283774035843972344420179717445); ------------------ | | 1056| 52|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 52|# define K(x) ((E) x) | | ------------------ ------------------ 298| 52| { 299| 52| INT i; 300| 371| for (i = v; i > 0; i = i - 1, ri = ri + ivs, ii = ii + ivs, ro = ro + ovs, io = io + ovs, MAKE_VOLATILE_STRIDE(44, is), MAKE_VOLATILE_STRIDE(44, os)) { ------------------ | | 851| 319| (nptr <= ESTIMATED_AVAILABLE_INDEX_REGISTERS ? \ | | ------------------ | | | | 849| 638|#define ESTIMATED_AVAILABLE_INDEX_REGISTERS 16 | | ------------------ | | | Branch (851:7): [Folded, False: 319] | | ------------------ | | 852| 319| 0 : \ | | 853| 319| ((x) = (x) ^ X(an_INT_guaranteed_to_be_zero))) | | ------------------ | | | | 78| 319|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 319|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ for (i = v; i > 0; i = i - 1, ri = ri + ivs, ii = ii + ivs, ro = ro + ovs, io = io + ovs, MAKE_VOLATILE_STRIDE(44, is), MAKE_VOLATILE_STRIDE(44, os)) { ------------------ | | 851| 319| (nptr <= ESTIMATED_AVAILABLE_INDEX_REGISTERS ? \ | | ------------------ | | | | 849| 638|#define ESTIMATED_AVAILABLE_INDEX_REGISTERS 16 | | ------------------ | | | Branch (851:7): [Folded, False: 319] | | ------------------ | | 852| 319| 0 : \ | | 853| 319| ((x) = (x) ^ X(an_INT_guaranteed_to_be_zero))) | | ------------------ | | | | 78| 319|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 319|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ | Branch (300:16): [True: 319, False: 52] ------------------ 301| 319| E T1, TM, T4, TG, Tk, TR, Tw, TN, T7, TK, Ta, TH, Tn, TQ, Td; 302| 319| E TJ, Tq, TO, Tt, TP, Tg, TI; 303| 319| { 304| 319| E T2, T3, Ti, Tj; 305| 319| T1 = ri[0]; 306| 319| TM = ii[0]; 307| 319| T2 = ri[WS(is, 1)]; ------------------ | | 825| 319|#define WS(stride, i) (stride * i) ------------------ 308| 319| T3 = ri[WS(is, 10)]; ------------------ | | 825| 319|#define WS(stride, i) (stride * i) ------------------ 309| 319| T4 = T2 + T3; 310| 319| TG = T3 - T2; 311| 319| Ti = ii[WS(is, 1)]; ------------------ | | 825| 319|#define WS(stride, i) (stride * i) ------------------ 312| 319| Tj = ii[WS(is, 10)]; ------------------ | | 825| 319|#define WS(stride, i) (stride * i) ------------------ 313| 319| Tk = Ti - Tj; 314| 319| TR = Ti + Tj; 315| 319| { 316| 319| E Tu, Tv, T5, T6; 317| 319| Tu = ii[WS(is, 2)]; ------------------ | | 825| 319|#define WS(stride, i) (stride * i) ------------------ 318| 319| Tv = ii[WS(is, 9)]; ------------------ | | 825| 319|#define WS(stride, i) (stride * i) ------------------ 319| 319| Tw = Tu - Tv; 320| 319| TN = Tu + Tv; 321| 319| T5 = ri[WS(is, 2)]; ------------------ | | 825| 319|#define WS(stride, i) (stride * i) ------------------ 322| 319| T6 = ri[WS(is, 9)]; ------------------ | | 825| 319|#define WS(stride, i) (stride * i) ------------------ 323| 319| T7 = T5 + T6; 324| 319| TK = T6 - T5; 325| 319| } 326| 319| } 327| 319| { 328| 319| E T8, T9, To, Tp; 329| 319| T8 = ri[WS(is, 3)]; ------------------ | | 825| 319|#define WS(stride, i) (stride * i) ------------------ 330| 319| T9 = ri[WS(is, 8)]; ------------------ | | 825| 319|#define WS(stride, i) (stride * i) ------------------ 331| 319| Ta = T8 + T9; 332| 319| TH = T9 - T8; 333| 319| { 334| 319| E Tl, Tm, Tb, Tc; 335| 319| Tl = ii[WS(is, 3)]; ------------------ | | 825| 319|#define WS(stride, i) (stride * i) ------------------ 336| 319| Tm = ii[WS(is, 8)]; ------------------ | | 825| 319|#define WS(stride, i) (stride * i) ------------------ 337| 319| Tn = Tl - Tm; 338| 319| TQ = Tl + Tm; 339| 319| Tb = ri[WS(is, 4)]; ------------------ | | 825| 319|#define WS(stride, i) (stride * i) ------------------ 340| 319| Tc = ri[WS(is, 7)]; ------------------ | | 825| 319|#define WS(stride, i) (stride * i) ------------------ 341| 319| Td = Tb + Tc; 342| 319| TJ = Tc - Tb; 343| 319| } 344| 319| To = ii[WS(is, 4)]; ------------------ | | 825| 319|#define WS(stride, i) (stride * i) ------------------ 345| 319| Tp = ii[WS(is, 7)]; ------------------ | | 825| 319|#define WS(stride, i) (stride * i) ------------------ 346| 319| Tq = To - Tp; 347| 319| TO = To + Tp; 348| 319| { 349| 319| E Tr, Ts, Te, Tf; 350| 319| Tr = ii[WS(is, 5)]; ------------------ | | 825| 319|#define WS(stride, i) (stride * i) ------------------ 351| 319| Ts = ii[WS(is, 6)]; ------------------ | | 825| 319|#define WS(stride, i) (stride * i) ------------------ 352| 319| Tt = Tr - Ts; 353| 319| TP = Tr + Ts; 354| 319| Te = ri[WS(is, 5)]; ------------------ | | 825| 319|#define WS(stride, i) (stride * i) ------------------ 355| 319| Tf = ri[WS(is, 6)]; ------------------ | | 825| 319|#define WS(stride, i) (stride * i) ------------------ 356| 319| Tg = Te + Tf; 357| 319| TI = Tf - Te; 358| 319| } 359| 319| } 360| 319| { 361| 319| E Tx, Th, TZ, T10; 362| 319| ro[0] = T1 + T4 + T7 + Ta + Td + Tg; 363| 319| io[0] = TM + TR + TN + TQ + TO + TP; 364| 319| Tx = FMA(KP755749574, Tk, KP540640817 * Tn) + FNMS(KP909631995, Tt, KP281732556 * Tq) - (KP989821441 * Tw); ------------------ | | 1131| 319|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ Tx = FMA(KP755749574, Tk, KP540640817 * Tn) + FNMS(KP909631995, Tt, KP281732556 * Tq) - (KP989821441 * Tw); ------------------ | | 1134| 319|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 365| 319| Th = FMA(KP841253532, Ta, T1) + FNMS(KP959492973, Td, KP415415013 * Tg) + FNMA(KP142314838, T7, KP654860733 * T4); ------------------ | | 1131| 319|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ Th = FMA(KP841253532, Ta, T1) + FNMS(KP959492973, Td, KP415415013 * Tg) + FNMA(KP142314838, T7, KP654860733 * T4); ------------------ | | 1134| 319|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ Th = FMA(KP841253532, Ta, T1) + FNMS(KP959492973, Td, KP415415013 * Tg) + FNMA(KP142314838, T7, KP654860733 * T4); ------------------ | | 1133| 319|#define FNMA(a, b, c) (- (((a) * (b)) + (c))) ------------------ 366| 319| ro[WS(os, 7)] = Th - Tx; ------------------ | | 825| 319|#define WS(stride, i) (stride * i) ------------------ 367| 319| ro[WS(os, 4)] = Th + Tx; ------------------ | | 825| 319|#define WS(stride, i) (stride * i) ------------------ 368| 319| TZ = FMA(KP755749574, TG, KP540640817 * TH) + FNMS(KP909631995, TI, KP281732556 * TJ) - (KP989821441 * TK); ------------------ | | 1131| 319|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ TZ = FMA(KP755749574, TG, KP540640817 * TH) + FNMS(KP909631995, TI, KP281732556 * TJ) - (KP989821441 * TK); ------------------ | | 1134| 319|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 369| 319| T10 = FMA(KP841253532, TQ, TM) + FNMS(KP959492973, TO, KP415415013 * TP) + FNMA(KP142314838, TN, KP654860733 * TR); ------------------ | | 1131| 319|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ T10 = FMA(KP841253532, TQ, TM) + FNMS(KP959492973, TO, KP415415013 * TP) + FNMA(KP142314838, TN, KP654860733 * TR); ------------------ | | 1134| 319|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ T10 = FMA(KP841253532, TQ, TM) + FNMS(KP959492973, TO, KP415415013 * TP) + FNMA(KP142314838, TN, KP654860733 * TR); ------------------ | | 1133| 319|#define FNMA(a, b, c) (- (((a) * (b)) + (c))) ------------------ 370| 319| io[WS(os, 4)] = TZ + T10; ------------------ | | 825| 319|#define WS(stride, i) (stride * i) ------------------ 371| 319| io[WS(os, 7)] = T10 - TZ; ------------------ | | 825| 319|#define WS(stride, i) (stride * i) ------------------ 372| 319| { 373| 319| E TX, TY, Tz, Ty; 374| 319| TX = FMA(KP909631995, TG, KP755749574 * TK) + FNMA(KP540640817, TI, KP989821441 * TJ) - (KP281732556 * TH); ------------------ | | 1131| 319|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ TX = FMA(KP909631995, TG, KP755749574 * TK) + FNMA(KP540640817, TI, KP989821441 * TJ) - (KP281732556 * TH); ------------------ | | 1133| 319|#define FNMA(a, b, c) (- (((a) * (b)) + (c))) ------------------ 375| 319| TY = FMA(KP415415013, TR, TM) + FNMS(KP142314838, TO, KP841253532 * TP) + FNMA(KP959492973, TQ, KP654860733 * TN); ------------------ | | 1131| 319|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ TY = FMA(KP415415013, TR, TM) + FNMS(KP142314838, TO, KP841253532 * TP) + FNMA(KP959492973, TQ, KP654860733 * TN); ------------------ | | 1134| 319|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ TY = FMA(KP415415013, TR, TM) + FNMS(KP142314838, TO, KP841253532 * TP) + FNMA(KP959492973, TQ, KP654860733 * TN); ------------------ | | 1133| 319|#define FNMA(a, b, c) (- (((a) * (b)) + (c))) ------------------ 376| 319| io[WS(os, 2)] = TX + TY; ------------------ | | 825| 319|#define WS(stride, i) (stride * i) ------------------ 377| 319| io[WS(os, 9)] = TY - TX; ------------------ | | 825| 319|#define WS(stride, i) (stride * i) ------------------ 378| 319| Tz = FMA(KP909631995, Tk, KP755749574 * Tw) + FNMA(KP540640817, Tt, KP989821441 * Tq) - (KP281732556 * Tn); ------------------ | | 1131| 319|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ Tz = FMA(KP909631995, Tk, KP755749574 * Tw) + FNMA(KP540640817, Tt, KP989821441 * Tq) - (KP281732556 * Tn); ------------------ | | 1133| 319|#define FNMA(a, b, c) (- (((a) * (b)) + (c))) ------------------ 379| 319| Ty = FMA(KP415415013, T4, T1) + FNMS(KP142314838, Td, KP841253532 * Tg) + FNMA(KP959492973, Ta, KP654860733 * T7); ------------------ | | 1131| 319|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ Ty = FMA(KP415415013, T4, T1) + FNMS(KP142314838, Td, KP841253532 * Tg) + FNMA(KP959492973, Ta, KP654860733 * T7); ------------------ | | 1134| 319|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ Ty = FMA(KP415415013, T4, T1) + FNMS(KP142314838, Td, KP841253532 * Tg) + FNMA(KP959492973, Ta, KP654860733 * T7); ------------------ | | 1133| 319|#define FNMA(a, b, c) (- (((a) * (b)) + (c))) ------------------ 380| 319| ro[WS(os, 9)] = Ty - Tz; ------------------ | | 825| 319|#define WS(stride, i) (stride * i) ------------------ 381| 319| ro[WS(os, 2)] = Ty + Tz; ------------------ | | 825| 319|#define WS(stride, i) (stride * i) ------------------ 382| 319| } 383| 319| } 384| 319| { 385| 319| E TB, TA, TT, TU; 386| 319| TB = FMA(KP540640817, Tk, KP909631995 * Tw) + FMA(KP989821441, Tn, KP755749574 * Tq) + (KP281732556 * Tt); ------------------ | | 1131| 319|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ TB = FMA(KP540640817, Tk, KP909631995 * Tw) + FMA(KP989821441, Tn, KP755749574 * Tq) + (KP281732556 * Tt); ------------------ | | 1131| 319|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 387| 319| TA = FMA(KP841253532, T4, T1) + FNMS(KP959492973, Tg, KP415415013 * T7) + FNMA(KP654860733, Td, KP142314838 * Ta); ------------------ | | 1131| 319|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ TA = FMA(KP841253532, T4, T1) + FNMS(KP959492973, Tg, KP415415013 * T7) + FNMA(KP654860733, Td, KP142314838 * Ta); ------------------ | | 1134| 319|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ TA = FMA(KP841253532, T4, T1) + FNMS(KP959492973, Tg, KP415415013 * T7) + FNMA(KP654860733, Td, KP142314838 * Ta); ------------------ | | 1133| 319|#define FNMA(a, b, c) (- (((a) * (b)) + (c))) ------------------ 388| 319| ro[WS(os, 10)] = TA - TB; ------------------ | | 825| 319|#define WS(stride, i) (stride * i) ------------------ 389| 319| ro[WS(os, 1)] = TA + TB; ------------------ | | 825| 319|#define WS(stride, i) (stride * i) ------------------ 390| 319| { 391| 319| E TV, TW, TD, TC; 392| 319| TV = FMA(KP540640817, TG, KP909631995 * TK) + FMA(KP989821441, TH, KP755749574 * TJ) + (KP281732556 * TI); ------------------ | | 1131| 319|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ TV = FMA(KP540640817, TG, KP909631995 * TK) + FMA(KP989821441, TH, KP755749574 * TJ) + (KP281732556 * TI); ------------------ | | 1131| 319|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 393| 319| TW = FMA(KP841253532, TR, TM) + FNMS(KP959492973, TP, KP415415013 * TN) + FNMA(KP654860733, TO, KP142314838 * TQ); ------------------ | | 1131| 319|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ TW = FMA(KP841253532, TR, TM) + FNMS(KP959492973, TP, KP415415013 * TN) + FNMA(KP654860733, TO, KP142314838 * TQ); ------------------ | | 1134| 319|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ TW = FMA(KP841253532, TR, TM) + FNMS(KP959492973, TP, KP415415013 * TN) + FNMA(KP654860733, TO, KP142314838 * TQ); ------------------ | | 1133| 319|#define FNMA(a, b, c) (- (((a) * (b)) + (c))) ------------------ 394| 319| io[WS(os, 1)] = TV + TW; ------------------ | | 825| 319|#define WS(stride, i) (stride * i) ------------------ 395| 319| io[WS(os, 10)] = TW - TV; ------------------ | | 825| 319|#define WS(stride, i) (stride * i) ------------------ 396| 319| TD = FMA(KP989821441, Tk, KP540640817 * Tq) + FNMS(KP909631995, Tn, KP755749574 * Tt) - (KP281732556 * Tw); ------------------ | | 1131| 319|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ TD = FMA(KP989821441, Tk, KP540640817 * Tq) + FNMS(KP909631995, Tn, KP755749574 * Tt) - (KP281732556 * Tw); ------------------ | | 1134| 319|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 397| 319| TC = FMA(KP415415013, Ta, T1) + FNMS(KP654860733, Tg, KP841253532 * Td) + FNMA(KP959492973, T7, KP142314838 * T4); ------------------ | | 1131| 319|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ TC = FMA(KP415415013, Ta, T1) + FNMS(KP654860733, Tg, KP841253532 * Td) + FNMA(KP959492973, T7, KP142314838 * T4); ------------------ | | 1134| 319|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ TC = FMA(KP415415013, Ta, T1) + FNMS(KP654860733, Tg, KP841253532 * Td) + FNMA(KP959492973, T7, KP142314838 * T4); ------------------ | | 1133| 319|#define FNMA(a, b, c) (- (((a) * (b)) + (c))) ------------------ 398| 319| ro[WS(os, 8)] = TC - TD; ------------------ | | 825| 319|#define WS(stride, i) (stride * i) ------------------ 399| 319| ro[WS(os, 3)] = TC + TD; ------------------ | | 825| 319|#define WS(stride, i) (stride * i) ------------------ 400| 319| } 401| 319| TT = FMA(KP989821441, TG, KP540640817 * TJ) + FNMS(KP909631995, TH, KP755749574 * TI) - (KP281732556 * TK); ------------------ | | 1131| 319|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ TT = FMA(KP989821441, TG, KP540640817 * TJ) + FNMS(KP909631995, TH, KP755749574 * TI) - (KP281732556 * TK); ------------------ | | 1134| 319|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 402| 319| TU = FMA(KP415415013, TQ, TM) + FNMS(KP654860733, TP, KP841253532 * TO) + FNMA(KP959492973, TN, KP142314838 * TR); ------------------ | | 1131| 319|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ TU = FMA(KP415415013, TQ, TM) + FNMS(KP654860733, TP, KP841253532 * TO) + FNMA(KP959492973, TN, KP142314838 * TR); ------------------ | | 1134| 319|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ TU = FMA(KP415415013, TQ, TM) + FNMS(KP654860733, TP, KP841253532 * TO) + FNMA(KP959492973, TN, KP142314838 * TR); ------------------ | | 1133| 319|#define FNMA(a, b, c) (- (((a) * (b)) + (c))) ------------------ 403| 319| io[WS(os, 3)] = TT + TU; ------------------ | | 825| 319|#define WS(stride, i) (stride * i) ------------------ 404| 319| io[WS(os, 8)] = TU - TT; ------------------ | | 825| 319|#define WS(stride, i) (stride * i) ------------------ 405| 319| { 406| 319| E TL, TS, TF, TE; 407| 319| TL = FMA(KP281732556, TG, KP755749574 * TH) + FNMS(KP909631995, TJ, KP989821441 * TI) - (KP540640817 * TK); ------------------ | | 1131| 319|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ TL = FMA(KP281732556, TG, KP755749574 * TH) + FNMS(KP909631995, TJ, KP989821441 * TI) - (KP540640817 * TK); ------------------ | | 1134| 319|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 408| 319| TS = FMA(KP841253532, TN, TM) + FNMS(KP142314838, TP, KP415415013 * TO) + FNMA(KP654860733, TQ, KP959492973 * TR); ------------------ | | 1131| 319|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ TS = FMA(KP841253532, TN, TM) + FNMS(KP142314838, TP, KP415415013 * TO) + FNMA(KP654860733, TQ, KP959492973 * TR); ------------------ | | 1134| 319|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ TS = FMA(KP841253532, TN, TM) + FNMS(KP142314838, TP, KP415415013 * TO) + FNMA(KP654860733, TQ, KP959492973 * TR); ------------------ | | 1133| 319|#define FNMA(a, b, c) (- (((a) * (b)) + (c))) ------------------ 409| 319| io[WS(os, 5)] = TL + TS; ------------------ | | 825| 319|#define WS(stride, i) (stride * i) ------------------ 410| 319| io[WS(os, 6)] = TS - TL; ------------------ | | 825| 319|#define WS(stride, i) (stride * i) ------------------ 411| 319| TF = FMA(KP281732556, Tk, KP755749574 * Tn) + FNMS(KP909631995, Tq, KP989821441 * Tt) - (KP540640817 * Tw); ------------------ | | 1131| 319|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ TF = FMA(KP281732556, Tk, KP755749574 * Tn) + FNMS(KP909631995, Tq, KP989821441 * Tt) - (KP540640817 * Tw); ------------------ | | 1134| 319|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 412| 319| TE = FMA(KP841253532, T7, T1) + FNMS(KP142314838, Tg, KP415415013 * Td) + FNMA(KP654860733, Ta, KP959492973 * T4); ------------------ | | 1131| 319|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ TE = FMA(KP841253532, T7, T1) + FNMS(KP142314838, Tg, KP415415013 * Td) + FNMA(KP654860733, Ta, KP959492973 * T4); ------------------ | | 1134| 319|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ TE = FMA(KP841253532, T7, T1) + FNMS(KP142314838, Tg, KP415415013 * Td) + FNMA(KP654860733, Ta, KP959492973 * T4); ------------------ | | 1133| 319|#define FNMA(a, b, c) (- (((a) * (b)) + (c))) ------------------ 413| 319| ro[WS(os, 6)] = TE - TF; ------------------ | | 825| 319|#define WS(stride, i) (stride * i) ------------------ 414| 319| ro[WS(os, 5)] = TE + TF; ------------------ | | 825| 319|#define WS(stride, i) (stride * i) ------------------ 415| 319| } 416| 319| } 417| 319| } 418| 52| } 419| 52|} fftw_codelet_n1_12: 417| 1|void X(codelet_n1_12) (planner *p) { X(kdft_register) (p, n1_12, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 418| 1|} n1_12.c:n1_12: 235| 181|{ 236| 181| DK(KP866025403, +0.866025403784438646763723170752936183471402627); ------------------ | | 1056| 181|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 181|# define K(x) ((E) x) | | ------------------ ------------------ 237| 181| DK(KP500000000, +0.500000000000000000000000000000000000000000000); ------------------ | | 1056| 181|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 181|# define K(x) ((E) x) | | ------------------ ------------------ 238| 181| { 239| 181| INT i; 240| 1.03k| for (i = v; i > 0; i = i - 1, ri = ri + ivs, ii = ii + ivs, ro = ro + ovs, io = io + ovs, MAKE_VOLATILE_STRIDE(48, is), MAKE_VOLATILE_STRIDE(48, os)) { ------------------ | | 851| 851| (nptr <= ESTIMATED_AVAILABLE_INDEX_REGISTERS ? \ | | ------------------ | | | | 849| 1.70k|#define ESTIMATED_AVAILABLE_INDEX_REGISTERS 16 | | ------------------ | | | Branch (851:7): [Folded, False: 851] | | ------------------ | | 852| 851| 0 : \ | | 853| 851| ((x) = (x) ^ X(an_INT_guaranteed_to_be_zero))) | | ------------------ | | | | 78| 851|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 851|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ for (i = v; i > 0; i = i - 1, ri = ri + ivs, ii = ii + ivs, ro = ro + ovs, io = io + ovs, MAKE_VOLATILE_STRIDE(48, is), MAKE_VOLATILE_STRIDE(48, os)) { ------------------ | | 851| 851| (nptr <= ESTIMATED_AVAILABLE_INDEX_REGISTERS ? \ | | ------------------ | | | | 849| 1.70k|#define ESTIMATED_AVAILABLE_INDEX_REGISTERS 16 | | ------------------ | | | Branch (851:7): [Folded, False: 851] | | ------------------ | | 852| 851| 0 : \ | | 853| 851| ((x) = (x) ^ X(an_INT_guaranteed_to_be_zero))) | | ------------------ | | | | 78| 851|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 851|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ | Branch (240:16): [True: 851, False: 181] ------------------ 241| 851| E T5, TR, TA, Ts, TS, Tz, Ta, TU, TD, Tx, TV, TC, Tg, T1a, TG; 242| 851| E TJ, T1u, T1d, Tl, T1f, TL, TO, T1v, T1i; 243| 851| { 244| 851| E T1, T2, T3, T4; 245| 851| T1 = ri[0]; 246| 851| T2 = ri[WS(is, 4)]; ------------------ | | 825| 851|#define WS(stride, i) (stride * i) ------------------ 247| 851| T3 = ri[WS(is, 8)]; ------------------ | | 825| 851|#define WS(stride, i) (stride * i) ------------------ 248| 851| T4 = T2 + T3; 249| 851| T5 = T1 + T4; 250| 851| TR = FNMS(KP500000000, T4, T1); ------------------ | | 1134| 851|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 251| 851| TA = KP866025403 * (T3 - T2); 252| 851| } 253| 851| { 254| 851| E To, Tp, Tq, Tr; 255| 851| To = ii[0]; 256| 851| Tp = ii[WS(is, 4)]; ------------------ | | 825| 851|#define WS(stride, i) (stride * i) ------------------ 257| 851| Tq = ii[WS(is, 8)]; ------------------ | | 825| 851|#define WS(stride, i) (stride * i) ------------------ 258| 851| Tr = Tp + Tq; 259| 851| Ts = To + Tr; 260| 851| TS = KP866025403 * (Tp - Tq); 261| 851| Tz = FNMS(KP500000000, Tr, To); ------------------ | | 1134| 851|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 262| 851| } 263| 851| { 264| 851| E T6, T7, T8, T9; 265| 851| T6 = ri[WS(is, 6)]; ------------------ | | 825| 851|#define WS(stride, i) (stride * i) ------------------ 266| 851| T7 = ri[WS(is, 10)]; ------------------ | | 825| 851|#define WS(stride, i) (stride * i) ------------------ 267| 851| T8 = ri[WS(is, 2)]; ------------------ | | 825| 851|#define WS(stride, i) (stride * i) ------------------ 268| 851| T9 = T7 + T8; 269| 851| Ta = T6 + T9; 270| 851| TU = FNMS(KP500000000, T9, T6); ------------------ | | 1134| 851|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 271| 851| TD = KP866025403 * (T8 - T7); 272| 851| } 273| 851| { 274| 851| E Tt, Tu, Tv, Tw; 275| 851| Tt = ii[WS(is, 6)]; ------------------ | | 825| 851|#define WS(stride, i) (stride * i) ------------------ 276| 851| Tu = ii[WS(is, 10)]; ------------------ | | 825| 851|#define WS(stride, i) (stride * i) ------------------ 277| 851| Tv = ii[WS(is, 2)]; ------------------ | | 825| 851|#define WS(stride, i) (stride * i) ------------------ 278| 851| Tw = Tu + Tv; 279| 851| Tx = Tt + Tw; 280| 851| TV = KP866025403 * (Tu - Tv); 281| 851| TC = FNMS(KP500000000, Tw, Tt); ------------------ | | 1134| 851|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 282| 851| } 283| 851| { 284| 851| E Tc, Td, Te, Tf; 285| 851| Tc = ri[WS(is, 3)]; ------------------ | | 825| 851|#define WS(stride, i) (stride * i) ------------------ 286| 851| Td = ri[WS(is, 7)]; ------------------ | | 825| 851|#define WS(stride, i) (stride * i) ------------------ 287| 851| Te = ri[WS(is, 11)]; ------------------ | | 825| 851|#define WS(stride, i) (stride * i) ------------------ 288| 851| Tf = Td + Te; 289| 851| Tg = Tc + Tf; 290| 851| T1a = KP866025403 * (Te - Td); 291| 851| TG = FNMS(KP500000000, Tf, Tc); ------------------ | | 1134| 851|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 292| 851| } 293| 851| { 294| 851| E T1b, TH, TI, T1c; 295| 851| T1b = ii[WS(is, 3)]; ------------------ | | 825| 851|#define WS(stride, i) (stride * i) ------------------ 296| 851| TH = ii[WS(is, 7)]; ------------------ | | 825| 851|#define WS(stride, i) (stride * i) ------------------ 297| 851| TI = ii[WS(is, 11)]; ------------------ | | 825| 851|#define WS(stride, i) (stride * i) ------------------ 298| 851| T1c = TH + TI; 299| 851| TJ = KP866025403 * (TH - TI); 300| 851| T1u = T1b + T1c; 301| 851| T1d = FNMS(KP500000000, T1c, T1b); ------------------ | | 1134| 851|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 302| 851| } 303| 851| { 304| 851| E Th, Ti, Tj, Tk; 305| 851| Th = ri[WS(is, 9)]; ------------------ | | 825| 851|#define WS(stride, i) (stride * i) ------------------ 306| 851| Ti = ri[WS(is, 1)]; ------------------ | | 825| 851|#define WS(stride, i) (stride * i) ------------------ 307| 851| Tj = ri[WS(is, 5)]; ------------------ | | 825| 851|#define WS(stride, i) (stride * i) ------------------ 308| 851| Tk = Ti + Tj; 309| 851| Tl = Th + Tk; 310| 851| T1f = KP866025403 * (Tj - Ti); 311| 851| TL = FNMS(KP500000000, Tk, Th); ------------------ | | 1134| 851|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 312| 851| } 313| 851| { 314| 851| E T1g, TM, TN, T1h; 315| 851| T1g = ii[WS(is, 9)]; ------------------ | | 825| 851|#define WS(stride, i) (stride * i) ------------------ 316| 851| TM = ii[WS(is, 1)]; ------------------ | | 825| 851|#define WS(stride, i) (stride * i) ------------------ 317| 851| TN = ii[WS(is, 5)]; ------------------ | | 825| 851|#define WS(stride, i) (stride * i) ------------------ 318| 851| T1h = TM + TN; 319| 851| TO = KP866025403 * (TM - TN); 320| 851| T1v = T1g + T1h; 321| 851| T1i = FNMS(KP500000000, T1h, T1g); ------------------ | | 1134| 851|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 322| 851| } 323| 851| { 324| 851| E Tb, Tm, T1t, T1w; 325| 851| Tb = T5 + Ta; 326| 851| Tm = Tg + Tl; 327| 851| ro[WS(os, 6)] = Tb - Tm; ------------------ | | 825| 851|#define WS(stride, i) (stride * i) ------------------ 328| 851| ro[0] = Tb + Tm; 329| 851| { 330| 851| E T1x, T1y, Tn, Ty; 331| 851| T1x = Ts + Tx; 332| 851| T1y = T1u + T1v; 333| 851| io[WS(os, 6)] = T1x - T1y; ------------------ | | 825| 851|#define WS(stride, i) (stride * i) ------------------ 334| 851| io[0] = T1x + T1y; 335| 851| Tn = Tg - Tl; 336| 851| Ty = Ts - Tx; 337| 851| io[WS(os, 3)] = Tn + Ty; ------------------ | | 825| 851|#define WS(stride, i) (stride * i) ------------------ 338| 851| io[WS(os, 9)] = Ty - Tn; ------------------ | | 825| 851|#define WS(stride, i) (stride * i) ------------------ 339| 851| } 340| 851| T1t = T5 - Ta; 341| 851| T1w = T1u - T1v; 342| 851| ro[WS(os, 3)] = T1t - T1w; ------------------ | | 825| 851|#define WS(stride, i) (stride * i) ------------------ 343| 851| ro[WS(os, 9)] = T1t + T1w; ------------------ | | 825| 851|#define WS(stride, i) (stride * i) ------------------ 344| 851| { 345| 851| E T11, T1l, T1k, T1m, T14, T18, T17, T19; 346| 851| { 347| 851| E TZ, T10, T1e, T1j; 348| 851| TZ = TA + Tz; 349| 851| T10 = TD + TC; 350| 851| T11 = TZ - T10; 351| 851| T1l = TZ + T10; 352| 851| T1e = T1a + T1d; 353| 851| T1j = T1f + T1i; 354| 851| T1k = T1e - T1j; 355| 851| T1m = T1e + T1j; 356| 851| } 357| 851| { 358| 851| E T12, T13, T15, T16; 359| 851| T12 = TG + TJ; 360| 851| T13 = TL + TO; 361| 851| T14 = T12 - T13; 362| 851| T18 = T12 + T13; 363| 851| T15 = TR + TS; 364| 851| T16 = TU + TV; 365| 851| T17 = T15 + T16; 366| 851| T19 = T15 - T16; 367| 851| } 368| 851| io[WS(os, 1)] = T11 - T14; ------------------ | | 825| 851|#define WS(stride, i) (stride * i) ------------------ 369| 851| ro[WS(os, 1)] = T19 + T1k; ------------------ | | 825| 851|#define WS(stride, i) (stride * i) ------------------ 370| 851| io[WS(os, 7)] = T11 + T14; ------------------ | | 825| 851|#define WS(stride, i) (stride * i) ------------------ 371| 851| ro[WS(os, 7)] = T19 - T1k; ------------------ | | 825| 851|#define WS(stride, i) (stride * i) ------------------ 372| 851| ro[WS(os, 10)] = T17 - T18; ------------------ | | 825| 851|#define WS(stride, i) (stride * i) ------------------ 373| 851| io[WS(os, 10)] = T1l - T1m; ------------------ | | 825| 851|#define WS(stride, i) (stride * i) ------------------ 374| 851| ro[WS(os, 4)] = T17 + T18; ------------------ | | 825| 851|#define WS(stride, i) (stride * i) ------------------ 375| 851| io[WS(os, 4)] = T1l + T1m; ------------------ | | 825| 851|#define WS(stride, i) (stride * i) ------------------ 376| 851| } 377| 851| { 378| 851| E TF, T1r, T1q, T1s, TQ, TY, TX, T1n; 379| 851| { 380| 851| E TB, TE, T1o, T1p; 381| 851| TB = Tz - TA; 382| 851| TE = TC - TD; 383| 851| TF = TB - TE; 384| 851| T1r = TB + TE; 385| 851| T1o = T1d - T1a; 386| 851| T1p = T1i - T1f; 387| 851| T1q = T1o - T1p; 388| 851| T1s = T1o + T1p; 389| 851| } 390| 851| { 391| 851| E TK, TP, TT, TW; 392| 851| TK = TG - TJ; 393| 851| TP = TL - TO; 394| 851| TQ = TK - TP; 395| 851| TY = TK + TP; 396| 851| TT = TR - TS; 397| 851| TW = TU - TV; 398| 851| TX = TT + TW; 399| 851| T1n = TT - TW; 400| 851| } 401| 851| io[WS(os, 5)] = TF - TQ; ------------------ | | 825| 851|#define WS(stride, i) (stride * i) ------------------ 402| 851| ro[WS(os, 5)] = T1n + T1q; ------------------ | | 825| 851|#define WS(stride, i) (stride * i) ------------------ 403| 851| io[WS(os, 11)] = TF + TQ; ------------------ | | 825| 851|#define WS(stride, i) (stride * i) ------------------ 404| 851| ro[WS(os, 11)] = T1n - T1q; ------------------ | | 825| 851|#define WS(stride, i) (stride * i) ------------------ 405| 851| ro[WS(os, 2)] = TX - TY; ------------------ | | 825| 851|#define WS(stride, i) (stride * i) ------------------ 406| 851| io[WS(os, 2)] = T1r - T1s; ------------------ | | 825| 851|#define WS(stride, i) (stride * i) ------------------ 407| 851| ro[WS(os, 8)] = TX + TY; ------------------ | | 825| 851|#define WS(stride, i) (stride * i) ------------------ 408| 851| io[WS(os, 8)] = T1r + T1s; ------------------ | | 825| 851|#define WS(stride, i) (stride * i) ------------------ 409| 851| } 410| 851| } 411| 851| } 412| 181| } 413| 181|} fftw_codelet_n1_13: 678| 1|void X(codelet_n1_13) (planner *p) { X(kdft_register) (p, n1_13, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 679| 1|} n1_13.c:n1_13: 368| 36|{ 369| 36| DK(KP2_000000000, +2.000000000000000000000000000000000000000000000); ------------------ | | 1056| 36|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 36|# define K(x) ((E) x) | | ------------------ ------------------ 370| 36| DK(KP083333333, +0.083333333333333333333333333333333333333333333); ------------------ | | 1056| 36|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 36|# define K(x) ((E) x) | | ------------------ ------------------ 371| 36| DK(KP251768516, +0.251768516431883313623436926934233488546674281); ------------------ | | 1056| 36|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 36|# define K(x) ((E) x) | | ------------------ ------------------ 372| 36| DK(KP075902986, +0.075902986037193865983102897245103540356428373); ------------------ | | 1056| 36|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 36|# define K(x) ((E) x) | | ------------------ ------------------ 373| 36| DK(KP132983124, +0.132983124607418643793760531921092974399165133); ------------------ | | 1056| 36|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 36|# define K(x) ((E) x) | | ------------------ ------------------ 374| 36| DK(KP258260390, +0.258260390311744861420450644284508567852516811); ------------------ | | 1056| 36|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 36|# define K(x) ((E) x) | | ------------------ ------------------ 375| 36| DK(KP1_732050807, +1.732050807568877293527446341505872366942805254); ------------------ | | 1056| 36|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 36|# define K(x) ((E) x) | | ------------------ ------------------ 376| 36| DK(KP300238635, +0.300238635966332641462884626667381504676006424); ------------------ | | 1056| 36|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 36|# define K(x) ((E) x) | | ------------------ ------------------ 377| 36| DK(KP011599105, +0.011599105605768290721655456654083252189827041); ------------------ | | 1056| 36|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 36|# define K(x) ((E) x) | | ------------------ ------------------ 378| 36| DK(KP156891391, +0.156891391051584611046832726756003269660212636); ------------------ | | 1056| 36|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 36|# define K(x) ((E) x) | | ------------------ ------------------ 379| 36| DK(KP256247671, +0.256247671582936600958684654061725059144125175); ------------------ | | 1056| 36|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 36|# define K(x) ((E) x) | | ------------------ ------------------ 380| 36| DK(KP174138601, +0.174138601152135905005660794929264742616964676); ------------------ | | 1056| 36|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 36|# define K(x) ((E) x) | | ------------------ ------------------ 381| 36| DK(KP575140729, +0.575140729474003121368385547455453388461001608); ------------------ | | 1056| 36|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 36|# define K(x) ((E) x) | | ------------------ ------------------ 382| 36| DK(KP503537032, +0.503537032863766627246873853868466977093348562); ------------------ | | 1056| 36|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 36|# define K(x) ((E) x) | | ------------------ ------------------ 383| 36| DK(KP113854479, +0.113854479055790798974654345867655310534642560); ------------------ | | 1056| 36|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 36|# define K(x) ((E) x) | | ------------------ ------------------ 384| 36| DK(KP265966249, +0.265966249214837287587521063842185948798330267); ------------------ | | 1056| 36|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 36|# define K(x) ((E) x) | | ------------------ ------------------ 385| 36| DK(KP387390585, +0.387390585467617292130675966426762851778775217); ------------------ | | 1056| 36|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 36|# define K(x) ((E) x) | | ------------------ ------------------ 386| 36| DK(KP866025403, +0.866025403784438646763723170752936183471402627); ------------------ | | 1056| 36|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 36|# define K(x) ((E) x) | | ------------------ ------------------ 387| 36| DK(KP300462606, +0.300462606288665774426601772289207995520941381); ------------------ | | 1056| 36|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 36|# define K(x) ((E) x) | | ------------------ ------------------ 388| 36| DK(KP500000000, +0.500000000000000000000000000000000000000000000); ------------------ | | 1056| 36|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 36|# define K(x) ((E) x) | | ------------------ ------------------ 389| 36| { 390| 36| INT i; 391| 201| for (i = v; i > 0; i = i - 1, ri = ri + ivs, ii = ii + ivs, ro = ro + ovs, io = io + ovs, MAKE_VOLATILE_STRIDE(52, is), MAKE_VOLATILE_STRIDE(52, os)) { ------------------ | | 851| 165| (nptr <= ESTIMATED_AVAILABLE_INDEX_REGISTERS ? \ | | ------------------ | | | | 849| 330|#define ESTIMATED_AVAILABLE_INDEX_REGISTERS 16 | | ------------------ | | | Branch (851:7): [Folded, False: 165] | | ------------------ | | 852| 165| 0 : \ | | 853| 165| ((x) = (x) ^ X(an_INT_guaranteed_to_be_zero))) | | ------------------ | | | | 78| 165|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 165|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ for (i = v; i > 0; i = i - 1, ri = ri + ivs, ii = ii + ivs, ro = ro + ovs, io = io + ovs, MAKE_VOLATILE_STRIDE(52, is), MAKE_VOLATILE_STRIDE(52, os)) { ------------------ | | 851| 165| (nptr <= ESTIMATED_AVAILABLE_INDEX_REGISTERS ? \ | | ------------------ | | | | 849| 330|#define ESTIMATED_AVAILABLE_INDEX_REGISTERS 16 | | ------------------ | | | Branch (851:7): [Folded, False: 165] | | ------------------ | | 852| 165| 0 : \ | | 853| 165| ((x) = (x) ^ X(an_INT_guaranteed_to_be_zero))) | | ------------------ | | | | 78| 165|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 165|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ | Branch (391:16): [True: 165, False: 36] ------------------ 392| 165| E T1, T1q, Tt, Tu, To, T22, T20, T24, TF, TH, TA, TI, T1X, T25, T2a; 393| 165| E T2d, T18, T1n, T2k, T2n, T1l, T1r, T1f, T1o, T2h, T2m; 394| 165| T1 = ri[0]; 395| 165| T1q = ii[0]; 396| 165| { 397| 165| E Tf, Tp, Tb, TC, Tx, T6, TB, Tw, Ti, Tq, Tl, Tr, Tm, Ts, Td; 398| 165| E Te, Tc, Tn; 399| 165| Td = ri[WS(is, 8)]; ------------------ | | 825| 165|#define WS(stride, i) (stride * i) ------------------ 400| 165| Te = ri[WS(is, 5)]; ------------------ | | 825| 165|#define WS(stride, i) (stride * i) ------------------ 401| 165| Tf = Td + Te; 402| 165| Tp = Td - Te; 403| 165| { 404| 165| E T7, T8, T9, Ta; 405| 165| T7 = ri[WS(is, 12)]; ------------------ | | 825| 165|#define WS(stride, i) (stride * i) ------------------ 406| 165| T8 = ri[WS(is, 10)]; ------------------ | | 825| 165|#define WS(stride, i) (stride * i) ------------------ 407| 165| T9 = ri[WS(is, 4)]; ------------------ | | 825| 165|#define WS(stride, i) (stride * i) ------------------ 408| 165| Ta = T8 + T9; 409| 165| Tb = T7 + Ta; 410| 165| TC = T8 - T9; 411| 165| Tx = FNMS(KP500000000, Ta, T7); ------------------ | | 1134| 165|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 412| 165| } 413| 165| { 414| 165| E T2, T3, T4, T5; 415| 165| T2 = ri[WS(is, 1)]; ------------------ | | 825| 165|#define WS(stride, i) (stride * i) ------------------ 416| 165| T3 = ri[WS(is, 3)]; ------------------ | | 825| 165|#define WS(stride, i) (stride * i) ------------------ 417| 165| T4 = ri[WS(is, 9)]; ------------------ | | 825| 165|#define WS(stride, i) (stride * i) ------------------ 418| 165| T5 = T3 + T4; 419| 165| T6 = T2 + T5; 420| 165| TB = T3 - T4; 421| 165| Tw = FNMS(KP500000000, T5, T2); ------------------ | | 1134| 165|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 422| 165| } 423| 165| { 424| 165| E Tg, Th, Tj, Tk; 425| 165| Tg = ri[WS(is, 11)]; ------------------ | | 825| 165|#define WS(stride, i) (stride * i) ------------------ 426| 165| Th = ri[WS(is, 6)]; ------------------ | | 825| 165|#define WS(stride, i) (stride * i) ------------------ 427| 165| Ti = Tg + Th; 428| 165| Tq = Tg - Th; 429| 165| Tj = ri[WS(is, 7)]; ------------------ | | 825| 165|#define WS(stride, i) (stride * i) ------------------ 430| 165| Tk = ri[WS(is, 2)]; ------------------ | | 825| 165|#define WS(stride, i) (stride * i) ------------------ 431| 165| Tl = Tj + Tk; 432| 165| Tr = Tj - Tk; 433| 165| } 434| 165| Tm = Ti + Tl; 435| 165| Ts = Tq + Tr; 436| 165| Tt = Tp + Ts; 437| 165| Tu = T6 - Tb; 438| 165| Tc = T6 + Tb; 439| 165| Tn = Tf + Tm; 440| 165| To = Tc + Tn; 441| 165| T22 = KP300462606 * (Tc - Tn); 442| 165| { 443| 165| E T1Y, T1Z, TD, TE; 444| 165| T1Y = TB + TC; 445| 165| T1Z = Tq - Tr; 446| 165| T20 = T1Y - T1Z; 447| 165| T24 = T1Y + T1Z; 448| 165| TD = KP866025403 * (TB - TC); 449| 165| TE = FNMS(KP500000000, Ts, Tp); ------------------ | | 1134| 165|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 450| 165| TF = TD - TE; 451| 165| TH = TD + TE; 452| 165| } 453| 165| { 454| 165| E Ty, Tz, T1V, T1W; 455| 165| Ty = Tw - Tx; 456| 165| Tz = KP866025403 * (Ti - Tl); 457| 165| TA = Ty + Tz; 458| 165| TI = Ty - Tz; 459| 165| T1V = Tw + Tx; 460| 165| T1W = FNMS(KP500000000, Tm, Tf); ------------------ | | 1134| 165|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 461| 165| T1X = T1V - T1W; 462| 165| T25 = T1V + T1W; 463| 165| } 464| 165| } 465| 165| { 466| 165| E TZ, T2b, TV, T1i, T1a, TQ, T1h, T19, T12, T1d, T15, T1c, T16, T2c, TX; 467| 165| E TY, TW, T17; 468| 165| TX = ii[WS(is, 8)]; ------------------ | | 825| 165|#define WS(stride, i) (stride * i) ------------------ 469| 165| TY = ii[WS(is, 5)]; ------------------ | | 825| 165|#define WS(stride, i) (stride * i) ------------------ 470| 165| TZ = TX + TY; 471| 165| T2b = TX - TY; 472| 165| { 473| 165| E TR, TS, TT, TU; 474| 165| TR = ii[WS(is, 12)]; ------------------ | | 825| 165|#define WS(stride, i) (stride * i) ------------------ 475| 165| TS = ii[WS(is, 10)]; ------------------ | | 825| 165|#define WS(stride, i) (stride * i) ------------------ 476| 165| TT = ii[WS(is, 4)]; ------------------ | | 825| 165|#define WS(stride, i) (stride * i) ------------------ 477| 165| TU = TS + TT; 478| 165| TV = FNMS(KP500000000, TU, TR); ------------------ | | 1134| 165|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 479| 165| T1i = TR + TU; 480| 165| T1a = TS - TT; 481| 165| } 482| 165| { 483| 165| E TM, TN, TO, TP; 484| 165| TM = ii[WS(is, 1)]; ------------------ | | 825| 165|#define WS(stride, i) (stride * i) ------------------ 485| 165| TN = ii[WS(is, 3)]; ------------------ | | 825| 165|#define WS(stride, i) (stride * i) ------------------ 486| 165| TO = ii[WS(is, 9)]; ------------------ | | 825| 165|#define WS(stride, i) (stride * i) ------------------ 487| 165| TP = TN + TO; 488| 165| TQ = FNMS(KP500000000, TP, TM); ------------------ | | 1134| 165|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 489| 165| T1h = TM + TP; 490| 165| T19 = TN - TO; 491| 165| } 492| 165| { 493| 165| E T10, T11, T13, T14; 494| 165| T10 = ii[WS(is, 11)]; ------------------ | | 825| 165|#define WS(stride, i) (stride * i) ------------------ 495| 165| T11 = ii[WS(is, 6)]; ------------------ | | 825| 165|#define WS(stride, i) (stride * i) ------------------ 496| 165| T12 = T10 + T11; 497| 165| T1d = T10 - T11; 498| 165| T13 = ii[WS(is, 7)]; ------------------ | | 825| 165|#define WS(stride, i) (stride * i) ------------------ 499| 165| T14 = ii[WS(is, 2)]; ------------------ | | 825| 165|#define WS(stride, i) (stride * i) ------------------ 500| 165| T15 = T13 + T14; 501| 165| T1c = T13 - T14; 502| 165| } 503| 165| T16 = T12 + T15; 504| 165| T2c = T1d + T1c; 505| 165| T2a = T1h - T1i; 506| 165| T2d = T2b + T2c; 507| 165| TW = TQ + TV; 508| 165| T17 = FNMS(KP500000000, T16, TZ); ------------------ | | 1134| 165|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 509| 165| T18 = TW - T17; 510| 165| T1n = TW + T17; 511| 165| { 512| 165| E T2i, T2j, T1j, T1k; 513| 165| T2i = TQ - TV; 514| 165| T2j = KP866025403 * (T15 - T12); 515| 165| T2k = T2i + T2j; 516| 165| T2n = T2i - T2j; 517| 165| T1j = T1h + T1i; 518| 165| T1k = TZ + T16; 519| 165| T1l = KP300462606 * (T1j - T1k); 520| 165| T1r = T1j + T1k; 521| 165| } 522| 165| { 523| 165| E T1b, T1e, T2f, T2g; 524| 165| T1b = T19 + T1a; 525| 165| T1e = T1c - T1d; 526| 165| T1f = T1b + T1e; 527| 165| T1o = T1e - T1b; 528| 165| T2f = FNMS(KP500000000, T2c, T2b); ------------------ | | 1134| 165|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 529| 165| T2g = KP866025403 * (T1a - T19); 530| 165| T2h = T2f - T2g; 531| 165| T2m = T2g + T2f; 532| 165| } 533| 165| } 534| 165| ro[0] = T1 + To; 535| 165| io[0] = T1q + T1r; 536| 165| { 537| 165| E T1D, T1N, T1y, T1x, T1E, T1O, Tv, TK, T1J, T1Q, T1m, T1R, T1t, T1I, TG; 538| 165| E TJ; 539| 165| { 540| 165| E T1B, T1C, T1v, T1w; 541| 165| T1B = FMA(KP387390585, T1f, KP265966249 * T18); ------------------ | | 1131| 165|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 542| 165| T1C = FMA(KP113854479, T1o, KP503537032 * T1n); ------------------ | | 1131| 165|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 543| 165| T1D = T1B + T1C; 544| 165| T1N = T1C - T1B; 545| 165| T1y = FMA(KP575140729, Tu, KP174138601 * Tt); ------------------ | | 1131| 165|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 546| 165| T1v = FNMS(KP156891391, TH, KP256247671 * TI); ------------------ | | 1134| 165|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 547| 165| T1w = FMA(KP011599105, TF, KP300238635 * TA); ------------------ | | 1131| 165|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 548| 165| T1x = T1v - T1w; 549| 165| T1E = T1y + T1x; 550| 165| T1O = KP1_732050807 * (T1v + T1w); 551| 165| } 552| 165| Tv = FNMS(KP174138601, Tu, KP575140729 * Tt); ------------------ | | 1134| 165|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 553| 165| TG = FNMS(KP300238635, TF, KP011599105 * TA); ------------------ | | 1134| 165|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 554| 165| TJ = FMA(KP256247671, TH, KP156891391 * TI); ------------------ | | 1131| 165|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 555| 165| TK = TG - TJ; 556| 165| T1J = KP1_732050807 * (TJ + TG); 557| 165| T1Q = Tv - TK; 558| 165| { 559| 165| E T1g, T1H, T1p, T1s, T1G; 560| 165| T1g = FNMS(KP132983124, T1f, KP258260390 * T18); ------------------ | | 1134| 165|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 561| 165| T1H = T1l - T1g; 562| 165| T1p = FNMS(KP251768516, T1o, KP075902986 * T1n); ------------------ | | 1134| 165|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 563| 165| T1s = FNMS(KP083333333, T1r, T1q); ------------------ | | 1134| 165|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 564| 165| T1G = T1s - T1p; 565| 165| T1m = FMA(KP2_000000000, T1g, T1l); ------------------ | | 1131| 165|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 566| 165| T1R = T1H + T1G; 567| 165| T1t = FMA(KP2_000000000, T1p, T1s); ------------------ | | 1131| 165|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 568| 165| T1I = T1G - T1H; 569| 165| } 570| 165| { 571| 165| E TL, T1u, T1P, T1S; 572| 165| TL = FMA(KP2_000000000, TK, Tv); ------------------ | | 1131| 165|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 573| 165| T1u = T1m + T1t; 574| 165| io[WS(os, 1)] = TL + T1u; ------------------ | | 825| 165|#define WS(stride, i) (stride * i) ------------------ 575| 165| io[WS(os, 12)] = T1u - TL; ------------------ | | 825| 165|#define WS(stride, i) (stride * i) ------------------ 576| 165| { 577| 165| E T1z, T1A, T1T, T1U; 578| 165| T1z = FMS(KP2_000000000, T1x, T1y); ------------------ | | 1132| 165|#define FMS(a, b, c) (((a) * (b)) - (c)) ------------------ 579| 165| T1A = T1t - T1m; 580| 165| io[WS(os, 5)] = T1z + T1A; ------------------ | | 825| 165|#define WS(stride, i) (stride * i) ------------------ 581| 165| io[WS(os, 8)] = T1A - T1z; ------------------ | | 825| 165|#define WS(stride, i) (stride * i) ------------------ 582| 165| T1T = T1R - T1Q; 583| 165| T1U = T1O + T1N; 584| 165| io[WS(os, 4)] = T1T - T1U; ------------------ | | 825| 165|#define WS(stride, i) (stride * i) ------------------ 585| 165| io[WS(os, 10)] = T1U + T1T; ------------------ | | 825| 165|#define WS(stride, i) (stride * i) ------------------ 586| 165| } 587| 165| T1P = T1N - T1O; 588| 165| T1S = T1Q + T1R; 589| 165| io[WS(os, 3)] = T1P + T1S; ------------------ | | 825| 165|#define WS(stride, i) (stride * i) ------------------ 590| 165| io[WS(os, 9)] = T1S - T1P; ------------------ | | 825| 165|#define WS(stride, i) (stride * i) ------------------ 591| 165| { 592| 165| E T1L, T1M, T1F, T1K; 593| 165| T1L = T1J + T1I; 594| 165| T1M = T1E + T1D; 595| 165| io[WS(os, 6)] = T1L - T1M; ------------------ | | 825| 165|#define WS(stride, i) (stride * i) ------------------ 596| 165| io[WS(os, 11)] = T1M + T1L; ------------------ | | 825| 165|#define WS(stride, i) (stride * i) ------------------ 597| 165| T1F = T1D - T1E; 598| 165| T1K = T1I - T1J; 599| 165| io[WS(os, 2)] = T1F + T1K; ------------------ | | 825| 165|#define WS(stride, i) (stride * i) ------------------ 600| 165| io[WS(os, 7)] = T1K - T1F; ------------------ | | 825| 165|#define WS(stride, i) (stride * i) ------------------ 601| 165| } 602| 165| } 603| 165| } 604| 165| { 605| 165| E T2y, T2I, T2J, T2K, T2B, T2L, T2e, T2p, T2u, T2G, T23, T2F, T28, T2t, T2l; 606| 165| E T2o; 607| 165| { 608| 165| E T2w, T2x, T2z, T2A; 609| 165| T2w = FMA(KP387390585, T20, KP265966249 * T1X); ------------------ | | 1131| 165|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 610| 165| T2x = FNMS(KP503537032, T25, KP113854479 * T24); ------------------ | | 1134| 165|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 611| 165| T2y = T2w + T2x; 612| 165| T2I = T2w - T2x; 613| 165| T2J = FMA(KP575140729, T2a, KP174138601 * T2d); ------------------ | | 1131| 165|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 614| 165| T2z = FNMS(KP300238635, T2n, KP011599105 * T2m); ------------------ | | 1134| 165|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 615| 165| T2A = FNMS(KP156891391, T2h, KP256247671 * T2k); ------------------ | | 1134| 165|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 616| 165| T2K = T2z + T2A; 617| 165| T2B = KP1_732050807 * (T2z - T2A); 618| 165| T2L = T2J + T2K; 619| 165| } 620| 165| T2e = FNMS(KP575140729, T2d, KP174138601 * T2a); ------------------ | | 1134| 165|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 621| 165| T2l = FMA(KP256247671, T2h, KP156891391 * T2k); ------------------ | | 1131| 165|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 622| 165| T2o = FMA(KP300238635, T2m, KP011599105 * T2n); ------------------ | | 1131| 165|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 623| 165| T2p = T2l - T2o; 624| 165| T2u = T2e - T2p; 625| 165| T2G = KP1_732050807 * (T2o + T2l); 626| 165| { 627| 165| E T21, T2r, T26, T27, T2s; 628| 165| T21 = FNMS(KP132983124, T20, KP258260390 * T1X); ------------------ | | 1134| 165|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 629| 165| T2r = T22 - T21; 630| 165| T26 = FMA(KP251768516, T24, KP075902986 * T25); ------------------ | | 1131| 165|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 631| 165| T27 = FNMS(KP083333333, To, T1); ------------------ | | 1134| 165|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 632| 165| T2s = T27 - T26; 633| 165| T23 = FMA(KP2_000000000, T21, T22); ------------------ | | 1131| 165|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 634| 165| T2F = T2s - T2r; 635| 165| T28 = FMA(KP2_000000000, T26, T27); ------------------ | | 1131| 165|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 636| 165| T2t = T2r + T2s; 637| 165| } 638| 165| { 639| 165| E T29, T2q, T2N, T2O; 640| 165| T29 = T23 + T28; 641| 165| T2q = FMA(KP2_000000000, T2p, T2e); ------------------ | | 1131| 165|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 642| 165| ro[WS(os, 12)] = T29 - T2q; ------------------ | | 825| 165|#define WS(stride, i) (stride * i) ------------------ 643| 165| ro[WS(os, 1)] = T29 + T2q; ------------------ | | 825| 165|#define WS(stride, i) (stride * i) ------------------ 644| 165| { 645| 165| E T2v, T2C, T2P, T2Q; 646| 165| T2v = T2t - T2u; 647| 165| T2C = T2y - T2B; 648| 165| ro[WS(os, 10)] = T2v - T2C; ------------------ | | 825| 165|#define WS(stride, i) (stride * i) ------------------ 649| 165| ro[WS(os, 4)] = T2v + T2C; ------------------ | | 825| 165|#define WS(stride, i) (stride * i) ------------------ 650| 165| T2P = T28 - T23; 651| 165| T2Q = FMS(KP2_000000000, T2K, T2J); ------------------ | | 1132| 165|#define FMS(a, b, c) (((a) * (b)) - (c)) ------------------ 652| 165| ro[WS(os, 5)] = T2P - T2Q; ------------------ | | 825| 165|#define WS(stride, i) (stride * i) ------------------ 653| 165| ro[WS(os, 8)] = T2P + T2Q; ------------------ | | 825| 165|#define WS(stride, i) (stride * i) ------------------ 654| 165| } 655| 165| T2N = T2F - T2G; 656| 165| T2O = T2L - T2I; 657| 165| ro[WS(os, 11)] = T2N - T2O; ------------------ | | 825| 165|#define WS(stride, i) (stride * i) ------------------ 658| 165| ro[WS(os, 6)] = T2N + T2O; ------------------ | | 825| 165|#define WS(stride, i) (stride * i) ------------------ 659| 165| { 660| 165| E T2H, T2M, T2D, T2E; 661| 165| T2H = T2F + T2G; 662| 165| T2M = T2I + T2L; 663| 165| ro[WS(os, 7)] = T2H - T2M; ------------------ | | 825| 165|#define WS(stride, i) (stride * i) ------------------ 664| 165| ro[WS(os, 2)] = T2H + T2M; ------------------ | | 825| 165|#define WS(stride, i) (stride * i) ------------------ 665| 165| T2D = T2t + T2u; 666| 165| T2E = T2y + T2B; 667| 165| ro[WS(os, 3)] = T2D - T2E; ------------------ | | 825| 165|#define WS(stride, i) (stride * i) ------------------ 668| 165| ro[WS(os, 9)] = T2D + T2E; ------------------ | | 825| 165|#define WS(stride, i) (stride * i) ------------------ 669| 165| } 670| 165| } 671| 165| } 672| 165| } 673| 36| } 674| 36|} fftw_codelet_n1_14: 510| 1|void X(codelet_n1_14) (planner *p) { X(kdft_register) (p, n1_14, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 511| 1|} n1_14.c:n1_14: 303| 15|{ 304| 15| DK(KP222520933, +0.222520933956314404288902564496794759466355569); ------------------ | | 1056| 15|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 15|# define K(x) ((E) x) | | ------------------ ------------------ 305| 15| DK(KP900968867, +0.900968867902419126236102319507445051165919162); ------------------ | | 1056| 15|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 15|# define K(x) ((E) x) | | ------------------ ------------------ 306| 15| DK(KP623489801, +0.623489801858733530525004884004239810632274731); ------------------ | | 1056| 15|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 15|# define K(x) ((E) x) | | ------------------ ------------------ 307| 15| DK(KP433883739, +0.433883739117558120475768332848358754609990728); ------------------ | | 1056| 15|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 15|# define K(x) ((E) x) | | ------------------ ------------------ 308| 15| DK(KP781831482, +0.781831482468029808708444526674057750232334519); ------------------ | | 1056| 15|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 15|# define K(x) ((E) x) | | ------------------ ------------------ 309| 15| DK(KP974927912, +0.974927912181823607018131682993931217232785801); ------------------ | | 1056| 15|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 15|# define K(x) ((E) x) | | ------------------ ------------------ 310| 15| { 311| 15| INT i; 312| 62| for (i = v; i > 0; i = i - 1, ri = ri + ivs, ii = ii + ivs, ro = ro + ovs, io = io + ovs, MAKE_VOLATILE_STRIDE(56, is), MAKE_VOLATILE_STRIDE(56, os)) { ------------------ | | 851| 47| (nptr <= ESTIMATED_AVAILABLE_INDEX_REGISTERS ? \ | | ------------------ | | | | 849| 94|#define ESTIMATED_AVAILABLE_INDEX_REGISTERS 16 | | ------------------ | | | Branch (851:7): [Folded, False: 47] | | ------------------ | | 852| 47| 0 : \ | | 853| 47| ((x) = (x) ^ X(an_INT_guaranteed_to_be_zero))) | | ------------------ | | | | 78| 47|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 47|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ for (i = v; i > 0; i = i - 1, ri = ri + ivs, ii = ii + ivs, ro = ro + ovs, io = io + ovs, MAKE_VOLATILE_STRIDE(56, is), MAKE_VOLATILE_STRIDE(56, os)) { ------------------ | | 851| 47| (nptr <= ESTIMATED_AVAILABLE_INDEX_REGISTERS ? \ | | ------------------ | | | | 849| 94|#define ESTIMATED_AVAILABLE_INDEX_REGISTERS 16 | | ------------------ | | | Branch (851:7): [Folded, False: 47] | | ------------------ | | 852| 47| 0 : \ | | 853| 47| ((x) = (x) ^ X(an_INT_guaranteed_to_be_zero))) | | ------------------ | | | | 78| 47|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 47|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ | Branch (312:16): [True: 47, False: 15] ------------------ 313| 47| E T3, Tp, T16, T1f, Ta, T1q, Ts, T10, TG, T1z, T19, T1i, Th, T1s, Tv; 314| 47| E T12, TU, T1B, T17, T1o, To, T1r, Ty, T11, TN, T1A, T18, T1l; 315| 47| { 316| 47| E T1, T2, T14, T15; 317| 47| T1 = ri[0]; 318| 47| T2 = ri[WS(is, 7)]; ------------------ | | 825| 47|#define WS(stride, i) (stride * i) ------------------ 319| 47| T3 = T1 - T2; 320| 47| Tp = T1 + T2; 321| 47| T14 = ii[0]; 322| 47| T15 = ii[WS(is, 7)]; ------------------ | | 825| 47|#define WS(stride, i) (stride * i) ------------------ 323| 47| T16 = T14 - T15; 324| 47| T1f = T14 + T15; 325| 47| } 326| 47| { 327| 47| E T6, Tq, T9, Tr; 328| 47| { 329| 47| E T4, T5, T7, T8; 330| 47| T4 = ri[WS(is, 2)]; ------------------ | | 825| 47|#define WS(stride, i) (stride * i) ------------------ 331| 47| T5 = ri[WS(is, 9)]; ------------------ | | 825| 47|#define WS(stride, i) (stride * i) ------------------ 332| 47| T6 = T4 - T5; 333| 47| Tq = T4 + T5; 334| 47| T7 = ri[WS(is, 12)]; ------------------ | | 825| 47|#define WS(stride, i) (stride * i) ------------------ 335| 47| T8 = ri[WS(is, 5)]; ------------------ | | 825| 47|#define WS(stride, i) (stride * i) ------------------ 336| 47| T9 = T7 - T8; 337| 47| Tr = T7 + T8; 338| 47| } 339| 47| Ta = T6 + T9; 340| 47| T1q = Tr - Tq; 341| 47| Ts = Tq + Tr; 342| 47| T10 = T9 - T6; 343| 47| } 344| 47| { 345| 47| E TC, T1g, TF, T1h; 346| 47| { 347| 47| E TA, TB, TD, TE; 348| 47| TA = ii[WS(is, 2)]; ------------------ | | 825| 47|#define WS(stride, i) (stride * i) ------------------ 349| 47| TB = ii[WS(is, 9)]; ------------------ | | 825| 47|#define WS(stride, i) (stride * i) ------------------ 350| 47| TC = TA - TB; 351| 47| T1g = TA + TB; 352| 47| TD = ii[WS(is, 12)]; ------------------ | | 825| 47|#define WS(stride, i) (stride * i) ------------------ 353| 47| TE = ii[WS(is, 5)]; ------------------ | | 825| 47|#define WS(stride, i) (stride * i) ------------------ 354| 47| TF = TD - TE; 355| 47| T1h = TD + TE; 356| 47| } 357| 47| TG = TC - TF; 358| 47| T1z = T1g - T1h; 359| 47| T19 = TC + TF; 360| 47| T1i = T1g + T1h; 361| 47| } 362| 47| { 363| 47| E Td, Tt, Tg, Tu; 364| 47| { 365| 47| E Tb, Tc, Te, Tf; 366| 47| Tb = ri[WS(is, 4)]; ------------------ | | 825| 47|#define WS(stride, i) (stride * i) ------------------ 367| 47| Tc = ri[WS(is, 11)]; ------------------ | | 825| 47|#define WS(stride, i) (stride * i) ------------------ 368| 47| Td = Tb - Tc; 369| 47| Tt = Tb + Tc; 370| 47| Te = ri[WS(is, 10)]; ------------------ | | 825| 47|#define WS(stride, i) (stride * i) ------------------ 371| 47| Tf = ri[WS(is, 3)]; ------------------ | | 825| 47|#define WS(stride, i) (stride * i) ------------------ 372| 47| Tg = Te - Tf; 373| 47| Tu = Te + Tf; 374| 47| } 375| 47| Th = Td + Tg; 376| 47| T1s = Tt - Tu; 377| 47| Tv = Tt + Tu; 378| 47| T12 = Tg - Td; 379| 47| } 380| 47| { 381| 47| E TQ, T1m, TT, T1n; 382| 47| { 383| 47| E TO, TP, TR, TS; 384| 47| TO = ii[WS(is, 4)]; ------------------ | | 825| 47|#define WS(stride, i) (stride * i) ------------------ 385| 47| TP = ii[WS(is, 11)]; ------------------ | | 825| 47|#define WS(stride, i) (stride * i) ------------------ 386| 47| TQ = TO - TP; 387| 47| T1m = TO + TP; 388| 47| TR = ii[WS(is, 10)]; ------------------ | | 825| 47|#define WS(stride, i) (stride * i) ------------------ 389| 47| TS = ii[WS(is, 3)]; ------------------ | | 825| 47|#define WS(stride, i) (stride * i) ------------------ 390| 47| TT = TR - TS; 391| 47| T1n = TR + TS; 392| 47| } 393| 47| TU = TQ - TT; 394| 47| T1B = T1n - T1m; 395| 47| T17 = TQ + TT; 396| 47| T1o = T1m + T1n; 397| 47| } 398| 47| { 399| 47| E Tk, Tw, Tn, Tx; 400| 47| { 401| 47| E Ti, Tj, Tl, Tm; 402| 47| Ti = ri[WS(is, 6)]; ------------------ | | 825| 47|#define WS(stride, i) (stride * i) ------------------ 403| 47| Tj = ri[WS(is, 13)]; ------------------ | | 825| 47|#define WS(stride, i) (stride * i) ------------------ 404| 47| Tk = Ti - Tj; 405| 47| Tw = Ti + Tj; 406| 47| Tl = ri[WS(is, 8)]; ------------------ | | 825| 47|#define WS(stride, i) (stride * i) ------------------ 407| 47| Tm = ri[WS(is, 1)]; ------------------ | | 825| 47|#define WS(stride, i) (stride * i) ------------------ 408| 47| Tn = Tl - Tm; 409| 47| Tx = Tl + Tm; 410| 47| } 411| 47| To = Tk + Tn; 412| 47| T1r = Tw - Tx; 413| 47| Ty = Tw + Tx; 414| 47| T11 = Tn - Tk; 415| 47| } 416| 47| { 417| 47| E TJ, T1j, TM, T1k; 418| 47| { 419| 47| E TH, TI, TK, TL; 420| 47| TH = ii[WS(is, 6)]; ------------------ | | 825| 47|#define WS(stride, i) (stride * i) ------------------ 421| 47| TI = ii[WS(is, 13)]; ------------------ | | 825| 47|#define WS(stride, i) (stride * i) ------------------ 422| 47| TJ = TH - TI; 423| 47| T1j = TH + TI; 424| 47| TK = ii[WS(is, 8)]; ------------------ | | 825| 47|#define WS(stride, i) (stride * i) ------------------ 425| 47| TL = ii[WS(is, 1)]; ------------------ | | 825| 47|#define WS(stride, i) (stride * i) ------------------ 426| 47| TM = TK - TL; 427| 47| T1k = TK + TL; 428| 47| } 429| 47| TN = TJ - TM; 430| 47| T1A = T1k - T1j; 431| 47| T18 = TJ + TM; 432| 47| T1l = T1j + T1k; 433| 47| } 434| 47| ro[WS(os, 7)] = T3 + Ta + Th + To; ------------------ | | 825| 47|#define WS(stride, i) (stride * i) ------------------ 435| 47| io[WS(os, 7)] = T16 + T19 + T17 + T18; ------------------ | | 825| 47|#define WS(stride, i) (stride * i) ------------------ 436| 47| ro[0] = Tp + Ts + Tv + Ty; 437| 47| io[0] = T1f + T1i + T1o + T1l; 438| 47| { 439| 47| E TV, Tz, T1e, T1d; 440| 47| TV = FNMS(KP781831482, TN, KP974927912 * TG) - (KP433883739 * TU); ------------------ | | 1134| 47|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 441| 47| Tz = FMA(KP623489801, To, T3) + FNMA(KP900968867, Th, KP222520933 * Ta); ------------------ | | 1131| 47|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ Tz = FMA(KP623489801, To, T3) + FNMA(KP900968867, Th, KP222520933 * Ta); ------------------ | | 1133| 47|#define FNMA(a, b, c) (- (((a) * (b)) + (c))) ------------------ 442| 47| ro[WS(os, 5)] = Tz - TV; ------------------ | | 825| 47|#define WS(stride, i) (stride * i) ------------------ 443| 47| ro[WS(os, 9)] = Tz + TV; ------------------ | | 825| 47|#define WS(stride, i) (stride * i) ------------------ 444| 47| T1e = FNMS(KP781831482, T11, KP974927912 * T10) - (KP433883739 * T12); ------------------ | | 1134| 47|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 445| 47| T1d = FMA(KP623489801, T18, T16) + FNMA(KP900968867, T17, KP222520933 * T19); ------------------ | | 1131| 47|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ T1d = FMA(KP623489801, T18, T16) + FNMA(KP900968867, T17, KP222520933 * T19); ------------------ | | 1133| 47|#define FNMA(a, b, c) (- (((a) * (b)) + (c))) ------------------ 446| 47| io[WS(os, 5)] = T1d - T1e; ------------------ | | 825| 47|#define WS(stride, i) (stride * i) ------------------ 447| 47| io[WS(os, 9)] = T1e + T1d; ------------------ | | 825| 47|#define WS(stride, i) (stride * i) ------------------ 448| 47| } 449| 47| { 450| 47| E TX, TW, T1b, T1c; 451| 47| TX = FMA(KP781831482, TG, KP974927912 * TU) + (KP433883739 * TN); ------------------ | | 1131| 47|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 452| 47| TW = FMA(KP623489801, Ta, T3) + FNMA(KP900968867, To, KP222520933 * Th); ------------------ | | 1131| 47|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ TW = FMA(KP623489801, Ta, T3) + FNMA(KP900968867, To, KP222520933 * Th); ------------------ | | 1133| 47|#define FNMA(a, b, c) (- (((a) * (b)) + (c))) ------------------ 453| 47| ro[WS(os, 13)] = TW - TX; ------------------ | | 825| 47|#define WS(stride, i) (stride * i) ------------------ 454| 47| ro[WS(os, 1)] = TW + TX; ------------------ | | 825| 47|#define WS(stride, i) (stride * i) ------------------ 455| 47| T1b = FMA(KP781831482, T10, KP974927912 * T12) + (KP433883739 * T11); ------------------ | | 1131| 47|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 456| 47| T1c = FMA(KP623489801, T19, T16) + FNMA(KP900968867, T18, KP222520933 * T17); ------------------ | | 1131| 47|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ T1c = FMA(KP623489801, T19, T16) + FNMA(KP900968867, T18, KP222520933 * T17); ------------------ | | 1133| 47|#define FNMA(a, b, c) (- (((a) * (b)) + (c))) ------------------ 457| 47| io[WS(os, 1)] = T1b + T1c; ------------------ | | 825| 47|#define WS(stride, i) (stride * i) ------------------ 458| 47| io[WS(os, 13)] = T1c - T1b; ------------------ | | 825| 47|#define WS(stride, i) (stride * i) ------------------ 459| 47| } 460| 47| { 461| 47| E TZ, TY, T13, T1a; 462| 47| TZ = FMA(KP433883739, TG, KP974927912 * TN) - (KP781831482 * TU); ------------------ | | 1131| 47|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 463| 47| TY = FMA(KP623489801, Th, T3) + FNMA(KP222520933, To, KP900968867 * Ta); ------------------ | | 1131| 47|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ TY = FMA(KP623489801, Th, T3) + FNMA(KP222520933, To, KP900968867 * Ta); ------------------ | | 1133| 47|#define FNMA(a, b, c) (- (((a) * (b)) + (c))) ------------------ 464| 47| ro[WS(os, 11)] = TY - TZ; ------------------ | | 825| 47|#define WS(stride, i) (stride * i) ------------------ 465| 47| ro[WS(os, 3)] = TY + TZ; ------------------ | | 825| 47|#define WS(stride, i) (stride * i) ------------------ 466| 47| T13 = FMA(KP433883739, T10, KP974927912 * T11) - (KP781831482 * T12); ------------------ | | 1131| 47|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 467| 47| T1a = FMA(KP623489801, T17, T16) + FNMA(KP222520933, T18, KP900968867 * T19); ------------------ | | 1131| 47|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ T1a = FMA(KP623489801, T17, T16) + FNMA(KP222520933, T18, KP900968867 * T19); ------------------ | | 1133| 47|#define FNMA(a, b, c) (- (((a) * (b)) + (c))) ------------------ 468| 47| io[WS(os, 3)] = T13 + T1a; ------------------ | | 825| 47|#define WS(stride, i) (stride * i) ------------------ 469| 47| io[WS(os, 11)] = T1a - T13; ------------------ | | 825| 47|#define WS(stride, i) (stride * i) ------------------ 470| 47| } 471| 47| { 472| 47| E T1t, T1p, T1C, T1y; 473| 47| T1t = FNMS(KP433883739, T1r, KP781831482 * T1q) - (KP974927912 * T1s); ------------------ | | 1134| 47|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 474| 47| T1p = FMA(KP623489801, T1i, T1f) + FNMA(KP900968867, T1l, KP222520933 * T1o); ------------------ | | 1131| 47|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ T1p = FMA(KP623489801, T1i, T1f) + FNMA(KP900968867, T1l, KP222520933 * T1o); ------------------ | | 1133| 47|#define FNMA(a, b, c) (- (((a) * (b)) + (c))) ------------------ 475| 47| io[WS(os, 6)] = T1p - T1t; ------------------ | | 825| 47|#define WS(stride, i) (stride * i) ------------------ 476| 47| io[WS(os, 8)] = T1t + T1p; ------------------ | | 825| 47|#define WS(stride, i) (stride * i) ------------------ 477| 47| T1C = FNMS(KP433883739, T1A, KP781831482 * T1z) - (KP974927912 * T1B); ------------------ | | 1134| 47|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 478| 47| T1y = FMA(KP623489801, Ts, Tp) + FNMA(KP900968867, Ty, KP222520933 * Tv); ------------------ | | 1131| 47|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ T1y = FMA(KP623489801, Ts, Tp) + FNMA(KP900968867, Ty, KP222520933 * Tv); ------------------ | | 1133| 47|#define FNMA(a, b, c) (- (((a) * (b)) + (c))) ------------------ 479| 47| ro[WS(os, 6)] = T1y - T1C; ------------------ | | 825| 47|#define WS(stride, i) (stride * i) ------------------ 480| 47| ro[WS(os, 8)] = T1y + T1C; ------------------ | | 825| 47|#define WS(stride, i) (stride * i) ------------------ 481| 47| } 482| 47| { 483| 47| E T1v, T1u, T1E, T1D; 484| 47| T1v = FMA(KP433883739, T1q, KP781831482 * T1s) - (KP974927912 * T1r); ------------------ | | 1131| 47|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 485| 47| T1u = FMA(KP623489801, T1o, T1f) + FNMA(KP222520933, T1l, KP900968867 * T1i); ------------------ | | 1131| 47|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ T1u = FMA(KP623489801, T1o, T1f) + FNMA(KP222520933, T1l, KP900968867 * T1i); ------------------ | | 1133| 47|#define FNMA(a, b, c) (- (((a) * (b)) + (c))) ------------------ 486| 47| io[WS(os, 4)] = T1u - T1v; ------------------ | | 825| 47|#define WS(stride, i) (stride * i) ------------------ 487| 47| io[WS(os, 10)] = T1v + T1u; ------------------ | | 825| 47|#define WS(stride, i) (stride * i) ------------------ 488| 47| T1E = FMA(KP433883739, T1z, KP781831482 * T1B) - (KP974927912 * T1A); ------------------ | | 1131| 47|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 489| 47| T1D = FMA(KP623489801, Tv, Tp) + FNMA(KP222520933, Ty, KP900968867 * Ts); ------------------ | | 1131| 47|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ T1D = FMA(KP623489801, Tv, Tp) + FNMA(KP222520933, Ty, KP900968867 * Ts); ------------------ | | 1133| 47|#define FNMA(a, b, c) (- (((a) * (b)) + (c))) ------------------ 490| 47| ro[WS(os, 4)] = T1D - T1E; ------------------ | | 825| 47|#define WS(stride, i) (stride * i) ------------------ 491| 47| ro[WS(os, 10)] = T1D + T1E; ------------------ | | 825| 47|#define WS(stride, i) (stride * i) ------------------ 492| 47| } 493| 47| { 494| 47| E T1w, T1x, T1G, T1F; 495| 47| T1w = FMA(KP974927912, T1q, KP433883739 * T1s) + (KP781831482 * T1r); ------------------ | | 1131| 47|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 496| 47| T1x = FMA(KP623489801, T1l, T1f) + FNMA(KP900968867, T1o, KP222520933 * T1i); ------------------ | | 1131| 47|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ T1x = FMA(KP623489801, T1l, T1f) + FNMA(KP900968867, T1o, KP222520933 * T1i); ------------------ | | 1133| 47|#define FNMA(a, b, c) (- (((a) * (b)) + (c))) ------------------ 497| 47| io[WS(os, 2)] = T1w + T1x; ------------------ | | 825| 47|#define WS(stride, i) (stride * i) ------------------ 498| 47| io[WS(os, 12)] = T1x - T1w; ------------------ | | 825| 47|#define WS(stride, i) (stride * i) ------------------ 499| 47| T1G = FMA(KP974927912, T1z, KP433883739 * T1B) + (KP781831482 * T1A); ------------------ | | 1131| 47|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 500| 47| T1F = FMA(KP623489801, Ty, Tp) + FNMA(KP900968867, Tv, KP222520933 * Ts); ------------------ | | 1131| 47|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ T1F = FMA(KP623489801, Ty, Tp) + FNMA(KP900968867, Tv, KP222520933 * Ts); ------------------ | | 1133| 47|#define FNMA(a, b, c) (- (((a) * (b)) + (c))) ------------------ 501| 47| ro[WS(os, 12)] = T1F - T1G; ------------------ | | 825| 47|#define WS(stride, i) (stride * i) ------------------ 502| 47| ro[WS(os, 2)] = T1F + T1G; ------------------ | | 825| 47|#define WS(stride, i) (stride * i) ------------------ 503| 47| } 504| 47| } 505| 15| } 506| 15|} fftw_codelet_n1_15: 551| 1|void X(codelet_n1_15) (planner *p) { X(kdft_register) (p, n1_15, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 552| 1|} n1_15.c:n1_15: 302| 9|{ 303| 9| DK(KP587785252, +0.587785252292473129168705954639072768597652438); ------------------ | | 1056| 9|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 9|# define K(x) ((E) x) | | ------------------ ------------------ 304| 9| DK(KP951056516, +0.951056516295153572116439333379382143405698634); ------------------ | | 1056| 9|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 9|# define K(x) ((E) x) | | ------------------ ------------------ 305| 9| DK(KP250000000, +0.250000000000000000000000000000000000000000000); ------------------ | | 1056| 9|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 9|# define K(x) ((E) x) | | ------------------ ------------------ 306| 9| DK(KP559016994, +0.559016994374947424102293417182819058860154590); ------------------ | | 1056| 9|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 9|# define K(x) ((E) x) | | ------------------ ------------------ 307| 9| DK(KP500000000, +0.500000000000000000000000000000000000000000000); ------------------ | | 1056| 9|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 9|# define K(x) ((E) x) | | ------------------ ------------------ 308| 9| DK(KP866025403, +0.866025403784438646763723170752936183471402627); ------------------ | | 1056| 9|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 9|# define K(x) ((E) x) | | ------------------ ------------------ 309| 9| { 310| 9| INT i; 311| 115| for (i = v; i > 0; i = i - 1, ri = ri + ivs, ii = ii + ivs, ro = ro + ovs, io = io + ovs, MAKE_VOLATILE_STRIDE(60, is), MAKE_VOLATILE_STRIDE(60, os)) { ------------------ | | 851| 106| (nptr <= ESTIMATED_AVAILABLE_INDEX_REGISTERS ? \ | | ------------------ | | | | 849| 212|#define ESTIMATED_AVAILABLE_INDEX_REGISTERS 16 | | ------------------ | | | Branch (851:7): [Folded, False: 106] | | ------------------ | | 852| 106| 0 : \ | | 853| 106| ((x) = (x) ^ X(an_INT_guaranteed_to_be_zero))) | | ------------------ | | | | 78| 106|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 106|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ for (i = v; i > 0; i = i - 1, ri = ri + ivs, ii = ii + ivs, ro = ro + ovs, io = io + ovs, MAKE_VOLATILE_STRIDE(60, is), MAKE_VOLATILE_STRIDE(60, os)) { ------------------ | | 851| 106| (nptr <= ESTIMATED_AVAILABLE_INDEX_REGISTERS ? \ | | ------------------ | | | | 849| 212|#define ESTIMATED_AVAILABLE_INDEX_REGISTERS 16 | | ------------------ | | | Branch (851:7): [Folded, False: 106] | | ------------------ | | 852| 106| 0 : \ | | 853| 106| ((x) = (x) ^ X(an_INT_guaranteed_to_be_zero))) | | ------------------ | | | | 78| 106|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 106|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ | Branch (311:16): [True: 106, False: 9] ------------------ 312| 106| E T5, T2l, Tx, TV, T1C, T20, Tl, Tq, Tr, TN, TS, TT, T2c, T2d, T2n; 313| 106| E T1O, T1P, T22, T1l, T1q, T1w, TZ, T10, T11, Ta, Tf, Tg, TC, TH, TI; 314| 106| E T2f, T2g, T2m, T1R, T1S, T21, T1a, T1f, T1v, TW, TX, TY; 315| 106| { 316| 106| E T1, T1z, T4, T1y, Tw, T1A, Tt, T1B; 317| 106| T1 = ri[0]; 318| 106| T1z = ii[0]; 319| 106| { 320| 106| E T2, T3, Tu, Tv; 321| 106| T2 = ri[WS(is, 5)]; ------------------ | | 825| 106|#define WS(stride, i) (stride * i) ------------------ 322| 106| T3 = ri[WS(is, 10)]; ------------------ | | 825| 106|#define WS(stride, i) (stride * i) ------------------ 323| 106| T4 = T2 + T3; 324| 106| T1y = KP866025403 * (T3 - T2); 325| 106| Tu = ii[WS(is, 5)]; ------------------ | | 825| 106|#define WS(stride, i) (stride * i) ------------------ 326| 106| Tv = ii[WS(is, 10)]; ------------------ | | 825| 106|#define WS(stride, i) (stride * i) ------------------ 327| 106| Tw = KP866025403 * (Tu - Tv); 328| 106| T1A = Tu + Tv; 329| 106| } 330| 106| T5 = T1 + T4; 331| 106| T2l = T1z + T1A; 332| 106| Tt = FNMS(KP500000000, T4, T1); ------------------ | | 1134| 106|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 333| 106| Tx = Tt - Tw; 334| 106| TV = Tt + Tw; 335| 106| T1B = FNMS(KP500000000, T1A, T1z); ------------------ | | 1134| 106|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 336| 106| T1C = T1y + T1B; 337| 106| T20 = T1B - T1y; 338| 106| } 339| 106| { 340| 106| E Th, Tk, TJ, T1h, T1i, T1j, TM, T1k, Tm, Tp, TO, T1m, T1n, T1o, TR; 341| 106| E T1p; 342| 106| { 343| 106| E Ti, Tj, TK, TL; 344| 106| Th = ri[WS(is, 6)]; ------------------ | | 825| 106|#define WS(stride, i) (stride * i) ------------------ 345| 106| Ti = ri[WS(is, 11)]; ------------------ | | 825| 106|#define WS(stride, i) (stride * i) ------------------ 346| 106| Tj = ri[WS(is, 1)]; ------------------ | | 825| 106|#define WS(stride, i) (stride * i) ------------------ 347| 106| Tk = Ti + Tj; 348| 106| TJ = FNMS(KP500000000, Tk, Th); ------------------ | | 1134| 106|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 349| 106| T1h = KP866025403 * (Tj - Ti); 350| 106| T1i = ii[WS(is, 6)]; ------------------ | | 825| 106|#define WS(stride, i) (stride * i) ------------------ 351| 106| TK = ii[WS(is, 11)]; ------------------ | | 825| 106|#define WS(stride, i) (stride * i) ------------------ 352| 106| TL = ii[WS(is, 1)]; ------------------ | | 825| 106|#define WS(stride, i) (stride * i) ------------------ 353| 106| T1j = TK + TL; 354| 106| TM = KP866025403 * (TK - TL); 355| 106| T1k = FNMS(KP500000000, T1j, T1i); ------------------ | | 1134| 106|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 356| 106| } 357| 106| { 358| 106| E Tn, To, TP, TQ; 359| 106| Tm = ri[WS(is, 9)]; ------------------ | | 825| 106|#define WS(stride, i) (stride * i) ------------------ 360| 106| Tn = ri[WS(is, 14)]; ------------------ | | 825| 106|#define WS(stride, i) (stride * i) ------------------ 361| 106| To = ri[WS(is, 4)]; ------------------ | | 825| 106|#define WS(stride, i) (stride * i) ------------------ 362| 106| Tp = Tn + To; 363| 106| TO = FNMS(KP500000000, Tp, Tm); ------------------ | | 1134| 106|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 364| 106| T1m = KP866025403 * (To - Tn); 365| 106| T1n = ii[WS(is, 9)]; ------------------ | | 825| 106|#define WS(stride, i) (stride * i) ------------------ 366| 106| TP = ii[WS(is, 14)]; ------------------ | | 825| 106|#define WS(stride, i) (stride * i) ------------------ 367| 106| TQ = ii[WS(is, 4)]; ------------------ | | 825| 106|#define WS(stride, i) (stride * i) ------------------ 368| 106| T1o = TP + TQ; 369| 106| TR = KP866025403 * (TP - TQ); 370| 106| T1p = FNMS(KP500000000, T1o, T1n); ------------------ | | 1134| 106|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 371| 106| } 372| 106| Tl = Th + Tk; 373| 106| Tq = Tm + Tp; 374| 106| Tr = Tl + Tq; 375| 106| TN = TJ - TM; 376| 106| TS = TO - TR; 377| 106| TT = TN + TS; 378| 106| T2c = T1i + T1j; 379| 106| T2d = T1n + T1o; 380| 106| T2n = T2c + T2d; 381| 106| T1O = T1k - T1h; 382| 106| T1P = T1p - T1m; 383| 106| T22 = T1O + T1P; 384| 106| T1l = T1h + T1k; 385| 106| T1q = T1m + T1p; 386| 106| T1w = T1l + T1q; 387| 106| TZ = TJ + TM; 388| 106| T10 = TO + TR; 389| 106| T11 = TZ + T10; 390| 106| } 391| 106| { 392| 106| E T6, T9, Ty, T16, T17, T18, TB, T19, Tb, Te, TD, T1b, T1c, T1d, TG; 393| 106| E T1e; 394| 106| { 395| 106| E T7, T8, Tz, TA; 396| 106| T6 = ri[WS(is, 3)]; ------------------ | | 825| 106|#define WS(stride, i) (stride * i) ------------------ 397| 106| T7 = ri[WS(is, 8)]; ------------------ | | 825| 106|#define WS(stride, i) (stride * i) ------------------ 398| 106| T8 = ri[WS(is, 13)]; ------------------ | | 825| 106|#define WS(stride, i) (stride * i) ------------------ 399| 106| T9 = T7 + T8; 400| 106| Ty = FNMS(KP500000000, T9, T6); ------------------ | | 1134| 106|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 401| 106| T16 = KP866025403 * (T8 - T7); 402| 106| T17 = ii[WS(is, 3)]; ------------------ | | 825| 106|#define WS(stride, i) (stride * i) ------------------ 403| 106| Tz = ii[WS(is, 8)]; ------------------ | | 825| 106|#define WS(stride, i) (stride * i) ------------------ 404| 106| TA = ii[WS(is, 13)]; ------------------ | | 825| 106|#define WS(stride, i) (stride * i) ------------------ 405| 106| T18 = Tz + TA; 406| 106| TB = KP866025403 * (Tz - TA); 407| 106| T19 = FNMS(KP500000000, T18, T17); ------------------ | | 1134| 106|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 408| 106| } 409| 106| { 410| 106| E Tc, Td, TE, TF; 411| 106| Tb = ri[WS(is, 12)]; ------------------ | | 825| 106|#define WS(stride, i) (stride * i) ------------------ 412| 106| Tc = ri[WS(is, 2)]; ------------------ | | 825| 106|#define WS(stride, i) (stride * i) ------------------ 413| 106| Td = ri[WS(is, 7)]; ------------------ | | 825| 106|#define WS(stride, i) (stride * i) ------------------ 414| 106| Te = Tc + Td; 415| 106| TD = FNMS(KP500000000, Te, Tb); ------------------ | | 1134| 106|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 416| 106| T1b = KP866025403 * (Td - Tc); 417| 106| T1c = ii[WS(is, 12)]; ------------------ | | 825| 106|#define WS(stride, i) (stride * i) ------------------ 418| 106| TE = ii[WS(is, 2)]; ------------------ | | 825| 106|#define WS(stride, i) (stride * i) ------------------ 419| 106| TF = ii[WS(is, 7)]; ------------------ | | 825| 106|#define WS(stride, i) (stride * i) ------------------ 420| 106| T1d = TE + TF; 421| 106| TG = KP866025403 * (TE - TF); 422| 106| T1e = FNMS(KP500000000, T1d, T1c); ------------------ | | 1134| 106|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 423| 106| } 424| 106| Ta = T6 + T9; 425| 106| Tf = Tb + Te; 426| 106| Tg = Ta + Tf; 427| 106| TC = Ty - TB; 428| 106| TH = TD - TG; 429| 106| TI = TC + TH; 430| 106| T2f = T17 + T18; 431| 106| T2g = T1c + T1d; 432| 106| T2m = T2f + T2g; 433| 106| T1R = T19 - T16; 434| 106| T1S = T1e - T1b; 435| 106| T21 = T1R + T1S; 436| 106| T1a = T16 + T19; 437| 106| T1f = T1b + T1e; 438| 106| T1v = T1a + T1f; 439| 106| TW = Ty + TB; 440| 106| TX = TD + TG; 441| 106| TY = TW + TX; 442| 106| } 443| 106| { 444| 106| E T2a, Ts, T29, T2i, T2k, T2e, T2h, T2j, T2b; 445| 106| T2a = KP559016994 * (Tg - Tr); 446| 106| Ts = Tg + Tr; 447| 106| T29 = FNMS(KP250000000, Ts, T5); ------------------ | | 1134| 106|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 448| 106| T2e = T2c - T2d; 449| 106| T2h = T2f - T2g; 450| 106| T2i = FNMS(KP587785252, T2h, KP951056516 * T2e); ------------------ | | 1134| 106|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 451| 106| T2k = FMA(KP951056516, T2h, KP587785252 * T2e); ------------------ | | 1131| 106|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 452| 106| ro[0] = T5 + Ts; 453| 106| T2j = T2a + T29; 454| 106| ro[WS(os, 9)] = T2j - T2k; ------------------ | | 825| 106|#define WS(stride, i) (stride * i) ------------------ 455| 106| ro[WS(os, 6)] = T2j + T2k; ------------------ | | 825| 106|#define WS(stride, i) (stride * i) ------------------ 456| 106| T2b = T29 - T2a; 457| 106| ro[WS(os, 12)] = T2b - T2i; ------------------ | | 825| 106|#define WS(stride, i) (stride * i) ------------------ 458| 106| ro[WS(os, 3)] = T2b + T2i; ------------------ | | 825| 106|#define WS(stride, i) (stride * i) ------------------ 459| 106| } 460| 106| { 461| 106| E T2q, T2o, T2p, T2u, T2w, T2s, T2t, T2v, T2r; 462| 106| T2q = KP559016994 * (T2m - T2n); 463| 106| T2o = T2m + T2n; 464| 106| T2p = FNMS(KP250000000, T2o, T2l); ------------------ | | 1134| 106|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 465| 106| T2s = Tl - Tq; 466| 106| T2t = Ta - Tf; 467| 106| T2u = FNMS(KP587785252, T2t, KP951056516 * T2s); ------------------ | | 1134| 106|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 468| 106| T2w = FMA(KP951056516, T2t, KP587785252 * T2s); ------------------ | | 1131| 106|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 469| 106| io[0] = T2l + T2o; 470| 106| T2v = T2q + T2p; 471| 106| io[WS(os, 6)] = T2v - T2w; ------------------ | | 825| 106|#define WS(stride, i) (stride * i) ------------------ 472| 106| io[WS(os, 9)] = T2w + T2v; ------------------ | | 825| 106|#define WS(stride, i) (stride * i) ------------------ 473| 106| T2r = T2p - T2q; 474| 106| io[WS(os, 3)] = T2r - T2u; ------------------ | | 825| 106|#define WS(stride, i) (stride * i) ------------------ 475| 106| io[WS(os, 12)] = T2u + T2r; ------------------ | | 825| 106|#define WS(stride, i) (stride * i) ------------------ 476| 106| } 477| 106| { 478| 106| E T1M, TU, T1L, T1U, T1W, T1Q, T1T, T1V, T1N; 479| 106| T1M = KP559016994 * (TI - TT); 480| 106| TU = TI + TT; 481| 106| T1L = FNMS(KP250000000, TU, Tx); ------------------ | | 1134| 106|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 482| 106| T1Q = T1O - T1P; 483| 106| T1T = T1R - T1S; 484| 106| T1U = FNMS(KP587785252, T1T, KP951056516 * T1Q); ------------------ | | 1134| 106|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 485| 106| T1W = FMA(KP951056516, T1T, KP587785252 * T1Q); ------------------ | | 1131| 106|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 486| 106| ro[WS(os, 5)] = Tx + TU; ------------------ | | 825| 106|#define WS(stride, i) (stride * i) ------------------ 487| 106| T1V = T1M + T1L; 488| 106| ro[WS(os, 14)] = T1V - T1W; ------------------ | | 825| 106|#define WS(stride, i) (stride * i) ------------------ 489| 106| ro[WS(os, 11)] = T1V + T1W; ------------------ | | 825| 106|#define WS(stride, i) (stride * i) ------------------ 490| 106| T1N = T1L - T1M; 491| 106| ro[WS(os, 2)] = T1N - T1U; ------------------ | | 825| 106|#define WS(stride, i) (stride * i) ------------------ 492| 106| ro[WS(os, 8)] = T1N + T1U; ------------------ | | 825| 106|#define WS(stride, i) (stride * i) ------------------ 493| 106| } 494| 106| { 495| 106| E T25, T23, T24, T1Z, T28, T1X, T1Y, T27, T26; 496| 106| T25 = KP559016994 * (T21 - T22); 497| 106| T23 = T21 + T22; 498| 106| T24 = FNMS(KP250000000, T23, T20); ------------------ | | 1134| 106|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 499| 106| T1X = TN - TS; 500| 106| T1Y = TC - TH; 501| 106| T1Z = FNMS(KP587785252, T1Y, KP951056516 * T1X); ------------------ | | 1134| 106|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 502| 106| T28 = FMA(KP951056516, T1Y, KP587785252 * T1X); ------------------ | | 1131| 106|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 503| 106| io[WS(os, 5)] = T20 + T23; ------------------ | | 825| 106|#define WS(stride, i) (stride * i) ------------------ 504| 106| T27 = T25 + T24; 505| 106| io[WS(os, 11)] = T27 - T28; ------------------ | | 825| 106|#define WS(stride, i) (stride * i) ------------------ 506| 106| io[WS(os, 14)] = T28 + T27; ------------------ | | 825| 106|#define WS(stride, i) (stride * i) ------------------ 507| 106| T26 = T24 - T25; 508| 106| io[WS(os, 2)] = T1Z + T26; ------------------ | | 825| 106|#define WS(stride, i) (stride * i) ------------------ 509| 106| io[WS(os, 8)] = T26 - T1Z; ------------------ | | 825| 106|#define WS(stride, i) (stride * i) ------------------ 510| 106| } 511| 106| { 512| 106| E T1x, T1D, T1E, T1I, T1J, T1G, T1H, T1K, T1F; 513| 106| T1x = KP559016994 * (T1v - T1w); 514| 106| T1D = T1v + T1w; 515| 106| T1E = FNMS(KP250000000, T1D, T1C); ------------------ | | 1134| 106|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 516| 106| T1G = TW - TX; 517| 106| T1H = TZ - T10; 518| 106| T1I = FMA(KP951056516, T1G, KP587785252 * T1H); ------------------ | | 1131| 106|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 519| 106| T1J = FNMS(KP587785252, T1G, KP951056516 * T1H); ------------------ | | 1134| 106|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 520| 106| io[WS(os, 10)] = T1C + T1D; ------------------ | | 825| 106|#define WS(stride, i) (stride * i) ------------------ 521| 106| T1K = T1E - T1x; 522| 106| io[WS(os, 7)] = T1J + T1K; ------------------ | | 825| 106|#define WS(stride, i) (stride * i) ------------------ 523| 106| io[WS(os, 13)] = T1K - T1J; ------------------ | | 825| 106|#define WS(stride, i) (stride * i) ------------------ 524| 106| T1F = T1x + T1E; 525| 106| io[WS(os, 1)] = T1F - T1I; ------------------ | | 825| 106|#define WS(stride, i) (stride * i) ------------------ 526| 106| io[WS(os, 4)] = T1I + T1F; ------------------ | | 825| 106|#define WS(stride, i) (stride * i) ------------------ 527| 106| } 528| 106| { 529| 106| E T13, T12, T14, T1s, T1u, T1g, T1r, T1t, T15; 530| 106| T13 = KP559016994 * (TY - T11); 531| 106| T12 = TY + T11; 532| 106| T14 = FNMS(KP250000000, T12, TV); ------------------ | | 1134| 106|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 533| 106| T1g = T1a - T1f; 534| 106| T1r = T1l - T1q; 535| 106| T1s = FMA(KP951056516, T1g, KP587785252 * T1r); ------------------ | | 1131| 106|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 536| 106| T1u = FNMS(KP587785252, T1g, KP951056516 * T1r); ------------------ | | 1134| 106|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 537| 106| ro[WS(os, 10)] = TV + T12; ------------------ | | 825| 106|#define WS(stride, i) (stride * i) ------------------ 538| 106| T1t = T14 - T13; 539| 106| ro[WS(os, 7)] = T1t - T1u; ------------------ | | 825| 106|#define WS(stride, i) (stride * i) ------------------ 540| 106| ro[WS(os, 13)] = T1t + T1u; ------------------ | | 825| 106|#define WS(stride, i) (stride * i) ------------------ 541| 106| T15 = T13 + T14; 542| 106| ro[WS(os, 4)] = T15 - T1s; ------------------ | | 825| 106|#define WS(stride, i) (stride * i) ------------------ 543| 106| ro[WS(os, 1)] = T15 + T1s; ------------------ | | 825| 106|#define WS(stride, i) (stride * i) ------------------ 544| 106| } 545| 106| } 546| 9| } 547| 9|} fftw_codelet_n1_16: 557| 1|void X(codelet_n1_16) (planner *p) { X(kdft_register) (p, n1_16, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 558| 1|} n1_16.c:n1_16: 305| 9|{ 306| 9| DK(KP382683432, +0.382683432365089771728459984030398866761344562); ------------------ | | 1056| 9|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 9|# define K(x) ((E) x) | | ------------------ ------------------ 307| 9| DK(KP923879532, +0.923879532511286756128183189396788286822416626); ------------------ | | 1056| 9|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 9|# define K(x) ((E) x) | | ------------------ ------------------ 308| 9| DK(KP707106781, +0.707106781186547524400844362104849039284835938); ------------------ | | 1056| 9|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 9|# define K(x) ((E) x) | | ------------------ ------------------ 309| 9| { 310| 9| INT i; 311| 124| for (i = v; i > 0; i = i - 1, ri = ri + ivs, ii = ii + ivs, ro = ro + ovs, io = io + ovs, MAKE_VOLATILE_STRIDE(64, is), MAKE_VOLATILE_STRIDE(64, os)) { ------------------ | | 851| 115| (nptr <= ESTIMATED_AVAILABLE_INDEX_REGISTERS ? \ | | ------------------ | | | | 849| 230|#define ESTIMATED_AVAILABLE_INDEX_REGISTERS 16 | | ------------------ | | | Branch (851:7): [Folded, False: 115] | | ------------------ | | 852| 115| 0 : \ | | 853| 115| ((x) = (x) ^ X(an_INT_guaranteed_to_be_zero))) | | ------------------ | | | | 78| 115|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 115|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ for (i = v; i > 0; i = i - 1, ri = ri + ivs, ii = ii + ivs, ro = ro + ovs, io = io + ovs, MAKE_VOLATILE_STRIDE(64, is), MAKE_VOLATILE_STRIDE(64, os)) { ------------------ | | 851| 115| (nptr <= ESTIMATED_AVAILABLE_INDEX_REGISTERS ? \ | | ------------------ | | | | 849| 230|#define ESTIMATED_AVAILABLE_INDEX_REGISTERS 16 | | ------------------ | | | Branch (851:7): [Folded, False: 115] | | ------------------ | | 852| 115| 0 : \ | | 853| 115| ((x) = (x) ^ X(an_INT_guaranteed_to_be_zero))) | | ------------------ | | | | 78| 115|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 115|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ | Branch (311:16): [True: 115, False: 9] ------------------ 312| 115| E T7, T1R, T25, TC, TN, T1x, T1H, T1l, Tt, T22, T2h, T1b, T1g, T1E, T1Z; 313| 115| E T1D, Te, T1S, T26, TJ, TQ, T1m, T1n, TT, Tm, T1X, T2g, T10, T15, T1B; 314| 115| E T1U, T1A; 315| 115| { 316| 115| E T3, TL, Ty, T1k, T6, T1j, TB, TM; 317| 115| { 318| 115| E T1, T2, Tw, Tx; 319| 115| T1 = ri[0]; 320| 115| T2 = ri[WS(is, 8)]; ------------------ | | 825| 115|#define WS(stride, i) (stride * i) ------------------ 321| 115| T3 = T1 + T2; 322| 115| TL = T1 - T2; 323| 115| Tw = ii[0]; 324| 115| Tx = ii[WS(is, 8)]; ------------------ | | 825| 115|#define WS(stride, i) (stride * i) ------------------ 325| 115| Ty = Tw + Tx; 326| 115| T1k = Tw - Tx; 327| 115| } 328| 115| { 329| 115| E T4, T5, Tz, TA; 330| 115| T4 = ri[WS(is, 4)]; ------------------ | | 825| 115|#define WS(stride, i) (stride * i) ------------------ 331| 115| T5 = ri[WS(is, 12)]; ------------------ | | 825| 115|#define WS(stride, i) (stride * i) ------------------ 332| 115| T6 = T4 + T5; 333| 115| T1j = T4 - T5; 334| 115| Tz = ii[WS(is, 4)]; ------------------ | | 825| 115|#define WS(stride, i) (stride * i) ------------------ 335| 115| TA = ii[WS(is, 12)]; ------------------ | | 825| 115|#define WS(stride, i) (stride * i) ------------------ 336| 115| TB = Tz + TA; 337| 115| TM = Tz - TA; 338| 115| } 339| 115| T7 = T3 + T6; 340| 115| T1R = T3 - T6; 341| 115| T25 = Ty - TB; 342| 115| TC = Ty + TB; 343| 115| TN = TL - TM; 344| 115| T1x = TL + TM; 345| 115| T1H = T1k - T1j; 346| 115| T1l = T1j + T1k; 347| 115| } 348| 115| { 349| 115| E Tp, T17, T1f, T20, Ts, T1c, T1a, T21; 350| 115| { 351| 115| E Tn, To, T1d, T1e; 352| 115| Tn = ri[WS(is, 15)]; ------------------ | | 825| 115|#define WS(stride, i) (stride * i) ------------------ 353| 115| To = ri[WS(is, 7)]; ------------------ | | 825| 115|#define WS(stride, i) (stride * i) ------------------ 354| 115| Tp = Tn + To; 355| 115| T17 = Tn - To; 356| 115| T1d = ii[WS(is, 15)]; ------------------ | | 825| 115|#define WS(stride, i) (stride * i) ------------------ 357| 115| T1e = ii[WS(is, 7)]; ------------------ | | 825| 115|#define WS(stride, i) (stride * i) ------------------ 358| 115| T1f = T1d - T1e; 359| 115| T20 = T1d + T1e; 360| 115| } 361| 115| { 362| 115| E Tq, Tr, T18, T19; 363| 115| Tq = ri[WS(is, 3)]; ------------------ | | 825| 115|#define WS(stride, i) (stride * i) ------------------ 364| 115| Tr = ri[WS(is, 11)]; ------------------ | | 825| 115|#define WS(stride, i) (stride * i) ------------------ 365| 115| Ts = Tq + Tr; 366| 115| T1c = Tq - Tr; 367| 115| T18 = ii[WS(is, 3)]; ------------------ | | 825| 115|#define WS(stride, i) (stride * i) ------------------ 368| 115| T19 = ii[WS(is, 11)]; ------------------ | | 825| 115|#define WS(stride, i) (stride * i) ------------------ 369| 115| T1a = T18 - T19; 370| 115| T21 = T18 + T19; 371| 115| } 372| 115| Tt = Tp + Ts; 373| 115| T22 = T20 - T21; 374| 115| T2h = T20 + T21; 375| 115| T1b = T17 - T1a; 376| 115| T1g = T1c + T1f; 377| 115| T1E = T1f - T1c; 378| 115| T1Z = Tp - Ts; 379| 115| T1D = T17 + T1a; 380| 115| } 381| 115| { 382| 115| E Ta, TP, TF, TO, Td, TR, TI, TS; 383| 115| { 384| 115| E T8, T9, TD, TE; 385| 115| T8 = ri[WS(is, 2)]; ------------------ | | 825| 115|#define WS(stride, i) (stride * i) ------------------ 386| 115| T9 = ri[WS(is, 10)]; ------------------ | | 825| 115|#define WS(stride, i) (stride * i) ------------------ 387| 115| Ta = T8 + T9; 388| 115| TP = T8 - T9; 389| 115| TD = ii[WS(is, 2)]; ------------------ | | 825| 115|#define WS(stride, i) (stride * i) ------------------ 390| 115| TE = ii[WS(is, 10)]; ------------------ | | 825| 115|#define WS(stride, i) (stride * i) ------------------ 391| 115| TF = TD + TE; 392| 115| TO = TD - TE; 393| 115| } 394| 115| { 395| 115| E Tb, Tc, TG, TH; 396| 115| Tb = ri[WS(is, 14)]; ------------------ | | 825| 115|#define WS(stride, i) (stride * i) ------------------ 397| 115| Tc = ri[WS(is, 6)]; ------------------ | | 825| 115|#define WS(stride, i) (stride * i) ------------------ 398| 115| Td = Tb + Tc; 399| 115| TR = Tb - Tc; 400| 115| TG = ii[WS(is, 14)]; ------------------ | | 825| 115|#define WS(stride, i) (stride * i) ------------------ 401| 115| TH = ii[WS(is, 6)]; ------------------ | | 825| 115|#define WS(stride, i) (stride * i) ------------------ 402| 115| TI = TG + TH; 403| 115| TS = TG - TH; 404| 115| } 405| 115| Te = Ta + Td; 406| 115| T1S = TF - TI; 407| 115| T26 = Td - Ta; 408| 115| TJ = TF + TI; 409| 115| TQ = TO - TP; 410| 115| T1m = TR - TS; 411| 115| T1n = TP + TO; 412| 115| TT = TR + TS; 413| 115| } 414| 115| { 415| 115| E Ti, T11, TZ, T1V, Tl, TW, T14, T1W; 416| 115| { 417| 115| E Tg, Th, TX, TY; 418| 115| Tg = ri[WS(is, 1)]; ------------------ | | 825| 115|#define WS(stride, i) (stride * i) ------------------ 419| 115| Th = ri[WS(is, 9)]; ------------------ | | 825| 115|#define WS(stride, i) (stride * i) ------------------ 420| 115| Ti = Tg + Th; 421| 115| T11 = Tg - Th; 422| 115| TX = ii[WS(is, 1)]; ------------------ | | 825| 115|#define WS(stride, i) (stride * i) ------------------ 423| 115| TY = ii[WS(is, 9)]; ------------------ | | 825| 115|#define WS(stride, i) (stride * i) ------------------ 424| 115| TZ = TX - TY; 425| 115| T1V = TX + TY; 426| 115| } 427| 115| { 428| 115| E Tj, Tk, T12, T13; 429| 115| Tj = ri[WS(is, 5)]; ------------------ | | 825| 115|#define WS(stride, i) (stride * i) ------------------ 430| 115| Tk = ri[WS(is, 13)]; ------------------ | | 825| 115|#define WS(stride, i) (stride * i) ------------------ 431| 115| Tl = Tj + Tk; 432| 115| TW = Tj - Tk; 433| 115| T12 = ii[WS(is, 5)]; ------------------ | | 825| 115|#define WS(stride, i) (stride * i) ------------------ 434| 115| T13 = ii[WS(is, 13)]; ------------------ | | 825| 115|#define WS(stride, i) (stride * i) ------------------ 435| 115| T14 = T12 - T13; 436| 115| T1W = T12 + T13; 437| 115| } 438| 115| Tm = Ti + Tl; 439| 115| T1X = T1V - T1W; 440| 115| T2g = T1V + T1W; 441| 115| T10 = TW + TZ; 442| 115| T15 = T11 - T14; 443| 115| T1B = T11 + T14; 444| 115| T1U = Ti - Tl; 445| 115| T1A = TZ - TW; 446| 115| } 447| 115| { 448| 115| E Tf, Tu, T2j, T2k; 449| 115| Tf = T7 + Te; 450| 115| Tu = Tm + Tt; 451| 115| ro[WS(os, 8)] = Tf - Tu; ------------------ | | 825| 115|#define WS(stride, i) (stride * i) ------------------ 452| 115| ro[0] = Tf + Tu; 453| 115| T2j = TC + TJ; 454| 115| T2k = T2g + T2h; 455| 115| io[WS(os, 8)] = T2j - T2k; ------------------ | | 825| 115|#define WS(stride, i) (stride * i) ------------------ 456| 115| io[0] = T2j + T2k; 457| 115| } 458| 115| { 459| 115| E Tv, TK, T2f, T2i; 460| 115| Tv = Tt - Tm; 461| 115| TK = TC - TJ; 462| 115| io[WS(os, 4)] = Tv + TK; ------------------ | | 825| 115|#define WS(stride, i) (stride * i) ------------------ 463| 115| io[WS(os, 12)] = TK - Tv; ------------------ | | 825| 115|#define WS(stride, i) (stride * i) ------------------ 464| 115| T2f = T7 - Te; 465| 115| T2i = T2g - T2h; 466| 115| ro[WS(os, 12)] = T2f - T2i; ------------------ | | 825| 115|#define WS(stride, i) (stride * i) ------------------ 467| 115| ro[WS(os, 4)] = T2f + T2i; ------------------ | | 825| 115|#define WS(stride, i) (stride * i) ------------------ 468| 115| } 469| 115| { 470| 115| E T1T, T27, T24, T28, T1Y, T23; 471| 115| T1T = T1R + T1S; 472| 115| T27 = T25 - T26; 473| 115| T1Y = T1U + T1X; 474| 115| T23 = T1Z - T22; 475| 115| T24 = KP707106781 * (T1Y + T23); 476| 115| T28 = KP707106781 * (T23 - T1Y); 477| 115| ro[WS(os, 10)] = T1T - T24; ------------------ | | 825| 115|#define WS(stride, i) (stride * i) ------------------ 478| 115| io[WS(os, 6)] = T27 + T28; ------------------ | | 825| 115|#define WS(stride, i) (stride * i) ------------------ 479| 115| ro[WS(os, 2)] = T1T + T24; ------------------ | | 825| 115|#define WS(stride, i) (stride * i) ------------------ 480| 115| io[WS(os, 14)] = T27 - T28; ------------------ | | 825| 115|#define WS(stride, i) (stride * i) ------------------ 481| 115| } 482| 115| { 483| 115| E T29, T2d, T2c, T2e, T2a, T2b; 484| 115| T29 = T1R - T1S; 485| 115| T2d = T26 + T25; 486| 115| T2a = T1X - T1U; 487| 115| T2b = T1Z + T22; 488| 115| T2c = KP707106781 * (T2a - T2b); 489| 115| T2e = KP707106781 * (T2a + T2b); 490| 115| ro[WS(os, 14)] = T29 - T2c; ------------------ | | 825| 115|#define WS(stride, i) (stride * i) ------------------ 491| 115| io[WS(os, 2)] = T2d + T2e; ------------------ | | 825| 115|#define WS(stride, i) (stride * i) ------------------ 492| 115| ro[WS(os, 6)] = T29 + T2c; ------------------ | | 825| 115|#define WS(stride, i) (stride * i) ------------------ 493| 115| io[WS(os, 10)] = T2d - T2e; ------------------ | | 825| 115|#define WS(stride, i) (stride * i) ------------------ 494| 115| } 495| 115| { 496| 115| E TV, T1r, T1p, T1v, T1i, T1q, T1u, T1w, TU, T1o; 497| 115| TU = KP707106781 * (TQ - TT); 498| 115| TV = TN + TU; 499| 115| T1r = TN - TU; 500| 115| T1o = KP707106781 * (T1m - T1n); 501| 115| T1p = T1l - T1o; 502| 115| T1v = T1l + T1o; 503| 115| { 504| 115| E T16, T1h, T1s, T1t; 505| 115| T16 = FMA(KP923879532, T10, KP382683432 * T15); ------------------ | | 1131| 115|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 506| 115| T1h = FNMS(KP923879532, T1g, KP382683432 * T1b); ------------------ | | 1134| 115|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 507| 115| T1i = T16 + T1h; 508| 115| T1q = T1h - T16; 509| 115| T1s = FNMS(KP923879532, T15, KP382683432 * T10); ------------------ | | 1134| 115|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 510| 115| T1t = FMA(KP382683432, T1g, KP923879532 * T1b); ------------------ | | 1131| 115|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 511| 115| T1u = T1s - T1t; 512| 115| T1w = T1s + T1t; 513| 115| } 514| 115| ro[WS(os, 11)] = TV - T1i; ------------------ | | 825| 115|#define WS(stride, i) (stride * i) ------------------ 515| 115| io[WS(os, 11)] = T1v - T1w; ------------------ | | 825| 115|#define WS(stride, i) (stride * i) ------------------ 516| 115| ro[WS(os, 3)] = TV + T1i; ------------------ | | 825| 115|#define WS(stride, i) (stride * i) ------------------ 517| 115| io[WS(os, 3)] = T1v + T1w; ------------------ | | 825| 115|#define WS(stride, i) (stride * i) ------------------ 518| 115| io[WS(os, 15)] = T1p - T1q; ------------------ | | 825| 115|#define WS(stride, i) (stride * i) ------------------ 519| 115| ro[WS(os, 15)] = T1r - T1u; ------------------ | | 825| 115|#define WS(stride, i) (stride * i) ------------------ 520| 115| io[WS(os, 7)] = T1p + T1q; ------------------ | | 825| 115|#define WS(stride, i) (stride * i) ------------------ 521| 115| ro[WS(os, 7)] = T1r + T1u; ------------------ | | 825| 115|#define WS(stride, i) (stride * i) ------------------ 522| 115| } 523| 115| { 524| 115| E T1z, T1L, T1J, T1P, T1G, T1K, T1O, T1Q, T1y, T1I; 525| 115| T1y = KP707106781 * (T1n + T1m); 526| 115| T1z = T1x + T1y; 527| 115| T1L = T1x - T1y; 528| 115| T1I = KP707106781 * (TQ + TT); 529| 115| T1J = T1H - T1I; 530| 115| T1P = T1H + T1I; 531| 115| { 532| 115| E T1C, T1F, T1M, T1N; 533| 115| T1C = FMA(KP382683432, T1A, KP923879532 * T1B); ------------------ | | 1131| 115|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 534| 115| T1F = FNMS(KP382683432, T1E, KP923879532 * T1D); ------------------ | | 1134| 115|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 535| 115| T1G = T1C + T1F; 536| 115| T1K = T1F - T1C; 537| 115| T1M = FNMS(KP382683432, T1B, KP923879532 * T1A); ------------------ | | 1134| 115|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 538| 115| T1N = FMA(KP923879532, T1E, KP382683432 * T1D); ------------------ | | 1131| 115|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 539| 115| T1O = T1M - T1N; 540| 115| T1Q = T1M + T1N; 541| 115| } 542| 115| ro[WS(os, 9)] = T1z - T1G; ------------------ | | 825| 115|#define WS(stride, i) (stride * i) ------------------ 543| 115| io[WS(os, 9)] = T1P - T1Q; ------------------ | | 825| 115|#define WS(stride, i) (stride * i) ------------------ 544| 115| ro[WS(os, 1)] = T1z + T1G; ------------------ | | 825| 115|#define WS(stride, i) (stride * i) ------------------ 545| 115| io[WS(os, 1)] = T1P + T1Q; ------------------ | | 825| 115|#define WS(stride, i) (stride * i) ------------------ 546| 115| io[WS(os, 13)] = T1J - T1K; ------------------ | | 825| 115|#define WS(stride, i) (stride * i) ------------------ 547| 115| ro[WS(os, 13)] = T1L - T1O; ------------------ | | 825| 115|#define WS(stride, i) (stride * i) ------------------ 548| 115| io[WS(os, 5)] = T1J + T1K; ------------------ | | 825| 115|#define WS(stride, i) (stride * i) ------------------ 549| 115| ro[WS(os, 5)] = T1L + T1O; ------------------ | | 825| 115|#define WS(stride, i) (stride * i) ------------------ 550| 115| } 551| 115| } 552| 9| } 553| 9|} fftw_codelet_n1_2: 91| 1|void X(codelet_n1_2) (planner *p) { X(kdft_register) (p, n1_2, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 92| 1|} n1_2.c:n1_2: 72| 6|{ 73| 6| { 74| 6| INT i; 75| 12| for (i = v; i > 0; i = i - 1, ri = ri + ivs, ii = ii + ivs, ro = ro + ovs, io = io + ovs, MAKE_VOLATILE_STRIDE(8, is), MAKE_VOLATILE_STRIDE(8, os)) { ------------------ | | 851| 6| (nptr <= ESTIMATED_AVAILABLE_INDEX_REGISTERS ? \ | | ------------------ | | | | 849| 12|#define ESTIMATED_AVAILABLE_INDEX_REGISTERS 16 | | ------------------ | | | Branch (851:7): [True: 6, Folded] | | ------------------ | | 852| 6| 0 : \ | | 853| 6| ((x) = (x) ^ X(an_INT_guaranteed_to_be_zero))) | | ------------------ | | | | 78| 0|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ for (i = v; i > 0; i = i - 1, ri = ri + ivs, ii = ii + ivs, ro = ro + ovs, io = io + ovs, MAKE_VOLATILE_STRIDE(8, is), MAKE_VOLATILE_STRIDE(8, os)) { ------------------ | | 851| 6| (nptr <= ESTIMATED_AVAILABLE_INDEX_REGISTERS ? \ | | ------------------ | | | | 849| 12|#define ESTIMATED_AVAILABLE_INDEX_REGISTERS 16 | | ------------------ | | | Branch (851:7): [True: 6, Folded] | | ------------------ | | 852| 6| 0 : \ | | 853| 6| ((x) = (x) ^ X(an_INT_guaranteed_to_be_zero))) | | ------------------ | | | | 78| 0|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ | Branch (75:16): [True: 6, False: 6] ------------------ 76| 6| E T1, T2, T3, T4; 77| 6| T1 = ri[0]; 78| 6| T2 = ri[WS(is, 1)]; ------------------ | | 825| 6|#define WS(stride, i) (stride * i) ------------------ 79| 6| ro[WS(os, 1)] = T1 - T2; ------------------ | | 825| 6|#define WS(stride, i) (stride * i) ------------------ 80| 6| ro[0] = T1 + T2; 81| 6| T3 = ii[0]; 82| 6| T4 = ii[WS(is, 1)]; ------------------ | | 825| 6|#define WS(stride, i) (stride * i) ------------------ 83| 6| io[WS(os, 1)] = T3 - T4; ------------------ | | 825| 6|#define WS(stride, i) (stride * i) ------------------ 84| 6| io[0] = T3 + T4; 85| 6| } 86| 6| } 87| 6|} fftw_codelet_n1_20: 715| 1|void X(codelet_n1_20) (planner *p) { X(kdft_register) (p, n1_20, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 716| 1|} n1_20.c:n1_20: 384| 5|{ 385| 5| DK(KP587785252, +0.587785252292473129168705954639072768597652438); ------------------ | | 1056| 5|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 5|# define K(x) ((E) x) | | ------------------ ------------------ 386| 5| DK(KP951056516, +0.951056516295153572116439333379382143405698634); ------------------ | | 1056| 5|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 5|# define K(x) ((E) x) | | ------------------ ------------------ 387| 5| DK(KP250000000, +0.250000000000000000000000000000000000000000000); ------------------ | | 1056| 5|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 5|# define K(x) ((E) x) | | ------------------ ------------------ 388| 5| DK(KP559016994, +0.559016994374947424102293417182819058860154590); ------------------ | | 1056| 5|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 5|# define K(x) ((E) x) | | ------------------ ------------------ 389| 5| { 390| 5| INT i; 391| 76| for (i = v; i > 0; i = i - 1, ri = ri + ivs, ii = ii + ivs, ro = ro + ovs, io = io + ovs, MAKE_VOLATILE_STRIDE(80, is), MAKE_VOLATILE_STRIDE(80, os)) { ------------------ | | 851| 71| (nptr <= ESTIMATED_AVAILABLE_INDEX_REGISTERS ? \ | | ------------------ | | | | 849| 142|#define ESTIMATED_AVAILABLE_INDEX_REGISTERS 16 | | ------------------ | | | Branch (851:7): [Folded, False: 71] | | ------------------ | | 852| 71| 0 : \ | | 853| 71| ((x) = (x) ^ X(an_INT_guaranteed_to_be_zero))) | | ------------------ | | | | 78| 71|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 71|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ for (i = v; i > 0; i = i - 1, ri = ri + ivs, ii = ii + ivs, ro = ro + ovs, io = io + ovs, MAKE_VOLATILE_STRIDE(80, is), MAKE_VOLATILE_STRIDE(80, os)) { ------------------ | | 851| 71| (nptr <= ESTIMATED_AVAILABLE_INDEX_REGISTERS ? \ | | ------------------ | | | | 849| 142|#define ESTIMATED_AVAILABLE_INDEX_REGISTERS 16 | | ------------------ | | | Branch (851:7): [Folded, False: 71] | | ------------------ | | 852| 71| 0 : \ | | 853| 71| ((x) = (x) ^ X(an_INT_guaranteed_to_be_zero))) | | ------------------ | | | | 78| 71|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 71|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ | Branch (391:16): [True: 71, False: 5] ------------------ 392| 71| E T7, T2Q, T3h, TD, TP, T1U, T2l, T1d, Tt, TA, TB, T2w, T2z, T2S, T35; 393| 71| E T36, T3f, TH, TI, TJ, T15, T1a, T1b, T1s, T1x, T1W, T29, T2a, T2j, T1h; 394| 71| E T1i, T1j, Te, Tl, Tm, T2D, T2G, T2R, T32, T33, T3e, TE, TF, TG, TU; 395| 71| E TZ, T10, T1D, T1I, T1V, T26, T27, T2i, T1e, T1f, T1g; 396| 71| { 397| 71| E T3, T1Q, TN, T2O, T6, TO, T1T, T2P; 398| 71| { 399| 71| E T1, T2, TL, TM; 400| 71| T1 = ri[0]; 401| 71| T2 = ri[WS(is, 10)]; ------------------ | | 825| 71|#define WS(stride, i) (stride * i) ------------------ 402| 71| T3 = T1 + T2; 403| 71| T1Q = T1 - T2; 404| 71| TL = ii[0]; 405| 71| TM = ii[WS(is, 10)]; ------------------ | | 825| 71|#define WS(stride, i) (stride * i) ------------------ 406| 71| TN = TL - TM; 407| 71| T2O = TL + TM; 408| 71| } 409| 71| { 410| 71| E T4, T5, T1R, T1S; 411| 71| T4 = ri[WS(is, 5)]; ------------------ | | 825| 71|#define WS(stride, i) (stride * i) ------------------ 412| 71| T5 = ri[WS(is, 15)]; ------------------ | | 825| 71|#define WS(stride, i) (stride * i) ------------------ 413| 71| T6 = T4 + T5; 414| 71| TO = T4 - T5; 415| 71| T1R = ii[WS(is, 5)]; ------------------ | | 825| 71|#define WS(stride, i) (stride * i) ------------------ 416| 71| T1S = ii[WS(is, 15)]; ------------------ | | 825| 71|#define WS(stride, i) (stride * i) ------------------ 417| 71| T1T = T1R - T1S; 418| 71| T2P = T1R + T1S; 419| 71| } 420| 71| T7 = T3 - T6; 421| 71| T2Q = T2O - T2P; 422| 71| T3h = T2O + T2P; 423| 71| TD = T3 + T6; 424| 71| TP = TN - TO; 425| 71| T1U = T1Q - T1T; 426| 71| T2l = T1Q + T1T; 427| 71| T1d = TO + TN; 428| 71| } 429| 71| { 430| 71| E Tp, T1o, T13, T2u, Ts, T14, T1r, T2v, Tw, T1t, T18, T2x, Tz, T19, T1w; 431| 71| E T2y; 432| 71| { 433| 71| E Tn, To, T11, T12; 434| 71| Tn = ri[WS(is, 8)]; ------------------ | | 825| 71|#define WS(stride, i) (stride * i) ------------------ 435| 71| To = ri[WS(is, 18)]; ------------------ | | 825| 71|#define WS(stride, i) (stride * i) ------------------ 436| 71| Tp = Tn + To; 437| 71| T1o = Tn - To; 438| 71| T11 = ii[WS(is, 8)]; ------------------ | | 825| 71|#define WS(stride, i) (stride * i) ------------------ 439| 71| T12 = ii[WS(is, 18)]; ------------------ | | 825| 71|#define WS(stride, i) (stride * i) ------------------ 440| 71| T13 = T11 - T12; 441| 71| T2u = T11 + T12; 442| 71| } 443| 71| { 444| 71| E Tq, Tr, T1p, T1q; 445| 71| Tq = ri[WS(is, 13)]; ------------------ | | 825| 71|#define WS(stride, i) (stride * i) ------------------ 446| 71| Tr = ri[WS(is, 3)]; ------------------ | | 825| 71|#define WS(stride, i) (stride * i) ------------------ 447| 71| Ts = Tq + Tr; 448| 71| T14 = Tq - Tr; 449| 71| T1p = ii[WS(is, 13)]; ------------------ | | 825| 71|#define WS(stride, i) (stride * i) ------------------ 450| 71| T1q = ii[WS(is, 3)]; ------------------ | | 825| 71|#define WS(stride, i) (stride * i) ------------------ 451| 71| T1r = T1p - T1q; 452| 71| T2v = T1p + T1q; 453| 71| } 454| 71| { 455| 71| E Tu, Tv, T16, T17; 456| 71| Tu = ri[WS(is, 12)]; ------------------ | | 825| 71|#define WS(stride, i) (stride * i) ------------------ 457| 71| Tv = ri[WS(is, 2)]; ------------------ | | 825| 71|#define WS(stride, i) (stride * i) ------------------ 458| 71| Tw = Tu + Tv; 459| 71| T1t = Tu - Tv; 460| 71| T16 = ii[WS(is, 12)]; ------------------ | | 825| 71|#define WS(stride, i) (stride * i) ------------------ 461| 71| T17 = ii[WS(is, 2)]; ------------------ | | 825| 71|#define WS(stride, i) (stride * i) ------------------ 462| 71| T18 = T16 - T17; 463| 71| T2x = T16 + T17; 464| 71| } 465| 71| { 466| 71| E Tx, Ty, T1u, T1v; 467| 71| Tx = ri[WS(is, 17)]; ------------------ | | 825| 71|#define WS(stride, i) (stride * i) ------------------ 468| 71| Ty = ri[WS(is, 7)]; ------------------ | | 825| 71|#define WS(stride, i) (stride * i) ------------------ 469| 71| Tz = Tx + Ty; 470| 71| T19 = Tx - Ty; 471| 71| T1u = ii[WS(is, 17)]; ------------------ | | 825| 71|#define WS(stride, i) (stride * i) ------------------ 472| 71| T1v = ii[WS(is, 7)]; ------------------ | | 825| 71|#define WS(stride, i) (stride * i) ------------------ 473| 71| T1w = T1u - T1v; 474| 71| T2y = T1u + T1v; 475| 71| } 476| 71| Tt = Tp - Ts; 477| 71| TA = Tw - Tz; 478| 71| TB = Tt + TA; 479| 71| T2w = T2u - T2v; 480| 71| T2z = T2x - T2y; 481| 71| T2S = T2w + T2z; 482| 71| T35 = T2u + T2v; 483| 71| T36 = T2x + T2y; 484| 71| T3f = T35 + T36; 485| 71| TH = Tp + Ts; 486| 71| TI = Tw + Tz; 487| 71| TJ = TH + TI; 488| 71| T15 = T13 - T14; 489| 71| T1a = T18 - T19; 490| 71| T1b = T15 + T1a; 491| 71| T1s = T1o - T1r; 492| 71| T1x = T1t - T1w; 493| 71| T1W = T1s + T1x; 494| 71| T29 = T1o + T1r; 495| 71| T2a = T1t + T1w; 496| 71| T2j = T29 + T2a; 497| 71| T1h = T14 + T13; 498| 71| T1i = T19 + T18; 499| 71| T1j = T1h + T1i; 500| 71| } 501| 71| { 502| 71| E Ta, T1z, TS, T2B, Td, TT, T1C, T2C, Th, T1E, TX, T2E, Tk, TY, T1H; 503| 71| E T2F; 504| 71| { 505| 71| E T8, T9, TQ, TR; 506| 71| T8 = ri[WS(is, 4)]; ------------------ | | 825| 71|#define WS(stride, i) (stride * i) ------------------ 507| 71| T9 = ri[WS(is, 14)]; ------------------ | | 825| 71|#define WS(stride, i) (stride * i) ------------------ 508| 71| Ta = T8 + T9; 509| 71| T1z = T8 - T9; 510| 71| TQ = ii[WS(is, 4)]; ------------------ | | 825| 71|#define WS(stride, i) (stride * i) ------------------ 511| 71| TR = ii[WS(is, 14)]; ------------------ | | 825| 71|#define WS(stride, i) (stride * i) ------------------ 512| 71| TS = TQ - TR; 513| 71| T2B = TQ + TR; 514| 71| } 515| 71| { 516| 71| E Tb, Tc, T1A, T1B; 517| 71| Tb = ri[WS(is, 9)]; ------------------ | | 825| 71|#define WS(stride, i) (stride * i) ------------------ 518| 71| Tc = ri[WS(is, 19)]; ------------------ | | 825| 71|#define WS(stride, i) (stride * i) ------------------ 519| 71| Td = Tb + Tc; 520| 71| TT = Tb - Tc; 521| 71| T1A = ii[WS(is, 9)]; ------------------ | | 825| 71|#define WS(stride, i) (stride * i) ------------------ 522| 71| T1B = ii[WS(is, 19)]; ------------------ | | 825| 71|#define WS(stride, i) (stride * i) ------------------ 523| 71| T1C = T1A - T1B; 524| 71| T2C = T1A + T1B; 525| 71| } 526| 71| { 527| 71| E Tf, Tg, TV, TW; 528| 71| Tf = ri[WS(is, 16)]; ------------------ | | 825| 71|#define WS(stride, i) (stride * i) ------------------ 529| 71| Tg = ri[WS(is, 6)]; ------------------ | | 825| 71|#define WS(stride, i) (stride * i) ------------------ 530| 71| Th = Tf + Tg; 531| 71| T1E = Tf - Tg; 532| 71| TV = ii[WS(is, 16)]; ------------------ | | 825| 71|#define WS(stride, i) (stride * i) ------------------ 533| 71| TW = ii[WS(is, 6)]; ------------------ | | 825| 71|#define WS(stride, i) (stride * i) ------------------ 534| 71| TX = TV - TW; 535| 71| T2E = TV + TW; 536| 71| } 537| 71| { 538| 71| E Ti, Tj, T1F, T1G; 539| 71| Ti = ri[WS(is, 1)]; ------------------ | | 825| 71|#define WS(stride, i) (stride * i) ------------------ 540| 71| Tj = ri[WS(is, 11)]; ------------------ | | 825| 71|#define WS(stride, i) (stride * i) ------------------ 541| 71| Tk = Ti + Tj; 542| 71| TY = Ti - Tj; 543| 71| T1F = ii[WS(is, 1)]; ------------------ | | 825| 71|#define WS(stride, i) (stride * i) ------------------ 544| 71| T1G = ii[WS(is, 11)]; ------------------ | | 825| 71|#define WS(stride, i) (stride * i) ------------------ 545| 71| T1H = T1F - T1G; 546| 71| T2F = T1F + T1G; 547| 71| } 548| 71| Te = Ta - Td; 549| 71| Tl = Th - Tk; 550| 71| Tm = Te + Tl; 551| 71| T2D = T2B - T2C; 552| 71| T2G = T2E - T2F; 553| 71| T2R = T2D + T2G; 554| 71| T32 = T2B + T2C; 555| 71| T33 = T2E + T2F; 556| 71| T3e = T32 + T33; 557| 71| TE = Ta + Td; 558| 71| TF = Th + Tk; 559| 71| TG = TE + TF; 560| 71| TU = TS - TT; 561| 71| TZ = TX - TY; 562| 71| T10 = TU + TZ; 563| 71| T1D = T1z - T1C; 564| 71| T1I = T1E - T1H; 565| 71| T1V = T1D + T1I; 566| 71| T26 = T1z + T1C; 567| 71| T27 = T1E + T1H; 568| 71| T2i = T26 + T27; 569| 71| T1e = TT + TS; 570| 71| T1f = TY + TX; 571| 71| T1g = T1e + T1f; 572| 71| } 573| 71| { 574| 71| E T2s, TC, T2r, T2I, T2K, T2A, T2H, T2J, T2t; 575| 71| T2s = KP559016994 * (Tm - TB); 576| 71| TC = Tm + TB; 577| 71| T2r = FNMS(KP250000000, TC, T7); ------------------ | | 1134| 71|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 578| 71| T2A = T2w - T2z; 579| 71| T2H = T2D - T2G; 580| 71| T2I = FNMS(KP587785252, T2H, KP951056516 * T2A); ------------------ | | 1134| 71|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 581| 71| T2K = FMA(KP951056516, T2H, KP587785252 * T2A); ------------------ | | 1131| 71|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 582| 71| ro[WS(os, 10)] = T7 + TC; ------------------ | | 825| 71|#define WS(stride, i) (stride * i) ------------------ 583| 71| T2J = T2s + T2r; 584| 71| ro[WS(os, 14)] = T2J - T2K; ------------------ | | 825| 71|#define WS(stride, i) (stride * i) ------------------ 585| 71| ro[WS(os, 6)] = T2J + T2K; ------------------ | | 825| 71|#define WS(stride, i) (stride * i) ------------------ 586| 71| T2t = T2r - T2s; 587| 71| ro[WS(os, 2)] = T2t - T2I; ------------------ | | 825| 71|#define WS(stride, i) (stride * i) ------------------ 588| 71| ro[WS(os, 18)] = T2t + T2I; ------------------ | | 825| 71|#define WS(stride, i) (stride * i) ------------------ 589| 71| } 590| 71| { 591| 71| E T2V, T2T, T2U, T2N, T2Y, T2L, T2M, T2X, T2W; 592| 71| T2V = KP559016994 * (T2R - T2S); 593| 71| T2T = T2R + T2S; 594| 71| T2U = FNMS(KP250000000, T2T, T2Q); ------------------ | | 1134| 71|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 595| 71| T2L = Tt - TA; 596| 71| T2M = Te - Tl; 597| 71| T2N = FNMS(KP587785252, T2M, KP951056516 * T2L); ------------------ | | 1134| 71|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 598| 71| T2Y = FMA(KP951056516, T2M, KP587785252 * T2L); ------------------ | | 1131| 71|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 599| 71| io[WS(os, 10)] = T2Q + T2T; ------------------ | | 825| 71|#define WS(stride, i) (stride * i) ------------------ 600| 71| T2X = T2V + T2U; 601| 71| io[WS(os, 6)] = T2X - T2Y; ------------------ | | 825| 71|#define WS(stride, i) (stride * i) ------------------ 602| 71| io[WS(os, 14)] = T2Y + T2X; ------------------ | | 825| 71|#define WS(stride, i) (stride * i) ------------------ 603| 71| T2W = T2U - T2V; 604| 71| io[WS(os, 2)] = T2N + T2W; ------------------ | | 825| 71|#define WS(stride, i) (stride * i) ------------------ 605| 71| io[WS(os, 18)] = T2W - T2N; ------------------ | | 825| 71|#define WS(stride, i) (stride * i) ------------------ 606| 71| } 607| 71| { 608| 71| E T2Z, TK, T30, T38, T3a, T34, T37, T39, T31; 609| 71| T2Z = KP559016994 * (TG - TJ); 610| 71| TK = TG + TJ; 611| 71| T30 = FNMS(KP250000000, TK, TD); ------------------ | | 1134| 71|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 612| 71| T34 = T32 - T33; 613| 71| T37 = T35 - T36; 614| 71| T38 = FMA(KP951056516, T34, KP587785252 * T37); ------------------ | | 1131| 71|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 615| 71| T3a = FNMS(KP587785252, T34, KP951056516 * T37); ------------------ | | 1134| 71|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 616| 71| ro[0] = TD + TK; 617| 71| T39 = T30 - T2Z; 618| 71| ro[WS(os, 12)] = T39 - T3a; ------------------ | | 825| 71|#define WS(stride, i) (stride * i) ------------------ 619| 71| ro[WS(os, 8)] = T39 + T3a; ------------------ | | 825| 71|#define WS(stride, i) (stride * i) ------------------ 620| 71| T31 = T2Z + T30; 621| 71| ro[WS(os, 4)] = T31 - T38; ------------------ | | 825| 71|#define WS(stride, i) (stride * i) ------------------ 622| 71| ro[WS(os, 16)] = T31 + T38; ------------------ | | 825| 71|#define WS(stride, i) (stride * i) ------------------ 623| 71| } 624| 71| { 625| 71| E T3g, T3i, T3j, T3d, T3m, T3b, T3c, T3l, T3k; 626| 71| T3g = KP559016994 * (T3e - T3f); 627| 71| T3i = T3e + T3f; 628| 71| T3j = FNMS(KP250000000, T3i, T3h); ------------------ | | 1134| 71|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 629| 71| T3b = TE - TF; 630| 71| T3c = TH - TI; 631| 71| T3d = FMA(KP951056516, T3b, KP587785252 * T3c); ------------------ | | 1131| 71|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 632| 71| T3m = FNMS(KP587785252, T3b, KP951056516 * T3c); ------------------ | | 1134| 71|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 633| 71| io[0] = T3h + T3i; 634| 71| T3l = T3j - T3g; 635| 71| io[WS(os, 8)] = T3l - T3m; ------------------ | | 825| 71|#define WS(stride, i) (stride * i) ------------------ 636| 71| io[WS(os, 12)] = T3m + T3l; ------------------ | | 825| 71|#define WS(stride, i) (stride * i) ------------------ 637| 71| T3k = T3g + T3j; 638| 71| io[WS(os, 4)] = T3d + T3k; ------------------ | | 825| 71|#define WS(stride, i) (stride * i) ------------------ 639| 71| io[WS(os, 16)] = T3k - T3d; ------------------ | | 825| 71|#define WS(stride, i) (stride * i) ------------------ 640| 71| } 641| 71| { 642| 71| E T23, T1c, T24, T2c, T2e, T28, T2b, T2d, T25; 643| 71| T23 = KP559016994 * (T10 - T1b); 644| 71| T1c = T10 + T1b; 645| 71| T24 = FNMS(KP250000000, T1c, TP); ------------------ | | 1134| 71|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 646| 71| T28 = T26 - T27; 647| 71| T2b = T29 - T2a; 648| 71| T2c = FMA(KP951056516, T28, KP587785252 * T2b); ------------------ | | 1131| 71|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 649| 71| T2e = FNMS(KP587785252, T28, KP951056516 * T2b); ------------------ | | 1134| 71|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 650| 71| io[WS(os, 5)] = TP + T1c; ------------------ | | 825| 71|#define WS(stride, i) (stride * i) ------------------ 651| 71| T2d = T24 - T23; 652| 71| io[WS(os, 13)] = T2d - T2e; ------------------ | | 825| 71|#define WS(stride, i) (stride * i) ------------------ 653| 71| io[WS(os, 17)] = T2d + T2e; ------------------ | | 825| 71|#define WS(stride, i) (stride * i) ------------------ 654| 71| T25 = T23 + T24; 655| 71| io[WS(os, 1)] = T25 - T2c; ------------------ | | 825| 71|#define WS(stride, i) (stride * i) ------------------ 656| 71| io[WS(os, 9)] = T25 + T2c; ------------------ | | 825| 71|#define WS(stride, i) (stride * i) ------------------ 657| 71| } 658| 71| { 659| 71| E T2k, T2m, T2n, T2h, T2p, T2f, T2g, T2q, T2o; 660| 71| T2k = KP559016994 * (T2i - T2j); 661| 71| T2m = T2i + T2j; 662| 71| T2n = FNMS(KP250000000, T2m, T2l); ------------------ | | 1134| 71|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 663| 71| T2f = TU - TZ; 664| 71| T2g = T15 - T1a; 665| 71| T2h = FMA(KP951056516, T2f, KP587785252 * T2g); ------------------ | | 1131| 71|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 666| 71| T2p = FNMS(KP587785252, T2f, KP951056516 * T2g); ------------------ | | 1134| 71|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 667| 71| ro[WS(os, 5)] = T2l + T2m; ------------------ | | 825| 71|#define WS(stride, i) (stride * i) ------------------ 668| 71| T2q = T2n - T2k; 669| 71| ro[WS(os, 13)] = T2p + T2q; ------------------ | | 825| 71|#define WS(stride, i) (stride * i) ------------------ 670| 71| ro[WS(os, 17)] = T2q - T2p; ------------------ | | 825| 71|#define WS(stride, i) (stride * i) ------------------ 671| 71| T2o = T2k + T2n; 672| 71| ro[WS(os, 1)] = T2h + T2o; ------------------ | | 825| 71|#define WS(stride, i) (stride * i) ------------------ 673| 71| ro[WS(os, 9)] = T2o - T2h; ------------------ | | 825| 71|#define WS(stride, i) (stride * i) ------------------ 674| 71| } 675| 71| { 676| 71| E T1m, T1k, T1l, T1K, T1M, T1y, T1J, T1L, T1n; 677| 71| T1m = KP559016994 * (T1g - T1j); 678| 71| T1k = T1g + T1j; 679| 71| T1l = FNMS(KP250000000, T1k, T1d); ------------------ | | 1134| 71|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 680| 71| T1y = T1s - T1x; 681| 71| T1J = T1D - T1I; 682| 71| T1K = FNMS(KP587785252, T1J, KP951056516 * T1y); ------------------ | | 1134| 71|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 683| 71| T1M = FMA(KP951056516, T1J, KP587785252 * T1y); ------------------ | | 1131| 71|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 684| 71| io[WS(os, 15)] = T1d + T1k; ------------------ | | 825| 71|#define WS(stride, i) (stride * i) ------------------ 685| 71| T1L = T1m + T1l; 686| 71| io[WS(os, 11)] = T1L - T1M; ------------------ | | 825| 71|#define WS(stride, i) (stride * i) ------------------ 687| 71| io[WS(os, 19)] = T1L + T1M; ------------------ | | 825| 71|#define WS(stride, i) (stride * i) ------------------ 688| 71| T1n = T1l - T1m; 689| 71| io[WS(os, 3)] = T1n - T1K; ------------------ | | 825| 71|#define WS(stride, i) (stride * i) ------------------ 690| 71| io[WS(os, 7)] = T1n + T1K; ------------------ | | 825| 71|#define WS(stride, i) (stride * i) ------------------ 691| 71| } 692| 71| { 693| 71| E T1Z, T1X, T1Y, T1P, T21, T1N, T1O, T22, T20; 694| 71| T1Z = KP559016994 * (T1V - T1W); 695| 71| T1X = T1V + T1W; 696| 71| T1Y = FNMS(KP250000000, T1X, T1U); ------------------ | | 1134| 71|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 697| 71| T1N = T1h - T1i; 698| 71| T1O = T1e - T1f; 699| 71| T1P = FNMS(KP587785252, T1O, KP951056516 * T1N); ------------------ | | 1134| 71|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 700| 71| T21 = FMA(KP951056516, T1O, KP587785252 * T1N); ------------------ | | 1131| 71|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 701| 71| ro[WS(os, 15)] = T1U + T1X; ------------------ | | 825| 71|#define WS(stride, i) (stride * i) ------------------ 702| 71| T22 = T1Z + T1Y; 703| 71| ro[WS(os, 11)] = T21 + T22; ------------------ | | 825| 71|#define WS(stride, i) (stride * i) ------------------ 704| 71| ro[WS(os, 19)] = T22 - T21; ------------------ | | 825| 71|#define WS(stride, i) (stride * i) ------------------ 705| 71| T20 = T1Y - T1Z; 706| 71| ro[WS(os, 3)] = T1P + T20; ------------------ | | 825| 71|#define WS(stride, i) (stride * i) ------------------ 707| 71| ro[WS(os, 7)] = T20 - T1P; ------------------ | | 825| 71|#define WS(stride, i) (stride * i) ------------------ 708| 71| } 709| 71| } 710| 5| } 711| 5|} fftw_codelet_n1_25: 1220| 1|void X(codelet_n1_25) (planner *p) { X(kdft_register) (p, n1_25, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 1221| 1|} n1_25.c:n1_25: 650| 5|{ 651| 5| DK(KP425779291, +0.425779291565072648862502445744251703979973042); ------------------ | | 1056| 5|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 5|# define K(x) ((E) x) | | ------------------ ------------------ 652| 5| DK(KP904827052, +0.904827052466019527713668647932697593970413911); ------------------ | | 1056| 5|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 5|# define K(x) ((E) x) | | ------------------ ------------------ 653| 5| DK(KP637423989, +0.637423989748689710176712811676016195434917298); ------------------ | | 1056| 5|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 5|# define K(x) ((E) x) | | ------------------ ------------------ 654| 5| DK(KP770513242, +0.770513242775789230803009636396177847271667672); ------------------ | | 1056| 5|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 5|# define K(x) ((E) x) | | ------------------ ------------------ 655| 5| DK(KP998026728, +0.998026728428271561952336806863450553336905220); ------------------ | | 1056| 5|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 5|# define K(x) ((E) x) | | ------------------ ------------------ 656| 5| DK(KP062790519, +0.062790519529313376076178224565631133122484832); ------------------ | | 1056| 5|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 5|# define K(x) ((E) x) | | ------------------ ------------------ 657| 5| DK(KP992114701, +0.992114701314477831049793042785778521453036709); ------------------ | | 1056| 5|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 5|# define K(x) ((E) x) | | ------------------ ------------------ 658| 5| DK(KP125333233, +0.125333233564304245373118759816508793942918247); ------------------ | | 1056| 5|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 5|# define K(x) ((E) x) | | ------------------ ------------------ 659| 5| DK(KP684547105, +0.684547105928688673732283357621209269889519233); ------------------ | | 1056| 5|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 5|# define K(x) ((E) x) | | ------------------ ------------------ 660| 5| DK(KP728968627, +0.728968627421411523146730319055259111372571664); ------------------ | | 1056| 5|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 5|# define K(x) ((E) x) | | ------------------ ------------------ 661| 5| DK(KP481753674, +0.481753674101715274987191502872129653528542010); ------------------ | | 1056| 5|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 5|# define K(x) ((E) x) | | ------------------ ------------------ 662| 5| DK(KP876306680, +0.876306680043863587308115903922062583399064238); ------------------ | | 1056| 5|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 5|# define K(x) ((E) x) | | ------------------ ------------------ 663| 5| DK(KP844327925, +0.844327925502015078548558063966681505381659241); ------------------ | | 1056| 5|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 5|# define K(x) ((E) x) | | ------------------ ------------------ 664| 5| DK(KP535826794, +0.535826794978996618271308767867639978063575346); ------------------ | | 1056| 5|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 5|# define K(x) ((E) x) | | ------------------ ------------------ 665| 5| DK(KP248689887, +0.248689887164854788242283746006447968417567406); ------------------ | | 1056| 5|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 5|# define K(x) ((E) x) | | ------------------ ------------------ 666| 5| DK(KP968583161, +0.968583161128631119490168375464735813836012403); ------------------ | | 1056| 5|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 5|# define K(x) ((E) x) | | ------------------ ------------------ 667| 5| DK(KP250000000, +0.250000000000000000000000000000000000000000000); ------------------ | | 1056| 5|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 5|# define K(x) ((E) x) | | ------------------ ------------------ 668| 5| DK(KP559016994, +0.559016994374947424102293417182819058860154590); ------------------ | | 1056| 5|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 5|# define K(x) ((E) x) | | ------------------ ------------------ 669| 5| DK(KP587785252, +0.587785252292473129168705954639072768597652438); ------------------ | | 1056| 5|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 5|# define K(x) ((E) x) | | ------------------ ------------------ 670| 5| DK(KP951056516, +0.951056516295153572116439333379382143405698634); ------------------ | | 1056| 5|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 5|# define K(x) ((E) x) | | ------------------ ------------------ 671| 5| { 672| 5| INT i; 673| 50| for (i = v; i > 0; i = i - 1, ri = ri + ivs, ii = ii + ivs, ro = ro + ovs, io = io + ovs, MAKE_VOLATILE_STRIDE(100, is), MAKE_VOLATILE_STRIDE(100, os)) { ------------------ | | 851| 45| (nptr <= ESTIMATED_AVAILABLE_INDEX_REGISTERS ? \ | | ------------------ | | | | 849| 90|#define ESTIMATED_AVAILABLE_INDEX_REGISTERS 16 | | ------------------ | | | Branch (851:7): [Folded, False: 45] | | ------------------ | | 852| 45| 0 : \ | | 853| 45| ((x) = (x) ^ X(an_INT_guaranteed_to_be_zero))) | | ------------------ | | | | 78| 45|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 45|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ for (i = v; i > 0; i = i - 1, ri = ri + ivs, ii = ii + ivs, ro = ro + ovs, io = io + ovs, MAKE_VOLATILE_STRIDE(100, is), MAKE_VOLATILE_STRIDE(100, os)) { ------------------ | | 851| 45| (nptr <= ESTIMATED_AVAILABLE_INDEX_REGISTERS ? \ | | ------------------ | | | | 849| 90|#define ESTIMATED_AVAILABLE_INDEX_REGISTERS 16 | | ------------------ | | | Branch (851:7): [Folded, False: 45] | | ------------------ | | 852| 45| 0 : \ | | 853| 45| ((x) = (x) ^ X(an_INT_guaranteed_to_be_zero))) | | ------------------ | | | | 78| 45|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 45|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ | Branch (673:16): [True: 45, False: 5] ------------------ 674| 45| E T9, T4u, T2T, TP, T3H, TW, T5y, T3I, T2Q, T4v, Ti, Tr, Ts, T5m, T5n; 675| 45| E T5v, T18, T4G, T34, T3M, T1G, T4J, T38, T3T, T1v, T4K, T37, T3W, T1j, T4H; 676| 45| E T35, T3P, TB, TK, TL, T5p, T5q, T5w, T1T, T4N, T3c, T41, T2r, T4Q, T3e; 677| 45| E T4b, T2g, T4R, T3f, T48, T24, T4O, T3b, T44; 678| 45| { 679| 45| E T1, T4, T7, T8, T2S, T2R, TN, TO; 680| 45| T1 = ri[0]; 681| 45| { 682| 45| E T2, T3, T5, T6; 683| 45| T2 = ri[WS(is, 5)]; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 684| 45| T3 = ri[WS(is, 20)]; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 685| 45| T4 = T2 + T3; 686| 45| T5 = ri[WS(is, 10)]; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 687| 45| T6 = ri[WS(is, 15)]; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 688| 45| T7 = T5 + T6; 689| 45| T8 = T4 + T7; 690| 45| T2S = T5 - T6; 691| 45| T2R = T2 - T3; 692| 45| } 693| 45| T9 = T1 + T8; 694| 45| T4u = FNMS(KP587785252, T2R, KP951056516 * T2S); ------------------ | | 1134| 45|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 695| 45| T2T = FMA(KP951056516, T2R, KP587785252 * T2S); ------------------ | | 1131| 45|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 696| 45| TN = KP559016994 * (T4 - T7); 697| 45| TO = FNMS(KP250000000, T8, T1); ------------------ | | 1134| 45|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 698| 45| TP = TN + TO; 699| 45| T3H = TO - TN; 700| 45| } 701| 45| { 702| 45| E T2N, T2K, T2L, TS, T2O, TV, T2M, T2P; 703| 45| T2N = ii[0]; 704| 45| { 705| 45| E TQ, TR, TT, TU; 706| 45| TQ = ii[WS(is, 5)]; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 707| 45| TR = ii[WS(is, 20)]; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 708| 45| T2K = TQ + TR; 709| 45| TT = ii[WS(is, 10)]; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 710| 45| TU = ii[WS(is, 15)]; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 711| 45| T2L = TT + TU; 712| 45| TS = TQ - TR; 713| 45| T2O = T2K + T2L; 714| 45| TV = TT - TU; 715| 45| } 716| 45| TW = FMA(KP951056516, TS, KP587785252 * TV); ------------------ | | 1131| 45|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 717| 45| T5y = T2N + T2O; 718| 45| T3I = FNMS(KP587785252, TS, KP951056516 * TV); ------------------ | | 1134| 45|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 719| 45| T2M = KP559016994 * (T2K - T2L); 720| 45| T2P = FNMS(KP250000000, T2O, T2N); ------------------ | | 1134| 45|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 721| 45| T2Q = T2M + T2P; 722| 45| T4v = T2P - T2M; 723| 45| } 724| 45| { 725| 45| E Ta, T1c, Tj, T1z, Th, T1h, TY, T1g, T13, T1d, T16, T1b, Tq, T1E, T1l; 726| 45| E T1D, T1q, T1A, T1t, T1y; 727| 45| Ta = ri[WS(is, 1)]; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 728| 45| T1c = ii[WS(is, 1)]; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 729| 45| Tj = ri[WS(is, 4)]; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 730| 45| T1z = ii[WS(is, 4)]; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 731| 45| { 732| 45| E Tb, Tc, Td, Te, Tf, Tg; 733| 45| Tb = ri[WS(is, 6)]; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 734| 45| Tc = ri[WS(is, 21)]; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 735| 45| Td = Tb + Tc; 736| 45| Te = ri[WS(is, 11)]; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 737| 45| Tf = ri[WS(is, 16)]; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 738| 45| Tg = Te + Tf; 739| 45| Th = Td + Tg; 740| 45| T1h = Te - Tf; 741| 45| TY = KP559016994 * (Td - Tg); 742| 45| T1g = Tb - Tc; 743| 45| } 744| 45| { 745| 45| E T11, T12, T19, T14, T15, T1a; 746| 45| T11 = ii[WS(is, 6)]; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 747| 45| T12 = ii[WS(is, 21)]; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 748| 45| T19 = T11 + T12; 749| 45| T14 = ii[WS(is, 11)]; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 750| 45| T15 = ii[WS(is, 16)]; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 751| 45| T1a = T14 + T15; 752| 45| T13 = T11 - T12; 753| 45| T1d = T19 + T1a; 754| 45| T16 = T14 - T15; 755| 45| T1b = KP559016994 * (T19 - T1a); 756| 45| } 757| 45| { 758| 45| E Tk, Tl, Tm, Tn, To, Tp; 759| 45| Tk = ri[WS(is, 9)]; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 760| 45| Tl = ri[WS(is, 24)]; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 761| 45| Tm = Tk + Tl; 762| 45| Tn = ri[WS(is, 14)]; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 763| 45| To = ri[WS(is, 19)]; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 764| 45| Tp = Tn + To; 765| 45| Tq = Tm + Tp; 766| 45| T1E = Tn - To; 767| 45| T1l = KP559016994 * (Tm - Tp); 768| 45| T1D = Tk - Tl; 769| 45| } 770| 45| { 771| 45| E T1o, T1p, T1w, T1r, T1s, T1x; 772| 45| T1o = ii[WS(is, 9)]; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 773| 45| T1p = ii[WS(is, 24)]; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 774| 45| T1w = T1o + T1p; 775| 45| T1r = ii[WS(is, 14)]; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 776| 45| T1s = ii[WS(is, 19)]; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 777| 45| T1x = T1r + T1s; 778| 45| T1q = T1o - T1p; 779| 45| T1A = T1w + T1x; 780| 45| T1t = T1r - T1s; 781| 45| T1y = KP559016994 * (T1w - T1x); 782| 45| } 783| 45| Ti = Ta + Th; 784| 45| Tr = Tj + Tq; 785| 45| Ts = Ti + Tr; 786| 45| T5m = T1c + T1d; 787| 45| T5n = T1z + T1A; 788| 45| T5v = T5m + T5n; 789| 45| { 790| 45| E T17, T3L, T10, T3K, TZ; 791| 45| T17 = FMA(KP951056516, T13, KP587785252 * T16); ------------------ | | 1131| 45|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 792| 45| T3L = FNMS(KP587785252, T13, KP951056516 * T16); ------------------ | | 1134| 45|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 793| 45| TZ = FNMS(KP250000000, Th, Ta); ------------------ | | 1134| 45|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 794| 45| T10 = TY + TZ; 795| 45| T3K = TZ - TY; 796| 45| T18 = T10 + T17; 797| 45| T4G = T3K + T3L; 798| 45| T34 = T10 - T17; 799| 45| T3M = T3K - T3L; 800| 45| } 801| 45| { 802| 45| E T1F, T3R, T1C, T3S, T1B; 803| 45| T1F = FMA(KP951056516, T1D, KP587785252 * T1E); ------------------ | | 1131| 45|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 804| 45| T3R = FNMS(KP587785252, T1D, KP951056516 * T1E); ------------------ | | 1134| 45|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 805| 45| T1B = FNMS(KP250000000, T1A, T1z); ------------------ | | 1134| 45|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 806| 45| T1C = T1y + T1B; 807| 45| T3S = T1B - T1y; 808| 45| T1G = T1C - T1F; 809| 45| T4J = T3S - T3R; 810| 45| T38 = T1F + T1C; 811| 45| T3T = T3R + T3S; 812| 45| } 813| 45| { 814| 45| E T1u, T3V, T1n, T3U, T1m; 815| 45| T1u = FMA(KP951056516, T1q, KP587785252 * T1t); ------------------ | | 1131| 45|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 816| 45| T3V = FNMS(KP587785252, T1q, KP951056516 * T1t); ------------------ | | 1134| 45|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 817| 45| T1m = FNMS(KP250000000, Tq, Tj); ------------------ | | 1134| 45|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 818| 45| T1n = T1l + T1m; 819| 45| T3U = T1m - T1l; 820| 45| T1v = T1n + T1u; 821| 45| T4K = T3U + T3V; 822| 45| T37 = T1n - T1u; 823| 45| T3W = T3U - T3V; 824| 45| } 825| 45| { 826| 45| E T1i, T3N, T1f, T3O, T1e; 827| 45| T1i = FMA(KP951056516, T1g, KP587785252 * T1h); ------------------ | | 1131| 45|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 828| 45| T3N = FNMS(KP587785252, T1g, KP951056516 * T1h); ------------------ | | 1134| 45|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 829| 45| T1e = FNMS(KP250000000, T1d, T1c); ------------------ | | 1134| 45|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 830| 45| T1f = T1b + T1e; 831| 45| T3O = T1e - T1b; 832| 45| T1j = T1f - T1i; 833| 45| T4H = T3O - T3N; 834| 45| T35 = T1i + T1f; 835| 45| T3P = T3N + T3O; 836| 45| } 837| 45| } 838| 45| { 839| 45| E Tt, T1X, TC, T2k, TA, T22, T1J, T21, T1O, T1Y, T1R, T1W, TJ, T2p, T26; 840| 45| E T2o, T2b, T2l, T2e, T2j; 841| 45| Tt = ri[WS(is, 2)]; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 842| 45| T1X = ii[WS(is, 2)]; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 843| 45| TC = ri[WS(is, 3)]; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 844| 45| T2k = ii[WS(is, 3)]; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 845| 45| { 846| 45| E Tu, Tv, Tw, Tx, Ty, Tz; 847| 45| Tu = ri[WS(is, 7)]; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 848| 45| Tv = ri[WS(is, 22)]; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 849| 45| Tw = Tu + Tv; 850| 45| Tx = ri[WS(is, 12)]; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 851| 45| Ty = ri[WS(is, 17)]; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 852| 45| Tz = Tx + Ty; 853| 45| TA = Tw + Tz; 854| 45| T22 = Tx - Ty; 855| 45| T1J = KP559016994 * (Tw - Tz); 856| 45| T21 = Tu - Tv; 857| 45| } 858| 45| { 859| 45| E T1M, T1N, T1U, T1P, T1Q, T1V; 860| 45| T1M = ii[WS(is, 7)]; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 861| 45| T1N = ii[WS(is, 22)]; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 862| 45| T1U = T1M + T1N; 863| 45| T1P = ii[WS(is, 12)]; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 864| 45| T1Q = ii[WS(is, 17)]; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 865| 45| T1V = T1P + T1Q; 866| 45| T1O = T1M - T1N; 867| 45| T1Y = T1U + T1V; 868| 45| T1R = T1P - T1Q; 869| 45| T1W = KP559016994 * (T1U - T1V); 870| 45| } 871| 45| { 872| 45| E TD, TE, TF, TG, TH, TI; 873| 45| TD = ri[WS(is, 8)]; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 874| 45| TE = ri[WS(is, 23)]; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 875| 45| TF = TD + TE; 876| 45| TG = ri[WS(is, 13)]; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 877| 45| TH = ri[WS(is, 18)]; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 878| 45| TI = TG + TH; 879| 45| TJ = TF + TI; 880| 45| T2p = TG - TH; 881| 45| T26 = KP559016994 * (TF - TI); 882| 45| T2o = TD - TE; 883| 45| } 884| 45| { 885| 45| E T29, T2a, T2h, T2c, T2d, T2i; 886| 45| T29 = ii[WS(is, 8)]; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 887| 45| T2a = ii[WS(is, 23)]; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 888| 45| T2h = T29 + T2a; 889| 45| T2c = ii[WS(is, 13)]; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 890| 45| T2d = ii[WS(is, 18)]; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 891| 45| T2i = T2c + T2d; 892| 45| T2b = T29 - T2a; 893| 45| T2l = T2h + T2i; 894| 45| T2e = T2c - T2d; 895| 45| T2j = KP559016994 * (T2h - T2i); 896| 45| } 897| 45| TB = Tt + TA; 898| 45| TK = TC + TJ; 899| 45| TL = TB + TK; 900| 45| T5p = T1X + T1Y; 901| 45| T5q = T2k + T2l; 902| 45| T5w = T5p + T5q; 903| 45| { 904| 45| E T1S, T40, T1L, T3Z, T1K; 905| 45| T1S = FMA(KP951056516, T1O, KP587785252 * T1R); ------------------ | | 1131| 45|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 906| 45| T40 = FNMS(KP587785252, T1O, KP951056516 * T1R); ------------------ | | 1134| 45|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 907| 45| T1K = FNMS(KP250000000, TA, Tt); ------------------ | | 1134| 45|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 908| 45| T1L = T1J + T1K; 909| 45| T3Z = T1K - T1J; 910| 45| T1T = T1L + T1S; 911| 45| T4N = T3Z + T40; 912| 45| T3c = T1L - T1S; 913| 45| T41 = T3Z - T40; 914| 45| } 915| 45| { 916| 45| E T2q, T49, T2n, T4a, T2m; 917| 45| T2q = FMA(KP951056516, T2o, KP587785252 * T2p); ------------------ | | 1131| 45|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 918| 45| T49 = FNMS(KP587785252, T2o, KP951056516 * T2p); ------------------ | | 1134| 45|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 919| 45| T2m = FNMS(KP250000000, T2l, T2k); ------------------ | | 1134| 45|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 920| 45| T2n = T2j + T2m; 921| 45| T4a = T2m - T2j; 922| 45| T2r = T2n - T2q; 923| 45| T4Q = T4a - T49; 924| 45| T3e = T2q + T2n; 925| 45| T4b = T49 + T4a; 926| 45| } 927| 45| { 928| 45| E T2f, T47, T28, T46, T27; 929| 45| T2f = FMA(KP951056516, T2b, KP587785252 * T2e); ------------------ | | 1131| 45|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 930| 45| T47 = FNMS(KP587785252, T2b, KP951056516 * T2e); ------------------ | | 1134| 45|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 931| 45| T27 = FNMS(KP250000000, TJ, TC); ------------------ | | 1134| 45|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 932| 45| T28 = T26 + T27; 933| 45| T46 = T27 - T26; 934| 45| T2g = T28 + T2f; 935| 45| T4R = T46 + T47; 936| 45| T3f = T28 - T2f; 937| 45| T48 = T46 - T47; 938| 45| } 939| 45| { 940| 45| E T23, T42, T20, T43, T1Z; 941| 45| T23 = FMA(KP951056516, T21, KP587785252 * T22); ------------------ | | 1131| 45|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 942| 45| T42 = FNMS(KP587785252, T21, KP951056516 * T22); ------------------ | | 1134| 45|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 943| 45| T1Z = FNMS(KP250000000, T1Y, T1X); ------------------ | | 1134| 45|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 944| 45| T20 = T1W + T1Z; 945| 45| T43 = T1Z - T1W; 946| 45| T24 = T20 - T23; 947| 45| T4O = T43 - T42; 948| 45| T3b = T23 + T20; 949| 45| T44 = T42 + T43; 950| 45| } 951| 45| } 952| 45| { 953| 45| E T5j, TM, T5k, T5s, T5u, T5o, T5r, T5t, T5l; 954| 45| T5j = KP559016994 * (Ts - TL); 955| 45| TM = Ts + TL; 956| 45| T5k = FNMS(KP250000000, TM, T9); ------------------ | | 1134| 45|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 957| 45| T5o = T5m - T5n; 958| 45| T5r = T5p - T5q; 959| 45| T5s = FMA(KP951056516, T5o, KP587785252 * T5r); ------------------ | | 1131| 45|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 960| 45| T5u = FNMS(KP587785252, T5o, KP951056516 * T5r); ------------------ | | 1134| 45|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 961| 45| ro[0] = T9 + TM; 962| 45| T5t = T5k - T5j; 963| 45| ro[WS(os, 10)] = T5t - T5u; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 964| 45| ro[WS(os, 15)] = T5t + T5u; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 965| 45| T5l = T5j + T5k; 966| 45| ro[WS(os, 20)] = T5l - T5s; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 967| 45| ro[WS(os, 5)] = T5l + T5s; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 968| 45| } 969| 45| { 970| 45| E T5x, T5z, T5A, T5E, T5F, T5C, T5D, T5G, T5B; 971| 45| T5x = KP559016994 * (T5v - T5w); 972| 45| T5z = T5v + T5w; 973| 45| T5A = FNMS(KP250000000, T5z, T5y); ------------------ | | 1134| 45|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 974| 45| T5C = Ti - Tr; 975| 45| T5D = TB - TK; 976| 45| T5E = FMA(KP951056516, T5C, KP587785252 * T5D); ------------------ | | 1131| 45|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 977| 45| T5F = FNMS(KP587785252, T5C, KP951056516 * T5D); ------------------ | | 1134| 45|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 978| 45| io[0] = T5y + T5z; 979| 45| T5G = T5A - T5x; 980| 45| io[WS(os, 10)] = T5F + T5G; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 981| 45| io[WS(os, 15)] = T5G - T5F; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 982| 45| T5B = T5x + T5A; 983| 45| io[WS(os, 5)] = T5B - T5E; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 984| 45| io[WS(os, 20)] = T5E + T5B; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 985| 45| } 986| 45| { 987| 45| E TX, T2U, T2u, T2Z, T2v, T2Y, T2A, T2V, T2D, T2J; 988| 45| TX = TP + TW; 989| 45| T2U = T2Q - T2T; 990| 45| { 991| 45| E T1k, T1H, T1I, T25, T2s, T2t; 992| 45| T1k = FMA(KP968583161, T18, KP248689887 * T1j); ------------------ | | 1131| 45|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 993| 45| T1H = FMA(KP535826794, T1v, KP844327925 * T1G); ------------------ | | 1131| 45|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 994| 45| T1I = T1k + T1H; 995| 45| T25 = FMA(KP876306680, T1T, KP481753674 * T24); ------------------ | | 1131| 45|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 996| 45| T2s = FMA(KP728968627, T2g, KP684547105 * T2r); ------------------ | | 1131| 45|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 997| 45| T2t = T25 + T2s; 998| 45| T2u = T1I + T2t; 999| 45| T2Z = T25 - T2s; 1000| 45| T2v = KP559016994 * (T1I - T2t); 1001| 45| T2Y = T1k - T1H; 1002| 45| } 1003| 45| { 1004| 45| E T2y, T2z, T2H, T2B, T2C, T2I; 1005| 45| T2y = FNMS(KP248689887, T18, KP968583161 * T1j); ------------------ | | 1134| 45|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1006| 45| T2z = FNMS(KP844327925, T1v, KP535826794 * T1G); ------------------ | | 1134| 45|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1007| 45| T2H = T2y + T2z; 1008| 45| T2B = FNMS(KP481753674, T1T, KP876306680 * T24); ------------------ | | 1134| 45|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1009| 45| T2C = FNMS(KP684547105, T2g, KP728968627 * T2r); ------------------ | | 1134| 45|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1010| 45| T2I = T2B + T2C; 1011| 45| T2A = T2y - T2z; 1012| 45| T2V = T2H + T2I; 1013| 45| T2D = T2B - T2C; 1014| 45| T2J = KP559016994 * (T2H - T2I); 1015| 45| } 1016| 45| ro[WS(os, 1)] = TX + T2u; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 1017| 45| io[WS(os, 1)] = T2U + T2V; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 1018| 45| { 1019| 45| E T2E, T2G, T2x, T2F, T2w; 1020| 45| T2E = FMA(KP951056516, T2A, KP587785252 * T2D); ------------------ | | 1131| 45|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1021| 45| T2G = FNMS(KP587785252, T2A, KP951056516 * T2D); ------------------ | | 1134| 45|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1022| 45| T2w = FNMS(KP250000000, T2u, TX); ------------------ | | 1134| 45|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1023| 45| T2x = T2v + T2w; 1024| 45| T2F = T2w - T2v; 1025| 45| ro[WS(os, 21)] = T2x - T2E; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 1026| 45| ro[WS(os, 16)] = T2F + T2G; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 1027| 45| ro[WS(os, 6)] = T2x + T2E; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 1028| 45| ro[WS(os, 11)] = T2F - T2G; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 1029| 45| } 1030| 45| { 1031| 45| E T30, T31, T2X, T32, T2W; 1032| 45| T30 = FMA(KP951056516, T2Y, KP587785252 * T2Z); ------------------ | | 1131| 45|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1033| 45| T31 = FNMS(KP587785252, T2Y, KP951056516 * T2Z); ------------------ | | 1134| 45|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1034| 45| T2W = FNMS(KP250000000, T2V, T2U); ------------------ | | 1134| 45|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1035| 45| T2X = T2J + T2W; 1036| 45| T32 = T2W - T2J; 1037| 45| io[WS(os, 6)] = T2X - T30; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 1038| 45| io[WS(os, 16)] = T32 - T31; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 1039| 45| io[WS(os, 21)] = T30 + T2X; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 1040| 45| io[WS(os, 11)] = T31 + T32; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 1041| 45| } 1042| 45| } 1043| 45| { 1044| 45| E T4F, T52, T4U, T5b, T56, T57, T51, T5f, T53, T5e; 1045| 45| T4F = T3H + T3I; 1046| 45| T52 = T4v - T4u; 1047| 45| { 1048| 45| E T4I, T4L, T4M, T4P, T4S, T4T; 1049| 45| T4I = FMA(KP728968627, T4G, KP684547105 * T4H); ------------------ | | 1131| 45|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1050| 45| T4L = FNMS(KP992114701, T4K, KP125333233 * T4J); ------------------ | | 1134| 45|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1051| 45| T4M = T4I + T4L; 1052| 45| T4P = FMA(KP062790519, T4N, KP998026728 * T4O); ------------------ | | 1131| 45|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1053| 45| T4S = FNMS(KP637423989, T4R, KP770513242 * T4Q); ------------------ | | 1134| 45|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1054| 45| T4T = T4P + T4S; 1055| 45| T4U = T4M + T4T; 1056| 45| T5b = KP559016994 * (T4M - T4T); 1057| 45| T56 = T4I - T4L; 1058| 45| T57 = T4P - T4S; 1059| 45| } 1060| 45| { 1061| 45| E T4V, T4W, T4X, T4Y, T4Z, T50; 1062| 45| T4V = FNMS(KP684547105, T4G, KP728968627 * T4H); ------------------ | | 1134| 45|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1063| 45| T4W = FMA(KP125333233, T4K, KP992114701 * T4J); ------------------ | | 1131| 45|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1064| 45| T4X = T4V - T4W; 1065| 45| T4Y = FNMS(KP998026728, T4N, KP062790519 * T4O); ------------------ | | 1134| 45|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1066| 45| T4Z = FMA(KP770513242, T4R, KP637423989 * T4Q); ------------------ | | 1131| 45|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1067| 45| T50 = T4Y - T4Z; 1068| 45| T51 = KP559016994 * (T4X - T50); 1069| 45| T5f = T4Y + T4Z; 1070| 45| T53 = T4X + T50; 1071| 45| T5e = T4V + T4W; 1072| 45| } 1073| 45| ro[WS(os, 3)] = T4F + T4U; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 1074| 45| io[WS(os, 3)] = T52 + T53; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 1075| 45| { 1076| 45| E T58, T59, T55, T5a, T54; 1077| 45| T58 = FMA(KP951056516, T56, KP587785252 * T57); ------------------ | | 1131| 45|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1078| 45| T59 = FNMS(KP587785252, T56, KP951056516 * T57); ------------------ | | 1134| 45|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1079| 45| T54 = FNMS(KP250000000, T53, T52); ------------------ | | 1134| 45|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1080| 45| T55 = T51 + T54; 1081| 45| T5a = T54 - T51; 1082| 45| io[WS(os, 8)] = T55 - T58; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 1083| 45| io[WS(os, 18)] = T5a - T59; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 1084| 45| io[WS(os, 23)] = T58 + T55; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 1085| 45| io[WS(os, 13)] = T59 + T5a; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 1086| 45| } 1087| 45| { 1088| 45| E T5g, T5i, T5d, T5h, T5c; 1089| 45| T5g = FMA(KP951056516, T5e, KP587785252 * T5f); ------------------ | | 1131| 45|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1090| 45| T5i = FNMS(KP587785252, T5e, KP951056516 * T5f); ------------------ | | 1134| 45|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1091| 45| T5c = FNMS(KP250000000, T4U, T4F); ------------------ | | 1134| 45|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1092| 45| T5d = T5b + T5c; 1093| 45| T5h = T5c - T5b; 1094| 45| ro[WS(os, 23)] = T5d - T5g; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 1095| 45| ro[WS(os, 18)] = T5h + T5i; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 1096| 45| ro[WS(os, 8)] = T5d + T5g; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 1097| 45| ro[WS(os, 13)] = T5h - T5i; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 1098| 45| } 1099| 45| } 1100| 45| { 1101| 45| E T3J, T4w, T4e, T4B, T4f, T4A, T4k, T4x, T4n, T4t; 1102| 45| T3J = T3H - T3I; 1103| 45| T4w = T4u + T4v; 1104| 45| { 1105| 45| E T3Q, T3X, T3Y, T45, T4c, T4d; 1106| 45| T3Q = FMA(KP876306680, T3M, KP481753674 * T3P); ------------------ | | 1131| 45|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1107| 45| T3X = FNMS(KP425779291, T3W, KP904827052 * T3T); ------------------ | | 1134| 45|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1108| 45| T3Y = T3Q + T3X; 1109| 45| T45 = FMA(KP535826794, T41, KP844327925 * T44); ------------------ | | 1131| 45|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1110| 45| T4c = FMA(KP062790519, T48, KP998026728 * T4b); ------------------ | | 1131| 45|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1111| 45| T4d = T45 + T4c; 1112| 45| T4e = T3Y + T4d; 1113| 45| T4B = T45 - T4c; 1114| 45| T4f = KP559016994 * (T3Y - T4d); 1115| 45| T4A = T3Q - T3X; 1116| 45| } 1117| 45| { 1118| 45| E T4i, T4j, T4r, T4l, T4m, T4s; 1119| 45| T4i = FNMS(KP481753674, T3M, KP876306680 * T3P); ------------------ | | 1134| 45|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1120| 45| T4j = FMA(KP904827052, T3W, KP425779291 * T3T); ------------------ | | 1131| 45|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1121| 45| T4r = T4i - T4j; 1122| 45| T4l = FNMS(KP844327925, T41, KP535826794 * T44); ------------------ | | 1134| 45|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1123| 45| T4m = FNMS(KP998026728, T48, KP062790519 * T4b); ------------------ | | 1134| 45|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1124| 45| T4s = T4l + T4m; 1125| 45| T4k = T4i + T4j; 1126| 45| T4x = T4r + T4s; 1127| 45| T4n = T4l - T4m; 1128| 45| T4t = KP559016994 * (T4r - T4s); 1129| 45| } 1130| 45| ro[WS(os, 2)] = T3J + T4e; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 1131| 45| io[WS(os, 2)] = T4w + T4x; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 1132| 45| { 1133| 45| E T4o, T4q, T4h, T4p, T4g; 1134| 45| T4o = FMA(KP951056516, T4k, KP587785252 * T4n); ------------------ | | 1131| 45|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1135| 45| T4q = FNMS(KP587785252, T4k, KP951056516 * T4n); ------------------ | | 1134| 45|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1136| 45| T4g = FNMS(KP250000000, T4e, T3J); ------------------ | | 1134| 45|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1137| 45| T4h = T4f + T4g; 1138| 45| T4p = T4g - T4f; 1139| 45| ro[WS(os, 22)] = T4h - T4o; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 1140| 45| ro[WS(os, 17)] = T4p + T4q; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 1141| 45| ro[WS(os, 7)] = T4h + T4o; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 1142| 45| ro[WS(os, 12)] = T4p - T4q; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 1143| 45| } 1144| 45| { 1145| 45| E T4C, T4D, T4z, T4E, T4y; 1146| 45| T4C = FMA(KP951056516, T4A, KP587785252 * T4B); ------------------ | | 1131| 45|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1147| 45| T4D = FNMS(KP587785252, T4A, KP951056516 * T4B); ------------------ | | 1134| 45|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1148| 45| T4y = FNMS(KP250000000, T4x, T4w); ------------------ | | 1134| 45|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1149| 45| T4z = T4t + T4y; 1150| 45| T4E = T4y - T4t; 1151| 45| io[WS(os, 7)] = T4z - T4C; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 1152| 45| io[WS(os, 17)] = T4E - T4D; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 1153| 45| io[WS(os, 22)] = T4C + T4z; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 1154| 45| io[WS(os, 12)] = T4D + T4E; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 1155| 45| } 1156| 45| } 1157| 45| { 1158| 45| E T33, T3j, T3i, T3z, T3r, T3s, T3q, T3D, T3v, T3C; 1159| 45| T33 = TP - TW; 1160| 45| T3j = T2T + T2Q; 1161| 45| { 1162| 45| E T36, T39, T3a, T3d, T3g, T3h; 1163| 45| T36 = FMA(KP535826794, T34, KP844327925 * T35); ------------------ | | 1131| 45|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1164| 45| T39 = FMA(KP637423989, T37, KP770513242 * T38); ------------------ | | 1131| 45|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1165| 45| T3a = T36 - T39; 1166| 45| T3d = FNMS(KP425779291, T3c, KP904827052 * T3b); ------------------ | | 1134| 45|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1167| 45| T3g = FNMS(KP992114701, T3f, KP125333233 * T3e); ------------------ | | 1134| 45|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1168| 45| T3h = T3d + T3g; 1169| 45| T3i = T3a + T3h; 1170| 45| T3z = KP559016994 * (T3a - T3h); 1171| 45| T3r = T3d - T3g; 1172| 45| T3s = T36 + T39; 1173| 45| } 1174| 45| { 1175| 45| E T3k, T3l, T3m, T3n, T3o, T3p; 1176| 45| T3k = FNMS(KP844327925, T34, KP535826794 * T35); ------------------ | | 1134| 45|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1177| 45| T3l = FNMS(KP637423989, T38, KP770513242 * T37); ------------------ | | 1134| 45|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1178| 45| T3m = T3k + T3l; 1179| 45| T3n = FMA(KP904827052, T3c, KP425779291 * T3b); ------------------ | | 1131| 45|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1180| 45| T3o = FMA(KP125333233, T3f, KP992114701 * T3e); ------------------ | | 1131| 45|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1181| 45| T3p = T3n + T3o; 1182| 45| T3q = T3m - T3p; 1183| 45| T3D = T3o - T3n; 1184| 45| T3v = KP559016994 * (T3m + T3p); 1185| 45| T3C = T3k - T3l; 1186| 45| } 1187| 45| ro[WS(os, 4)] = T33 + T3i; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 1188| 45| io[WS(os, 4)] = T3j + T3q; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 1189| 45| { 1190| 45| E T3t, T3y, T3w, T3x, T3u; 1191| 45| T3t = FNMS(KP587785252, T3s, KP951056516 * T3r); ------------------ | | 1134| 45|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1192| 45| T3y = FMA(KP951056516, T3s, KP587785252 * T3r); ------------------ | | 1131| 45|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1193| 45| T3u = FNMS(KP250000000, T3q, T3j); ------------------ | | 1134| 45|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1194| 45| T3w = T3u - T3v; 1195| 45| T3x = T3u + T3v; 1196| 45| io[WS(os, 14)] = T3t + T3w; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 1197| 45| io[WS(os, 24)] = T3y + T3x; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 1198| 45| io[WS(os, 19)] = T3w - T3t; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 1199| 45| io[WS(os, 9)] = T3x - T3y; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 1200| 45| } 1201| 45| { 1202| 45| E T3E, T3G, T3B, T3F, T3A; 1203| 45| T3E = FMA(KP951056516, T3C, KP587785252 * T3D); ------------------ | | 1131| 45|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1204| 45| T3G = FNMS(KP587785252, T3C, KP951056516 * T3D); ------------------ | | 1134| 45|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1205| 45| T3A = FNMS(KP250000000, T3i, T33); ------------------ | | 1134| 45|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1206| 45| T3B = T3z + T3A; 1207| 45| T3F = T3A - T3z; 1208| 45| ro[WS(os, 24)] = T3B - T3E; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 1209| 45| ro[WS(os, 19)] = T3F + T3G; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 1210| 45| ro[WS(os, 9)] = T3B + T3E; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 1211| 45| ro[WS(os, 14)] = T3F - T3G; ------------------ | | 825| 45|#define WS(stride, i) (stride * i) ------------------ 1212| 45| } 1213| 45| } 1214| 45| } 1215| 5| } 1216| 5|} fftw_codelet_n1_3: 121| 1|void X(codelet_n1_3) (planner *p) { X(kdft_register) (p, n1_3, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 122| 1|} n1_3.c:n1_3: 87| 3|{ 88| 3| DK(KP500000000, +0.500000000000000000000000000000000000000000000); ------------------ | | 1056| 3|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 3|# define K(x) ((E) x) | | ------------------ ------------------ 89| 3| DK(KP866025403, +0.866025403784438646763723170752936183471402627); ------------------ | | 1056| 3|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 3|# define K(x) ((E) x) | | ------------------ ------------------ 90| 3| { 91| 3| INT i; 92| 6| for (i = v; i > 0; i = i - 1, ri = ri + ivs, ii = ii + ivs, ro = ro + ovs, io = io + ovs, MAKE_VOLATILE_STRIDE(12, is), MAKE_VOLATILE_STRIDE(12, os)) { ------------------ | | 851| 3| (nptr <= ESTIMATED_AVAILABLE_INDEX_REGISTERS ? \ | | ------------------ | | | | 849| 6|#define ESTIMATED_AVAILABLE_INDEX_REGISTERS 16 | | ------------------ | | | Branch (851:7): [True: 3, Folded] | | ------------------ | | 852| 3| 0 : \ | | 853| 3| ((x) = (x) ^ X(an_INT_guaranteed_to_be_zero))) | | ------------------ | | | | 78| 0|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ for (i = v; i > 0; i = i - 1, ri = ri + ivs, ii = ii + ivs, ro = ro + ovs, io = io + ovs, MAKE_VOLATILE_STRIDE(12, is), MAKE_VOLATILE_STRIDE(12, os)) { ------------------ | | 851| 3| (nptr <= ESTIMATED_AVAILABLE_INDEX_REGISTERS ? \ | | ------------------ | | | | 849| 6|#define ESTIMATED_AVAILABLE_INDEX_REGISTERS 16 | | ------------------ | | | Branch (851:7): [True: 3, Folded] | | ------------------ | | 852| 3| 0 : \ | | 853| 3| ((x) = (x) ^ X(an_INT_guaranteed_to_be_zero))) | | ------------------ | | | | 78| 0|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ | Branch (92:16): [True: 3, False: 3] ------------------ 93| 3| E T1, Ta, T4, T9, T8, Tb, T5, Tc; 94| 3| T1 = ri[0]; 95| 3| Ta = ii[0]; 96| 3| { 97| 3| E T2, T3, T6, T7; 98| 3| T2 = ri[WS(is, 1)]; ------------------ | | 825| 3|#define WS(stride, i) (stride * i) ------------------ 99| 3| T3 = ri[WS(is, 2)]; ------------------ | | 825| 3|#define WS(stride, i) (stride * i) ------------------ 100| 3| T4 = T2 + T3; 101| 3| T9 = KP866025403 * (T3 - T2); 102| 3| T6 = ii[WS(is, 1)]; ------------------ | | 825| 3|#define WS(stride, i) (stride * i) ------------------ 103| 3| T7 = ii[WS(is, 2)]; ------------------ | | 825| 3|#define WS(stride, i) (stride * i) ------------------ 104| 3| T8 = KP866025403 * (T6 - T7); 105| 3| Tb = T6 + T7; 106| 3| } 107| 3| ro[0] = T1 + T4; 108| 3| io[0] = Ta + Tb; 109| 3| T5 = FNMS(KP500000000, T4, T1); ------------------ | | 1134| 3|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 110| 3| ro[WS(os, 2)] = T5 - T8; ------------------ | | 825| 3|#define WS(stride, i) (stride * i) ------------------ 111| 3| ro[WS(os, 1)] = T5 + T8; ------------------ | | 825| 3|#define WS(stride, i) (stride * i) ------------------ 112| 3| Tc = FNMS(KP500000000, Tb, Ta); ------------------ | | 1134| 3|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 113| 3| io[WS(os, 1)] = T9 + Tc; ------------------ | | 825| 3|#define WS(stride, i) (stride * i) ------------------ 114| 3| io[WS(os, 2)] = Tc - T9; ------------------ | | 825| 3|#define WS(stride, i) (stride * i) ------------------ 115| 3| } 116| 3| } 117| 3|} fftw_codelet_n1_32: 1315| 1|void X(codelet_n1_32) (planner *p) { X(kdft_register) (p, n1_32, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 1316| 1|} n1_32.c:n1_32: 684| 2|{ 685| 2| DK(KP831469612, +0.831469612302545237078788377617905756738560812); ------------------ | | 1056| 2|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 2|# define K(x) ((E) x) | | ------------------ ------------------ 686| 2| DK(KP555570233, +0.555570233019602224742830813948532874374937191); ------------------ | | 1056| 2|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 2|# define K(x) ((E) x) | | ------------------ ------------------ 687| 2| DK(KP195090322, +0.195090322016128267848284868477022240927691618); ------------------ | | 1056| 2|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 2|# define K(x) ((E) x) | | ------------------ ------------------ 688| 2| DK(KP980785280, +0.980785280403230449126182236134239036973933731); ------------------ | | 1056| 2|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 2|# define K(x) ((E) x) | | ------------------ ------------------ 689| 2| DK(KP923879532, +0.923879532511286756128183189396788286822416626); ------------------ | | 1056| 2|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 2|# define K(x) ((E) x) | | ------------------ ------------------ 690| 2| DK(KP382683432, +0.382683432365089771728459984030398866761344562); ------------------ | | 1056| 2|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 2|# define K(x) ((E) x) | | ------------------ ------------------ 691| 2| DK(KP707106781, +0.707106781186547524400844362104849039284835938); ------------------ | | 1056| 2|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 2|# define K(x) ((E) x) | | ------------------ ------------------ 692| 2| { 693| 2| INT i; 694| 4| for (i = v; i > 0; i = i - 1, ri = ri + ivs, ii = ii + ivs, ro = ro + ovs, io = io + ovs, MAKE_VOLATILE_STRIDE(128, is), MAKE_VOLATILE_STRIDE(128, os)) { ------------------ | | 851| 2| (nptr <= ESTIMATED_AVAILABLE_INDEX_REGISTERS ? \ | | ------------------ | | | | 849| 4|#define ESTIMATED_AVAILABLE_INDEX_REGISTERS 16 | | ------------------ | | | Branch (851:7): [Folded, False: 2] | | ------------------ | | 852| 2| 0 : \ | | 853| 2| ((x) = (x) ^ X(an_INT_guaranteed_to_be_zero))) | | ------------------ | | | | 78| 2|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 2|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ for (i = v; i > 0; i = i - 1, ri = ri + ivs, ii = ii + ivs, ro = ro + ovs, io = io + ovs, MAKE_VOLATILE_STRIDE(128, is), MAKE_VOLATILE_STRIDE(128, os)) { ------------------ | | 851| 2| (nptr <= ESTIMATED_AVAILABLE_INDEX_REGISTERS ? \ | | ------------------ | | | | 849| 4|#define ESTIMATED_AVAILABLE_INDEX_REGISTERS 16 | | ------------------ | | | Branch (851:7): [Folded, False: 2] | | ------------------ | | 852| 2| 0 : \ | | 853| 2| ((x) = (x) ^ X(an_INT_guaranteed_to_be_zero))) | | ------------------ | | | | 78| 2|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 2|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ | Branch (694:16): [True: 2, False: 2] ------------------ 695| 2| E T7, T4r, T4Z, T18, T1z, T3t, T3T, T2T, Te, T1f, T50, T4s, T2W, T3u, T1G; 696| 2| E T3U, Tm, T1n, T1O, T2Z, T3y, T3X, T4w, T53, Tt, T1u, T1V, T2Y, T3B, T3W; 697| 2| E T4z, T52, T2t, T3L, T3O, T2K, TR, TY, T5F, T5G, T5H, T5I, T4R, T5j, T2E; 698| 2| E T3P, T4W, T5k, T2N, T3M, T22, T3E, T3H, T2j, TC, TJ, T5A, T5B, T5C, T5D; 699| 2| E T4G, T5g, T2d, T3F, T4L, T5h, T2m, T3I; 700| 2| { 701| 2| E T3, T1x, T14, T2S, T6, T2R, T17, T1y; 702| 2| { 703| 2| E T1, T2, T12, T13; 704| 2| T1 = ri[0]; 705| 2| T2 = ri[WS(is, 16)]; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 706| 2| T3 = T1 + T2; 707| 2| T1x = T1 - T2; 708| 2| T12 = ii[0]; 709| 2| T13 = ii[WS(is, 16)]; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 710| 2| T14 = T12 + T13; 711| 2| T2S = T12 - T13; 712| 2| } 713| 2| { 714| 2| E T4, T5, T15, T16; 715| 2| T4 = ri[WS(is, 8)]; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 716| 2| T5 = ri[WS(is, 24)]; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 717| 2| T6 = T4 + T5; 718| 2| T2R = T4 - T5; 719| 2| T15 = ii[WS(is, 8)]; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 720| 2| T16 = ii[WS(is, 24)]; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 721| 2| T17 = T15 + T16; 722| 2| T1y = T15 - T16; 723| 2| } 724| 2| T7 = T3 + T6; 725| 2| T4r = T3 - T6; 726| 2| T4Z = T14 - T17; 727| 2| T18 = T14 + T17; 728| 2| T1z = T1x - T1y; 729| 2| T3t = T1x + T1y; 730| 2| T3T = T2S - T2R; 731| 2| T2T = T2R + T2S; 732| 2| } 733| 2| { 734| 2| E Ta, T1B, T1b, T1A, Td, T1D, T1e, T1E; 735| 2| { 736| 2| E T8, T9, T19, T1a; 737| 2| T8 = ri[WS(is, 4)]; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 738| 2| T9 = ri[WS(is, 20)]; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 739| 2| Ta = T8 + T9; 740| 2| T1B = T8 - T9; 741| 2| T19 = ii[WS(is, 4)]; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 742| 2| T1a = ii[WS(is, 20)]; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 743| 2| T1b = T19 + T1a; 744| 2| T1A = T19 - T1a; 745| 2| } 746| 2| { 747| 2| E Tb, Tc, T1c, T1d; 748| 2| Tb = ri[WS(is, 28)]; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 749| 2| Tc = ri[WS(is, 12)]; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 750| 2| Td = Tb + Tc; 751| 2| T1D = Tb - Tc; 752| 2| T1c = ii[WS(is, 28)]; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 753| 2| T1d = ii[WS(is, 12)]; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 754| 2| T1e = T1c + T1d; 755| 2| T1E = T1c - T1d; 756| 2| } 757| 2| Te = Ta + Td; 758| 2| T1f = T1b + T1e; 759| 2| T50 = Td - Ta; 760| 2| T4s = T1b - T1e; 761| 2| { 762| 2| E T2U, T2V, T1C, T1F; 763| 2| T2U = T1D - T1E; 764| 2| T2V = T1B + T1A; 765| 2| T2W = KP707106781 * (T2U - T2V); 766| 2| T3u = KP707106781 * (T2V + T2U); 767| 2| T1C = T1A - T1B; 768| 2| T1F = T1D + T1E; 769| 2| T1G = KP707106781 * (T1C - T1F); 770| 2| T3U = KP707106781 * (T1C + T1F); 771| 2| } 772| 2| } 773| 2| { 774| 2| E Ti, T1L, T1j, T1J, Tl, T1I, T1m, T1M, T1K, T1N; 775| 2| { 776| 2| E Tg, Th, T1h, T1i; 777| 2| Tg = ri[WS(is, 2)]; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 778| 2| Th = ri[WS(is, 18)]; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 779| 2| Ti = Tg + Th; 780| 2| T1L = Tg - Th; 781| 2| T1h = ii[WS(is, 2)]; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 782| 2| T1i = ii[WS(is, 18)]; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 783| 2| T1j = T1h + T1i; 784| 2| T1J = T1h - T1i; 785| 2| } 786| 2| { 787| 2| E Tj, Tk, T1k, T1l; 788| 2| Tj = ri[WS(is, 10)]; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 789| 2| Tk = ri[WS(is, 26)]; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 790| 2| Tl = Tj + Tk; 791| 2| T1I = Tj - Tk; 792| 2| T1k = ii[WS(is, 10)]; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 793| 2| T1l = ii[WS(is, 26)]; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 794| 2| T1m = T1k + T1l; 795| 2| T1M = T1k - T1l; 796| 2| } 797| 2| Tm = Ti + Tl; 798| 2| T1n = T1j + T1m; 799| 2| T1K = T1I + T1J; 800| 2| T1N = T1L - T1M; 801| 2| T1O = FNMS(KP923879532, T1N, KP382683432 * T1K); ------------------ | | 1134| 2|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 802| 2| T2Z = FMA(KP923879532, T1K, KP382683432 * T1N); ------------------ | | 1131| 2|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 803| 2| { 804| 2| E T3w, T3x, T4u, T4v; 805| 2| T3w = T1J - T1I; 806| 2| T3x = T1L + T1M; 807| 2| T3y = FNMS(KP382683432, T3x, KP923879532 * T3w); ------------------ | | 1134| 2|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 808| 2| T3X = FMA(KP382683432, T3w, KP923879532 * T3x); ------------------ | | 1131| 2|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 809| 2| T4u = T1j - T1m; 810| 2| T4v = Ti - Tl; 811| 2| T4w = T4u - T4v; 812| 2| T53 = T4v + T4u; 813| 2| } 814| 2| } 815| 2| { 816| 2| E Tp, T1S, T1q, T1Q, Ts, T1P, T1t, T1T, T1R, T1U; 817| 2| { 818| 2| E Tn, To, T1o, T1p; 819| 2| Tn = ri[WS(is, 30)]; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 820| 2| To = ri[WS(is, 14)]; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 821| 2| Tp = Tn + To; 822| 2| T1S = Tn - To; 823| 2| T1o = ii[WS(is, 30)]; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 824| 2| T1p = ii[WS(is, 14)]; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 825| 2| T1q = T1o + T1p; 826| 2| T1Q = T1o - T1p; 827| 2| } 828| 2| { 829| 2| E Tq, Tr, T1r, T1s; 830| 2| Tq = ri[WS(is, 6)]; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 831| 2| Tr = ri[WS(is, 22)]; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 832| 2| Ts = Tq + Tr; 833| 2| T1P = Tq - Tr; 834| 2| T1r = ii[WS(is, 6)]; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 835| 2| T1s = ii[WS(is, 22)]; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 836| 2| T1t = T1r + T1s; 837| 2| T1T = T1r - T1s; 838| 2| } 839| 2| Tt = Tp + Ts; 840| 2| T1u = T1q + T1t; 841| 2| T1R = T1P + T1Q; 842| 2| T1U = T1S - T1T; 843| 2| T1V = FMA(KP382683432, T1R, KP923879532 * T1U); ------------------ | | 1131| 2|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 844| 2| T2Y = FNMS(KP923879532, T1R, KP382683432 * T1U); ------------------ | | 1134| 2|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 845| 2| { 846| 2| E T3z, T3A, T4x, T4y; 847| 2| T3z = T1Q - T1P; 848| 2| T3A = T1S + T1T; 849| 2| T3B = FMA(KP923879532, T3z, KP382683432 * T3A); ------------------ | | 1131| 2|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 850| 2| T3W = FNMS(KP382683432, T3z, KP923879532 * T3A); ------------------ | | 1134| 2|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 851| 2| T4x = Tp - Ts; 852| 2| T4y = T1q - T1t; 853| 2| T4z = T4x + T4y; 854| 2| T52 = T4x - T4y; 855| 2| } 856| 2| } 857| 2| { 858| 2| E TN, T2p, T2J, T4S, TQ, T2G, T2s, T4T, TU, T2x, T2w, T4O, TX, T2z, T2C; 859| 2| E T4P; 860| 2| { 861| 2| E TL, TM, T2H, T2I; 862| 2| TL = ri[WS(is, 31)]; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 863| 2| TM = ri[WS(is, 15)]; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 864| 2| TN = TL + TM; 865| 2| T2p = TL - TM; 866| 2| T2H = ii[WS(is, 31)]; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 867| 2| T2I = ii[WS(is, 15)]; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 868| 2| T2J = T2H - T2I; 869| 2| T4S = T2H + T2I; 870| 2| } 871| 2| { 872| 2| E TO, TP, T2q, T2r; 873| 2| TO = ri[WS(is, 7)]; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 874| 2| TP = ri[WS(is, 23)]; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 875| 2| TQ = TO + TP; 876| 2| T2G = TO - TP; 877| 2| T2q = ii[WS(is, 7)]; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 878| 2| T2r = ii[WS(is, 23)]; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 879| 2| T2s = T2q - T2r; 880| 2| T4T = T2q + T2r; 881| 2| } 882| 2| { 883| 2| E TS, TT, T2u, T2v; 884| 2| TS = ri[WS(is, 3)]; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 885| 2| TT = ri[WS(is, 19)]; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 886| 2| TU = TS + TT; 887| 2| T2x = TS - TT; 888| 2| T2u = ii[WS(is, 3)]; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 889| 2| T2v = ii[WS(is, 19)]; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 890| 2| T2w = T2u - T2v; 891| 2| T4O = T2u + T2v; 892| 2| } 893| 2| { 894| 2| E TV, TW, T2A, T2B; 895| 2| TV = ri[WS(is, 27)]; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 896| 2| TW = ri[WS(is, 11)]; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 897| 2| TX = TV + TW; 898| 2| T2z = TV - TW; 899| 2| T2A = ii[WS(is, 27)]; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 900| 2| T2B = ii[WS(is, 11)]; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 901| 2| T2C = T2A - T2B; 902| 2| T4P = T2A + T2B; 903| 2| } 904| 2| T2t = T2p - T2s; 905| 2| T3L = T2p + T2s; 906| 2| T3O = T2J - T2G; 907| 2| T2K = T2G + T2J; 908| 2| TR = TN + TQ; 909| 2| TY = TU + TX; 910| 2| T5F = TR - TY; 911| 2| { 912| 2| E T4N, T4Q, T2y, T2D; 913| 2| T5G = T4S + T4T; 914| 2| T5H = T4O + T4P; 915| 2| T5I = T5G - T5H; 916| 2| T4N = TN - TQ; 917| 2| T4Q = T4O - T4P; 918| 2| T4R = T4N - T4Q; 919| 2| T5j = T4N + T4Q; 920| 2| T2y = T2w - T2x; 921| 2| T2D = T2z + T2C; 922| 2| T2E = KP707106781 * (T2y - T2D); 923| 2| T3P = KP707106781 * (T2y + T2D); 924| 2| { 925| 2| E T4U, T4V, T2L, T2M; 926| 2| T4U = T4S - T4T; 927| 2| T4V = TX - TU; 928| 2| T4W = T4U - T4V; 929| 2| T5k = T4V + T4U; 930| 2| T2L = T2z - T2C; 931| 2| T2M = T2x + T2w; 932| 2| T2N = KP707106781 * (T2L - T2M); 933| 2| T3M = KP707106781 * (T2M + T2L); 934| 2| } 935| 2| } 936| 2| } 937| 2| { 938| 2| E Ty, T2f, T21, T4C, TB, T1Y, T2i, T4D, TF, T28, T2b, T4I, TI, T23, T26; 939| 2| E T4J; 940| 2| { 941| 2| E Tw, Tx, T1Z, T20; 942| 2| Tw = ri[WS(is, 1)]; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 943| 2| Tx = ri[WS(is, 17)]; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 944| 2| Ty = Tw + Tx; 945| 2| T2f = Tw - Tx; 946| 2| T1Z = ii[WS(is, 1)]; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 947| 2| T20 = ii[WS(is, 17)]; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 948| 2| T21 = T1Z - T20; 949| 2| T4C = T1Z + T20; 950| 2| } 951| 2| { 952| 2| E Tz, TA, T2g, T2h; 953| 2| Tz = ri[WS(is, 9)]; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 954| 2| TA = ri[WS(is, 25)]; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 955| 2| TB = Tz + TA; 956| 2| T1Y = Tz - TA; 957| 2| T2g = ii[WS(is, 9)]; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 958| 2| T2h = ii[WS(is, 25)]; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 959| 2| T2i = T2g - T2h; 960| 2| T4D = T2g + T2h; 961| 2| } 962| 2| { 963| 2| E TD, TE, T29, T2a; 964| 2| TD = ri[WS(is, 5)]; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 965| 2| TE = ri[WS(is, 21)]; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 966| 2| TF = TD + TE; 967| 2| T28 = TD - TE; 968| 2| T29 = ii[WS(is, 5)]; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 969| 2| T2a = ii[WS(is, 21)]; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 970| 2| T2b = T29 - T2a; 971| 2| T4I = T29 + T2a; 972| 2| } 973| 2| { 974| 2| E TG, TH, T24, T25; 975| 2| TG = ri[WS(is, 29)]; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 976| 2| TH = ri[WS(is, 13)]; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 977| 2| TI = TG + TH; 978| 2| T23 = TG - TH; 979| 2| T24 = ii[WS(is, 29)]; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 980| 2| T25 = ii[WS(is, 13)]; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 981| 2| T26 = T24 - T25; 982| 2| T4J = T24 + T25; 983| 2| } 984| 2| T22 = T1Y + T21; 985| 2| T3E = T2f + T2i; 986| 2| T3H = T21 - T1Y; 987| 2| T2j = T2f - T2i; 988| 2| TC = Ty + TB; 989| 2| TJ = TF + TI; 990| 2| T5A = TC - TJ; 991| 2| { 992| 2| E T4E, T4F, T27, T2c; 993| 2| T5B = T4C + T4D; 994| 2| T5C = T4I + T4J; 995| 2| T5D = T5B - T5C; 996| 2| T4E = T4C - T4D; 997| 2| T4F = TI - TF; 998| 2| T4G = T4E - T4F; 999| 2| T5g = T4F + T4E; 1000| 2| T27 = T23 - T26; 1001| 2| T2c = T28 + T2b; 1002| 2| T2d = KP707106781 * (T27 - T2c); 1003| 2| T3F = KP707106781 * (T2c + T27); 1004| 2| { 1005| 2| E T4H, T4K, T2k, T2l; 1006| 2| T4H = Ty - TB; 1007| 2| T4K = T4I - T4J; 1008| 2| T4L = T4H - T4K; 1009| 2| T5h = T4H + T4K; 1010| 2| T2k = T2b - T28; 1011| 2| T2l = T23 + T26; 1012| 2| T2m = KP707106781 * (T2k - T2l); 1013| 2| T3I = KP707106781 * (T2k + T2l); 1014| 2| } 1015| 2| } 1016| 2| } 1017| 2| { 1018| 2| E T4B, T57, T5a, T5c, T4Y, T56, T55, T5b; 1019| 2| { 1020| 2| E T4t, T4A, T58, T59; 1021| 2| T4t = T4r - T4s; 1022| 2| T4A = KP707106781 * (T4w - T4z); 1023| 2| T4B = T4t + T4A; 1024| 2| T57 = T4t - T4A; 1025| 2| T58 = FNMS(KP923879532, T4L, KP382683432 * T4G); ------------------ | | 1134| 2|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1026| 2| T59 = FMA(KP382683432, T4W, KP923879532 * T4R); ------------------ | | 1131| 2|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1027| 2| T5a = T58 - T59; 1028| 2| T5c = T58 + T59; 1029| 2| } 1030| 2| { 1031| 2| E T4M, T4X, T51, T54; 1032| 2| T4M = FMA(KP923879532, T4G, KP382683432 * T4L); ------------------ | | 1131| 2|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1033| 2| T4X = FNMS(KP923879532, T4W, KP382683432 * T4R); ------------------ | | 1134| 2|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1034| 2| T4Y = T4M + T4X; 1035| 2| T56 = T4X - T4M; 1036| 2| T51 = T4Z - T50; 1037| 2| T54 = KP707106781 * (T52 - T53); 1038| 2| T55 = T51 - T54; 1039| 2| T5b = T51 + T54; 1040| 2| } 1041| 2| ro[WS(os, 22)] = T4B - T4Y; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 1042| 2| io[WS(os, 22)] = T5b - T5c; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 1043| 2| ro[WS(os, 6)] = T4B + T4Y; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 1044| 2| io[WS(os, 6)] = T5b + T5c; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 1045| 2| io[WS(os, 30)] = T55 - T56; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 1046| 2| ro[WS(os, 30)] = T57 - T5a; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 1047| 2| io[WS(os, 14)] = T55 + T56; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 1048| 2| ro[WS(os, 14)] = T57 + T5a; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 1049| 2| } 1050| 2| { 1051| 2| E T5f, T5r, T5u, T5w, T5m, T5q, T5p, T5v; 1052| 2| { 1053| 2| E T5d, T5e, T5s, T5t; 1054| 2| T5d = T4r + T4s; 1055| 2| T5e = KP707106781 * (T53 + T52); 1056| 2| T5f = T5d + T5e; 1057| 2| T5r = T5d - T5e; 1058| 2| T5s = FNMS(KP382683432, T5h, KP923879532 * T5g); ------------------ | | 1134| 2|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1059| 2| T5t = FMA(KP923879532, T5k, KP382683432 * T5j); ------------------ | | 1131| 2|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1060| 2| T5u = T5s - T5t; 1061| 2| T5w = T5s + T5t; 1062| 2| } 1063| 2| { 1064| 2| E T5i, T5l, T5n, T5o; 1065| 2| T5i = FMA(KP382683432, T5g, KP923879532 * T5h); ------------------ | | 1131| 2|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1066| 2| T5l = FNMS(KP382683432, T5k, KP923879532 * T5j); ------------------ | | 1134| 2|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1067| 2| T5m = T5i + T5l; 1068| 2| T5q = T5l - T5i; 1069| 2| T5n = T50 + T4Z; 1070| 2| T5o = KP707106781 * (T4w + T4z); 1071| 2| T5p = T5n - T5o; 1072| 2| T5v = T5n + T5o; 1073| 2| } 1074| 2| ro[WS(os, 18)] = T5f - T5m; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 1075| 2| io[WS(os, 18)] = T5v - T5w; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 1076| 2| ro[WS(os, 2)] = T5f + T5m; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 1077| 2| io[WS(os, 2)] = T5v + T5w; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 1078| 2| io[WS(os, 26)] = T5p - T5q; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 1079| 2| ro[WS(os, 26)] = T5r - T5u; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 1080| 2| io[WS(os, 10)] = T5p + T5q; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 1081| 2| ro[WS(os, 10)] = T5r + T5u; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 1082| 2| } 1083| 2| { 1084| 2| E T5z, T5P, T5S, T5U, T5K, T5O, T5N, T5T; 1085| 2| { 1086| 2| E T5x, T5y, T5Q, T5R; 1087| 2| T5x = T7 - Te; 1088| 2| T5y = T1n - T1u; 1089| 2| T5z = T5x + T5y; 1090| 2| T5P = T5x - T5y; 1091| 2| T5Q = T5D - T5A; 1092| 2| T5R = T5F + T5I; 1093| 2| T5S = KP707106781 * (T5Q - T5R); 1094| 2| T5U = KP707106781 * (T5Q + T5R); 1095| 2| } 1096| 2| { 1097| 2| E T5E, T5J, T5L, T5M; 1098| 2| T5E = T5A + T5D; 1099| 2| T5J = T5F - T5I; 1100| 2| T5K = KP707106781 * (T5E + T5J); 1101| 2| T5O = KP707106781 * (T5J - T5E); 1102| 2| T5L = T18 - T1f; 1103| 2| T5M = Tt - Tm; 1104| 2| T5N = T5L - T5M; 1105| 2| T5T = T5M + T5L; 1106| 2| } 1107| 2| ro[WS(os, 20)] = T5z - T5K; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 1108| 2| io[WS(os, 20)] = T5T - T5U; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 1109| 2| ro[WS(os, 4)] = T5z + T5K; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 1110| 2| io[WS(os, 4)] = T5T + T5U; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 1111| 2| io[WS(os, 28)] = T5N - T5O; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 1112| 2| ro[WS(os, 28)] = T5P - T5S; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 1113| 2| io[WS(os, 12)] = T5N + T5O; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 1114| 2| ro[WS(os, 12)] = T5P + T5S; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 1115| 2| } 1116| 2| { 1117| 2| E Tv, T5V, T5Y, T60, T10, T11, T1w, T5Z; 1118| 2| { 1119| 2| E Tf, Tu, T5W, T5X; 1120| 2| Tf = T7 + Te; 1121| 2| Tu = Tm + Tt; 1122| 2| Tv = Tf + Tu; 1123| 2| T5V = Tf - Tu; 1124| 2| T5W = T5B + T5C; 1125| 2| T5X = T5G + T5H; 1126| 2| T5Y = T5W - T5X; 1127| 2| T60 = T5W + T5X; 1128| 2| } 1129| 2| { 1130| 2| E TK, TZ, T1g, T1v; 1131| 2| TK = TC + TJ; 1132| 2| TZ = TR + TY; 1133| 2| T10 = TK + TZ; 1134| 2| T11 = TZ - TK; 1135| 2| T1g = T18 + T1f; 1136| 2| T1v = T1n + T1u; 1137| 2| T1w = T1g - T1v; 1138| 2| T5Z = T1g + T1v; 1139| 2| } 1140| 2| ro[WS(os, 16)] = Tv - T10; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 1141| 2| io[WS(os, 16)] = T5Z - T60; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 1142| 2| ro[0] = Tv + T10; 1143| 2| io[0] = T5Z + T60; 1144| 2| io[WS(os, 8)] = T11 + T1w; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 1145| 2| ro[WS(os, 8)] = T5V + T5Y; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 1146| 2| io[WS(os, 24)] = T1w - T11; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 1147| 2| ro[WS(os, 24)] = T5V - T5Y; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 1148| 2| } 1149| 2| { 1150| 2| E T1X, T33, T31, T37, T2o, T34, T2P, T35; 1151| 2| { 1152| 2| E T1H, T1W, T2X, T30; 1153| 2| T1H = T1z - T1G; 1154| 2| T1W = T1O - T1V; 1155| 2| T1X = T1H + T1W; 1156| 2| T33 = T1H - T1W; 1157| 2| T2X = T2T - T2W; 1158| 2| T30 = T2Y - T2Z; 1159| 2| T31 = T2X - T30; 1160| 2| T37 = T2X + T30; 1161| 2| } 1162| 2| { 1163| 2| E T2e, T2n, T2F, T2O; 1164| 2| T2e = T22 - T2d; 1165| 2| T2n = T2j - T2m; 1166| 2| T2o = FMA(KP980785280, T2e, KP195090322 * T2n); ------------------ | | 1131| 2|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1167| 2| T34 = FNMS(KP980785280, T2n, KP195090322 * T2e); ------------------ | | 1134| 2|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1168| 2| T2F = T2t - T2E; 1169| 2| T2O = T2K - T2N; 1170| 2| T2P = FNMS(KP980785280, T2O, KP195090322 * T2F); ------------------ | | 1134| 2|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1171| 2| T35 = FMA(KP195090322, T2O, KP980785280 * T2F); ------------------ | | 1131| 2|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1172| 2| } 1173| 2| { 1174| 2| E T2Q, T38, T32, T36; 1175| 2| T2Q = T2o + T2P; 1176| 2| ro[WS(os, 23)] = T1X - T2Q; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 1177| 2| ro[WS(os, 7)] = T1X + T2Q; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 1178| 2| T38 = T34 + T35; 1179| 2| io[WS(os, 23)] = T37 - T38; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 1180| 2| io[WS(os, 7)] = T37 + T38; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 1181| 2| T32 = T2P - T2o; 1182| 2| io[WS(os, 31)] = T31 - T32; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 1183| 2| io[WS(os, 15)] = T31 + T32; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 1184| 2| T36 = T34 - T35; 1185| 2| ro[WS(os, 31)] = T33 - T36; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 1186| 2| ro[WS(os, 15)] = T33 + T36; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 1187| 2| } 1188| 2| } 1189| 2| { 1190| 2| E T3D, T41, T3Z, T45, T3K, T42, T3R, T43; 1191| 2| { 1192| 2| E T3v, T3C, T3V, T3Y; 1193| 2| T3v = T3t - T3u; 1194| 2| T3C = T3y - T3B; 1195| 2| T3D = T3v + T3C; 1196| 2| T41 = T3v - T3C; 1197| 2| T3V = T3T - T3U; 1198| 2| T3Y = T3W - T3X; 1199| 2| T3Z = T3V - T3Y; 1200| 2| T45 = T3V + T3Y; 1201| 2| } 1202| 2| { 1203| 2| E T3G, T3J, T3N, T3Q; 1204| 2| T3G = T3E - T3F; 1205| 2| T3J = T3H - T3I; 1206| 2| T3K = FMA(KP555570233, T3G, KP831469612 * T3J); ------------------ | | 1131| 2|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1207| 2| T42 = FNMS(KP831469612, T3G, KP555570233 * T3J); ------------------ | | 1134| 2|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1208| 2| T3N = T3L - T3M; 1209| 2| T3Q = T3O - T3P; 1210| 2| T3R = FNMS(KP831469612, T3Q, KP555570233 * T3N); ------------------ | | 1134| 2|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1211| 2| T43 = FMA(KP831469612, T3N, KP555570233 * T3Q); ------------------ | | 1131| 2|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1212| 2| } 1213| 2| { 1214| 2| E T3S, T46, T40, T44; 1215| 2| T3S = T3K + T3R; 1216| 2| ro[WS(os, 21)] = T3D - T3S; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 1217| 2| ro[WS(os, 5)] = T3D + T3S; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 1218| 2| T46 = T42 + T43; 1219| 2| io[WS(os, 21)] = T45 - T46; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 1220| 2| io[WS(os, 5)] = T45 + T46; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 1221| 2| T40 = T3R - T3K; 1222| 2| io[WS(os, 29)] = T3Z - T40; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 1223| 2| io[WS(os, 13)] = T3Z + T40; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 1224| 2| T44 = T42 - T43; 1225| 2| ro[WS(os, 29)] = T41 - T44; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 1226| 2| ro[WS(os, 13)] = T41 + T44; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 1227| 2| } 1228| 2| } 1229| 2| { 1230| 2| E T49, T4l, T4j, T4p, T4c, T4m, T4f, T4n; 1231| 2| { 1232| 2| E T47, T48, T4h, T4i; 1233| 2| T47 = T3t + T3u; 1234| 2| T48 = T3X + T3W; 1235| 2| T49 = T47 + T48; 1236| 2| T4l = T47 - T48; 1237| 2| T4h = T3T + T3U; 1238| 2| T4i = T3y + T3B; 1239| 2| T4j = T4h - T4i; 1240| 2| T4p = T4h + T4i; 1241| 2| } 1242| 2| { 1243| 2| E T4a, T4b, T4d, T4e; 1244| 2| T4a = T3E + T3F; 1245| 2| T4b = T3H + T3I; 1246| 2| T4c = FMA(KP980785280, T4a, KP195090322 * T4b); ------------------ | | 1131| 2|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1247| 2| T4m = FNMS(KP195090322, T4a, KP980785280 * T4b); ------------------ | | 1134| 2|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1248| 2| T4d = T3L + T3M; 1249| 2| T4e = T3O + T3P; 1250| 2| T4f = FNMS(KP195090322, T4e, KP980785280 * T4d); ------------------ | | 1134| 2|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1251| 2| T4n = FMA(KP195090322, T4d, KP980785280 * T4e); ------------------ | | 1131| 2|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1252| 2| } 1253| 2| { 1254| 2| E T4g, T4q, T4k, T4o; 1255| 2| T4g = T4c + T4f; 1256| 2| ro[WS(os, 17)] = T49 - T4g; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 1257| 2| ro[WS(os, 1)] = T49 + T4g; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 1258| 2| T4q = T4m + T4n; 1259| 2| io[WS(os, 17)] = T4p - T4q; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 1260| 2| io[WS(os, 1)] = T4p + T4q; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 1261| 2| T4k = T4f - T4c; 1262| 2| io[WS(os, 25)] = T4j - T4k; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 1263| 2| io[WS(os, 9)] = T4j + T4k; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 1264| 2| T4o = T4m - T4n; 1265| 2| ro[WS(os, 25)] = T4l - T4o; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 1266| 2| ro[WS(os, 9)] = T4l + T4o; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 1267| 2| } 1268| 2| } 1269| 2| { 1270| 2| E T3b, T3n, T3l, T3r, T3e, T3o, T3h, T3p; 1271| 2| { 1272| 2| E T39, T3a, T3j, T3k; 1273| 2| T39 = T1z + T1G; 1274| 2| T3a = T2Z + T2Y; 1275| 2| T3b = T39 + T3a; 1276| 2| T3n = T39 - T3a; 1277| 2| T3j = T2T + T2W; 1278| 2| T3k = T1O + T1V; 1279| 2| T3l = T3j - T3k; 1280| 2| T3r = T3j + T3k; 1281| 2| } 1282| 2| { 1283| 2| E T3c, T3d, T3f, T3g; 1284| 2| T3c = T22 + T2d; 1285| 2| T3d = T2j + T2m; 1286| 2| T3e = FMA(KP555570233, T3c, KP831469612 * T3d); ------------------ | | 1131| 2|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1287| 2| T3o = FNMS(KP555570233, T3d, KP831469612 * T3c); ------------------ | | 1134| 2|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1288| 2| T3f = T2t + T2E; 1289| 2| T3g = T2K + T2N; 1290| 2| T3h = FNMS(KP555570233, T3g, KP831469612 * T3f); ------------------ | | 1134| 2|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1291| 2| T3p = FMA(KP831469612, T3g, KP555570233 * T3f); ------------------ | | 1131| 2|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1292| 2| } 1293| 2| { 1294| 2| E T3i, T3s, T3m, T3q; 1295| 2| T3i = T3e + T3h; 1296| 2| ro[WS(os, 19)] = T3b - T3i; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 1297| 2| ro[WS(os, 3)] = T3b + T3i; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 1298| 2| T3s = T3o + T3p; 1299| 2| io[WS(os, 19)] = T3r - T3s; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 1300| 2| io[WS(os, 3)] = T3r + T3s; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 1301| 2| T3m = T3h - T3e; 1302| 2| io[WS(os, 27)] = T3l - T3m; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 1303| 2| io[WS(os, 11)] = T3l + T3m; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 1304| 2| T3q = T3o - T3p; 1305| 2| ro[WS(os, 27)] = T3n - T3q; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 1306| 2| ro[WS(os, 11)] = T3n + T3q; ------------------ | | 825| 2|#define WS(stride, i) (stride * i) ------------------ 1307| 2| } 1308| 2| } 1309| 2| } 1310| 2| } 1311| 2|} fftw_codelet_n1_4: 135| 1|void X(codelet_n1_4) (planner *p) { X(kdft_register) (p, n1_4, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 136| 1|} n1_4.c:n1_4: 94| 4|{ 95| 4| { 96| 4| INT i; 97| 30| for (i = v; i > 0; i = i - 1, ri = ri + ivs, ii = ii + ivs, ro = ro + ovs, io = io + ovs, MAKE_VOLATILE_STRIDE(16, is), MAKE_VOLATILE_STRIDE(16, os)) { ------------------ | | 851| 26| (nptr <= ESTIMATED_AVAILABLE_INDEX_REGISTERS ? \ | | ------------------ | | | | 849| 52|#define ESTIMATED_AVAILABLE_INDEX_REGISTERS 16 | | ------------------ | | | Branch (851:7): [True: 26, Folded] | | ------------------ | | 852| 26| 0 : \ | | 853| 26| ((x) = (x) ^ X(an_INT_guaranteed_to_be_zero))) | | ------------------ | | | | 78| 0|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ for (i = v; i > 0; i = i - 1, ri = ri + ivs, ii = ii + ivs, ro = ro + ovs, io = io + ovs, MAKE_VOLATILE_STRIDE(16, is), MAKE_VOLATILE_STRIDE(16, os)) { ------------------ | | 851| 26| (nptr <= ESTIMATED_AVAILABLE_INDEX_REGISTERS ? \ | | ------------------ | | | | 849| 52|#define ESTIMATED_AVAILABLE_INDEX_REGISTERS 16 | | ------------------ | | | Branch (851:7): [True: 26, Folded] | | ------------------ | | 852| 26| 0 : \ | | 853| 26| ((x) = (x) ^ X(an_INT_guaranteed_to_be_zero))) | | ------------------ | | | | 78| 0|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ | Branch (97:16): [True: 26, False: 4] ------------------ 98| 26| E T3, Tb, T9, Tf, T6, Ta, Te, Tg; 99| 26| { 100| 26| E T1, T2, T7, T8; 101| 26| T1 = ri[0]; 102| 26| T2 = ri[WS(is, 2)]; ------------------ | | 825| 26|#define WS(stride, i) (stride * i) ------------------ 103| 26| T3 = T1 + T2; 104| 26| Tb = T1 - T2; 105| 26| T7 = ii[0]; 106| 26| T8 = ii[WS(is, 2)]; ------------------ | | 825| 26|#define WS(stride, i) (stride * i) ------------------ 107| 26| T9 = T7 - T8; 108| 26| Tf = T7 + T8; 109| 26| } 110| 26| { 111| 26| E T4, T5, Tc, Td; 112| 26| T4 = ri[WS(is, 1)]; ------------------ | | 825| 26|#define WS(stride, i) (stride * i) ------------------ 113| 26| T5 = ri[WS(is, 3)]; ------------------ | | 825| 26|#define WS(stride, i) (stride * i) ------------------ 114| 26| T6 = T4 + T5; 115| 26| Ta = T4 - T5; 116| 26| Tc = ii[WS(is, 1)]; ------------------ | | 825| 26|#define WS(stride, i) (stride * i) ------------------ 117| 26| Td = ii[WS(is, 3)]; ------------------ | | 825| 26|#define WS(stride, i) (stride * i) ------------------ 118| 26| Te = Tc - Td; 119| 26| Tg = Tc + Td; 120| 26| } 121| 26| ro[WS(os, 2)] = T3 - T6; ------------------ | | 825| 26|#define WS(stride, i) (stride * i) ------------------ 122| 26| io[WS(os, 2)] = Tf - Tg; ------------------ | | 825| 26|#define WS(stride, i) (stride * i) ------------------ 123| 26| ro[0] = T3 + T6; 124| 26| io[0] = Tf + Tg; 125| 26| io[WS(os, 1)] = T9 - Ta; ------------------ | | 825| 26|#define WS(stride, i) (stride * i) ------------------ 126| 26| ro[WS(os, 1)] = Tb + Te; ------------------ | | 825| 26|#define WS(stride, i) (stride * i) ------------------ 127| 26| io[WS(os, 3)] = Ta + T9; ------------------ | | 825| 26|#define WS(stride, i) (stride * i) ------------------ 128| 26| ro[WS(os, 3)] = Tb - Te; ------------------ | | 825| 26|#define WS(stride, i) (stride * i) ------------------ 129| 26| } 130| 4| } 131| 4|} fftw_codelet_n1_5: 191| 1|void X(codelet_n1_5) (planner *p) { X(kdft_register) (p, n1_5, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 192| 1|} n1_5.c:n1_5: 122| 6|{ 123| 6| DK(KP250000000, +0.250000000000000000000000000000000000000000000); ------------------ | | 1056| 6|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 6|# define K(x) ((E) x) | | ------------------ ------------------ 124| 6| DK(KP587785252, +0.587785252292473129168705954639072768597652438); ------------------ | | 1056| 6|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 6|# define K(x) ((E) x) | | ------------------ ------------------ 125| 6| DK(KP951056516, +0.951056516295153572116439333379382143405698634); ------------------ | | 1056| 6|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 6|# define K(x) ((E) x) | | ------------------ ------------------ 126| 6| DK(KP559016994, +0.559016994374947424102293417182819058860154590); ------------------ | | 1056| 6|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 6|# define K(x) ((E) x) | | ------------------ ------------------ 127| 6| { 128| 6| INT i; 129| 24| for (i = v; i > 0; i = i - 1, ri = ri + ivs, ii = ii + ivs, ro = ro + ovs, io = io + ovs, MAKE_VOLATILE_STRIDE(20, is), MAKE_VOLATILE_STRIDE(20, os)) { ------------------ | | 851| 18| (nptr <= ESTIMATED_AVAILABLE_INDEX_REGISTERS ? \ | | ------------------ | | | | 849| 36|#define ESTIMATED_AVAILABLE_INDEX_REGISTERS 16 | | ------------------ | | | Branch (851:7): [Folded, False: 18] | | ------------------ | | 852| 18| 0 : \ | | 853| 18| ((x) = (x) ^ X(an_INT_guaranteed_to_be_zero))) | | ------------------ | | | | 78| 18|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 18|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ for (i = v; i > 0; i = i - 1, ri = ri + ivs, ii = ii + ivs, ro = ro + ovs, io = io + ovs, MAKE_VOLATILE_STRIDE(20, is), MAKE_VOLATILE_STRIDE(20, os)) { ------------------ | | 851| 18| (nptr <= ESTIMATED_AVAILABLE_INDEX_REGISTERS ? \ | | ------------------ | | | | 849| 36|#define ESTIMATED_AVAILABLE_INDEX_REGISTERS 16 | | ------------------ | | | Branch (851:7): [Folded, False: 18] | | ------------------ | | 852| 18| 0 : \ | | 853| 18| ((x) = (x) ^ X(an_INT_guaranteed_to_be_zero))) | | ------------------ | | | | 78| 18|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 18|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ | Branch (129:16): [True: 18, False: 6] ------------------ 130| 18| E T1, To, T8, Tt, T9, Ts, Te, Tp, Th, Tn; 131| 18| T1 = ri[0]; 132| 18| To = ii[0]; 133| 18| { 134| 18| E T2, T3, T4, T5, T6, T7; 135| 18| T2 = ri[WS(is, 1)]; ------------------ | | 825| 18|#define WS(stride, i) (stride * i) ------------------ 136| 18| T3 = ri[WS(is, 4)]; ------------------ | | 825| 18|#define WS(stride, i) (stride * i) ------------------ 137| 18| T4 = T2 + T3; 138| 18| T5 = ri[WS(is, 2)]; ------------------ | | 825| 18|#define WS(stride, i) (stride * i) ------------------ 139| 18| T6 = ri[WS(is, 3)]; ------------------ | | 825| 18|#define WS(stride, i) (stride * i) ------------------ 140| 18| T7 = T5 + T6; 141| 18| T8 = T4 + T7; 142| 18| Tt = T5 - T6; 143| 18| T9 = KP559016994 * (T4 - T7); 144| 18| Ts = T2 - T3; 145| 18| } 146| 18| { 147| 18| E Tc, Td, Tl, Tf, Tg, Tm; 148| 18| Tc = ii[WS(is, 1)]; ------------------ | | 825| 18|#define WS(stride, i) (stride * i) ------------------ 149| 18| Td = ii[WS(is, 4)]; ------------------ | | 825| 18|#define WS(stride, i) (stride * i) ------------------ 150| 18| Tl = Tc + Td; 151| 18| Tf = ii[WS(is, 2)]; ------------------ | | 825| 18|#define WS(stride, i) (stride * i) ------------------ 152| 18| Tg = ii[WS(is, 3)]; ------------------ | | 825| 18|#define WS(stride, i) (stride * i) ------------------ 153| 18| Tm = Tf + Tg; 154| 18| Te = Tc - Td; 155| 18| Tp = Tl + Tm; 156| 18| Th = Tf - Tg; 157| 18| Tn = KP559016994 * (Tl - Tm); 158| 18| } 159| 18| ro[0] = T1 + T8; 160| 18| io[0] = To + Tp; 161| 18| { 162| 18| E Ti, Tk, Tb, Tj, Ta; 163| 18| Ti = FMA(KP951056516, Te, KP587785252 * Th); ------------------ | | 1131| 18|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 164| 18| Tk = FNMS(KP587785252, Te, KP951056516 * Th); ------------------ | | 1134| 18|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 165| 18| Ta = FNMS(KP250000000, T8, T1); ------------------ | | 1134| 18|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 166| 18| Tb = T9 + Ta; 167| 18| Tj = Ta - T9; 168| 18| ro[WS(os, 4)] = Tb - Ti; ------------------ | | 825| 18|#define WS(stride, i) (stride * i) ------------------ 169| 18| ro[WS(os, 3)] = Tj + Tk; ------------------ | | 825| 18|#define WS(stride, i) (stride * i) ------------------ 170| 18| ro[WS(os, 1)] = Tb + Ti; ------------------ | | 825| 18|#define WS(stride, i) (stride * i) ------------------ 171| 18| ro[WS(os, 2)] = Tj - Tk; ------------------ | | 825| 18|#define WS(stride, i) (stride * i) ------------------ 172| 18| } 173| 18| { 174| 18| E Tu, Tv, Tr, Tw, Tq; 175| 18| Tu = FMA(KP951056516, Ts, KP587785252 * Tt); ------------------ | | 1131| 18|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 176| 18| Tv = FNMS(KP587785252, Ts, KP951056516 * Tt); ------------------ | | 1134| 18|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 177| 18| Tq = FNMS(KP250000000, Tp, To); ------------------ | | 1134| 18|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 178| 18| Tr = Tn + Tq; 179| 18| Tw = Tq - Tn; 180| 18| io[WS(os, 1)] = Tr - Tu; ------------------ | | 825| 18|#define WS(stride, i) (stride * i) ------------------ 181| 18| io[WS(os, 3)] = Tw - Tv; ------------------ | | 825| 18|#define WS(stride, i) (stride * i) ------------------ 182| 18| io[WS(os, 4)] = Tu + Tr; ------------------ | | 825| 18|#define WS(stride, i) (stride * i) ------------------ 183| 18| io[WS(os, 2)] = Tv + Tw; ------------------ | | 825| 18|#define WS(stride, i) (stride * i) ------------------ 184| 18| } 185| 18| } 186| 6| } 187| 6|} fftw_codelet_n1_6: 207| 1|void X(codelet_n1_6) (planner *p) { X(kdft_register) (p, n1_6, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 208| 1|} n1_6.c:n1_6: 130| 608|{ 131| 608| DK(KP866025403, +0.866025403784438646763723170752936183471402627); ------------------ | | 1056| 608|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 608|# define K(x) ((E) x) | | ------------------ ------------------ 132| 608| DK(KP500000000, +0.500000000000000000000000000000000000000000000); ------------------ | | 1056| 608|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 608|# define K(x) ((E) x) | | ------------------ ------------------ 133| 608| { 134| 608| INT i; 135| 2.66k| for (i = v; i > 0; i = i - 1, ri = ri + ivs, ii = ii + ivs, ro = ro + ovs, io = io + ovs, MAKE_VOLATILE_STRIDE(24, is), MAKE_VOLATILE_STRIDE(24, os)) { ------------------ | | 851| 2.06k| (nptr <= ESTIMATED_AVAILABLE_INDEX_REGISTERS ? \ | | ------------------ | | | | 849| 4.12k|#define ESTIMATED_AVAILABLE_INDEX_REGISTERS 16 | | ------------------ | | | Branch (851:7): [Folded, False: 2.06k] | | ------------------ | | 852| 2.06k| 0 : \ | | 853| 2.06k| ((x) = (x) ^ X(an_INT_guaranteed_to_be_zero))) | | ------------------ | | | | 78| 2.06k|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 2.06k|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ for (i = v; i > 0; i = i - 1, ri = ri + ivs, ii = ii + ivs, ro = ro + ovs, io = io + ovs, MAKE_VOLATILE_STRIDE(24, is), MAKE_VOLATILE_STRIDE(24, os)) { ------------------ | | 851| 2.06k| (nptr <= ESTIMATED_AVAILABLE_INDEX_REGISTERS ? \ | | ------------------ | | | | 849| 4.12k|#define ESTIMATED_AVAILABLE_INDEX_REGISTERS 16 | | ------------------ | | | Branch (851:7): [Folded, False: 2.06k] | | ------------------ | | 852| 2.06k| 0 : \ | | 853| 2.06k| ((x) = (x) ^ X(an_INT_guaranteed_to_be_zero))) | | ------------------ | | | | 78| 2.06k|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 2.06k|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ | Branch (135:16): [True: 2.06k, False: 608] ------------------ 136| 2.06k| E T3, Tb, Tq, Tx, T6, Tc, T9, Td, Ta, Te, Ti, Tu, Tl, Tv, Tr; 137| 2.06k| E Ty; 138| 2.06k| { 139| 2.06k| E T1, T2, To, Tp; 140| 2.06k| T1 = ri[0]; 141| 2.06k| T2 = ri[WS(is, 3)]; ------------------ | | 825| 2.06k|#define WS(stride, i) (stride * i) ------------------ 142| 2.06k| T3 = T1 - T2; 143| 2.06k| Tb = T1 + T2; 144| 2.06k| To = ii[0]; 145| 2.06k| Tp = ii[WS(is, 3)]; ------------------ | | 825| 2.06k|#define WS(stride, i) (stride * i) ------------------ 146| 2.06k| Tq = To - Tp; 147| 2.06k| Tx = To + Tp; 148| 2.06k| } 149| 2.06k| { 150| 2.06k| E T4, T5, T7, T8; 151| 2.06k| T4 = ri[WS(is, 2)]; ------------------ | | 825| 2.06k|#define WS(stride, i) (stride * i) ------------------ 152| 2.06k| T5 = ri[WS(is, 5)]; ------------------ | | 825| 2.06k|#define WS(stride, i) (stride * i) ------------------ 153| 2.06k| T6 = T4 - T5; 154| 2.06k| Tc = T4 + T5; 155| 2.06k| T7 = ri[WS(is, 4)]; ------------------ | | 825| 2.06k|#define WS(stride, i) (stride * i) ------------------ 156| 2.06k| T8 = ri[WS(is, 1)]; ------------------ | | 825| 2.06k|#define WS(stride, i) (stride * i) ------------------ 157| 2.06k| T9 = T7 - T8; 158| 2.06k| Td = T7 + T8; 159| 2.06k| } 160| 2.06k| Ta = T6 + T9; 161| 2.06k| Te = Tc + Td; 162| 2.06k| { 163| 2.06k| E Tg, Th, Tj, Tk; 164| 2.06k| Tg = ii[WS(is, 2)]; ------------------ | | 825| 2.06k|#define WS(stride, i) (stride * i) ------------------ 165| 2.06k| Th = ii[WS(is, 5)]; ------------------ | | 825| 2.06k|#define WS(stride, i) (stride * i) ------------------ 166| 2.06k| Ti = Tg - Th; 167| 2.06k| Tu = Tg + Th; 168| 2.06k| Tj = ii[WS(is, 4)]; ------------------ | | 825| 2.06k|#define WS(stride, i) (stride * i) ------------------ 169| 2.06k| Tk = ii[WS(is, 1)]; ------------------ | | 825| 2.06k|#define WS(stride, i) (stride * i) ------------------ 170| 2.06k| Tl = Tj - Tk; 171| 2.06k| Tv = Tj + Tk; 172| 2.06k| } 173| 2.06k| Tr = Ti + Tl; 174| 2.06k| Ty = Tu + Tv; 175| 2.06k| ro[WS(os, 3)] = T3 + Ta; ------------------ | | 825| 2.06k|#define WS(stride, i) (stride * i) ------------------ 176| 2.06k| io[WS(os, 3)] = Tq + Tr; ------------------ | | 825| 2.06k|#define WS(stride, i) (stride * i) ------------------ 177| 2.06k| ro[0] = Tb + Te; 178| 2.06k| io[0] = Tx + Ty; 179| 2.06k| { 180| 2.06k| E Tf, Tm, Tn, Ts; 181| 2.06k| Tf = FNMS(KP500000000, Ta, T3); ------------------ | | 1134| 2.06k|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 182| 2.06k| Tm = KP866025403 * (Ti - Tl); 183| 2.06k| ro[WS(os, 5)] = Tf - Tm; ------------------ | | 825| 2.06k|#define WS(stride, i) (stride * i) ------------------ 184| 2.06k| ro[WS(os, 1)] = Tf + Tm; ------------------ | | 825| 2.06k|#define WS(stride, i) (stride * i) ------------------ 185| 2.06k| Tn = KP866025403 * (T9 - T6); 186| 2.06k| Ts = FNMS(KP500000000, Tr, Tq); ------------------ | | 1134| 2.06k|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 187| 2.06k| io[WS(os, 1)] = Tn + Ts; ------------------ | | 825| 2.06k|#define WS(stride, i) (stride * i) ------------------ 188| 2.06k| io[WS(os, 5)] = Ts - Tn; ------------------ | | 825| 2.06k|#define WS(stride, i) (stride * i) ------------------ 189| 2.06k| } 190| 2.06k| { 191| 2.06k| E Tt, Tw, Tz, TA; 192| 2.06k| Tt = FNMS(KP500000000, Te, Tb); ------------------ | | 1134| 2.06k|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 193| 2.06k| Tw = KP866025403 * (Tu - Tv); 194| 2.06k| ro[WS(os, 2)] = Tt - Tw; ------------------ | | 825| 2.06k|#define WS(stride, i) (stride * i) ------------------ 195| 2.06k| ro[WS(os, 4)] = Tt + Tw; ------------------ | | 825| 2.06k|#define WS(stride, i) (stride * i) ------------------ 196| 2.06k| Tz = FNMS(KP500000000, Ty, Tx); ------------------ | | 1134| 2.06k|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 197| 2.06k| TA = KP866025403 * (Td - Tc); 198| 2.06k| io[WS(os, 2)] = Tz - TA; ------------------ | | 825| 2.06k|#define WS(stride, i) (stride * i) ------------------ 199| 2.06k| io[WS(os, 4)] = TA + Tz; ------------------ | | 825| 2.06k|#define WS(stride, i) (stride * i) ------------------ 200| 2.06k| } 201| 2.06k| } 202| 608| } 203| 608|} fftw_codelet_n1_64: 3083| 1|void X(codelet_n1_64) (planner *p) { X(kdft_register) (p, n1_64, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 3084| 1|} fftw_codelet_n1_7: 246| 1|void X(codelet_n1_7) (planner *p) { X(kdft_register) (p, n1_7, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 247| 1|} n1_7.c:n1_7: 159| 36|{ 160| 36| DK(KP222520933, +0.222520933956314404288902564496794759466355569); ------------------ | | 1056| 36|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 36|# define K(x) ((E) x) | | ------------------ ------------------ 161| 36| DK(KP900968867, +0.900968867902419126236102319507445051165919162); ------------------ | | 1056| 36|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 36|# define K(x) ((E) x) | | ------------------ ------------------ 162| 36| DK(KP623489801, +0.623489801858733530525004884004239810632274731); ------------------ | | 1056| 36|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 36|# define K(x) ((E) x) | | ------------------ ------------------ 163| 36| DK(KP433883739, +0.433883739117558120475768332848358754609990728); ------------------ | | 1056| 36|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 36|# define K(x) ((E) x) | | ------------------ ------------------ 164| 36| DK(KP781831482, +0.781831482468029808708444526674057750232334519); ------------------ | | 1056| 36|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 36|# define K(x) ((E) x) | | ------------------ ------------------ 165| 36| DK(KP974927912, +0.974927912181823607018131682993931217232785801); ------------------ | | 1056| 36|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 36|# define K(x) ((E) x) | | ------------------ ------------------ 166| 36| { 167| 36| INT i; 168| 178| for (i = v; i > 0; i = i - 1, ri = ri + ivs, ii = ii + ivs, ro = ro + ovs, io = io + ovs, MAKE_VOLATILE_STRIDE(28, is), MAKE_VOLATILE_STRIDE(28, os)) { ------------------ | | 851| 142| (nptr <= ESTIMATED_AVAILABLE_INDEX_REGISTERS ? \ | | ------------------ | | | | 849| 284|#define ESTIMATED_AVAILABLE_INDEX_REGISTERS 16 | | ------------------ | | | Branch (851:7): [Folded, False: 142] | | ------------------ | | 852| 142| 0 : \ | | 853| 142| ((x) = (x) ^ X(an_INT_guaranteed_to_be_zero))) | | ------------------ | | | | 78| 142|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 142|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ for (i = v; i > 0; i = i - 1, ri = ri + ivs, ii = ii + ivs, ro = ro + ovs, io = io + ovs, MAKE_VOLATILE_STRIDE(28, is), MAKE_VOLATILE_STRIDE(28, os)) { ------------------ | | 851| 142| (nptr <= ESTIMATED_AVAILABLE_INDEX_REGISTERS ? \ | | ------------------ | | | | 849| 284|#define ESTIMATED_AVAILABLE_INDEX_REGISTERS 16 | | ------------------ | | | Branch (851:7): [Folded, False: 142] | | ------------------ | | 852| 142| 0 : \ | | 853| 142| ((x) = (x) ^ X(an_INT_guaranteed_to_be_zero))) | | ------------------ | | | | 78| 142|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 142|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ | Branch (168:16): [True: 142, False: 36] ------------------ 169| 142| E T1, Tu, T4, Tq, Te, Tx, T7, Ts, Tk, Tv, Ta, Tr, Th, Tw; 170| 142| T1 = ri[0]; 171| 142| Tu = ii[0]; 172| 142| { 173| 142| E T2, T3, Tc, Td; 174| 142| T2 = ri[WS(is, 1)]; ------------------ | | 825| 142|#define WS(stride, i) (stride * i) ------------------ 175| 142| T3 = ri[WS(is, 6)]; ------------------ | | 825| 142|#define WS(stride, i) (stride * i) ------------------ 176| 142| T4 = T2 + T3; 177| 142| Tq = T3 - T2; 178| 142| Tc = ii[WS(is, 1)]; ------------------ | | 825| 142|#define WS(stride, i) (stride * i) ------------------ 179| 142| Td = ii[WS(is, 6)]; ------------------ | | 825| 142|#define WS(stride, i) (stride * i) ------------------ 180| 142| Te = Tc - Td; 181| 142| Tx = Tc + Td; 182| 142| } 183| 142| { 184| 142| E T5, T6, Ti, Tj; 185| 142| T5 = ri[WS(is, 2)]; ------------------ | | 825| 142|#define WS(stride, i) (stride * i) ------------------ 186| 142| T6 = ri[WS(is, 5)]; ------------------ | | 825| 142|#define WS(stride, i) (stride * i) ------------------ 187| 142| T7 = T5 + T6; 188| 142| Ts = T6 - T5; 189| 142| Ti = ii[WS(is, 2)]; ------------------ | | 825| 142|#define WS(stride, i) (stride * i) ------------------ 190| 142| Tj = ii[WS(is, 5)]; ------------------ | | 825| 142|#define WS(stride, i) (stride * i) ------------------ 191| 142| Tk = Ti - Tj; 192| 142| Tv = Ti + Tj; 193| 142| } 194| 142| { 195| 142| E T8, T9, Tf, Tg; 196| 142| T8 = ri[WS(is, 3)]; ------------------ | | 825| 142|#define WS(stride, i) (stride * i) ------------------ 197| 142| T9 = ri[WS(is, 4)]; ------------------ | | 825| 142|#define WS(stride, i) (stride * i) ------------------ 198| 142| Ta = T8 + T9; 199| 142| Tr = T9 - T8; 200| 142| Tf = ii[WS(is, 3)]; ------------------ | | 825| 142|#define WS(stride, i) (stride * i) ------------------ 201| 142| Tg = ii[WS(is, 4)]; ------------------ | | 825| 142|#define WS(stride, i) (stride * i) ------------------ 202| 142| Th = Tf - Tg; 203| 142| Tw = Tf + Tg; 204| 142| } 205| 142| ro[0] = T1 + T4 + T7 + Ta; 206| 142| io[0] = Tu + Tx + Tv + Tw; 207| 142| { 208| 142| E Tl, Tb, TB, TC; 209| 142| Tl = FNMS(KP781831482, Th, KP974927912 * Te) - (KP433883739 * Tk); ------------------ | | 1134| 142|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 210| 142| Tb = FMA(KP623489801, Ta, T1) + FNMA(KP900968867, T7, KP222520933 * T4); ------------------ | | 1131| 142|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ Tb = FMA(KP623489801, Ta, T1) + FNMA(KP900968867, T7, KP222520933 * T4); ------------------ | | 1133| 142|#define FNMA(a, b, c) (- (((a) * (b)) + (c))) ------------------ 211| 142| ro[WS(os, 5)] = Tb - Tl; ------------------ | | 825| 142|#define WS(stride, i) (stride * i) ------------------ 212| 142| ro[WS(os, 2)] = Tb + Tl; ------------------ | | 825| 142|#define WS(stride, i) (stride * i) ------------------ 213| 142| TB = FNMS(KP781831482, Tr, KP974927912 * Tq) - (KP433883739 * Ts); ------------------ | | 1134| 142|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 214| 142| TC = FMA(KP623489801, Tw, Tu) + FNMA(KP900968867, Tv, KP222520933 * Tx); ------------------ | | 1131| 142|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ TC = FMA(KP623489801, Tw, Tu) + FNMA(KP900968867, Tv, KP222520933 * Tx); ------------------ | | 1133| 142|#define FNMA(a, b, c) (- (((a) * (b)) + (c))) ------------------ 215| 142| io[WS(os, 2)] = TB + TC; ------------------ | | 825| 142|#define WS(stride, i) (stride * i) ------------------ 216| 142| io[WS(os, 5)] = TC - TB; ------------------ | | 825| 142|#define WS(stride, i) (stride * i) ------------------ 217| 142| } 218| 142| { 219| 142| E Tn, Tm, Tz, TA; 220| 142| Tn = FMA(KP781831482, Te, KP974927912 * Tk) + (KP433883739 * Th); ------------------ | | 1131| 142|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 221| 142| Tm = FMA(KP623489801, T4, T1) + FNMA(KP900968867, Ta, KP222520933 * T7); ------------------ | | 1131| 142|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ Tm = FMA(KP623489801, T4, T1) + FNMA(KP900968867, Ta, KP222520933 * T7); ------------------ | | 1133| 142|#define FNMA(a, b, c) (- (((a) * (b)) + (c))) ------------------ 222| 142| ro[WS(os, 6)] = Tm - Tn; ------------------ | | 825| 142|#define WS(stride, i) (stride * i) ------------------ 223| 142| ro[WS(os, 1)] = Tm + Tn; ------------------ | | 825| 142|#define WS(stride, i) (stride * i) ------------------ 224| 142| Tz = FMA(KP781831482, Tq, KP974927912 * Ts) + (KP433883739 * Tr); ------------------ | | 1131| 142|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 225| 142| TA = FMA(KP623489801, Tx, Tu) + FNMA(KP900968867, Tw, KP222520933 * Tv); ------------------ | | 1131| 142|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ TA = FMA(KP623489801, Tx, Tu) + FNMA(KP900968867, Tw, KP222520933 * Tv); ------------------ | | 1133| 142|#define FNMA(a, b, c) (- (((a) * (b)) + (c))) ------------------ 226| 142| io[WS(os, 1)] = Tz + TA; ------------------ | | 825| 142|#define WS(stride, i) (stride * i) ------------------ 227| 142| io[WS(os, 6)] = TA - Tz; ------------------ | | 825| 142|#define WS(stride, i) (stride * i) ------------------ 228| 142| } 229| 142| { 230| 142| E Tp, To, Tt, Ty; 231| 142| Tp = FMA(KP433883739, Te, KP974927912 * Th) - (KP781831482 * Tk); ------------------ | | 1131| 142|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 232| 142| To = FMA(KP623489801, T7, T1) + FNMA(KP222520933, Ta, KP900968867 * T4); ------------------ | | 1131| 142|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ To = FMA(KP623489801, T7, T1) + FNMA(KP222520933, Ta, KP900968867 * T4); ------------------ | | 1133| 142|#define FNMA(a, b, c) (- (((a) * (b)) + (c))) ------------------ 233| 142| ro[WS(os, 4)] = To - Tp; ------------------ | | 825| 142|#define WS(stride, i) (stride * i) ------------------ 234| 142| ro[WS(os, 3)] = To + Tp; ------------------ | | 825| 142|#define WS(stride, i) (stride * i) ------------------ 235| 142| Tt = FMA(KP433883739, Tq, KP974927912 * Tr) - (KP781831482 * Ts); ------------------ | | 1131| 142|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 236| 142| Ty = FMA(KP623489801, Tv, Tu) + FNMA(KP222520933, Tw, KP900968867 * Tx); ------------------ | | 1131| 142|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ Ty = FMA(KP623489801, Tv, Tu) + FNMA(KP222520933, Tw, KP900968867 * Tx); ------------------ | | 1133| 142|#define FNMA(a, b, c) (- (((a) * (b)) + (c))) ------------------ 237| 142| io[WS(os, 3)] = Tt + Ty; ------------------ | | 825| 142|#define WS(stride, i) (stride * i) ------------------ 238| 142| io[WS(os, 4)] = Ty - Tt; ------------------ | | 825| 142|#define WS(stride, i) (stride * i) ------------------ 239| 142| } 240| 142| } 241| 36| } 242| 36|} fftw_codelet_n1_8: 263| 1|void X(codelet_n1_8) (planner *p) { X(kdft_register) (p, n1_8, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 264| 1|} n1_8.c:n1_8: 158| 41|{ 159| 41| DK(KP707106781, +0.707106781186547524400844362104849039284835938); ------------------ | | 1056| 41|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 41|# define K(x) ((E) x) | | ------------------ ------------------ 160| 41| { 161| 41| INT i; 162| 251| for (i = v; i > 0; i = i - 1, ri = ri + ivs, ii = ii + ivs, ro = ro + ovs, io = io + ovs, MAKE_VOLATILE_STRIDE(32, is), MAKE_VOLATILE_STRIDE(32, os)) { ------------------ | | 851| 210| (nptr <= ESTIMATED_AVAILABLE_INDEX_REGISTERS ? \ | | ------------------ | | | | 849| 420|#define ESTIMATED_AVAILABLE_INDEX_REGISTERS 16 | | ------------------ | | | Branch (851:7): [Folded, False: 210] | | ------------------ | | 852| 210| 0 : \ | | 853| 210| ((x) = (x) ^ X(an_INT_guaranteed_to_be_zero))) | | ------------------ | | | | 78| 210|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 210|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ for (i = v; i > 0; i = i - 1, ri = ri + ivs, ii = ii + ivs, ro = ro + ovs, io = io + ovs, MAKE_VOLATILE_STRIDE(32, is), MAKE_VOLATILE_STRIDE(32, os)) { ------------------ | | 851| 210| (nptr <= ESTIMATED_AVAILABLE_INDEX_REGISTERS ? \ | | ------------------ | | | | 849| 420|#define ESTIMATED_AVAILABLE_INDEX_REGISTERS 16 | | ------------------ | | | Branch (851:7): [Folded, False: 210] | | ------------------ | | 852| 210| 0 : \ | | 853| 210| ((x) = (x) ^ X(an_INT_guaranteed_to_be_zero))) | | ------------------ | | | | 78| 210|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 210|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ | Branch (162:16): [True: 210, False: 41] ------------------ 163| 210| E T3, Tn, Ti, TC, T6, TB, Tl, To, Td, TN, Tz, TH, Ta, TM, Tu; 164| 210| E TG; 165| 210| { 166| 210| E T1, T2, Tj, Tk; 167| 210| T1 = ri[0]; 168| 210| T2 = ri[WS(is, 4)]; ------------------ | | 825| 210|#define WS(stride, i) (stride * i) ------------------ 169| 210| T3 = T1 + T2; 170| 210| Tn = T1 - T2; 171| 210| { 172| 210| E Tg, Th, T4, T5; 173| 210| Tg = ii[0]; 174| 210| Th = ii[WS(is, 4)]; ------------------ | | 825| 210|#define WS(stride, i) (stride * i) ------------------ 175| 210| Ti = Tg + Th; 176| 210| TC = Tg - Th; 177| 210| T4 = ri[WS(is, 2)]; ------------------ | | 825| 210|#define WS(stride, i) (stride * i) ------------------ 178| 210| T5 = ri[WS(is, 6)]; ------------------ | | 825| 210|#define WS(stride, i) (stride * i) ------------------ 179| 210| T6 = T4 + T5; 180| 210| TB = T4 - T5; 181| 210| } 182| 210| Tj = ii[WS(is, 2)]; ------------------ | | 825| 210|#define WS(stride, i) (stride * i) ------------------ 183| 210| Tk = ii[WS(is, 6)]; ------------------ | | 825| 210|#define WS(stride, i) (stride * i) ------------------ 184| 210| Tl = Tj + Tk; 185| 210| To = Tj - Tk; 186| 210| { 187| 210| E Tb, Tc, Tv, Tw, Tx, Ty; 188| 210| Tb = ri[WS(is, 7)]; ------------------ | | 825| 210|#define WS(stride, i) (stride * i) ------------------ 189| 210| Tc = ri[WS(is, 3)]; ------------------ | | 825| 210|#define WS(stride, i) (stride * i) ------------------ 190| 210| Tv = Tb - Tc; 191| 210| Tw = ii[WS(is, 7)]; ------------------ | | 825| 210|#define WS(stride, i) (stride * i) ------------------ 192| 210| Tx = ii[WS(is, 3)]; ------------------ | | 825| 210|#define WS(stride, i) (stride * i) ------------------ 193| 210| Ty = Tw - Tx; 194| 210| Td = Tb + Tc; 195| 210| TN = Tw + Tx; 196| 210| Tz = Tv - Ty; 197| 210| TH = Tv + Ty; 198| 210| } 199| 210| { 200| 210| E T8, T9, Tq, Tr, Ts, Tt; 201| 210| T8 = ri[WS(is, 1)]; ------------------ | | 825| 210|#define WS(stride, i) (stride * i) ------------------ 202| 210| T9 = ri[WS(is, 5)]; ------------------ | | 825| 210|#define WS(stride, i) (stride * i) ------------------ 203| 210| Tq = T8 - T9; 204| 210| Tr = ii[WS(is, 1)]; ------------------ | | 825| 210|#define WS(stride, i) (stride * i) ------------------ 205| 210| Ts = ii[WS(is, 5)]; ------------------ | | 825| 210|#define WS(stride, i) (stride * i) ------------------ 206| 210| Tt = Tr - Ts; 207| 210| Ta = T8 + T9; 208| 210| TM = Tr + Ts; 209| 210| Tu = Tq + Tt; 210| 210| TG = Tt - Tq; 211| 210| } 212| 210| } 213| 210| { 214| 210| E T7, Te, TP, TQ; 215| 210| T7 = T3 + T6; 216| 210| Te = Ta + Td; 217| 210| ro[WS(os, 4)] = T7 - Te; ------------------ | | 825| 210|#define WS(stride, i) (stride * i) ------------------ 218| 210| ro[0] = T7 + Te; 219| 210| TP = Ti + Tl; 220| 210| TQ = TM + TN; 221| 210| io[WS(os, 4)] = TP - TQ; ------------------ | | 825| 210|#define WS(stride, i) (stride * i) ------------------ 222| 210| io[0] = TP + TQ; 223| 210| } 224| 210| { 225| 210| E Tf, Tm, TL, TO; 226| 210| Tf = Td - Ta; 227| 210| Tm = Ti - Tl; 228| 210| io[WS(os, 2)] = Tf + Tm; ------------------ | | 825| 210|#define WS(stride, i) (stride * i) ------------------ 229| 210| io[WS(os, 6)] = Tm - Tf; ------------------ | | 825| 210|#define WS(stride, i) (stride * i) ------------------ 230| 210| TL = T3 - T6; 231| 210| TO = TM - TN; 232| 210| ro[WS(os, 6)] = TL - TO; ------------------ | | 825| 210|#define WS(stride, i) (stride * i) ------------------ 233| 210| ro[WS(os, 2)] = TL + TO; ------------------ | | 825| 210|#define WS(stride, i) (stride * i) ------------------ 234| 210| } 235| 210| { 236| 210| E Tp, TA, TJ, TK; 237| 210| Tp = Tn + To; 238| 210| TA = KP707106781 * (Tu + Tz); 239| 210| ro[WS(os, 5)] = Tp - TA; ------------------ | | 825| 210|#define WS(stride, i) (stride * i) ------------------ 240| 210| ro[WS(os, 1)] = Tp + TA; ------------------ | | 825| 210|#define WS(stride, i) (stride * i) ------------------ 241| 210| TJ = TC - TB; 242| 210| TK = KP707106781 * (TG + TH); 243| 210| io[WS(os, 5)] = TJ - TK; ------------------ | | 825| 210|#define WS(stride, i) (stride * i) ------------------ 244| 210| io[WS(os, 1)] = TJ + TK; ------------------ | | 825| 210|#define WS(stride, i) (stride * i) ------------------ 245| 210| } 246| 210| { 247| 210| E TD, TE, TF, TI; 248| 210| TD = TB + TC; 249| 210| TE = KP707106781 * (Tz - Tu); 250| 210| io[WS(os, 7)] = TD - TE; ------------------ | | 825| 210|#define WS(stride, i) (stride * i) ------------------ 251| 210| io[WS(os, 3)] = TD + TE; ------------------ | | 825| 210|#define WS(stride, i) (stride * i) ------------------ 252| 210| TF = Tn - To; 253| 210| TI = KP707106781 * (TG - TH); 254| 210| ro[WS(os, 7)] = TF - TI; ------------------ | | 825| 210|#define WS(stride, i) (stride * i) ------------------ 255| 210| ro[WS(os, 3)] = TF + TI; ------------------ | | 825| 210|#define WS(stride, i) (stride * i) ------------------ 256| 210| } 257| 210| } 258| 41| } 259| 41|} fftw_codelet_n1_9: 357| 1|void X(codelet_n1_9) (planner *p) { X(kdft_register) (p, n1_9, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 358| 1|} n1_9.c:n1_9: 206| 317|{ 207| 317| DK(KP939692620, +0.939692620785908384054109277324731469936208134); ------------------ | | 1056| 317|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 317|# define K(x) ((E) x) | | ------------------ ------------------ 208| 317| DK(KP342020143, +0.342020143325668733044099614682259580763083368); ------------------ | | 1056| 317|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 317|# define K(x) ((E) x) | | ------------------ ------------------ 209| 317| DK(KP984807753, +0.984807753012208059366743024589523013670643252); ------------------ | | 1056| 317|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 317|# define K(x) ((E) x) | | ------------------ ------------------ 210| 317| DK(KP173648177, +0.173648177666930348851716626769314796000375677); ------------------ | | 1056| 317|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 317|# define K(x) ((E) x) | | ------------------ ------------------ 211| 317| DK(KP642787609, +0.642787609686539326322643409907263432907559884); ------------------ | | 1056| 317|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 317|# define K(x) ((E) x) | | ------------------ ------------------ 212| 317| DK(KP766044443, +0.766044443118978035202392650555416673935832457); ------------------ | | 1056| 317|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 317|# define K(x) ((E) x) | | ------------------ ------------------ 213| 317| DK(KP500000000, +0.500000000000000000000000000000000000000000000); ------------------ | | 1056| 317|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 317|# define K(x) ((E) x) | | ------------------ ------------------ 214| 317| DK(KP866025403, +0.866025403784438646763723170752936183471402627); ------------------ | | 1056| 317|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 317|# define K(x) ((E) x) | | ------------------ ------------------ 215| 317| { 216| 317| INT i; 217| 1.60k| for (i = v; i > 0; i = i - 1, ri = ri + ivs, ii = ii + ivs, ro = ro + ovs, io = io + ovs, MAKE_VOLATILE_STRIDE(36, is), MAKE_VOLATILE_STRIDE(36, os)) { ------------------ | | 851| 1.28k| (nptr <= ESTIMATED_AVAILABLE_INDEX_REGISTERS ? \ | | ------------------ | | | | 849| 2.57k|#define ESTIMATED_AVAILABLE_INDEX_REGISTERS 16 | | ------------------ | | | Branch (851:7): [Folded, False: 1.28k] | | ------------------ | | 852| 1.28k| 0 : \ | | 853| 1.28k| ((x) = (x) ^ X(an_INT_guaranteed_to_be_zero))) | | ------------------ | | | | 78| 1.28k|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 1.28k|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ for (i = v; i > 0; i = i - 1, ri = ri + ivs, ii = ii + ivs, ro = ro + ovs, io = io + ovs, MAKE_VOLATILE_STRIDE(36, is), MAKE_VOLATILE_STRIDE(36, os)) { ------------------ | | 851| 1.28k| (nptr <= ESTIMATED_AVAILABLE_INDEX_REGISTERS ? \ | | ------------------ | | | | 849| 2.57k|#define ESTIMATED_AVAILABLE_INDEX_REGISTERS 16 | | ------------------ | | | Branch (851:7): [Folded, False: 1.28k] | | ------------------ | | 852| 1.28k| 0 : \ | | 853| 1.28k| ((x) = (x) ^ X(an_INT_guaranteed_to_be_zero))) | | ------------------ | | | | 78| 1.28k|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 1.28k|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ | Branch (217:16): [True: 1.28k, False: 317] ------------------ 218| 1.28k| E T5, TO, Th, Tk, T1g, TR, Ta, T1c, Tq, TW, Tv, TX, Tf, T1d, TB; 219| 1.28k| E T10, TG, TZ; 220| 1.28k| { 221| 1.28k| E T1, T2, T3, T4; 222| 1.28k| T1 = ri[0]; 223| 1.28k| T2 = ri[WS(is, 3)]; ------------------ | | 825| 1.28k|#define WS(stride, i) (stride * i) ------------------ 224| 1.28k| T3 = ri[WS(is, 6)]; ------------------ | | 825| 1.28k|#define WS(stride, i) (stride * i) ------------------ 225| 1.28k| T4 = T2 + T3; 226| 1.28k| T5 = T1 + T4; 227| 1.28k| TO = KP866025403 * (T3 - T2); 228| 1.28k| Th = FNMS(KP500000000, T4, T1); ------------------ | | 1134| 1.28k|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 229| 1.28k| } 230| 1.28k| { 231| 1.28k| E TP, Ti, Tj, TQ; 232| 1.28k| TP = ii[0]; 233| 1.28k| Ti = ii[WS(is, 3)]; ------------------ | | 825| 1.28k|#define WS(stride, i) (stride * i) ------------------ 234| 1.28k| Tj = ii[WS(is, 6)]; ------------------ | | 825| 1.28k|#define WS(stride, i) (stride * i) ------------------ 235| 1.28k| TQ = Ti + Tj; 236| 1.28k| Tk = KP866025403 * (Ti - Tj); 237| 1.28k| T1g = TP + TQ; 238| 1.28k| TR = FNMS(KP500000000, TQ, TP); ------------------ | | 1134| 1.28k|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 239| 1.28k| } 240| 1.28k| { 241| 1.28k| E T6, Ts, T9, Tr, Tp, Tt, Tm, Tu; 242| 1.28k| T6 = ri[WS(is, 1)]; ------------------ | | 825| 1.28k|#define WS(stride, i) (stride * i) ------------------ 243| 1.28k| Ts = ii[WS(is, 1)]; ------------------ | | 825| 1.28k|#define WS(stride, i) (stride * i) ------------------ 244| 1.28k| { 245| 1.28k| E T7, T8, Tn, To; 246| 1.28k| T7 = ri[WS(is, 4)]; ------------------ | | 825| 1.28k|#define WS(stride, i) (stride * i) ------------------ 247| 1.28k| T8 = ri[WS(is, 7)]; ------------------ | | 825| 1.28k|#define WS(stride, i) (stride * i) ------------------ 248| 1.28k| T9 = T7 + T8; 249| 1.28k| Tr = KP866025403 * (T8 - T7); 250| 1.28k| Tn = ii[WS(is, 4)]; ------------------ | | 825| 1.28k|#define WS(stride, i) (stride * i) ------------------ 251| 1.28k| To = ii[WS(is, 7)]; ------------------ | | 825| 1.28k|#define WS(stride, i) (stride * i) ------------------ 252| 1.28k| Tp = KP866025403 * (Tn - To); 253| 1.28k| Tt = Tn + To; 254| 1.28k| } 255| 1.28k| Ta = T6 + T9; 256| 1.28k| T1c = Ts + Tt; 257| 1.28k| Tm = FNMS(KP500000000, T9, T6); ------------------ | | 1134| 1.28k|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 258| 1.28k| Tq = Tm + Tp; 259| 1.28k| TW = Tm - Tp; 260| 1.28k| Tu = FNMS(KP500000000, Tt, Ts); ------------------ | | 1134| 1.28k|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 261| 1.28k| Tv = Tr + Tu; 262| 1.28k| TX = Tu - Tr; 263| 1.28k| } 264| 1.28k| { 265| 1.28k| E Tb, TD, Te, TC, TA, TE, Tx, TF; 266| 1.28k| Tb = ri[WS(is, 2)]; ------------------ | | 825| 1.28k|#define WS(stride, i) (stride * i) ------------------ 267| 1.28k| TD = ii[WS(is, 2)]; ------------------ | | 825| 1.28k|#define WS(stride, i) (stride * i) ------------------ 268| 1.28k| { 269| 1.28k| E Tc, Td, Ty, Tz; 270| 1.28k| Tc = ri[WS(is, 5)]; ------------------ | | 825| 1.28k|#define WS(stride, i) (stride * i) ------------------ 271| 1.28k| Td = ri[WS(is, 8)]; ------------------ | | 825| 1.28k|#define WS(stride, i) (stride * i) ------------------ 272| 1.28k| Te = Tc + Td; 273| 1.28k| TC = KP866025403 * (Td - Tc); 274| 1.28k| Ty = ii[WS(is, 5)]; ------------------ | | 825| 1.28k|#define WS(stride, i) (stride * i) ------------------ 275| 1.28k| Tz = ii[WS(is, 8)]; ------------------ | | 825| 1.28k|#define WS(stride, i) (stride * i) ------------------ 276| 1.28k| TA = KP866025403 * (Ty - Tz); 277| 1.28k| TE = Ty + Tz; 278| 1.28k| } 279| 1.28k| Tf = Tb + Te; 280| 1.28k| T1d = TD + TE; 281| 1.28k| Tx = FNMS(KP500000000, Te, Tb); ------------------ | | 1134| 1.28k|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 282| 1.28k| TB = Tx + TA; 283| 1.28k| T10 = Tx - TA; 284| 1.28k| TF = FNMS(KP500000000, TE, TD); ------------------ | | 1134| 1.28k|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 285| 1.28k| TG = TC + TF; 286| 1.28k| TZ = TF - TC; 287| 1.28k| } 288| 1.28k| { 289| 1.28k| E T1e, Tg, T1b, T1f, T1h, T1i; 290| 1.28k| T1e = KP866025403 * (T1c - T1d); 291| 1.28k| Tg = Ta + Tf; 292| 1.28k| T1b = FNMS(KP500000000, Tg, T5); ------------------ | | 1134| 1.28k|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 293| 1.28k| ro[0] = T5 + Tg; 294| 1.28k| ro[WS(os, 3)] = T1b + T1e; ------------------ | | 825| 1.28k|#define WS(stride, i) (stride * i) ------------------ 295| 1.28k| ro[WS(os, 6)] = T1b - T1e; ------------------ | | 825| 1.28k|#define WS(stride, i) (stride * i) ------------------ 296| 1.28k| T1f = KP866025403 * (Tf - Ta); 297| 1.28k| T1h = T1c + T1d; 298| 1.28k| T1i = FNMS(KP500000000, T1h, T1g); ------------------ | | 1134| 1.28k|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 299| 1.28k| io[WS(os, 3)] = T1f + T1i; ------------------ | | 825| 1.28k|#define WS(stride, i) (stride * i) ------------------ 300| 1.28k| io[0] = T1g + T1h; 301| 1.28k| io[WS(os, 6)] = T1i - T1f; ------------------ | | 825| 1.28k|#define WS(stride, i) (stride * i) ------------------ 302| 1.28k| } 303| 1.28k| { 304| 1.28k| E Tl, TS, TI, TN, TM, TT, TJ, TU; 305| 1.28k| Tl = Th + Tk; 306| 1.28k| TS = TO + TR; 307| 1.28k| { 308| 1.28k| E Tw, TH, TK, TL; 309| 1.28k| Tw = FMA(KP766044443, Tq, KP642787609 * Tv); ------------------ | | 1131| 1.28k|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 310| 1.28k| TH = FMA(KP173648177, TB, KP984807753 * TG); ------------------ | | 1131| 1.28k|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 311| 1.28k| TI = Tw + TH; 312| 1.28k| TN = KP866025403 * (TH - Tw); 313| 1.28k| TK = FNMS(KP642787609, Tq, KP766044443 * Tv); ------------------ | | 1134| 1.28k|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 314| 1.28k| TL = FNMS(KP984807753, TB, KP173648177 * TG); ------------------ | | 1134| 1.28k|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 315| 1.28k| TM = KP866025403 * (TK - TL); 316| 1.28k| TT = TK + TL; 317| 1.28k| } 318| 1.28k| ro[WS(os, 1)] = Tl + TI; ------------------ | | 825| 1.28k|#define WS(stride, i) (stride * i) ------------------ 319| 1.28k| io[WS(os, 1)] = TS + TT; ------------------ | | 825| 1.28k|#define WS(stride, i) (stride * i) ------------------ 320| 1.28k| TJ = FNMS(KP500000000, TI, Tl); ------------------ | | 1134| 1.28k|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 321| 1.28k| ro[WS(os, 7)] = TJ - TM; ------------------ | | 825| 1.28k|#define WS(stride, i) (stride * i) ------------------ 322| 1.28k| ro[WS(os, 4)] = TJ + TM; ------------------ | | 825| 1.28k|#define WS(stride, i) (stride * i) ------------------ 323| 1.28k| TU = FNMS(KP500000000, TT, TS); ------------------ | | 1134| 1.28k|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 324| 1.28k| io[WS(os, 4)] = TN + TU; ------------------ | | 825| 1.28k|#define WS(stride, i) (stride * i) ------------------ 325| 1.28k| io[WS(os, 7)] = TU - TN; ------------------ | | 825| 1.28k|#define WS(stride, i) (stride * i) ------------------ 326| 1.28k| } 327| 1.28k| { 328| 1.28k| E TV, T14, T12, T13, T17, T1a, T18, T19; 329| 1.28k| TV = Th - Tk; 330| 1.28k| T14 = TR - TO; 331| 1.28k| { 332| 1.28k| E TY, T11, T15, T16; 333| 1.28k| TY = FMA(KP173648177, TW, KP984807753 * TX); ------------------ | | 1131| 1.28k|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 334| 1.28k| T11 = FNMS(KP939692620, T10, KP342020143 * TZ); ------------------ | | 1134| 1.28k|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 335| 1.28k| T12 = TY + T11; 336| 1.28k| T13 = KP866025403 * (T11 - TY); 337| 1.28k| T15 = FNMS(KP984807753, TW, KP173648177 * TX); ------------------ | | 1134| 1.28k|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 338| 1.28k| T16 = FMA(KP342020143, T10, KP939692620 * TZ); ------------------ | | 1131| 1.28k|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 339| 1.28k| T17 = T15 - T16; 340| 1.28k| T1a = KP866025403 * (T15 + T16); 341| 1.28k| } 342| 1.28k| ro[WS(os, 2)] = TV + T12; ------------------ | | 825| 1.28k|#define WS(stride, i) (stride * i) ------------------ 343| 1.28k| io[WS(os, 2)] = T14 + T17; ------------------ | | 825| 1.28k|#define WS(stride, i) (stride * i) ------------------ 344| 1.28k| T18 = FNMS(KP500000000, T17, T14); ------------------ | | 1134| 1.28k|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 345| 1.28k| io[WS(os, 5)] = T13 + T18; ------------------ | | 825| 1.28k|#define WS(stride, i) (stride * i) ------------------ 346| 1.28k| io[WS(os, 8)] = T18 - T13; ------------------ | | 825| 1.28k|#define WS(stride, i) (stride * i) ------------------ 347| 1.28k| T19 = FNMS(KP500000000, T12, TV); ------------------ | | 1134| 1.28k|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 348| 1.28k| ro[WS(os, 8)] = T19 - T1a; ------------------ | | 825| 1.28k|#define WS(stride, i) (stride * i) ------------------ 349| 1.28k| ro[WS(os, 5)] = T19 + T1a; ------------------ | | 825| 1.28k|#define WS(stride, i) (stride * i) ------------------ 350| 1.28k| } 351| 1.28k| } 352| 317| } 353| 317|} fftw_codelet_q1_2: 146| 1|void X(codelet_q1_2) (planner *p) { 147| 1| X(kdft_difsq_register) (p, q1_2, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 148| 1|} fftw_codelet_q1_3: 313| 1|void X(codelet_q1_3) (planner *p) { 314| 1| X(kdft_difsq_register) (p, q1_3, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 315| 1|} fftw_codelet_q1_4: 521| 1|void X(codelet_q1_4) (planner *p) { 522| 1| X(kdft_difsq_register) (p, q1_4, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 523| 1|} fftw_codelet_q1_5: 989| 1|void X(codelet_q1_5) (planner *p) { 990| 1| X(kdft_difsq_register) (p, q1_5, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 991| 1|} q1_5.c:q1_5: 556| 11|{ 557| 11| DK(KP250000000, +0.250000000000000000000000000000000000000000000); ------------------ | | 1056| 11|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 11|# define K(x) ((E) x) | | ------------------ ------------------ 558| 11| DK(KP587785252, +0.587785252292473129168705954639072768597652438); ------------------ | | 1056| 11|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 11|# define K(x) ((E) x) | | ------------------ ------------------ 559| 11| DK(KP951056516, +0.951056516295153572116439333379382143405698634); ------------------ | | 1056| 11|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 11|# define K(x) ((E) x) | | ------------------ ------------------ 560| 11| DK(KP559016994, +0.559016994374947424102293417182819058860154590); ------------------ | | 1056| 11|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 11|# define K(x) ((E) x) | | ------------------ ------------------ 561| 11| { 562| 11| INT m; 563| 110| for (m = mb, W = W + (mb * 8); m < me; m = m + 1, rio = rio + ms, iio = iio + ms, W = W + 8, MAKE_VOLATILE_STRIDE(10, rs), MAKE_VOLATILE_STRIDE(0, vs)) { ------------------ | | 851| 99| (nptr <= ESTIMATED_AVAILABLE_INDEX_REGISTERS ? \ | | ------------------ | | | | 849| 198|#define ESTIMATED_AVAILABLE_INDEX_REGISTERS 16 | | ------------------ | | | Branch (851:7): [True: 99, Folded] | | ------------------ | | 852| 99| 0 : \ | | 853| 99| ((x) = (x) ^ X(an_INT_guaranteed_to_be_zero))) | | ------------------ | | | | 78| 0|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ for (m = mb, W = W + (mb * 8); m < me; m = m + 1, rio = rio + ms, iio = iio + ms, W = W + 8, MAKE_VOLATILE_STRIDE(10, rs), MAKE_VOLATILE_STRIDE(0, vs)) { ------------------ | | 851| 99| (nptr <= ESTIMATED_AVAILABLE_INDEX_REGISTERS ? \ | | ------------------ | | | | 849| 198|#define ESTIMATED_AVAILABLE_INDEX_REGISTERS 16 | | ------------------ | | | Branch (851:7): [True: 99, Folded] | | ------------------ | | 852| 99| 0 : \ | | 853| 99| ((x) = (x) ^ X(an_INT_guaranteed_to_be_zero))) | | ------------------ | | | | 78| 0|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ | Branch (563:35): [True: 99, False: 11] ------------------ 564| 99| E T1, Ta, TG, Tv, T8, Tb, Tp, Tj, TD, To, Tq, Tr, TN, TW, T1s; 565| 99| E T1h, TU, TX, T1b, T15, T1p, T1a, T1c, T1d, T1z, T1I, T2e, T23, T1G, T1J; 566| 99| E T1X, T1R, T2b, T1W, T1Y, T1Z, T3v, T3p, T3J, T3u, T3w, T3x, T37, T3g, T3M; 567| 99| E T3B, T3e, T3h, T2l, T2u, T30, T2P, T2s, T2v, T2J, T2D, T2X, T2I, T2K, T2L; 568| 99| { 569| 99| E T7, Tu, T4, Tt; 570| 99| T1 = rio[0]; 571| 99| { 572| 99| E T5, T6, T2, T3; 573| 99| T5 = rio[WS(rs, 2)]; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ 574| 99| T6 = rio[WS(rs, 3)]; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ 575| 99| T7 = T5 + T6; 576| 99| Tu = T5 - T6; 577| 99| T2 = rio[WS(rs, 1)]; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ 578| 99| T3 = rio[WS(rs, 4)]; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ 579| 99| T4 = T2 + T3; 580| 99| Tt = T2 - T3; 581| 99| } 582| 99| Ta = KP559016994 * (T4 - T7); 583| 99| TG = FNMS(KP587785252, Tt, KP951056516 * Tu); ------------------ | | 1134| 99|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 584| 99| Tv = FMA(KP951056516, Tt, KP587785252 * Tu); ------------------ | | 1131| 99|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 585| 99| T8 = T4 + T7; 586| 99| Tb = FNMS(KP250000000, T8, T1); ------------------ | | 1134| 99|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 587| 99| } 588| 99| { 589| 99| E Ti, Tn, Tf, Tm; 590| 99| Tp = iio[0]; 591| 99| { 592| 99| E Tg, Th, Td, Te; 593| 99| Tg = iio[WS(rs, 2)]; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ 594| 99| Th = iio[WS(rs, 3)]; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ 595| 99| Ti = Tg - Th; 596| 99| Tn = Tg + Th; 597| 99| Td = iio[WS(rs, 1)]; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ 598| 99| Te = iio[WS(rs, 4)]; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ 599| 99| Tf = Td - Te; 600| 99| Tm = Td + Te; 601| 99| } 602| 99| Tj = FMA(KP951056516, Tf, KP587785252 * Ti); ------------------ | | 1131| 99|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 603| 99| TD = FNMS(KP587785252, Tf, KP951056516 * Ti); ------------------ | | 1134| 99|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 604| 99| To = KP559016994 * (Tm - Tn); 605| 99| Tq = Tm + Tn; 606| 99| Tr = FNMS(KP250000000, Tq, Tp); ------------------ | | 1134| 99|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 607| 99| } 608| 99| { 609| 99| E TT, T1g, TQ, T1f; 610| 99| TN = rio[WS(vs, 1)]; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ 611| 99| { 612| 99| E TR, TS, TO, TP; 613| 99| TR = rio[WS(vs, 1) + WS(rs, 2)]; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ TR = rio[WS(vs, 1) + WS(rs, 2)]; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ 614| 99| TS = rio[WS(vs, 1) + WS(rs, 3)]; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ TS = rio[WS(vs, 1) + WS(rs, 3)]; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ 615| 99| TT = TR + TS; 616| 99| T1g = TR - TS; 617| 99| TO = rio[WS(vs, 1) + WS(rs, 1)]; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ TO = rio[WS(vs, 1) + WS(rs, 1)]; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ 618| 99| TP = rio[WS(vs, 1) + WS(rs, 4)]; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ TP = rio[WS(vs, 1) + WS(rs, 4)]; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ 619| 99| TQ = TO + TP; 620| 99| T1f = TO - TP; 621| 99| } 622| 99| TW = KP559016994 * (TQ - TT); 623| 99| T1s = FNMS(KP587785252, T1f, KP951056516 * T1g); ------------------ | | 1134| 99|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 624| 99| T1h = FMA(KP951056516, T1f, KP587785252 * T1g); ------------------ | | 1131| 99|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 625| 99| TU = TQ + TT; 626| 99| TX = FNMS(KP250000000, TU, TN); ------------------ | | 1134| 99|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 627| 99| } 628| 99| { 629| 99| E T14, T19, T11, T18; 630| 99| T1b = iio[WS(vs, 1)]; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ 631| 99| { 632| 99| E T12, T13, TZ, T10; 633| 99| T12 = iio[WS(vs, 1) + WS(rs, 2)]; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ T12 = iio[WS(vs, 1) + WS(rs, 2)]; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ 634| 99| T13 = iio[WS(vs, 1) + WS(rs, 3)]; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ T13 = iio[WS(vs, 1) + WS(rs, 3)]; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ 635| 99| T14 = T12 - T13; 636| 99| T19 = T12 + T13; 637| 99| TZ = iio[WS(vs, 1) + WS(rs, 1)]; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ TZ = iio[WS(vs, 1) + WS(rs, 1)]; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ 638| 99| T10 = iio[WS(vs, 1) + WS(rs, 4)]; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ T10 = iio[WS(vs, 1) + WS(rs, 4)]; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ 639| 99| T11 = TZ - T10; 640| 99| T18 = TZ + T10; 641| 99| } 642| 99| T15 = FMA(KP951056516, T11, KP587785252 * T14); ------------------ | | 1131| 99|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 643| 99| T1p = FNMS(KP587785252, T11, KP951056516 * T14); ------------------ | | 1134| 99|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 644| 99| T1a = KP559016994 * (T18 - T19); 645| 99| T1c = T18 + T19; 646| 99| T1d = FNMS(KP250000000, T1c, T1b); ------------------ | | 1134| 99|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 647| 99| } 648| 99| { 649| 99| E T1F, T22, T1C, T21; 650| 99| T1z = rio[WS(vs, 2)]; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ 651| 99| { 652| 99| E T1D, T1E, T1A, T1B; 653| 99| T1D = rio[WS(vs, 2) + WS(rs, 2)]; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ T1D = rio[WS(vs, 2) + WS(rs, 2)]; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ 654| 99| T1E = rio[WS(vs, 2) + WS(rs, 3)]; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ T1E = rio[WS(vs, 2) + WS(rs, 3)]; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ 655| 99| T1F = T1D + T1E; 656| 99| T22 = T1D - T1E; 657| 99| T1A = rio[WS(vs, 2) + WS(rs, 1)]; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ T1A = rio[WS(vs, 2) + WS(rs, 1)]; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ 658| 99| T1B = rio[WS(vs, 2) + WS(rs, 4)]; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ T1B = rio[WS(vs, 2) + WS(rs, 4)]; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ 659| 99| T1C = T1A + T1B; 660| 99| T21 = T1A - T1B; 661| 99| } 662| 99| T1I = KP559016994 * (T1C - T1F); 663| 99| T2e = FNMS(KP587785252, T21, KP951056516 * T22); ------------------ | | 1134| 99|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 664| 99| T23 = FMA(KP951056516, T21, KP587785252 * T22); ------------------ | | 1131| 99|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 665| 99| T1G = T1C + T1F; 666| 99| T1J = FNMS(KP250000000, T1G, T1z); ------------------ | | 1134| 99|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 667| 99| } 668| 99| { 669| 99| E T1Q, T1V, T1N, T1U; 670| 99| T1X = iio[WS(vs, 2)]; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ 671| 99| { 672| 99| E T1O, T1P, T1L, T1M; 673| 99| T1O = iio[WS(vs, 2) + WS(rs, 2)]; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ T1O = iio[WS(vs, 2) + WS(rs, 2)]; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ 674| 99| T1P = iio[WS(vs, 2) + WS(rs, 3)]; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ T1P = iio[WS(vs, 2) + WS(rs, 3)]; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ 675| 99| T1Q = T1O - T1P; 676| 99| T1V = T1O + T1P; 677| 99| T1L = iio[WS(vs, 2) + WS(rs, 1)]; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ T1L = iio[WS(vs, 2) + WS(rs, 1)]; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ 678| 99| T1M = iio[WS(vs, 2) + WS(rs, 4)]; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ T1M = iio[WS(vs, 2) + WS(rs, 4)]; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ 679| 99| T1N = T1L - T1M; 680| 99| T1U = T1L + T1M; 681| 99| } 682| 99| T1R = FMA(KP951056516, T1N, KP587785252 * T1Q); ------------------ | | 1131| 99|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 683| 99| T2b = FNMS(KP587785252, T1N, KP951056516 * T1Q); ------------------ | | 1134| 99|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 684| 99| T1W = KP559016994 * (T1U - T1V); 685| 99| T1Y = T1U + T1V; 686| 99| T1Z = FNMS(KP250000000, T1Y, T1X); ------------------ | | 1134| 99|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 687| 99| } 688| 99| { 689| 99| E T3o, T3t, T3l, T3s; 690| 99| T3v = iio[WS(vs, 4)]; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ 691| 99| { 692| 99| E T3m, T3n, T3j, T3k; 693| 99| T3m = iio[WS(vs, 4) + WS(rs, 2)]; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ T3m = iio[WS(vs, 4) + WS(rs, 2)]; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ 694| 99| T3n = iio[WS(vs, 4) + WS(rs, 3)]; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ T3n = iio[WS(vs, 4) + WS(rs, 3)]; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ 695| 99| T3o = T3m - T3n; 696| 99| T3t = T3m + T3n; 697| 99| T3j = iio[WS(vs, 4) + WS(rs, 1)]; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ T3j = iio[WS(vs, 4) + WS(rs, 1)]; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ 698| 99| T3k = iio[WS(vs, 4) + WS(rs, 4)]; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ T3k = iio[WS(vs, 4) + WS(rs, 4)]; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ 699| 99| T3l = T3j - T3k; 700| 99| T3s = T3j + T3k; 701| 99| } 702| 99| T3p = FMA(KP951056516, T3l, KP587785252 * T3o); ------------------ | | 1131| 99|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 703| 99| T3J = FNMS(KP587785252, T3l, KP951056516 * T3o); ------------------ | | 1134| 99|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 704| 99| T3u = KP559016994 * (T3s - T3t); 705| 99| T3w = T3s + T3t; 706| 99| T3x = FNMS(KP250000000, T3w, T3v); ------------------ | | 1134| 99|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 707| 99| } 708| 99| { 709| 99| E T3d, T3A, T3a, T3z; 710| 99| T37 = rio[WS(vs, 4)]; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ 711| 99| { 712| 99| E T3b, T3c, T38, T39; 713| 99| T3b = rio[WS(vs, 4) + WS(rs, 2)]; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ T3b = rio[WS(vs, 4) + WS(rs, 2)]; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ 714| 99| T3c = rio[WS(vs, 4) + WS(rs, 3)]; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ T3c = rio[WS(vs, 4) + WS(rs, 3)]; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ 715| 99| T3d = T3b + T3c; 716| 99| T3A = T3b - T3c; 717| 99| T38 = rio[WS(vs, 4) + WS(rs, 1)]; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ T38 = rio[WS(vs, 4) + WS(rs, 1)]; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ 718| 99| T39 = rio[WS(vs, 4) + WS(rs, 4)]; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ T39 = rio[WS(vs, 4) + WS(rs, 4)]; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ 719| 99| T3a = T38 + T39; 720| 99| T3z = T38 - T39; 721| 99| } 722| 99| T3g = KP559016994 * (T3a - T3d); 723| 99| T3M = FNMS(KP587785252, T3z, KP951056516 * T3A); ------------------ | | 1134| 99|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 724| 99| T3B = FMA(KP951056516, T3z, KP587785252 * T3A); ------------------ | | 1131| 99|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 725| 99| T3e = T3a + T3d; 726| 99| T3h = FNMS(KP250000000, T3e, T37); ------------------ | | 1134| 99|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 727| 99| } 728| 99| { 729| 99| E T2r, T2O, T2o, T2N; 730| 99| T2l = rio[WS(vs, 3)]; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ 731| 99| { 732| 99| E T2p, T2q, T2m, T2n; 733| 99| T2p = rio[WS(vs, 3) + WS(rs, 2)]; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ T2p = rio[WS(vs, 3) + WS(rs, 2)]; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ 734| 99| T2q = rio[WS(vs, 3) + WS(rs, 3)]; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ T2q = rio[WS(vs, 3) + WS(rs, 3)]; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ 735| 99| T2r = T2p + T2q; 736| 99| T2O = T2p - T2q; 737| 99| T2m = rio[WS(vs, 3) + WS(rs, 1)]; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ T2m = rio[WS(vs, 3) + WS(rs, 1)]; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ 738| 99| T2n = rio[WS(vs, 3) + WS(rs, 4)]; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ T2n = rio[WS(vs, 3) + WS(rs, 4)]; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ 739| 99| T2o = T2m + T2n; 740| 99| T2N = T2m - T2n; 741| 99| } 742| 99| T2u = KP559016994 * (T2o - T2r); 743| 99| T30 = FNMS(KP587785252, T2N, KP951056516 * T2O); ------------------ | | 1134| 99|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 744| 99| T2P = FMA(KP951056516, T2N, KP587785252 * T2O); ------------------ | | 1131| 99|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 745| 99| T2s = T2o + T2r; 746| 99| T2v = FNMS(KP250000000, T2s, T2l); ------------------ | | 1134| 99|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 747| 99| } 748| 99| { 749| 99| E T2C, T2H, T2z, T2G; 750| 99| T2J = iio[WS(vs, 3)]; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ 751| 99| { 752| 99| E T2A, T2B, T2x, T2y; 753| 99| T2A = iio[WS(vs, 3) + WS(rs, 2)]; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ T2A = iio[WS(vs, 3) + WS(rs, 2)]; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ 754| 99| T2B = iio[WS(vs, 3) + WS(rs, 3)]; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ T2B = iio[WS(vs, 3) + WS(rs, 3)]; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ 755| 99| T2C = T2A - T2B; 756| 99| T2H = T2A + T2B; 757| 99| T2x = iio[WS(vs, 3) + WS(rs, 1)]; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ T2x = iio[WS(vs, 3) + WS(rs, 1)]; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ 758| 99| T2y = iio[WS(vs, 3) + WS(rs, 4)]; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ T2y = iio[WS(vs, 3) + WS(rs, 4)]; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ 759| 99| T2z = T2x - T2y; 760| 99| T2G = T2x + T2y; 761| 99| } 762| 99| T2D = FMA(KP951056516, T2z, KP587785252 * T2C); ------------------ | | 1131| 99|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 763| 99| T2X = FNMS(KP587785252, T2z, KP951056516 * T2C); ------------------ | | 1134| 99|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 764| 99| T2I = KP559016994 * (T2G - T2H); 765| 99| T2K = T2G + T2H; 766| 99| T2L = FNMS(KP250000000, T2K, T2J); ------------------ | | 1134| 99|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 767| 99| } 768| 99| rio[0] = T1 + T8; 769| 99| iio[0] = Tp + Tq; 770| 99| rio[WS(rs, 1)] = TN + TU; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ 771| 99| iio[WS(rs, 1)] = T1b + T1c; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ 772| 99| rio[WS(rs, 2)] = T1z + T1G; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ 773| 99| iio[WS(rs, 2)] = T1X + T1Y; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ 774| 99| iio[WS(rs, 4)] = T3v + T3w; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ 775| 99| rio[WS(rs, 4)] = T37 + T3e; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ 776| 99| rio[WS(rs, 3)] = T2l + T2s; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ 777| 99| iio[WS(rs, 3)] = T2J + T2K; ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ 778| 99| { 779| 99| E Tk, Ty, Tw, TA, Tc, Ts; 780| 99| Tc = Ta + Tb; 781| 99| Tk = Tc + Tj; 782| 99| Ty = Tc - Tj; 783| 99| Ts = To + Tr; 784| 99| Tw = Ts - Tv; 785| 99| TA = Tv + Ts; 786| 99| { 787| 99| E T9, Tl, Tx, Tz; 788| 99| T9 = W[0]; 789| 99| Tl = W[1]; 790| 99| rio[WS(vs, 1)] = FMA(T9, Tk, Tl * Tw); ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ rio[WS(vs, 1)] = FMA(T9, Tk, Tl * Tw); ------------------ | | 1131| 99|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 791| 99| iio[WS(vs, 1)] = FNMS(Tl, Tk, T9 * Tw); ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ iio[WS(vs, 1)] = FNMS(Tl, Tk, T9 * Tw); ------------------ | | 1134| 99|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 792| 99| Tx = W[6]; 793| 99| Tz = W[7]; 794| 99| rio[WS(vs, 4)] = FMA(Tx, Ty, Tz * TA); ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ rio[WS(vs, 4)] = FMA(Tx, Ty, Tz * TA); ------------------ | | 1131| 99|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 795| 99| iio[WS(vs, 4)] = FNMS(Tz, Ty, Tx * TA); ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ iio[WS(vs, 4)] = FNMS(Tz, Ty, Tx * TA); ------------------ | | 1134| 99|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 796| 99| } 797| 99| } 798| 99| { 799| 99| E TE, TK, TI, TM, TC, TH; 800| 99| TC = Tb - Ta; 801| 99| TE = TC - TD; 802| 99| TK = TC + TD; 803| 99| TH = Tr - To; 804| 99| TI = TG + TH; 805| 99| TM = TH - TG; 806| 99| { 807| 99| E TB, TF, TJ, TL; 808| 99| TB = W[2]; 809| 99| TF = W[3]; 810| 99| rio[WS(vs, 2)] = FMA(TB, TE, TF * TI); ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ rio[WS(vs, 2)] = FMA(TB, TE, TF * TI); ------------------ | | 1131| 99|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 811| 99| iio[WS(vs, 2)] = FNMS(TF, TE, TB * TI); ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ iio[WS(vs, 2)] = FNMS(TF, TE, TB * TI); ------------------ | | 1134| 99|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 812| 99| TJ = W[4]; 813| 99| TL = W[5]; 814| 99| rio[WS(vs, 3)] = FMA(TJ, TK, TL * TM); ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ rio[WS(vs, 3)] = FMA(TJ, TK, TL * TM); ------------------ | | 1131| 99|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 815| 99| iio[WS(vs, 3)] = FNMS(TL, TK, TJ * TM); ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ iio[WS(vs, 3)] = FNMS(TL, TK, TJ * TM); ------------------ | | 1134| 99|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 816| 99| } 817| 99| } 818| 99| { 819| 99| E T2c, T2i, T2g, T2k, T2a, T2f; 820| 99| T2a = T1J - T1I; 821| 99| T2c = T2a - T2b; 822| 99| T2i = T2a + T2b; 823| 99| T2f = T1Z - T1W; 824| 99| T2g = T2e + T2f; 825| 99| T2k = T2f - T2e; 826| 99| { 827| 99| E T29, T2d, T2h, T2j; 828| 99| T29 = W[2]; 829| 99| T2d = W[3]; 830| 99| rio[WS(vs, 2) + WS(rs, 2)] = FMA(T29, T2c, T2d * T2g); ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ rio[WS(vs, 2) + WS(rs, 2)] = FMA(T29, T2c, T2d * T2g); ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ rio[WS(vs, 2) + WS(rs, 2)] = FMA(T29, T2c, T2d * T2g); ------------------ | | 1131| 99|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 831| 99| iio[WS(vs, 2) + WS(rs, 2)] = FNMS(T2d, T2c, T29 * T2g); ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ iio[WS(vs, 2) + WS(rs, 2)] = FNMS(T2d, T2c, T29 * T2g); ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ iio[WS(vs, 2) + WS(rs, 2)] = FNMS(T2d, T2c, T29 * T2g); ------------------ | | 1134| 99|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 832| 99| T2h = W[4]; 833| 99| T2j = W[5]; 834| 99| rio[WS(vs, 3) + WS(rs, 2)] = FMA(T2h, T2i, T2j * T2k); ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ rio[WS(vs, 3) + WS(rs, 2)] = FMA(T2h, T2i, T2j * T2k); ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ rio[WS(vs, 3) + WS(rs, 2)] = FMA(T2h, T2i, T2j * T2k); ------------------ | | 1131| 99|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 835| 99| iio[WS(vs, 3) + WS(rs, 2)] = FNMS(T2j, T2i, T2h * T2k); ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ iio[WS(vs, 3) + WS(rs, 2)] = FNMS(T2j, T2i, T2h * T2k); ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ iio[WS(vs, 3) + WS(rs, 2)] = FNMS(T2j, T2i, T2h * T2k); ------------------ | | 1134| 99|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 836| 99| } 837| 99| } 838| 99| { 839| 99| E T3K, T3Q, T3O, T3S, T3I, T3N; 840| 99| T3I = T3h - T3g; 841| 99| T3K = T3I - T3J; 842| 99| T3Q = T3I + T3J; 843| 99| T3N = T3x - T3u; 844| 99| T3O = T3M + T3N; 845| 99| T3S = T3N - T3M; 846| 99| { 847| 99| E T3H, T3L, T3P, T3R; 848| 99| T3H = W[2]; 849| 99| T3L = W[3]; 850| 99| rio[WS(vs, 2) + WS(rs, 4)] = FMA(T3H, T3K, T3L * T3O); ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ rio[WS(vs, 2) + WS(rs, 4)] = FMA(T3H, T3K, T3L * T3O); ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ rio[WS(vs, 2) + WS(rs, 4)] = FMA(T3H, T3K, T3L * T3O); ------------------ | | 1131| 99|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 851| 99| iio[WS(vs, 2) + WS(rs, 4)] = FNMS(T3L, T3K, T3H * T3O); ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ iio[WS(vs, 2) + WS(rs, 4)] = FNMS(T3L, T3K, T3H * T3O); ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ iio[WS(vs, 2) + WS(rs, 4)] = FNMS(T3L, T3K, T3H * T3O); ------------------ | | 1134| 99|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 852| 99| T3P = W[4]; 853| 99| T3R = W[5]; 854| 99| rio[WS(vs, 3) + WS(rs, 4)] = FMA(T3P, T3Q, T3R * T3S); ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ rio[WS(vs, 3) + WS(rs, 4)] = FMA(T3P, T3Q, T3R * T3S); ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ rio[WS(vs, 3) + WS(rs, 4)] = FMA(T3P, T3Q, T3R * T3S); ------------------ | | 1131| 99|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 855| 99| iio[WS(vs, 3) + WS(rs, 4)] = FNMS(T3R, T3Q, T3P * T3S); ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ iio[WS(vs, 3) + WS(rs, 4)] = FNMS(T3R, T3Q, T3P * T3S); ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ iio[WS(vs, 3) + WS(rs, 4)] = FNMS(T3R, T3Q, T3P * T3S); ------------------ | | 1134| 99|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 856| 99| } 857| 99| } 858| 99| { 859| 99| E T1S, T26, T24, T28, T1K, T20; 860| 99| T1K = T1I + T1J; 861| 99| T1S = T1K + T1R; 862| 99| T26 = T1K - T1R; 863| 99| T20 = T1W + T1Z; 864| 99| T24 = T20 - T23; 865| 99| T28 = T23 + T20; 866| 99| { 867| 99| E T1H, T1T, T25, T27; 868| 99| T1H = W[0]; 869| 99| T1T = W[1]; 870| 99| rio[WS(vs, 1) + WS(rs, 2)] = FMA(T1H, T1S, T1T * T24); ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ rio[WS(vs, 1) + WS(rs, 2)] = FMA(T1H, T1S, T1T * T24); ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ rio[WS(vs, 1) + WS(rs, 2)] = FMA(T1H, T1S, T1T * T24); ------------------ | | 1131| 99|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 871| 99| iio[WS(vs, 1) + WS(rs, 2)] = FNMS(T1T, T1S, T1H * T24); ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ iio[WS(vs, 1) + WS(rs, 2)] = FNMS(T1T, T1S, T1H * T24); ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ iio[WS(vs, 1) + WS(rs, 2)] = FNMS(T1T, T1S, T1H * T24); ------------------ | | 1134| 99|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 872| 99| T25 = W[6]; 873| 99| T27 = W[7]; 874| 99| rio[WS(vs, 4) + WS(rs, 2)] = FMA(T25, T26, T27 * T28); ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ rio[WS(vs, 4) + WS(rs, 2)] = FMA(T25, T26, T27 * T28); ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ rio[WS(vs, 4) + WS(rs, 2)] = FMA(T25, T26, T27 * T28); ------------------ | | 1131| 99|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 875| 99| iio[WS(vs, 4) + WS(rs, 2)] = FNMS(T27, T26, T25 * T28); ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ iio[WS(vs, 4) + WS(rs, 2)] = FNMS(T27, T26, T25 * T28); ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ iio[WS(vs, 4) + WS(rs, 2)] = FNMS(T27, T26, T25 * T28); ------------------ | | 1134| 99|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 876| 99| } 877| 99| } 878| 99| { 879| 99| E T2E, T2S, T2Q, T2U, T2w, T2M; 880| 99| T2w = T2u + T2v; 881| 99| T2E = T2w + T2D; 882| 99| T2S = T2w - T2D; 883| 99| T2M = T2I + T2L; 884| 99| T2Q = T2M - T2P; 885| 99| T2U = T2P + T2M; 886| 99| { 887| 99| E T2t, T2F, T2R, T2T; 888| 99| T2t = W[0]; 889| 99| T2F = W[1]; 890| 99| rio[WS(vs, 1) + WS(rs, 3)] = FMA(T2t, T2E, T2F * T2Q); ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ rio[WS(vs, 1) + WS(rs, 3)] = FMA(T2t, T2E, T2F * T2Q); ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ rio[WS(vs, 1) + WS(rs, 3)] = FMA(T2t, T2E, T2F * T2Q); ------------------ | | 1131| 99|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 891| 99| iio[WS(vs, 1) + WS(rs, 3)] = FNMS(T2F, T2E, T2t * T2Q); ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ iio[WS(vs, 1) + WS(rs, 3)] = FNMS(T2F, T2E, T2t * T2Q); ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ iio[WS(vs, 1) + WS(rs, 3)] = FNMS(T2F, T2E, T2t * T2Q); ------------------ | | 1134| 99|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 892| 99| T2R = W[6]; 893| 99| T2T = W[7]; 894| 99| rio[WS(vs, 4) + WS(rs, 3)] = FMA(T2R, T2S, T2T * T2U); ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ rio[WS(vs, 4) + WS(rs, 3)] = FMA(T2R, T2S, T2T * T2U); ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ rio[WS(vs, 4) + WS(rs, 3)] = FMA(T2R, T2S, T2T * T2U); ------------------ | | 1131| 99|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 895| 99| iio[WS(vs, 4) + WS(rs, 3)] = FNMS(T2T, T2S, T2R * T2U); ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ iio[WS(vs, 4) + WS(rs, 3)] = FNMS(T2T, T2S, T2R * T2U); ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ iio[WS(vs, 4) + WS(rs, 3)] = FNMS(T2T, T2S, T2R * T2U); ------------------ | | 1134| 99|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 896| 99| } 897| 99| } 898| 99| { 899| 99| E T2Y, T34, T32, T36, T2W, T31; 900| 99| T2W = T2v - T2u; 901| 99| T2Y = T2W - T2X; 902| 99| T34 = T2W + T2X; 903| 99| T31 = T2L - T2I; 904| 99| T32 = T30 + T31; 905| 99| T36 = T31 - T30; 906| 99| { 907| 99| E T2V, T2Z, T33, T35; 908| 99| T2V = W[2]; 909| 99| T2Z = W[3]; 910| 99| rio[WS(vs, 2) + WS(rs, 3)] = FMA(T2V, T2Y, T2Z * T32); ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ rio[WS(vs, 2) + WS(rs, 3)] = FMA(T2V, T2Y, T2Z * T32); ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ rio[WS(vs, 2) + WS(rs, 3)] = FMA(T2V, T2Y, T2Z * T32); ------------------ | | 1131| 99|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 911| 99| iio[WS(vs, 2) + WS(rs, 3)] = FNMS(T2Z, T2Y, T2V * T32); ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ iio[WS(vs, 2) + WS(rs, 3)] = FNMS(T2Z, T2Y, T2V * T32); ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ iio[WS(vs, 2) + WS(rs, 3)] = FNMS(T2Z, T2Y, T2V * T32); ------------------ | | 1134| 99|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 912| 99| T33 = W[4]; 913| 99| T35 = W[5]; 914| 99| rio[WS(vs, 3) + WS(rs, 3)] = FMA(T33, T34, T35 * T36); ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ rio[WS(vs, 3) + WS(rs, 3)] = FMA(T33, T34, T35 * T36); ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ rio[WS(vs, 3) + WS(rs, 3)] = FMA(T33, T34, T35 * T36); ------------------ | | 1131| 99|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 915| 99| iio[WS(vs, 3) + WS(rs, 3)] = FNMS(T35, T34, T33 * T36); ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ iio[WS(vs, 3) + WS(rs, 3)] = FNMS(T35, T34, T33 * T36); ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ iio[WS(vs, 3) + WS(rs, 3)] = FNMS(T35, T34, T33 * T36); ------------------ | | 1134| 99|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 916| 99| } 917| 99| } 918| 99| { 919| 99| E T3q, T3E, T3C, T3G, T3i, T3y; 920| 99| T3i = T3g + T3h; 921| 99| T3q = T3i + T3p; 922| 99| T3E = T3i - T3p; 923| 99| T3y = T3u + T3x; 924| 99| T3C = T3y - T3B; 925| 99| T3G = T3B + T3y; 926| 99| { 927| 99| E T3f, T3r, T3D, T3F; 928| 99| T3f = W[0]; 929| 99| T3r = W[1]; 930| 99| rio[WS(vs, 1) + WS(rs, 4)] = FMA(T3f, T3q, T3r * T3C); ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ rio[WS(vs, 1) + WS(rs, 4)] = FMA(T3f, T3q, T3r * T3C); ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ rio[WS(vs, 1) + WS(rs, 4)] = FMA(T3f, T3q, T3r * T3C); ------------------ | | 1131| 99|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 931| 99| iio[WS(vs, 1) + WS(rs, 4)] = FNMS(T3r, T3q, T3f * T3C); ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ iio[WS(vs, 1) + WS(rs, 4)] = FNMS(T3r, T3q, T3f * T3C); ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ iio[WS(vs, 1) + WS(rs, 4)] = FNMS(T3r, T3q, T3f * T3C); ------------------ | | 1134| 99|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 932| 99| T3D = W[6]; 933| 99| T3F = W[7]; 934| 99| rio[WS(vs, 4) + WS(rs, 4)] = FMA(T3D, T3E, T3F * T3G); ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ rio[WS(vs, 4) + WS(rs, 4)] = FMA(T3D, T3E, T3F * T3G); ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ rio[WS(vs, 4) + WS(rs, 4)] = FMA(T3D, T3E, T3F * T3G); ------------------ | | 1131| 99|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 935| 99| iio[WS(vs, 4) + WS(rs, 4)] = FNMS(T3F, T3E, T3D * T3G); ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ iio[WS(vs, 4) + WS(rs, 4)] = FNMS(T3F, T3E, T3D * T3G); ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ iio[WS(vs, 4) + WS(rs, 4)] = FNMS(T3F, T3E, T3D * T3G); ------------------ | | 1134| 99|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 936| 99| } 937| 99| } 938| 99| { 939| 99| E T1q, T1w, T1u, T1y, T1o, T1t; 940| 99| T1o = TX - TW; 941| 99| T1q = T1o - T1p; 942| 99| T1w = T1o + T1p; 943| 99| T1t = T1d - T1a; 944| 99| T1u = T1s + T1t; 945| 99| T1y = T1t - T1s; 946| 99| { 947| 99| E T1n, T1r, T1v, T1x; 948| 99| T1n = W[2]; 949| 99| T1r = W[3]; 950| 99| rio[WS(vs, 2) + WS(rs, 1)] = FMA(T1n, T1q, T1r * T1u); ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ rio[WS(vs, 2) + WS(rs, 1)] = FMA(T1n, T1q, T1r * T1u); ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ rio[WS(vs, 2) + WS(rs, 1)] = FMA(T1n, T1q, T1r * T1u); ------------------ | | 1131| 99|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 951| 99| iio[WS(vs, 2) + WS(rs, 1)] = FNMS(T1r, T1q, T1n * T1u); ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ iio[WS(vs, 2) + WS(rs, 1)] = FNMS(T1r, T1q, T1n * T1u); ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ iio[WS(vs, 2) + WS(rs, 1)] = FNMS(T1r, T1q, T1n * T1u); ------------------ | | 1134| 99|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 952| 99| T1v = W[4]; 953| 99| T1x = W[5]; 954| 99| rio[WS(vs, 3) + WS(rs, 1)] = FMA(T1v, T1w, T1x * T1y); ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ rio[WS(vs, 3) + WS(rs, 1)] = FMA(T1v, T1w, T1x * T1y); ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ rio[WS(vs, 3) + WS(rs, 1)] = FMA(T1v, T1w, T1x * T1y); ------------------ | | 1131| 99|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 955| 99| iio[WS(vs, 3) + WS(rs, 1)] = FNMS(T1x, T1w, T1v * T1y); ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ iio[WS(vs, 3) + WS(rs, 1)] = FNMS(T1x, T1w, T1v * T1y); ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ iio[WS(vs, 3) + WS(rs, 1)] = FNMS(T1x, T1w, T1v * T1y); ------------------ | | 1134| 99|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 956| 99| } 957| 99| } 958| 99| { 959| 99| E T16, T1k, T1i, T1m, TY, T1e; 960| 99| TY = TW + TX; 961| 99| T16 = TY + T15; 962| 99| T1k = TY - T15; 963| 99| T1e = T1a + T1d; 964| 99| T1i = T1e - T1h; 965| 99| T1m = T1h + T1e; 966| 99| { 967| 99| E TV, T17, T1j, T1l; 968| 99| TV = W[0]; 969| 99| T17 = W[1]; 970| 99| rio[WS(vs, 1) + WS(rs, 1)] = FMA(TV, T16, T17 * T1i); ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ rio[WS(vs, 1) + WS(rs, 1)] = FMA(TV, T16, T17 * T1i); ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ rio[WS(vs, 1) + WS(rs, 1)] = FMA(TV, T16, T17 * T1i); ------------------ | | 1131| 99|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 971| 99| iio[WS(vs, 1) + WS(rs, 1)] = FNMS(T17, T16, TV * T1i); ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ iio[WS(vs, 1) + WS(rs, 1)] = FNMS(T17, T16, TV * T1i); ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ iio[WS(vs, 1) + WS(rs, 1)] = FNMS(T17, T16, TV * T1i); ------------------ | | 1134| 99|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 972| 99| T1j = W[6]; 973| 99| T1l = W[7]; 974| 99| rio[WS(vs, 4) + WS(rs, 1)] = FMA(T1j, T1k, T1l * T1m); ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ rio[WS(vs, 4) + WS(rs, 1)] = FMA(T1j, T1k, T1l * T1m); ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ rio[WS(vs, 4) + WS(rs, 1)] = FMA(T1j, T1k, T1l * T1m); ------------------ | | 1131| 99|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 975| 99| iio[WS(vs, 4) + WS(rs, 1)] = FNMS(T1l, T1k, T1j * T1m); ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ iio[WS(vs, 4) + WS(rs, 1)] = FNMS(T1l, T1k, T1j * T1m); ------------------ | | 825| 99|#define WS(stride, i) (stride * i) ------------------ iio[WS(vs, 4) + WS(rs, 1)] = FNMS(T1l, T1k, T1j * T1m); ------------------ | | 1134| 99|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 976| 99| } 977| 99| } 978| 99| } 979| 11| } 980| 11|} fftw_codelet_q1_6: 1317| 1|void X(codelet_q1_6) (planner *p) { 1318| 1| X(kdft_difsq_register) (p, q1_6, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 1319| 1|} fftw_codelet_q1_8: 2352| 1|void X(codelet_q1_8) (planner *p) { 2353| 1| X(kdft_difsq_register) (p, q1_8, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 2354| 1|} fftw_codelet_t1_10: 486| 1|void X(codelet_t1_10) (planner *p) { 487| 1| X(kdft_dit_register) (p, t1_10, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 488| 1|} fftw_codelet_t1_12: 578| 1|void X(codelet_t1_12) (planner *p) { 579| 1| X(kdft_dit_register) (p, t1_12, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 580| 1|} t1_12.c:t1_12: 322| 5|{ 323| 5| DK(KP500000000, +0.500000000000000000000000000000000000000000000); ------------------ | | 1056| 5|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 5|# define K(x) ((E) x) | | ------------------ ------------------ 324| 5| DK(KP866025403, +0.866025403784438646763723170752936183471402627); ------------------ | | 1056| 5|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 5|# define K(x) ((E) x) | | ------------------ ------------------ 325| 5| { 326| 5| INT m; 327| 54| for (m = mb, W = W + (mb * 22); m < me; m = m + 1, ri = ri + ms, ii = ii + ms, W = W + 22, MAKE_VOLATILE_STRIDE(24, rs)) { ------------------ | | 851| 49| (nptr <= ESTIMATED_AVAILABLE_INDEX_REGISTERS ? \ | | ------------------ | | | | 849| 98|#define ESTIMATED_AVAILABLE_INDEX_REGISTERS 16 | | ------------------ | | | Branch (851:7): [Folded, False: 49] | | ------------------ | | 852| 49| 0 : \ | | 853| 49| ((x) = (x) ^ X(an_INT_guaranteed_to_be_zero))) | | ------------------ | | | | 78| 49|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 49|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ | Branch (327:36): [True: 49, False: 5] ------------------ 328| 49| E T1, T1W, T18, T21, Tc, T15, T1V, T22, TR, T1E, T1o, T1D, T12, T1l, T1F; 329| 49| E T1G, Ti, T1S, T1d, T24, Tt, T1a, T1T, T25, TA, T1z, T1j, T1y, TL, T1g; 330| 49| E T1A, T1B; 331| 49| { 332| 49| E T6, T16, Tb, T17; 333| 49| T1 = ri[0]; 334| 49| T1W = ii[0]; 335| 49| { 336| 49| E T3, T5, T2, T4; 337| 49| T3 = ri[WS(rs, 4)]; ------------------ | | 825| 49|#define WS(stride, i) (stride * i) ------------------ 338| 49| T5 = ii[WS(rs, 4)]; ------------------ | | 825| 49|#define WS(stride, i) (stride * i) ------------------ 339| 49| T2 = W[6]; 340| 49| T4 = W[7]; 341| 49| T6 = FMA(T2, T3, T4 * T5); ------------------ | | 1131| 49|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 342| 49| T16 = FNMS(T4, T3, T2 * T5); ------------------ | | 1134| 49|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 343| 49| } 344| 49| { 345| 49| E T8, Ta, T7, T9; 346| 49| T8 = ri[WS(rs, 8)]; ------------------ | | 825| 49|#define WS(stride, i) (stride * i) ------------------ 347| 49| Ta = ii[WS(rs, 8)]; ------------------ | | 825| 49|#define WS(stride, i) (stride * i) ------------------ 348| 49| T7 = W[14]; 349| 49| T9 = W[15]; 350| 49| Tb = FMA(T7, T8, T9 * Ta); ------------------ | | 1131| 49|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 351| 49| T17 = FNMS(T9, T8, T7 * Ta); ------------------ | | 1134| 49|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 352| 49| } 353| 49| T18 = KP866025403 * (T16 - T17); 354| 49| T21 = KP866025403 * (Tb - T6); 355| 49| Tc = T6 + Tb; 356| 49| T15 = FNMS(KP500000000, Tc, T1); ------------------ | | 1134| 49|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 357| 49| T1V = T16 + T17; 358| 49| T22 = FNMS(KP500000000, T1V, T1W); ------------------ | | 1134| 49|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 359| 49| } 360| 49| { 361| 49| E T11, T1n, TW, T1m; 362| 49| { 363| 49| E TO, TQ, TN, TP; 364| 49| TO = ri[WS(rs, 9)]; ------------------ | | 825| 49|#define WS(stride, i) (stride * i) ------------------ 365| 49| TQ = ii[WS(rs, 9)]; ------------------ | | 825| 49|#define WS(stride, i) (stride * i) ------------------ 366| 49| TN = W[16]; 367| 49| TP = W[17]; 368| 49| TR = FMA(TN, TO, TP * TQ); ------------------ | | 1131| 49|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 369| 49| T1E = FNMS(TP, TO, TN * TQ); ------------------ | | 1134| 49|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 370| 49| } 371| 49| { 372| 49| E TY, T10, TX, TZ; 373| 49| TY = ri[WS(rs, 5)]; ------------------ | | 825| 49|#define WS(stride, i) (stride * i) ------------------ 374| 49| T10 = ii[WS(rs, 5)]; ------------------ | | 825| 49|#define WS(stride, i) (stride * i) ------------------ 375| 49| TX = W[8]; 376| 49| TZ = W[9]; 377| 49| T11 = FMA(TX, TY, TZ * T10); ------------------ | | 1131| 49|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 378| 49| T1n = FNMS(TZ, TY, TX * T10); ------------------ | | 1134| 49|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 379| 49| } 380| 49| { 381| 49| E TT, TV, TS, TU; 382| 49| TT = ri[WS(rs, 1)]; ------------------ | | 825| 49|#define WS(stride, i) (stride * i) ------------------ 383| 49| TV = ii[WS(rs, 1)]; ------------------ | | 825| 49|#define WS(stride, i) (stride * i) ------------------ 384| 49| TS = W[0]; 385| 49| TU = W[1]; 386| 49| TW = FMA(TS, TT, TU * TV); ------------------ | | 1131| 49|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 387| 49| T1m = FNMS(TU, TT, TS * TV); ------------------ | | 1134| 49|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 388| 49| } 389| 49| T1o = KP866025403 * (T1m - T1n); 390| 49| T1D = KP866025403 * (T11 - TW); 391| 49| T12 = TW + T11; 392| 49| T1l = FNMS(KP500000000, T12, TR); ------------------ | | 1134| 49|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 393| 49| T1F = T1m + T1n; 394| 49| T1G = FNMS(KP500000000, T1F, T1E); ------------------ | | 1134| 49|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 395| 49| } 396| 49| { 397| 49| E Ts, T1c, Tn, T1b; 398| 49| { 399| 49| E Tf, Th, Te, Tg; 400| 49| Tf = ri[WS(rs, 6)]; ------------------ | | 825| 49|#define WS(stride, i) (stride * i) ------------------ 401| 49| Th = ii[WS(rs, 6)]; ------------------ | | 825| 49|#define WS(stride, i) (stride * i) ------------------ 402| 49| Te = W[10]; 403| 49| Tg = W[11]; 404| 49| Ti = FMA(Te, Tf, Tg * Th); ------------------ | | 1131| 49|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 405| 49| T1S = FNMS(Tg, Tf, Te * Th); ------------------ | | 1134| 49|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 406| 49| } 407| 49| { 408| 49| E Tp, Tr, To, Tq; 409| 49| Tp = ri[WS(rs, 2)]; ------------------ | | 825| 49|#define WS(stride, i) (stride * i) ------------------ 410| 49| Tr = ii[WS(rs, 2)]; ------------------ | | 825| 49|#define WS(stride, i) (stride * i) ------------------ 411| 49| To = W[2]; 412| 49| Tq = W[3]; 413| 49| Ts = FMA(To, Tp, Tq * Tr); ------------------ | | 1131| 49|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 414| 49| T1c = FNMS(Tq, Tp, To * Tr); ------------------ | | 1134| 49|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 415| 49| } 416| 49| { 417| 49| E Tk, Tm, Tj, Tl; 418| 49| Tk = ri[WS(rs, 10)]; ------------------ | | 825| 49|#define WS(stride, i) (stride * i) ------------------ 419| 49| Tm = ii[WS(rs, 10)]; ------------------ | | 825| 49|#define WS(stride, i) (stride * i) ------------------ 420| 49| Tj = W[18]; 421| 49| Tl = W[19]; 422| 49| Tn = FMA(Tj, Tk, Tl * Tm); ------------------ | | 1131| 49|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 423| 49| T1b = FNMS(Tl, Tk, Tj * Tm); ------------------ | | 1134| 49|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 424| 49| } 425| 49| T1d = KP866025403 * (T1b - T1c); 426| 49| T24 = KP866025403 * (Ts - Tn); 427| 49| Tt = Tn + Ts; 428| 49| T1a = FNMS(KP500000000, Tt, Ti); ------------------ | | 1134| 49|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 429| 49| T1T = T1b + T1c; 430| 49| T25 = FNMS(KP500000000, T1T, T1S); ------------------ | | 1134| 49|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 431| 49| } 432| 49| { 433| 49| E TK, T1i, TF, T1h; 434| 49| { 435| 49| E Tx, Tz, Tw, Ty; 436| 49| Tx = ri[WS(rs, 3)]; ------------------ | | 825| 49|#define WS(stride, i) (stride * i) ------------------ 437| 49| Tz = ii[WS(rs, 3)]; ------------------ | | 825| 49|#define WS(stride, i) (stride * i) ------------------ 438| 49| Tw = W[4]; 439| 49| Ty = W[5]; 440| 49| TA = FMA(Tw, Tx, Ty * Tz); ------------------ | | 1131| 49|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 441| 49| T1z = FNMS(Ty, Tx, Tw * Tz); ------------------ | | 1134| 49|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 442| 49| } 443| 49| { 444| 49| E TH, TJ, TG, TI; 445| 49| TH = ri[WS(rs, 11)]; ------------------ | | 825| 49|#define WS(stride, i) (stride * i) ------------------ 446| 49| TJ = ii[WS(rs, 11)]; ------------------ | | 825| 49|#define WS(stride, i) (stride * i) ------------------ 447| 49| TG = W[20]; 448| 49| TI = W[21]; 449| 49| TK = FMA(TG, TH, TI * TJ); ------------------ | | 1131| 49|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 450| 49| T1i = FNMS(TI, TH, TG * TJ); ------------------ | | 1134| 49|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 451| 49| } 452| 49| { 453| 49| E TC, TE, TB, TD; 454| 49| TC = ri[WS(rs, 7)]; ------------------ | | 825| 49|#define WS(stride, i) (stride * i) ------------------ 455| 49| TE = ii[WS(rs, 7)]; ------------------ | | 825| 49|#define WS(stride, i) (stride * i) ------------------ 456| 49| TB = W[12]; 457| 49| TD = W[13]; 458| 49| TF = FMA(TB, TC, TD * TE); ------------------ | | 1131| 49|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 459| 49| T1h = FNMS(TD, TC, TB * TE); ------------------ | | 1134| 49|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 460| 49| } 461| 49| T1j = KP866025403 * (T1h - T1i); 462| 49| T1y = KP866025403 * (TK - TF); 463| 49| TL = TF + TK; 464| 49| T1g = FNMS(KP500000000, TL, TA); ------------------ | | 1134| 49|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 465| 49| T1A = T1h + T1i; 466| 49| T1B = FNMS(KP500000000, T1A, T1z); ------------------ | | 1134| 49|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 467| 49| } 468| 49| { 469| 49| E Tv, T1N, T1Y, T20, T14, T1Z, T1Q, T1R; 470| 49| { 471| 49| E Td, Tu, T1U, T1X; 472| 49| Td = T1 + Tc; 473| 49| Tu = Ti + Tt; 474| 49| Tv = Td + Tu; 475| 49| T1N = Td - Tu; 476| 49| T1U = T1S + T1T; 477| 49| T1X = T1V + T1W; 478| 49| T1Y = T1U + T1X; 479| 49| T20 = T1X - T1U; 480| 49| } 481| 49| { 482| 49| E TM, T13, T1O, T1P; 483| 49| TM = TA + TL; 484| 49| T13 = TR + T12; 485| 49| T14 = TM + T13; 486| 49| T1Z = TM - T13; 487| 49| T1O = T1z + T1A; 488| 49| T1P = T1E + T1F; 489| 49| T1Q = T1O - T1P; 490| 49| T1R = T1O + T1P; 491| 49| } 492| 49| ri[WS(rs, 6)] = Tv - T14; ------------------ | | 825| 49|#define WS(stride, i) (stride * i) ------------------ 493| 49| ii[WS(rs, 6)] = T1Y - T1R; ------------------ | | 825| 49|#define WS(stride, i) (stride * i) ------------------ 494| 49| ri[0] = Tv + T14; 495| 49| ii[0] = T1R + T1Y; 496| 49| ri[WS(rs, 3)] = T1N - T1Q; ------------------ | | 825| 49|#define WS(stride, i) (stride * i) ------------------ 497| 49| ii[WS(rs, 3)] = T1Z + T20; ------------------ | | 825| 49|#define WS(stride, i) (stride * i) ------------------ 498| 49| ri[WS(rs, 9)] = T1N + T1Q; ------------------ | | 825| 49|#define WS(stride, i) (stride * i) ------------------ 499| 49| ii[WS(rs, 9)] = T20 - T1Z; ------------------ | | 825| 49|#define WS(stride, i) (stride * i) ------------------ 500| 49| } 501| 49| { 502| 49| E T1t, T1x, T27, T2a, T1w, T28, T1I, T29; 503| 49| { 504| 49| E T1r, T1s, T23, T26; 505| 49| T1r = T15 + T18; 506| 49| T1s = T1a + T1d; 507| 49| T1t = T1r + T1s; 508| 49| T1x = T1r - T1s; 509| 49| T23 = T21 + T22; 510| 49| T26 = T24 + T25; 511| 49| T27 = T23 - T26; 512| 49| T2a = T26 + T23; 513| 49| } 514| 49| { 515| 49| E T1u, T1v, T1C, T1H; 516| 49| T1u = T1g + T1j; 517| 49| T1v = T1l + T1o; 518| 49| T1w = T1u + T1v; 519| 49| T28 = T1u - T1v; 520| 49| T1C = T1y + T1B; 521| 49| T1H = T1D + T1G; 522| 49| T1I = T1C - T1H; 523| 49| T29 = T1C + T1H; 524| 49| } 525| 49| ri[WS(rs, 10)] = T1t - T1w; ------------------ | | 825| 49|#define WS(stride, i) (stride * i) ------------------ 526| 49| ii[WS(rs, 10)] = T2a - T29; ------------------ | | 825| 49|#define WS(stride, i) (stride * i) ------------------ 527| 49| ri[WS(rs, 4)] = T1t + T1w; ------------------ | | 825| 49|#define WS(stride, i) (stride * i) ------------------ 528| 49| ii[WS(rs, 4)] = T29 + T2a; ------------------ | | 825| 49|#define WS(stride, i) (stride * i) ------------------ 529| 49| ri[WS(rs, 7)] = T1x - T1I; ------------------ | | 825| 49|#define WS(stride, i) (stride * i) ------------------ 530| 49| ii[WS(rs, 7)] = T28 + T27; ------------------ | | 825| 49|#define WS(stride, i) (stride * i) ------------------ 531| 49| ri[WS(rs, 1)] = T1x + T1I; ------------------ | | 825| 49|#define WS(stride, i) (stride * i) ------------------ 532| 49| ii[WS(rs, 1)] = T27 - T28; ------------------ | | 825| 49|#define WS(stride, i) (stride * i) ------------------ 533| 49| } 534| 49| { 535| 49| E T1f, T1J, T2d, T2f, T1q, T2g, T1M, T2e; 536| 49| { 537| 49| E T19, T1e, T2b, T2c; 538| 49| T19 = T15 - T18; 539| 49| T1e = T1a - T1d; 540| 49| T1f = T19 + T1e; 541| 49| T1J = T19 - T1e; 542| 49| T2b = T25 - T24; 543| 49| T2c = T22 - T21; 544| 49| T2d = T2b + T2c; 545| 49| T2f = T2c - T2b; 546| 49| } 547| 49| { 548| 49| E T1k, T1p, T1K, T1L; 549| 49| T1k = T1g - T1j; 550| 49| T1p = T1l - T1o; 551| 49| T1q = T1k + T1p; 552| 49| T2g = T1k - T1p; 553| 49| T1K = T1B - T1y; 554| 49| T1L = T1G - T1D; 555| 49| T1M = T1K - T1L; 556| 49| T2e = T1K + T1L; 557| 49| } 558| 49| ri[WS(rs, 2)] = T1f - T1q; ------------------ | | 825| 49|#define WS(stride, i) (stride * i) ------------------ 559| 49| ii[WS(rs, 2)] = T2d - T2e; ------------------ | | 825| 49|#define WS(stride, i) (stride * i) ------------------ 560| 49| ri[WS(rs, 8)] = T1f + T1q; ------------------ | | 825| 49|#define WS(stride, i) (stride * i) ------------------ 561| 49| ii[WS(rs, 8)] = T2e + T2d; ------------------ | | 825| 49|#define WS(stride, i) (stride * i) ------------------ 562| 49| ri[WS(rs, 11)] = T1J - T1M; ------------------ | | 825| 49|#define WS(stride, i) (stride * i) ------------------ 563| 49| ii[WS(rs, 11)] = T2g + T2f; ------------------ | | 825| 49|#define WS(stride, i) (stride * i) ------------------ 564| 49| ri[WS(rs, 5)] = T1J + T1M; ------------------ | | 825| 49|#define WS(stride, i) (stride * i) ------------------ 565| 49| ii[WS(rs, 5)] = T2f - T2g; ------------------ | | 825| 49|#define WS(stride, i) (stride * i) ------------------ 566| 49| } 567| 49| } 568| 5| } 569| 5|} fftw_codelet_t1_15: 813| 1|void X(codelet_t1_15) (planner *p) { 814| 1| X(kdft_dit_register) (p, t1_15, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 815| 1|} fftw_codelet_t1_16: 793| 1|void X(codelet_t1_16) (planner *p) { 794| 1| X(kdft_dit_register) (p, t1_16, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 795| 1|} fftw_codelet_t1_2: 114| 1|void X(codelet_t1_2) (planner *p) { 115| 1| X(kdft_dit_register) (p, t1_2, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 116| 1|} t1_2.c:t1_2: 83| 199|{ 84| 199| { 85| 199| INT m; 86| 3.98k| for (m = mb, W = W + (mb * 2); m < me; m = m + 1, ri = ri + ms, ii = ii + ms, W = W + 2, MAKE_VOLATILE_STRIDE(4, rs)) { ------------------ | | 851| 3.78k| (nptr <= ESTIMATED_AVAILABLE_INDEX_REGISTERS ? \ | | ------------------ | | | | 849| 7.56k|#define ESTIMATED_AVAILABLE_INDEX_REGISTERS 16 | | ------------------ | | | Branch (851:7): [True: 3.78k, Folded] | | ------------------ | | 852| 3.78k| 0 : \ | | 853| 3.78k| ((x) = (x) ^ X(an_INT_guaranteed_to_be_zero))) | | ------------------ | | | | 78| 0|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ | Branch (86:35): [True: 3.78k, False: 199] ------------------ 87| 3.78k| E T1, T8, T6, T7; 88| 3.78k| T1 = ri[0]; 89| 3.78k| T8 = ii[0]; 90| 3.78k| { 91| 3.78k| E T3, T5, T2, T4; 92| 3.78k| T3 = ri[WS(rs, 1)]; ------------------ | | 825| 3.78k|#define WS(stride, i) (stride * i) ------------------ 93| 3.78k| T5 = ii[WS(rs, 1)]; ------------------ | | 825| 3.78k|#define WS(stride, i) (stride * i) ------------------ 94| 3.78k| T2 = W[0]; 95| 3.78k| T4 = W[1]; 96| 3.78k| T6 = FMA(T2, T3, T4 * T5); ------------------ | | 1131| 3.78k|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 97| 3.78k| T7 = FNMS(T4, T3, T2 * T5); ------------------ | | 1134| 3.78k|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 98| 3.78k| } 99| 3.78k| ri[WS(rs, 1)] = T1 - T6; ------------------ | | 825| 3.78k|#define WS(stride, i) (stride * i) ------------------ 100| 3.78k| ii[WS(rs, 1)] = T8 - T7; ------------------ | | 825| 3.78k|#define WS(stride, i) (stride * i) ------------------ 101| 3.78k| ri[0] = T1 + T6; 102| 3.78k| ii[0] = T7 + T8; 103| 3.78k| } 104| 199| } 105| 199|} fftw_codelet_t1_20: 1047| 1|void X(codelet_t1_20) (planner *p) { 1048| 1| X(kdft_dit_register) (p, t1_20, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 1049| 1|} fftw_codelet_t1_25: 1569| 1|void X(codelet_t1_25) (planner *p) { 1570| 1| X(kdft_dit_register) (p, t1_25, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 1571| 1|} fftw_codelet_t1_3: 163| 1|void X(codelet_t1_3) (planner *p) { 164| 1| X(kdft_dit_register) (p, t1_3, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 165| 1|} t1_3.c:t1_3: 110| 804|{ 111| 804| DK(KP866025403, +0.866025403784438646763723170752936183471402627); ------------------ | | 1056| 804|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 804|# define K(x) ((E) x) | | ------------------ ------------------ 112| 804| DK(KP500000000, +0.500000000000000000000000000000000000000000000); ------------------ | | 1056| 804|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 804|# define K(x) ((E) x) | | ------------------ ------------------ 113| 804| { 114| 804| INT m; 115| 6.91k| for (m = mb, W = W + (mb * 4); m < me; m = m + 1, ri = ri + ms, ii = ii + ms, W = W + 4, MAKE_VOLATILE_STRIDE(6, rs)) { ------------------ | | 851| 6.11k| (nptr <= ESTIMATED_AVAILABLE_INDEX_REGISTERS ? \ | | ------------------ | | | | 849| 12.2k|#define ESTIMATED_AVAILABLE_INDEX_REGISTERS 16 | | ------------------ | | | Branch (851:7): [True: 6.11k, Folded] | | ------------------ | | 852| 6.11k| 0 : \ | | 853| 6.11k| ((x) = (x) ^ X(an_INT_guaranteed_to_be_zero))) | | ------------------ | | | | 78| 0|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ | Branch (115:35): [True: 6.11k, False: 804] ------------------ 116| 6.11k| E T1, Ti, T6, Te, Tb, Tf, Tc, Th; 117| 6.11k| T1 = ri[0]; 118| 6.11k| Ti = ii[0]; 119| 6.11k| { 120| 6.11k| E T3, T5, T2, T4; 121| 6.11k| T3 = ri[WS(rs, 1)]; ------------------ | | 825| 6.11k|#define WS(stride, i) (stride * i) ------------------ 122| 6.11k| T5 = ii[WS(rs, 1)]; ------------------ | | 825| 6.11k|#define WS(stride, i) (stride * i) ------------------ 123| 6.11k| T2 = W[0]; 124| 6.11k| T4 = W[1]; 125| 6.11k| T6 = FMA(T2, T3, T4 * T5); ------------------ | | 1131| 6.11k|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 126| 6.11k| Te = FNMS(T4, T3, T2 * T5); ------------------ | | 1134| 6.11k|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 127| 6.11k| } 128| 6.11k| { 129| 6.11k| E T8, Ta, T7, T9; 130| 6.11k| T8 = ri[WS(rs, 2)]; ------------------ | | 825| 6.11k|#define WS(stride, i) (stride * i) ------------------ 131| 6.11k| Ta = ii[WS(rs, 2)]; ------------------ | | 825| 6.11k|#define WS(stride, i) (stride * i) ------------------ 132| 6.11k| T7 = W[2]; 133| 6.11k| T9 = W[3]; 134| 6.11k| Tb = FMA(T7, T8, T9 * Ta); ------------------ | | 1131| 6.11k|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 135| 6.11k| Tf = FNMS(T9, T8, T7 * Ta); ------------------ | | 1134| 6.11k|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 136| 6.11k| } 137| 6.11k| Tc = T6 + Tb; 138| 6.11k| Th = Te + Tf; 139| 6.11k| ri[0] = T1 + Tc; 140| 6.11k| ii[0] = Th + Ti; 141| 6.11k| { 142| 6.11k| E Td, Tg, Tj, Tk; 143| 6.11k| Td = FNMS(KP500000000, Tc, T1); ------------------ | | 1134| 6.11k|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 144| 6.11k| Tg = KP866025403 * (Te - Tf); 145| 6.11k| ri[WS(rs, 2)] = Td - Tg; ------------------ | | 825| 6.11k|#define WS(stride, i) (stride * i) ------------------ 146| 6.11k| ri[WS(rs, 1)] = Td + Tg; ------------------ | | 825| 6.11k|#define WS(stride, i) (stride * i) ------------------ 147| 6.11k| Tj = KP866025403 * (Tb - T6); 148| 6.11k| Tk = FNMS(KP500000000, Th, Ti); ------------------ | | 1134| 6.11k|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 149| 6.11k| ii[WS(rs, 1)] = Tj + Tk; ------------------ | | 825| 6.11k|#define WS(stride, i) (stride * i) ------------------ 150| 6.11k| ii[WS(rs, 2)] = Tk - Tj; ------------------ | | 825| 6.11k|#define WS(stride, i) (stride * i) ------------------ 151| 6.11k| } 152| 6.11k| } 153| 804| } 154| 804|} fftw_codelet_t1_32: 1806| 1|void X(codelet_t1_32) (planner *p) { 1807| 1| X(kdft_dit_register) (p, t1_32, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 1808| 1|} fftw_codelet_t1_4: 193| 1|void X(codelet_t1_4) (planner *p) { 194| 1| X(kdft_dit_register) (p, t1_4, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 195| 1|} t1_4.c:t1_4: 126| 14|{ 127| 14| { 128| 14| INT m; 129| 528| for (m = mb, W = W + (mb * 6); m < me; m = m + 1, ri = ri + ms, ii = ii + ms, W = W + 6, MAKE_VOLATILE_STRIDE(8, rs)) { ------------------ | | 851| 514| (nptr <= ESTIMATED_AVAILABLE_INDEX_REGISTERS ? \ | | ------------------ | | | | 849| 1.02k|#define ESTIMATED_AVAILABLE_INDEX_REGISTERS 16 | | ------------------ | | | Branch (851:7): [True: 514, Folded] | | ------------------ | | 852| 514| 0 : \ | | 853| 514| ((x) = (x) ^ X(an_INT_guaranteed_to_be_zero))) | | ------------------ | | | | 78| 0|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ | Branch (129:35): [True: 514, False: 14] ------------------ 130| 514| E T1, Tp, T6, To, Tc, Tk, Th, Tl; 131| 514| T1 = ri[0]; 132| 514| Tp = ii[0]; 133| 514| { 134| 514| E T3, T5, T2, T4; 135| 514| T3 = ri[WS(rs, 2)]; ------------------ | | 825| 514|#define WS(stride, i) (stride * i) ------------------ 136| 514| T5 = ii[WS(rs, 2)]; ------------------ | | 825| 514|#define WS(stride, i) (stride * i) ------------------ 137| 514| T2 = W[2]; 138| 514| T4 = W[3]; 139| 514| T6 = FMA(T2, T3, T4 * T5); ------------------ | | 1131| 514|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 140| 514| To = FNMS(T4, T3, T2 * T5); ------------------ | | 1134| 514|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 141| 514| } 142| 514| { 143| 514| E T9, Tb, T8, Ta; 144| 514| T9 = ri[WS(rs, 1)]; ------------------ | | 825| 514|#define WS(stride, i) (stride * i) ------------------ 145| 514| Tb = ii[WS(rs, 1)]; ------------------ | | 825| 514|#define WS(stride, i) (stride * i) ------------------ 146| 514| T8 = W[0]; 147| 514| Ta = W[1]; 148| 514| Tc = FMA(T8, T9, Ta * Tb); ------------------ | | 1131| 514|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 149| 514| Tk = FNMS(Ta, T9, T8 * Tb); ------------------ | | 1134| 514|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 150| 514| } 151| 514| { 152| 514| E Te, Tg, Td, Tf; 153| 514| Te = ri[WS(rs, 3)]; ------------------ | | 825| 514|#define WS(stride, i) (stride * i) ------------------ 154| 514| Tg = ii[WS(rs, 3)]; ------------------ | | 825| 514|#define WS(stride, i) (stride * i) ------------------ 155| 514| Td = W[4]; 156| 514| Tf = W[5]; 157| 514| Th = FMA(Td, Te, Tf * Tg); ------------------ | | 1131| 514|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 158| 514| Tl = FNMS(Tf, Te, Td * Tg); ------------------ | | 1134| 514|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 159| 514| } 160| 514| { 161| 514| E T7, Ti, Tn, Tq; 162| 514| T7 = T1 + T6; 163| 514| Ti = Tc + Th; 164| 514| ri[WS(rs, 2)] = T7 - Ti; ------------------ | | 825| 514|#define WS(stride, i) (stride * i) ------------------ 165| 514| ri[0] = T7 + Ti; 166| 514| Tn = Tk + Tl; 167| 514| Tq = To + Tp; 168| 514| ii[0] = Tn + Tq; 169| 514| ii[WS(rs, 2)] = Tq - Tn; ------------------ | | 825| 514|#define WS(stride, i) (stride * i) ------------------ 170| 514| } 171| 514| { 172| 514| E Tj, Tm, Tr, Ts; 173| 514| Tj = T1 - T6; 174| 514| Tm = Tk - Tl; 175| 514| ri[WS(rs, 3)] = Tj - Tm; ------------------ | | 825| 514|#define WS(stride, i) (stride * i) ------------------ 176| 514| ri[WS(rs, 1)] = Tj + Tm; ------------------ | | 825| 514|#define WS(stride, i) (stride * i) ------------------ 177| 514| Tr = Tp - To; 178| 514| Ts = Tc - Th; 179| 514| ii[WS(rs, 1)] = Tr - Ts; ------------------ | | 825| 514|#define WS(stride, i) (stride * i) ------------------ 180| 514| ii[WS(rs, 3)] = Ts + Tr; ------------------ | | 825| 514|#define WS(stride, i) (stride * i) ------------------ 181| 514| } 182| 514| } 183| 14| } 184| 14|} fftw_codelet_t1_5: 250| 1|void X(codelet_t1_5) (planner *p) { 251| 1| X(kdft_dit_register) (p, t1_5, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 252| 1|} fftw_codelet_t1_6: 292| 1|void X(codelet_t1_6) (planner *p) { 293| 1| X(kdft_dit_register) (p, t1_6, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 294| 1|} t1_6.c:t1_6: 176| 84|{ 177| 84| DK(KP500000000, +0.500000000000000000000000000000000000000000000); ------------------ | | 1056| 84|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 84|# define K(x) ((E) x) | | ------------------ ------------------ 178| 84| DK(KP866025403, +0.866025403784438646763723170752936183471402627); ------------------ | | 1056| 84|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 84|# define K(x) ((E) x) | | ------------------ ------------------ 179| 84| { 180| 84| INT m; 181| 687| for (m = mb, W = W + (mb * 10); m < me; m = m + 1, ri = ri + ms, ii = ii + ms, W = W + 10, MAKE_VOLATILE_STRIDE(12, rs)) { ------------------ | | 851| 603| (nptr <= ESTIMATED_AVAILABLE_INDEX_REGISTERS ? \ | | ------------------ | | | | 849| 1.20k|#define ESTIMATED_AVAILABLE_INDEX_REGISTERS 16 | | ------------------ | | | Branch (851:7): [True: 603, Folded] | | ------------------ | | 852| 603| 0 : \ | | 853| 603| ((x) = (x) ^ X(an_INT_guaranteed_to_be_zero))) | | ------------------ | | | | 78| 0|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ | Branch (181:36): [True: 603, False: 84] ------------------ 182| 603| E T7, TS, Tv, TO, Tt, TJ, Tx, TF, Ti, TI, Tw, TC; 183| 603| { 184| 603| E T1, TN, T6, TM; 185| 603| T1 = ri[0]; 186| 603| TN = ii[0]; 187| 603| { 188| 603| E T3, T5, T2, T4; 189| 603| T3 = ri[WS(rs, 3)]; ------------------ | | 825| 603|#define WS(stride, i) (stride * i) ------------------ 190| 603| T5 = ii[WS(rs, 3)]; ------------------ | | 825| 603|#define WS(stride, i) (stride * i) ------------------ 191| 603| T2 = W[4]; 192| 603| T4 = W[5]; 193| 603| T6 = FMA(T2, T3, T4 * T5); ------------------ | | 1131| 603|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 194| 603| TM = FNMS(T4, T3, T2 * T5); ------------------ | | 1134| 603|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 195| 603| } 196| 603| T7 = T1 - T6; 197| 603| TS = TN - TM; 198| 603| Tv = T1 + T6; 199| 603| TO = TM + TN; 200| 603| } 201| 603| { 202| 603| E Tn, TD, Ts, TE; 203| 603| { 204| 603| E Tk, Tm, Tj, Tl; 205| 603| Tk = ri[WS(rs, 4)]; ------------------ | | 825| 603|#define WS(stride, i) (stride * i) ------------------ 206| 603| Tm = ii[WS(rs, 4)]; ------------------ | | 825| 603|#define WS(stride, i) (stride * i) ------------------ 207| 603| Tj = W[6]; 208| 603| Tl = W[7]; 209| 603| Tn = FMA(Tj, Tk, Tl * Tm); ------------------ | | 1131| 603|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 210| 603| TD = FNMS(Tl, Tk, Tj * Tm); ------------------ | | 1134| 603|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 211| 603| } 212| 603| { 213| 603| E Tp, Tr, To, Tq; 214| 603| Tp = ri[WS(rs, 1)]; ------------------ | | 825| 603|#define WS(stride, i) (stride * i) ------------------ 215| 603| Tr = ii[WS(rs, 1)]; ------------------ | | 825| 603|#define WS(stride, i) (stride * i) ------------------ 216| 603| To = W[0]; 217| 603| Tq = W[1]; 218| 603| Ts = FMA(To, Tp, Tq * Tr); ------------------ | | 1131| 603|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 219| 603| TE = FNMS(Tq, Tp, To * Tr); ------------------ | | 1134| 603|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 220| 603| } 221| 603| Tt = Tn - Ts; 222| 603| TJ = TD + TE; 223| 603| Tx = Tn + Ts; 224| 603| TF = TD - TE; 225| 603| } 226| 603| { 227| 603| E Tc, TA, Th, TB; 228| 603| { 229| 603| E T9, Tb, T8, Ta; 230| 603| T9 = ri[WS(rs, 2)]; ------------------ | | 825| 603|#define WS(stride, i) (stride * i) ------------------ 231| 603| Tb = ii[WS(rs, 2)]; ------------------ | | 825| 603|#define WS(stride, i) (stride * i) ------------------ 232| 603| T8 = W[2]; 233| 603| Ta = W[3]; 234| 603| Tc = FMA(T8, T9, Ta * Tb); ------------------ | | 1131| 603|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 235| 603| TA = FNMS(Ta, T9, T8 * Tb); ------------------ | | 1134| 603|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 236| 603| } 237| 603| { 238| 603| E Te, Tg, Td, Tf; 239| 603| Te = ri[WS(rs, 5)]; ------------------ | | 825| 603|#define WS(stride, i) (stride * i) ------------------ 240| 603| Tg = ii[WS(rs, 5)]; ------------------ | | 825| 603|#define WS(stride, i) (stride * i) ------------------ 241| 603| Td = W[8]; 242| 603| Tf = W[9]; 243| 603| Th = FMA(Td, Te, Tf * Tg); ------------------ | | 1131| 603|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 244| 603| TB = FNMS(Tf, Te, Td * Tg); ------------------ | | 1134| 603|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 245| 603| } 246| 603| Ti = Tc - Th; 247| 603| TI = TA + TB; 248| 603| Tw = Tc + Th; 249| 603| TC = TA - TB; 250| 603| } 251| 603| { 252| 603| E TG, Tu, Tz, TR, TT, TU; 253| 603| TG = KP866025403 * (TC - TF); 254| 603| Tu = Ti + Tt; 255| 603| Tz = FNMS(KP500000000, Tu, T7); ------------------ | | 1134| 603|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 256| 603| ri[WS(rs, 3)] = T7 + Tu; ------------------ | | 825| 603|#define WS(stride, i) (stride * i) ------------------ 257| 603| ri[WS(rs, 1)] = Tz + TG; ------------------ | | 825| 603|#define WS(stride, i) (stride * i) ------------------ 258| 603| ri[WS(rs, 5)] = Tz - TG; ------------------ | | 825| 603|#define WS(stride, i) (stride * i) ------------------ 259| 603| TR = KP866025403 * (Tt - Ti); 260| 603| TT = TC + TF; 261| 603| TU = FNMS(KP500000000, TT, TS); ------------------ | | 1134| 603|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 262| 603| ii[WS(rs, 1)] = TR + TU; ------------------ | | 825| 603|#define WS(stride, i) (stride * i) ------------------ 263| 603| ii[WS(rs, 3)] = TT + TS; ------------------ | | 825| 603|#define WS(stride, i) (stride * i) ------------------ 264| 603| ii[WS(rs, 5)] = TU - TR; ------------------ | | 825| 603|#define WS(stride, i) (stride * i) ------------------ 265| 603| } 266| 603| { 267| 603| E TK, Ty, TH, TQ, TL, TP; 268| 603| TK = KP866025403 * (TI - TJ); 269| 603| Ty = Tw + Tx; 270| 603| TH = FNMS(KP500000000, Ty, Tv); ------------------ | | 1134| 603|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 271| 603| ri[0] = Tv + Ty; 272| 603| ri[WS(rs, 4)] = TH + TK; ------------------ | | 825| 603|#define WS(stride, i) (stride * i) ------------------ 273| 603| ri[WS(rs, 2)] = TH - TK; ------------------ | | 825| 603|#define WS(stride, i) (stride * i) ------------------ 274| 603| TQ = KP866025403 * (Tx - Tw); 275| 603| TL = TI + TJ; 276| 603| TP = FNMS(KP500000000, TL, TO); ------------------ | | 1134| 603|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 277| 603| ii[0] = TL + TO; 278| 603| ii[WS(rs, 4)] = TQ + TP; ------------------ | | 825| 603|#define WS(stride, i) (stride * i) ------------------ 279| 603| ii[WS(rs, 2)] = TP - TQ; ------------------ | | 825| 603|#define WS(stride, i) (stride * i) ------------------ 280| 603| } 281| 603| } 282| 84| } 283| 84|} fftw_codelet_t1_64: 4102| 1|void X(codelet_t1_64) (planner *p) { 4103| 1| X(kdft_dit_register) (p, t1_64, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 4104| 1|} fftw_codelet_t1_7: 351| 1|void X(codelet_t1_7) (planner *p) { 352| 1| X(kdft_dit_register) (p, t1_7, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 353| 1|} t1_7.c:t1_7: 217| 19|{ 218| 19| DK(KP222520933, +0.222520933956314404288902564496794759466355569); ------------------ | | 1056| 19|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 19|# define K(x) ((E) x) | | ------------------ ------------------ 219| 19| DK(KP900968867, +0.900968867902419126236102319507445051165919162); ------------------ | | 1056| 19|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 19|# define K(x) ((E) x) | | ------------------ ------------------ 220| 19| DK(KP623489801, +0.623489801858733530525004884004239810632274731); ------------------ | | 1056| 19|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 19|# define K(x) ((E) x) | | ------------------ ------------------ 221| 19| DK(KP433883739, +0.433883739117558120475768332848358754609990728); ------------------ | | 1056| 19|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 19|# define K(x) ((E) x) | | ------------------ ------------------ 222| 19| DK(KP781831482, +0.781831482468029808708444526674057750232334519); ------------------ | | 1056| 19|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 19|# define K(x) ((E) x) | | ------------------ ------------------ 223| 19| DK(KP974927912, +0.974927912181823607018131682993931217232785801); ------------------ | | 1056| 19|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 19|# define K(x) ((E) x) | | ------------------ ------------------ 224| 19| { 225| 19| INT m; 226| 324| for (m = mb, W = W + (mb * 12); m < me; m = m + 1, ri = ri + ms, ii = ii + ms, W = W + 12, MAKE_VOLATILE_STRIDE(14, rs)) { ------------------ | | 851| 305| (nptr <= ESTIMATED_AVAILABLE_INDEX_REGISTERS ? \ | | ------------------ | | | | 849| 610|#define ESTIMATED_AVAILABLE_INDEX_REGISTERS 16 | | ------------------ | | | Branch (851:7): [True: 305, Folded] | | ------------------ | | 852| 305| 0 : \ | | 853| 305| ((x) = (x) ^ X(an_INT_guaranteed_to_be_zero))) | | ------------------ | | | | 78| 0|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ | Branch (226:36): [True: 305, False: 19] ------------------ 227| 305| E T1, TR, Tc, TS, TC, TO, Tn, TT, TI, TP, Ty, TU, TF, TQ; 228| 305| T1 = ri[0]; 229| 305| TR = ii[0]; 230| 305| { 231| 305| E T6, TA, Tb, TB; 232| 305| { 233| 305| E T3, T5, T2, T4; 234| 305| T3 = ri[WS(rs, 1)]; ------------------ | | 825| 305|#define WS(stride, i) (stride * i) ------------------ 235| 305| T5 = ii[WS(rs, 1)]; ------------------ | | 825| 305|#define WS(stride, i) (stride * i) ------------------ 236| 305| T2 = W[0]; 237| 305| T4 = W[1]; 238| 305| T6 = FMA(T2, T3, T4 * T5); ------------------ | | 1131| 305|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 239| 305| TA = FNMS(T4, T3, T2 * T5); ------------------ | | 1134| 305|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 240| 305| } 241| 305| { 242| 305| E T8, Ta, T7, T9; 243| 305| T8 = ri[WS(rs, 6)]; ------------------ | | 825| 305|#define WS(stride, i) (stride * i) ------------------ 244| 305| Ta = ii[WS(rs, 6)]; ------------------ | | 825| 305|#define WS(stride, i) (stride * i) ------------------ 245| 305| T7 = W[10]; 246| 305| T9 = W[11]; 247| 305| Tb = FMA(T7, T8, T9 * Ta); ------------------ | | 1131| 305|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 248| 305| TB = FNMS(T9, T8, T7 * Ta); ------------------ | | 1134| 305|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 249| 305| } 250| 305| Tc = T6 + Tb; 251| 305| TS = Tb - T6; 252| 305| TC = TA - TB; 253| 305| TO = TA + TB; 254| 305| } 255| 305| { 256| 305| E Th, TG, Tm, TH; 257| 305| { 258| 305| E Te, Tg, Td, Tf; 259| 305| Te = ri[WS(rs, 2)]; ------------------ | | 825| 305|#define WS(stride, i) (stride * i) ------------------ 260| 305| Tg = ii[WS(rs, 2)]; ------------------ | | 825| 305|#define WS(stride, i) (stride * i) ------------------ 261| 305| Td = W[2]; 262| 305| Tf = W[3]; 263| 305| Th = FMA(Td, Te, Tf * Tg); ------------------ | | 1131| 305|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 264| 305| TG = FNMS(Tf, Te, Td * Tg); ------------------ | | 1134| 305|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 265| 305| } 266| 305| { 267| 305| E Tj, Tl, Ti, Tk; 268| 305| Tj = ri[WS(rs, 5)]; ------------------ | | 825| 305|#define WS(stride, i) (stride * i) ------------------ 269| 305| Tl = ii[WS(rs, 5)]; ------------------ | | 825| 305|#define WS(stride, i) (stride * i) ------------------ 270| 305| Ti = W[8]; 271| 305| Tk = W[9]; 272| 305| Tm = FMA(Ti, Tj, Tk * Tl); ------------------ | | 1131| 305|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 273| 305| TH = FNMS(Tk, Tj, Ti * Tl); ------------------ | | 1134| 305|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 274| 305| } 275| 305| Tn = Th + Tm; 276| 305| TT = Tm - Th; 277| 305| TI = TG - TH; 278| 305| TP = TG + TH; 279| 305| } 280| 305| { 281| 305| E Ts, TD, Tx, TE; 282| 305| { 283| 305| E Tp, Tr, To, Tq; 284| 305| Tp = ri[WS(rs, 3)]; ------------------ | | 825| 305|#define WS(stride, i) (stride * i) ------------------ 285| 305| Tr = ii[WS(rs, 3)]; ------------------ | | 825| 305|#define WS(stride, i) (stride * i) ------------------ 286| 305| To = W[4]; 287| 305| Tq = W[5]; 288| 305| Ts = FMA(To, Tp, Tq * Tr); ------------------ | | 1131| 305|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 289| 305| TD = FNMS(Tq, Tp, To * Tr); ------------------ | | 1134| 305|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 290| 305| } 291| 305| { 292| 305| E Tu, Tw, Tt, Tv; 293| 305| Tu = ri[WS(rs, 4)]; ------------------ | | 825| 305|#define WS(stride, i) (stride * i) ------------------ 294| 305| Tw = ii[WS(rs, 4)]; ------------------ | | 825| 305|#define WS(stride, i) (stride * i) ------------------ 295| 305| Tt = W[6]; 296| 305| Tv = W[7]; 297| 305| Tx = FMA(Tt, Tu, Tv * Tw); ------------------ | | 1131| 305|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 298| 305| TE = FNMS(Tv, Tu, Tt * Tw); ------------------ | | 1134| 305|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 299| 305| } 300| 305| Ty = Ts + Tx; 301| 305| TU = Tx - Ts; 302| 305| TF = TD - TE; 303| 305| TQ = TD + TE; 304| 305| } 305| 305| ri[0] = T1 + Tc + Tn + Ty; 306| 305| ii[0] = TO + TP + TQ + TR; 307| 305| { 308| 305| E TJ, Tz, TX, TY; 309| 305| TJ = FNMS(KP781831482, TF, KP974927912 * TC) - (KP433883739 * TI); ------------------ | | 1134| 305|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 310| 305| Tz = FMA(KP623489801, Ty, T1) + FNMA(KP900968867, Tn, KP222520933 * Tc); ------------------ | | 1131| 305|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ Tz = FMA(KP623489801, Ty, T1) + FNMA(KP900968867, Tn, KP222520933 * Tc); ------------------ | | 1133| 305|#define FNMA(a, b, c) (- (((a) * (b)) + (c))) ------------------ 311| 305| ri[WS(rs, 5)] = Tz - TJ; ------------------ | | 825| 305|#define WS(stride, i) (stride * i) ------------------ 312| 305| ri[WS(rs, 2)] = Tz + TJ; ------------------ | | 825| 305|#define WS(stride, i) (stride * i) ------------------ 313| 305| TX = FNMS(KP781831482, TU, KP974927912 * TS) - (KP433883739 * TT); ------------------ | | 1134| 305|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 314| 305| TY = FMA(KP623489801, TQ, TR) + FNMA(KP900968867, TP, KP222520933 * TO); ------------------ | | 1131| 305|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ TY = FMA(KP623489801, TQ, TR) + FNMA(KP900968867, TP, KP222520933 * TO); ------------------ | | 1133| 305|#define FNMA(a, b, c) (- (((a) * (b)) + (c))) ------------------ 315| 305| ii[WS(rs, 2)] = TX + TY; ------------------ | | 825| 305|#define WS(stride, i) (stride * i) ------------------ 316| 305| ii[WS(rs, 5)] = TY - TX; ------------------ | | 825| 305|#define WS(stride, i) (stride * i) ------------------ 317| 305| } 318| 305| { 319| 305| E TL, TK, TV, TW; 320| 305| TL = FMA(KP781831482, TC, KP974927912 * TI) + (KP433883739 * TF); ------------------ | | 1131| 305|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 321| 305| TK = FMA(KP623489801, Tc, T1) + FNMA(KP900968867, Ty, KP222520933 * Tn); ------------------ | | 1131| 305|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ TK = FMA(KP623489801, Tc, T1) + FNMA(KP900968867, Ty, KP222520933 * Tn); ------------------ | | 1133| 305|#define FNMA(a, b, c) (- (((a) * (b)) + (c))) ------------------ 322| 305| ri[WS(rs, 6)] = TK - TL; ------------------ | | 825| 305|#define WS(stride, i) (stride * i) ------------------ 323| 305| ri[WS(rs, 1)] = TK + TL; ------------------ | | 825| 305|#define WS(stride, i) (stride * i) ------------------ 324| 305| TV = FMA(KP781831482, TS, KP974927912 * TT) + (KP433883739 * TU); ------------------ | | 1131| 305|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 325| 305| TW = FMA(KP623489801, TO, TR) + FNMA(KP900968867, TQ, KP222520933 * TP); ------------------ | | 1131| 305|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ TW = FMA(KP623489801, TO, TR) + FNMA(KP900968867, TQ, KP222520933 * TP); ------------------ | | 1133| 305|#define FNMA(a, b, c) (- (((a) * (b)) + (c))) ------------------ 326| 305| ii[WS(rs, 1)] = TV + TW; ------------------ | | 825| 305|#define WS(stride, i) (stride * i) ------------------ 327| 305| ii[WS(rs, 6)] = TW - TV; ------------------ | | 825| 305|#define WS(stride, i) (stride * i) ------------------ 328| 305| } 329| 305| { 330| 305| E TN, TM, TZ, T10; 331| 305| TN = FMA(KP433883739, TC, KP974927912 * TF) - (KP781831482 * TI); ------------------ | | 1131| 305|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 332| 305| TM = FMA(KP623489801, Tn, T1) + FNMA(KP222520933, Ty, KP900968867 * Tc); ------------------ | | 1131| 305|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ TM = FMA(KP623489801, Tn, T1) + FNMA(KP222520933, Ty, KP900968867 * Tc); ------------------ | | 1133| 305|#define FNMA(a, b, c) (- (((a) * (b)) + (c))) ------------------ 333| 305| ri[WS(rs, 4)] = TM - TN; ------------------ | | 825| 305|#define WS(stride, i) (stride * i) ------------------ 334| 305| ri[WS(rs, 3)] = TM + TN; ------------------ | | 825| 305|#define WS(stride, i) (stride * i) ------------------ 335| 305| TZ = FMA(KP433883739, TS, KP974927912 * TU) - (KP781831482 * TT); ------------------ | | 1131| 305|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 336| 305| T10 = FMA(KP623489801, TP, TR) + FNMA(KP222520933, TQ, KP900968867 * TO); ------------------ | | 1131| 305|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ T10 = FMA(KP623489801, TP, TR) + FNMA(KP222520933, TQ, KP900968867 * TO); ------------------ | | 1133| 305|#define FNMA(a, b, c) (- (((a) * (b)) + (c))) ------------------ 337| 305| ii[WS(rs, 3)] = TZ + T10; ------------------ | | 825| 305|#define WS(stride, i) (stride * i) ------------------ 338| 305| ii[WS(rs, 4)] = T10 - TZ; ------------------ | | 825| 305|#define WS(stride, i) (stride * i) ------------------ 339| 305| } 340| 305| } 341| 19| } 342| 19|} fftw_codelet_t1_8: 373| 1|void X(codelet_t1_8) (planner *p) { 374| 1| X(kdft_dit_register) (p, t1_8, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 375| 1|} fftw_codelet_t1_9: 484| 1|void X(codelet_t1_9) (planner *p) { 485| 1| X(kdft_dit_register) (p, t1_9, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 486| 1|} t1_9.c:t1_9: 272| 27|{ 273| 27| DK(KP939692620, +0.939692620785908384054109277324731469936208134); ------------------ | | 1056| 27|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 27|# define K(x) ((E) x) | | ------------------ ------------------ 274| 27| DK(KP342020143, +0.342020143325668733044099614682259580763083368); ------------------ | | 1056| 27|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 27|# define K(x) ((E) x) | | ------------------ ------------------ 275| 27| DK(KP984807753, +0.984807753012208059366743024589523013670643252); ------------------ | | 1056| 27|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 27|# define K(x) ((E) x) | | ------------------ ------------------ 276| 27| DK(KP173648177, +0.173648177666930348851716626769314796000375677); ------------------ | | 1056| 27|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 27|# define K(x) ((E) x) | | ------------------ ------------------ 277| 27| DK(KP642787609, +0.642787609686539326322643409907263432907559884); ------------------ | | 1056| 27|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 27|# define K(x) ((E) x) | | ------------------ ------------------ 278| 27| DK(KP766044443, +0.766044443118978035202392650555416673935832457); ------------------ | | 1056| 27|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 27|# define K(x) ((E) x) | | ------------------ ------------------ 279| 27| DK(KP500000000, +0.500000000000000000000000000000000000000000000); ------------------ | | 1056| 27|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 27|# define K(x) ((E) x) | | ------------------ ------------------ 280| 27| DK(KP866025403, +0.866025403784438646763723170752936183471402627); ------------------ | | 1056| 27|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 27|# define K(x) ((E) x) | | ------------------ ------------------ 281| 27| { 282| 27| INT m; 283| 456| for (m = mb, W = W + (mb * 16); m < me; m = m + 1, ri = ri + ms, ii = ii + ms, W = W + 16, MAKE_VOLATILE_STRIDE(18, rs)) { ------------------ | | 851| 429| (nptr <= ESTIMATED_AVAILABLE_INDEX_REGISTERS ? \ | | ------------------ | | | | 849| 858|#define ESTIMATED_AVAILABLE_INDEX_REGISTERS 16 | | ------------------ | | | Branch (851:7): [Folded, False: 429] | | ------------------ | | 852| 429| 0 : \ | | 853| 429| ((x) = (x) ^ X(an_INT_guaranteed_to_be_zero))) | | ------------------ | | | | 78| 429|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 429|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ | Branch (283:36): [True: 429, False: 27] ------------------ 284| 429| E T1, T1B, TQ, T1G, Tc, TN, T1A, T1H, TL, T1x, T17, T1o, T1c, T1n, Tu; 285| 429| E T1w, TW, T1k, T11, T1l; 286| 429| { 287| 429| E T6, TO, Tb, TP; 288| 429| T1 = ri[0]; 289| 429| T1B = ii[0]; 290| 429| { 291| 429| E T3, T5, T2, T4; 292| 429| T3 = ri[WS(rs, 3)]; ------------------ | | 825| 429|#define WS(stride, i) (stride * i) ------------------ 293| 429| T5 = ii[WS(rs, 3)]; ------------------ | | 825| 429|#define WS(stride, i) (stride * i) ------------------ 294| 429| T2 = W[4]; 295| 429| T4 = W[5]; 296| 429| T6 = FMA(T2, T3, T4 * T5); ------------------ | | 1131| 429|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 297| 429| TO = FNMS(T4, T3, T2 * T5); ------------------ | | 1134| 429|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 298| 429| } 299| 429| { 300| 429| E T8, Ta, T7, T9; 301| 429| T8 = ri[WS(rs, 6)]; ------------------ | | 825| 429|#define WS(stride, i) (stride * i) ------------------ 302| 429| Ta = ii[WS(rs, 6)]; ------------------ | | 825| 429|#define WS(stride, i) (stride * i) ------------------ 303| 429| T7 = W[10]; 304| 429| T9 = W[11]; 305| 429| Tb = FMA(T7, T8, T9 * Ta); ------------------ | | 1131| 429|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 306| 429| TP = FNMS(T9, T8, T7 * Ta); ------------------ | | 1134| 429|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 307| 429| } 308| 429| TQ = KP866025403 * (TO - TP); 309| 429| T1G = KP866025403 * (Tb - T6); 310| 429| Tc = T6 + Tb; 311| 429| TN = FNMS(KP500000000, Tc, T1); ------------------ | | 1134| 429|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 312| 429| T1A = TO + TP; 313| 429| T1H = FNMS(KP500000000, T1A, T1B); ------------------ | | 1134| 429|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 314| 429| } 315| 429| { 316| 429| E Tz, T19, TE, T14, TJ, T15, TK, T1a; 317| 429| { 318| 429| E Tw, Ty, Tv, Tx; 319| 429| Tw = ri[WS(rs, 2)]; ------------------ | | 825| 429|#define WS(stride, i) (stride * i) ------------------ 320| 429| Ty = ii[WS(rs, 2)]; ------------------ | | 825| 429|#define WS(stride, i) (stride * i) ------------------ 321| 429| Tv = W[2]; 322| 429| Tx = W[3]; 323| 429| Tz = FMA(Tv, Tw, Tx * Ty); ------------------ | | 1131| 429|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 324| 429| T19 = FNMS(Tx, Tw, Tv * Ty); ------------------ | | 1134| 429|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 325| 429| } 326| 429| { 327| 429| E TB, TD, TA, TC; 328| 429| TB = ri[WS(rs, 5)]; ------------------ | | 825| 429|#define WS(stride, i) (stride * i) ------------------ 329| 429| TD = ii[WS(rs, 5)]; ------------------ | | 825| 429|#define WS(stride, i) (stride * i) ------------------ 330| 429| TA = W[8]; 331| 429| TC = W[9]; 332| 429| TE = FMA(TA, TB, TC * TD); ------------------ | | 1131| 429|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 333| 429| T14 = FNMS(TC, TB, TA * TD); ------------------ | | 1134| 429|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 334| 429| } 335| 429| { 336| 429| E TG, TI, TF, TH; 337| 429| TG = ri[WS(rs, 8)]; ------------------ | | 825| 429|#define WS(stride, i) (stride * i) ------------------ 338| 429| TI = ii[WS(rs, 8)]; ------------------ | | 825| 429|#define WS(stride, i) (stride * i) ------------------ 339| 429| TF = W[14]; 340| 429| TH = W[15]; 341| 429| TJ = FMA(TF, TG, TH * TI); ------------------ | | 1131| 429|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 342| 429| T15 = FNMS(TH, TG, TF * TI); ------------------ | | 1134| 429|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 343| 429| } 344| 429| TK = TE + TJ; 345| 429| T1a = T14 + T15; 346| 429| TL = Tz + TK; 347| 429| T1x = T19 + T1a; 348| 429| { 349| 429| E T13, T16, T18, T1b; 350| 429| T13 = FNMS(KP500000000, TK, Tz); ------------------ | | 1134| 429|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 351| 429| T16 = KP866025403 * (T14 - T15); 352| 429| T17 = T13 + T16; 353| 429| T1o = T13 - T16; 354| 429| T18 = KP866025403 * (TJ - TE); 355| 429| T1b = FNMS(KP500000000, T1a, T19); ------------------ | | 1134| 429|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 356| 429| T1c = T18 + T1b; 357| 429| T1n = T1b - T18; 358| 429| } 359| 429| } 360| 429| { 361| 429| E Ti, TY, Tn, TT, Ts, TU, Tt, TZ; 362| 429| { 363| 429| E Tf, Th, Te, Tg; 364| 429| Tf = ri[WS(rs, 1)]; ------------------ | | 825| 429|#define WS(stride, i) (stride * i) ------------------ 365| 429| Th = ii[WS(rs, 1)]; ------------------ | | 825| 429|#define WS(stride, i) (stride * i) ------------------ 366| 429| Te = W[0]; 367| 429| Tg = W[1]; 368| 429| Ti = FMA(Te, Tf, Tg * Th); ------------------ | | 1131| 429|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 369| 429| TY = FNMS(Tg, Tf, Te * Th); ------------------ | | 1134| 429|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 370| 429| } 371| 429| { 372| 429| E Tk, Tm, Tj, Tl; 373| 429| Tk = ri[WS(rs, 4)]; ------------------ | | 825| 429|#define WS(stride, i) (stride * i) ------------------ 374| 429| Tm = ii[WS(rs, 4)]; ------------------ | | 825| 429|#define WS(stride, i) (stride * i) ------------------ 375| 429| Tj = W[6]; 376| 429| Tl = W[7]; 377| 429| Tn = FMA(Tj, Tk, Tl * Tm); ------------------ | | 1131| 429|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 378| 429| TT = FNMS(Tl, Tk, Tj * Tm); ------------------ | | 1134| 429|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 379| 429| } 380| 429| { 381| 429| E Tp, Tr, To, Tq; 382| 429| Tp = ri[WS(rs, 7)]; ------------------ | | 825| 429|#define WS(stride, i) (stride * i) ------------------ 383| 429| Tr = ii[WS(rs, 7)]; ------------------ | | 825| 429|#define WS(stride, i) (stride * i) ------------------ 384| 429| To = W[12]; 385| 429| Tq = W[13]; 386| 429| Ts = FMA(To, Tp, Tq * Tr); ------------------ | | 1131| 429|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 387| 429| TU = FNMS(Tq, Tp, To * Tr); ------------------ | | 1134| 429|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 388| 429| } 389| 429| Tt = Tn + Ts; 390| 429| TZ = TT + TU; 391| 429| Tu = Ti + Tt; 392| 429| T1w = TY + TZ; 393| 429| { 394| 429| E TS, TV, TX, T10; 395| 429| TS = FNMS(KP500000000, Tt, Ti); ------------------ | | 1134| 429|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 396| 429| TV = KP866025403 * (TT - TU); 397| 429| TW = TS + TV; 398| 429| T1k = TS - TV; 399| 429| TX = KP866025403 * (Ts - Tn); 400| 429| T10 = FNMS(KP500000000, TZ, TY); ------------------ | | 1134| 429|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 401| 429| T11 = TX + T10; 402| 429| T1l = T10 - TX; 403| 429| } 404| 429| } 405| 429| { 406| 429| E T1y, Td, TM, T1v; 407| 429| T1y = KP866025403 * (T1w - T1x); 408| 429| Td = T1 + Tc; 409| 429| TM = Tu + TL; 410| 429| T1v = FNMS(KP500000000, TM, Td); ------------------ | | 1134| 429|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 411| 429| ri[0] = Td + TM; 412| 429| ri[WS(rs, 3)] = T1v + T1y; ------------------ | | 825| 429|#define WS(stride, i) (stride * i) ------------------ 413| 429| ri[WS(rs, 6)] = T1v - T1y; ------------------ | | 825| 429|#define WS(stride, i) (stride * i) ------------------ 414| 429| } 415| 429| { 416| 429| E T1D, T1z, T1C, T1E; 417| 429| T1D = KP866025403 * (TL - Tu); 418| 429| T1z = T1w + T1x; 419| 429| T1C = T1A + T1B; 420| 429| T1E = FNMS(KP500000000, T1z, T1C); ------------------ | | 1134| 429|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 421| 429| ii[0] = T1z + T1C; 422| 429| ii[WS(rs, 6)] = T1E - T1D; ------------------ | | 825| 429|#define WS(stride, i) (stride * i) ------------------ 423| 429| ii[WS(rs, 3)] = T1D + T1E; ------------------ | | 825| 429|#define WS(stride, i) (stride * i) ------------------ 424| 429| } 425| 429| { 426| 429| E TR, T1I, T1e, T1J, T1i, T1F, T1f, T1K; 427| 429| TR = TN + TQ; 428| 429| T1I = T1G + T1H; 429| 429| { 430| 429| E T12, T1d, T1g, T1h; 431| 429| T12 = FMA(KP766044443, TW, KP642787609 * T11); ------------------ | | 1131| 429|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 432| 429| T1d = FMA(KP173648177, T17, KP984807753 * T1c); ------------------ | | 1131| 429|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 433| 429| T1e = T12 + T1d; 434| 429| T1J = KP866025403 * (T1d - T12); 435| 429| T1g = FNMS(KP642787609, TW, KP766044443 * T11); ------------------ | | 1134| 429|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 436| 429| T1h = FNMS(KP984807753, T17, KP173648177 * T1c); ------------------ | | 1134| 429|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 437| 429| T1i = KP866025403 * (T1g - T1h); 438| 429| T1F = T1g + T1h; 439| 429| } 440| 429| ri[WS(rs, 1)] = TR + T1e; ------------------ | | 825| 429|#define WS(stride, i) (stride * i) ------------------ 441| 429| ii[WS(rs, 1)] = T1F + T1I; ------------------ | | 825| 429|#define WS(stride, i) (stride * i) ------------------ 442| 429| T1f = FNMS(KP500000000, T1e, TR); ------------------ | | 1134| 429|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 443| 429| ri[WS(rs, 7)] = T1f - T1i; ------------------ | | 825| 429|#define WS(stride, i) (stride * i) ------------------ 444| 429| ri[WS(rs, 4)] = T1f + T1i; ------------------ | | 825| 429|#define WS(stride, i) (stride * i) ------------------ 445| 429| T1K = FNMS(KP500000000, T1F, T1I); ------------------ | | 1134| 429|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 446| 429| ii[WS(rs, 4)] = T1J + T1K; ------------------ | | 825| 429|#define WS(stride, i) (stride * i) ------------------ 447| 429| ii[WS(rs, 7)] = T1K - T1J; ------------------ | | 825| 429|#define WS(stride, i) (stride * i) ------------------ 448| 429| } 449| 429| { 450| 429| E T1j, T1M, T1q, T1N, T1u, T1L, T1r, T1O; 451| 429| T1j = TN - TQ; 452| 429| T1M = T1H - T1G; 453| 429| { 454| 429| E T1m, T1p, T1s, T1t; 455| 429| T1m = FMA(KP173648177, T1k, KP984807753 * T1l); ------------------ | | 1131| 429|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 456| 429| T1p = FNMS(KP939692620, T1o, KP342020143 * T1n); ------------------ | | 1134| 429|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 457| 429| T1q = T1m + T1p; 458| 429| T1N = KP866025403 * (T1p - T1m); 459| 429| T1s = FNMS(KP984807753, T1k, KP173648177 * T1l); ------------------ | | 1134| 429|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 460| 429| T1t = FMA(KP342020143, T1o, KP939692620 * T1n); ------------------ | | 1131| 429|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 461| 429| T1u = KP866025403 * (T1s + T1t); 462| 429| T1L = T1s - T1t; 463| 429| } 464| 429| ri[WS(rs, 2)] = T1j + T1q; ------------------ | | 825| 429|#define WS(stride, i) (stride * i) ------------------ 465| 429| ii[WS(rs, 2)] = T1L + T1M; ------------------ | | 825| 429|#define WS(stride, i) (stride * i) ------------------ 466| 429| T1r = FNMS(KP500000000, T1q, T1j); ------------------ | | 1134| 429|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 467| 429| ri[WS(rs, 8)] = T1r - T1u; ------------------ | | 825| 429|#define WS(stride, i) (stride * i) ------------------ 468| 429| ri[WS(rs, 5)] = T1r + T1u; ------------------ | | 825| 429|#define WS(stride, i) (stride * i) ------------------ 469| 429| T1O = FNMS(KP500000000, T1L, T1M); ------------------ | | 1134| 429|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 470| 429| ii[WS(rs, 5)] = T1N + T1O; ------------------ | | 825| 429|#define WS(stride, i) (stride * i) ------------------ 471| 429| ii[WS(rs, 8)] = T1O - T1N; ------------------ | | 825| 429|#define WS(stride, i) (stride * i) ------------------ 472| 429| } 473| 429| } 474| 27| } 475| 27|} fftw_codelet_t2_10: 506| 1|void X(codelet_t2_10) (planner *p) { 507| 1| X(kdft_dit_register) (p, t2_10, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 508| 1|} t2_10.c:t2_10: 299| 28|{ 300| 28| DK(KP587785252, +0.587785252292473129168705954639072768597652438); ------------------ | | 1056| 28|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 28|# define K(x) ((E) x) | | ------------------ ------------------ 301| 28| DK(KP951056516, +0.951056516295153572116439333379382143405698634); ------------------ | | 1056| 28|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 28|# define K(x) ((E) x) | | ------------------ ------------------ 302| 28| DK(KP250000000, +0.250000000000000000000000000000000000000000000); ------------------ | | 1056| 28|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 28|# define K(x) ((E) x) | | ------------------ ------------------ 303| 28| DK(KP559016994, +0.559016994374947424102293417182819058860154590); ------------------ | | 1056| 28|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 28|# define K(x) ((E) x) | | ------------------ ------------------ 304| 28| { 305| 28| INT m; 306| 450| for (m = mb, W = W + (mb * 6); m < me; m = m + 1, ri = ri + ms, ii = ii + ms, W = W + 6, MAKE_VOLATILE_STRIDE(20, rs)) { ------------------ | | 851| 422| (nptr <= ESTIMATED_AVAILABLE_INDEX_REGISTERS ? \ | | ------------------ | | | | 849| 844|#define ESTIMATED_AVAILABLE_INDEX_REGISTERS 16 | | ------------------ | | | Branch (851:7): [Folded, False: 422] | | ------------------ | | 852| 422| 0 : \ | | 853| 422| ((x) = (x) ^ X(an_INT_guaranteed_to_be_zero))) | | ------------------ | | | | 78| 422|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 422|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ | Branch (306:35): [True: 422, False: 28] ------------------ 307| 422| E T2, T5, T3, T6, T8, Tm, Tc, Tk, T9, Td, Te, TM, TO, Tg, Tp; 308| 422| E Tv, Tx, Tr; 309| 422| { 310| 422| E T4, Tb, T7, Ta; 311| 422| T2 = W[0]; 312| 422| T5 = W[1]; 313| 422| T3 = W[2]; 314| 422| T6 = W[3]; 315| 422| T4 = T2 * T3; 316| 422| Tb = T5 * T3; 317| 422| T7 = T5 * T6; 318| 422| Ta = T2 * T6; 319| 422| T8 = T4 - T7; 320| 422| Tm = Ta - Tb; 321| 422| Tc = Ta + Tb; 322| 422| Tk = T4 + T7; 323| 422| T9 = W[4]; 324| 422| Td = W[5]; 325| 422| Te = FMA(T8, T9, Tc * Td); ------------------ | | 1131| 422|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 326| 422| TM = FMA(T3, T9, T6 * Td); ------------------ | | 1131| 422|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 327| 422| TO = FNMS(T6, T9, T3 * Td); ------------------ | | 1134| 422|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 328| 422| Tg = FNMS(Tc, T9, T8 * Td); ------------------ | | 1134| 422|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 329| 422| Tp = FMA(Tk, T9, Tm * Td); ------------------ | | 1131| 422|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 330| 422| Tv = FMA(T2, T9, T5 * Td); ------------------ | | 1131| 422|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 331| 422| Tx = FNMS(T5, T9, T2 * Td); ------------------ | | 1134| 422|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 332| 422| Tr = FNMS(Tm, T9, Tk * Td); ------------------ | | 1134| 422|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 333| 422| } 334| 422| { 335| 422| E Tj, T1S, TX, T1G, TL, TU, TV, T1s, T1t, T1C, T11, T12, T13, T1h, T1k; 336| 422| E T1Q, Tu, TD, TE, T1v, T1w, T1B, TY, TZ, T10, T1a, T1d, T1P; 337| 422| { 338| 422| E T1, T1F, Ti, T1E, Tf, Th; 339| 422| T1 = ri[0]; 340| 422| T1F = ii[0]; 341| 422| Tf = ri[WS(rs, 5)]; ------------------ | | 825| 422|#define WS(stride, i) (stride * i) ------------------ 342| 422| Th = ii[WS(rs, 5)]; ------------------ | | 825| 422|#define WS(stride, i) (stride * i) ------------------ 343| 422| Ti = FMA(Te, Tf, Tg * Th); ------------------ | | 1131| 422|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 344| 422| T1E = FNMS(Tg, Tf, Te * Th); ------------------ | | 1134| 422|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 345| 422| Tj = T1 - Ti; 346| 422| T1S = T1F - T1E; 347| 422| TX = T1 + Ti; 348| 422| T1G = T1E + T1F; 349| 422| } 350| 422| { 351| 422| E TH, T1f, TT, T1j, TK, T1g, TQ, T1i; 352| 422| { 353| 422| E TF, TG, TR, TS; 354| 422| TF = ri[WS(rs, 4)]; ------------------ | | 825| 422|#define WS(stride, i) (stride * i) ------------------ 355| 422| TG = ii[WS(rs, 4)]; ------------------ | | 825| 422|#define WS(stride, i) (stride * i) ------------------ 356| 422| TH = FMA(T8, TF, Tc * TG); ------------------ | | 1131| 422|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 357| 422| T1f = FNMS(Tc, TF, T8 * TG); ------------------ | | 1134| 422|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 358| 422| TR = ri[WS(rs, 1)]; ------------------ | | 825| 422|#define WS(stride, i) (stride * i) ------------------ 359| 422| TS = ii[WS(rs, 1)]; ------------------ | | 825| 422|#define WS(stride, i) (stride * i) ------------------ 360| 422| TT = FMA(T2, TR, T5 * TS); ------------------ | | 1131| 422|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 361| 422| T1j = FNMS(T5, TR, T2 * TS); ------------------ | | 1134| 422|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 362| 422| } 363| 422| { 364| 422| E TI, TJ, TN, TP; 365| 422| TI = ri[WS(rs, 9)]; ------------------ | | 825| 422|#define WS(stride, i) (stride * i) ------------------ 366| 422| TJ = ii[WS(rs, 9)]; ------------------ | | 825| 422|#define WS(stride, i) (stride * i) ------------------ 367| 422| TK = FMA(T9, TI, Td * TJ); ------------------ | | 1131| 422|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 368| 422| T1g = FNMS(Td, TI, T9 * TJ); ------------------ | | 1134| 422|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 369| 422| TN = ri[WS(rs, 6)]; ------------------ | | 825| 422|#define WS(stride, i) (stride * i) ------------------ 370| 422| TP = ii[WS(rs, 6)]; ------------------ | | 825| 422|#define WS(stride, i) (stride * i) ------------------ 371| 422| TQ = FMA(TM, TN, TO * TP); ------------------ | | 1131| 422|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 372| 422| T1i = FNMS(TO, TN, TM * TP); ------------------ | | 1134| 422|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 373| 422| } 374| 422| TL = TH - TK; 375| 422| TU = TQ - TT; 376| 422| TV = TL + TU; 377| 422| T1s = T1f + T1g; 378| 422| T1t = T1i + T1j; 379| 422| T1C = T1s + T1t; 380| 422| T11 = TH + TK; 381| 422| T12 = TQ + TT; 382| 422| T13 = T11 + T12; 383| 422| T1h = T1f - T1g; 384| 422| T1k = T1i - T1j; 385| 422| T1Q = T1h + T1k; 386| 422| } 387| 422| { 388| 422| E To, T18, TC, T1c, Tt, T19, Tz, T1b; 389| 422| { 390| 422| E Tl, Tn, TA, TB; 391| 422| Tl = ri[WS(rs, 2)]; ------------------ | | 825| 422|#define WS(stride, i) (stride * i) ------------------ 392| 422| Tn = ii[WS(rs, 2)]; ------------------ | | 825| 422|#define WS(stride, i) (stride * i) ------------------ 393| 422| To = FMA(Tk, Tl, Tm * Tn); ------------------ | | 1131| 422|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 394| 422| T18 = FNMS(Tm, Tl, Tk * Tn); ------------------ | | 1134| 422|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 395| 422| TA = ri[WS(rs, 3)]; ------------------ | | 825| 422|#define WS(stride, i) (stride * i) ------------------ 396| 422| TB = ii[WS(rs, 3)]; ------------------ | | 825| 422|#define WS(stride, i) (stride * i) ------------------ 397| 422| TC = FMA(T3, TA, T6 * TB); ------------------ | | 1131| 422|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 398| 422| T1c = FNMS(T6, TA, T3 * TB); ------------------ | | 1134| 422|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 399| 422| } 400| 422| { 401| 422| E Tq, Ts, Tw, Ty; 402| 422| Tq = ri[WS(rs, 7)]; ------------------ | | 825| 422|#define WS(stride, i) (stride * i) ------------------ 403| 422| Ts = ii[WS(rs, 7)]; ------------------ | | 825| 422|#define WS(stride, i) (stride * i) ------------------ 404| 422| Tt = FMA(Tp, Tq, Tr * Ts); ------------------ | | 1131| 422|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 405| 422| T19 = FNMS(Tr, Tq, Tp * Ts); ------------------ | | 1134| 422|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 406| 422| Tw = ri[WS(rs, 8)]; ------------------ | | 825| 422|#define WS(stride, i) (stride * i) ------------------ 407| 422| Ty = ii[WS(rs, 8)]; ------------------ | | 825| 422|#define WS(stride, i) (stride * i) ------------------ 408| 422| Tz = FMA(Tv, Tw, Tx * Ty); ------------------ | | 1131| 422|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 409| 422| T1b = FNMS(Tx, Tw, Tv * Ty); ------------------ | | 1134| 422|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 410| 422| } 411| 422| Tu = To - Tt; 412| 422| TD = Tz - TC; 413| 422| TE = Tu + TD; 414| 422| T1v = T18 + T19; 415| 422| T1w = T1b + T1c; 416| 422| T1B = T1v + T1w; 417| 422| TY = To + Tt; 418| 422| TZ = Tz + TC; 419| 422| T10 = TY + TZ; 420| 422| T1a = T18 - T19; 421| 422| T1d = T1b - T1c; 422| 422| T1P = T1a + T1d; 423| 422| } 424| 422| { 425| 422| E T15, TW, T16, T1m, T1o, T1e, T1l, T1n, T17; 426| 422| T15 = KP559016994 * (TE - TV); 427| 422| TW = TE + TV; 428| 422| T16 = FNMS(KP250000000, TW, Tj); ------------------ | | 1134| 422|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 429| 422| T1e = T1a - T1d; 430| 422| T1l = T1h - T1k; 431| 422| T1m = FMA(KP951056516, T1e, KP587785252 * T1l); ------------------ | | 1131| 422|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 432| 422| T1o = FNMS(KP587785252, T1e, KP951056516 * T1l); ------------------ | | 1134| 422|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 433| 422| ri[WS(rs, 5)] = Tj + TW; ------------------ | | 825| 422|#define WS(stride, i) (stride * i) ------------------ 434| 422| T1n = T16 - T15; 435| 422| ri[WS(rs, 7)] = T1n - T1o; ------------------ | | 825| 422|#define WS(stride, i) (stride * i) ------------------ 436| 422| ri[WS(rs, 3)] = T1n + T1o; ------------------ | | 825| 422|#define WS(stride, i) (stride * i) ------------------ 437| 422| T17 = T15 + T16; 438| 422| ri[WS(rs, 9)] = T17 - T1m; ------------------ | | 825| 422|#define WS(stride, i) (stride * i) ------------------ 439| 422| ri[WS(rs, 1)] = T17 + T1m; ------------------ | | 825| 422|#define WS(stride, i) (stride * i) ------------------ 440| 422| } 441| 422| { 442| 422| E T1R, T1T, T1U, T1Y, T20, T1W, T1X, T1Z, T1V; 443| 422| T1R = KP559016994 * (T1P - T1Q); 444| 422| T1T = T1P + T1Q; 445| 422| T1U = FNMS(KP250000000, T1T, T1S); ------------------ | | 1134| 422|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 446| 422| T1W = Tu - TD; 447| 422| T1X = TL - TU; 448| 422| T1Y = FMA(KP951056516, T1W, KP587785252 * T1X); ------------------ | | 1131| 422|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 449| 422| T20 = FNMS(KP587785252, T1W, KP951056516 * T1X); ------------------ | | 1134| 422|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 450| 422| ii[WS(rs, 5)] = T1T + T1S; ------------------ | | 825| 422|#define WS(stride, i) (stride * i) ------------------ 451| 422| T1Z = T1U - T1R; 452| 422| ii[WS(rs, 3)] = T1Z - T20; ------------------ | | 825| 422|#define WS(stride, i) (stride * i) ------------------ 453| 422| ii[WS(rs, 7)] = T20 + T1Z; ------------------ | | 825| 422|#define WS(stride, i) (stride * i) ------------------ 454| 422| T1V = T1R + T1U; 455| 422| ii[WS(rs, 1)] = T1V - T1Y; ------------------ | | 825| 422|#define WS(stride, i) (stride * i) ------------------ 456| 422| ii[WS(rs, 9)] = T1Y + T1V; ------------------ | | 825| 422|#define WS(stride, i) (stride * i) ------------------ 457| 422| } 458| 422| { 459| 422| E T1q, T14, T1p, T1y, T1A, T1u, T1x, T1z, T1r; 460| 422| T1q = KP559016994 * (T10 - T13); 461| 422| T14 = T10 + T13; 462| 422| T1p = FNMS(KP250000000, T14, TX); ------------------ | | 1134| 422|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 463| 422| T1u = T1s - T1t; 464| 422| T1x = T1v - T1w; 465| 422| T1y = FNMS(KP587785252, T1x, KP951056516 * T1u); ------------------ | | 1134| 422|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 466| 422| T1A = FMA(KP951056516, T1x, KP587785252 * T1u); ------------------ | | 1131| 422|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 467| 422| ri[0] = TX + T14; 468| 422| T1z = T1q + T1p; 469| 422| ri[WS(rs, 4)] = T1z - T1A; ------------------ | | 825| 422|#define WS(stride, i) (stride * i) ------------------ 470| 422| ri[WS(rs, 6)] = T1z + T1A; ------------------ | | 825| 422|#define WS(stride, i) (stride * i) ------------------ 471| 422| T1r = T1p - T1q; 472| 422| ri[WS(rs, 2)] = T1r - T1y; ------------------ | | 825| 422|#define WS(stride, i) (stride * i) ------------------ 473| 422| ri[WS(rs, 8)] = T1r + T1y; ------------------ | | 825| 422|#define WS(stride, i) (stride * i) ------------------ 474| 422| } 475| 422| { 476| 422| E T1L, T1D, T1K, T1J, T1N, T1H, T1I, T1O, T1M; 477| 422| T1L = KP559016994 * (T1B - T1C); 478| 422| T1D = T1B + T1C; 479| 422| T1K = FNMS(KP250000000, T1D, T1G); ------------------ | | 1134| 422|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 480| 422| T1H = T11 - T12; 481| 422| T1I = TY - TZ; 482| 422| T1J = FNMS(KP587785252, T1I, KP951056516 * T1H); ------------------ | | 1134| 422|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 483| 422| T1N = FMA(KP951056516, T1I, KP587785252 * T1H); ------------------ | | 1131| 422|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 484| 422| ii[0] = T1D + T1G; 485| 422| T1O = T1L + T1K; 486| 422| ii[WS(rs, 4)] = T1N + T1O; ------------------ | | 825| 422|#define WS(stride, i) (stride * i) ------------------ 487| 422| ii[WS(rs, 6)] = T1O - T1N; ------------------ | | 825| 422|#define WS(stride, i) (stride * i) ------------------ 488| 422| T1M = T1K - T1L; 489| 422| ii[WS(rs, 2)] = T1J + T1M; ------------------ | | 825| 422|#define WS(stride, i) (stride * i) ------------------ 490| 422| ii[WS(rs, 8)] = T1M - T1J; ------------------ | | 825| 422|#define WS(stride, i) (stride * i) ------------------ 491| 422| } 492| 422| } 493| 422| } 494| 28| } 495| 28|} fftw_codelet_t2_16: 833| 1|void X(codelet_t2_16) (planner *p) { 834| 1| X(kdft_dit_register) (p, t2_16, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 835| 1|} t2_16.c:t2_16: 456| 19|{ 457| 19| DK(KP382683432, +0.382683432365089771728459984030398866761344562); ------------------ | | 1056| 19|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 19|# define K(x) ((E) x) | | ------------------ ------------------ 458| 19| DK(KP923879532, +0.923879532511286756128183189396788286822416626); ------------------ | | 1056| 19|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 19|# define K(x) ((E) x) | | ------------------ ------------------ 459| 19| DK(KP707106781, +0.707106781186547524400844362104849039284835938); ------------------ | | 1056| 19|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 19|# define K(x) ((E) x) | | ------------------ ------------------ 460| 19| { 461| 19| INT m; 462| 339| for (m = mb, W = W + (mb * 8); m < me; m = m + 1, ri = ri + ms, ii = ii + ms, W = W + 8, MAKE_VOLATILE_STRIDE(32, rs)) { ------------------ | | 851| 320| (nptr <= ESTIMATED_AVAILABLE_INDEX_REGISTERS ? \ | | ------------------ | | | | 849| 640|#define ESTIMATED_AVAILABLE_INDEX_REGISTERS 16 | | ------------------ | | | Branch (851:7): [Folded, False: 320] | | ------------------ | | 852| 320| 0 : \ | | 853| 320| ((x) = (x) ^ X(an_INT_guaranteed_to_be_zero))) | | ------------------ | | | | 78| 320|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 320|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ | Branch (462:35): [True: 320, False: 19] ------------------ 463| 320| E T2, T5, Tg, Ti, Tk, To, TE, TC, T6, T3, T8, TW, TJ, Tt, TU; 464| 320| E Tc, Tx, TH, TN, TO, TP, TR, T1f, T1k, T1b, T1i, T1y, T1H, T1u, T1F; 465| 320| { 466| 320| E T7, Tv, Ta, Ts, T4, Tw, Tb, Tr; 467| 320| { 468| 320| E Th, Tn, Tj, Tm; 469| 320| T2 = W[0]; 470| 320| T5 = W[1]; 471| 320| Tg = W[2]; 472| 320| Ti = W[3]; 473| 320| Th = T2 * Tg; 474| 320| Tn = T5 * Tg; 475| 320| Tj = T5 * Ti; 476| 320| Tm = T2 * Ti; 477| 320| Tk = Th - Tj; 478| 320| To = Tm + Tn; 479| 320| TE = Tm - Tn; 480| 320| TC = Th + Tj; 481| 320| T6 = W[5]; 482| 320| T7 = T5 * T6; 483| 320| Tv = Tg * T6; 484| 320| Ta = T2 * T6; 485| 320| Ts = Ti * T6; 486| 320| T3 = W[4]; 487| 320| T4 = T2 * T3; 488| 320| Tw = Ti * T3; 489| 320| Tb = T5 * T3; 490| 320| Tr = Tg * T3; 491| 320| } 492| 320| T8 = T4 + T7; 493| 320| TW = Tv - Tw; 494| 320| TJ = Ta + Tb; 495| 320| Tt = Tr - Ts; 496| 320| TU = Tr + Ts; 497| 320| Tc = Ta - Tb; 498| 320| Tx = Tv + Tw; 499| 320| TH = T4 - T7; 500| 320| TN = W[6]; 501| 320| TO = W[7]; 502| 320| TP = FMA(T2, TN, T5 * TO); ------------------ | | 1131| 320|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 503| 320| TR = FNMS(T5, TN, T2 * TO); ------------------ | | 1134| 320|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 504| 320| { 505| 320| E T1d, T1e, T19, T1a; 506| 320| T1d = Tk * T6; 507| 320| T1e = To * T3; 508| 320| T1f = T1d - T1e; 509| 320| T1k = T1d + T1e; 510| 320| T19 = Tk * T3; 511| 320| T1a = To * T6; 512| 320| T1b = T19 + T1a; 513| 320| T1i = T19 - T1a; 514| 320| } 515| 320| { 516| 320| E T1w, T1x, T1s, T1t; 517| 320| T1w = TC * T6; 518| 320| T1x = TE * T3; 519| 320| T1y = T1w - T1x; 520| 320| T1H = T1w + T1x; 521| 320| T1s = TC * T3; 522| 320| T1t = TE * T6; 523| 320| T1u = T1s + T1t; 524| 320| T1F = T1s - T1t; 525| 320| } 526| 320| } 527| 320| { 528| 320| E Tf, T3r, T1N, T3e, TA, T3s, T1Q, T3b, TM, T2M, T1W, T2w, TZ, T2N, T21; 529| 320| E T2x, T1B, T1K, T2V, T2W, T2X, T2Y, T2j, T2D, T2o, T2E, T18, T1n, T2Q, T2R; 530| 320| E T2S, T2T, T28, T2A, T2d, T2B; 531| 320| { 532| 320| E T1, T3d, Te, T3c, T9, Td; 533| 320| T1 = ri[0]; 534| 320| T3d = ii[0]; 535| 320| T9 = ri[WS(rs, 8)]; ------------------ | | 825| 320|#define WS(stride, i) (stride * i) ------------------ 536| 320| Td = ii[WS(rs, 8)]; ------------------ | | 825| 320|#define WS(stride, i) (stride * i) ------------------ 537| 320| Te = FMA(T8, T9, Tc * Td); ------------------ | | 1131| 320|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 538| 320| T3c = FNMS(Tc, T9, T8 * Td); ------------------ | | 1134| 320|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 539| 320| Tf = T1 + Te; 540| 320| T3r = T3d - T3c; 541| 320| T1N = T1 - Te; 542| 320| T3e = T3c + T3d; 543| 320| } 544| 320| { 545| 320| E Tq, T1O, Tz, T1P; 546| 320| { 547| 320| E Tl, Tp, Tu, Ty; 548| 320| Tl = ri[WS(rs, 4)]; ------------------ | | 825| 320|#define WS(stride, i) (stride * i) ------------------ 549| 320| Tp = ii[WS(rs, 4)]; ------------------ | | 825| 320|#define WS(stride, i) (stride * i) ------------------ 550| 320| Tq = FMA(Tk, Tl, To * Tp); ------------------ | | 1131| 320|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 551| 320| T1O = FNMS(To, Tl, Tk * Tp); ------------------ | | 1134| 320|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 552| 320| Tu = ri[WS(rs, 12)]; ------------------ | | 825| 320|#define WS(stride, i) (stride * i) ------------------ 553| 320| Ty = ii[WS(rs, 12)]; ------------------ | | 825| 320|#define WS(stride, i) (stride * i) ------------------ 554| 320| Tz = FMA(Tt, Tu, Tx * Ty); ------------------ | | 1131| 320|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 555| 320| T1P = FNMS(Tx, Tu, Tt * Ty); ------------------ | | 1134| 320|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 556| 320| } 557| 320| TA = Tq + Tz; 558| 320| T3s = Tq - Tz; 559| 320| T1Q = T1O - T1P; 560| 320| T3b = T1O + T1P; 561| 320| } 562| 320| { 563| 320| E TG, T1S, TL, T1T, T1U, T1V; 564| 320| { 565| 320| E TD, TF, TI, TK; 566| 320| TD = ri[WS(rs, 2)]; ------------------ | | 825| 320|#define WS(stride, i) (stride * i) ------------------ 567| 320| TF = ii[WS(rs, 2)]; ------------------ | | 825| 320|#define WS(stride, i) (stride * i) ------------------ 568| 320| TG = FMA(TC, TD, TE * TF); ------------------ | | 1131| 320|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 569| 320| T1S = FNMS(TE, TD, TC * TF); ------------------ | | 1134| 320|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 570| 320| TI = ri[WS(rs, 10)]; ------------------ | | 825| 320|#define WS(stride, i) (stride * i) ------------------ 571| 320| TK = ii[WS(rs, 10)]; ------------------ | | 825| 320|#define WS(stride, i) (stride * i) ------------------ 572| 320| TL = FMA(TH, TI, TJ * TK); ------------------ | | 1131| 320|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 573| 320| T1T = FNMS(TJ, TI, TH * TK); ------------------ | | 1134| 320|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 574| 320| } 575| 320| TM = TG + TL; 576| 320| T2M = T1S + T1T; 577| 320| T1U = T1S - T1T; 578| 320| T1V = TG - TL; 579| 320| T1W = T1U - T1V; 580| 320| T2w = T1V + T1U; 581| 320| } 582| 320| { 583| 320| E TT, T1Y, TY, T1Z, T1X, T20; 584| 320| { 585| 320| E TQ, TS, TV, TX; 586| 320| TQ = ri[WS(rs, 14)]; ------------------ | | 825| 320|#define WS(stride, i) (stride * i) ------------------ 587| 320| TS = ii[WS(rs, 14)]; ------------------ | | 825| 320|#define WS(stride, i) (stride * i) ------------------ 588| 320| TT = FMA(TP, TQ, TR * TS); ------------------ | | 1131| 320|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 589| 320| T1Y = FNMS(TR, TQ, TP * TS); ------------------ | | 1134| 320|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 590| 320| TV = ri[WS(rs, 6)]; ------------------ | | 825| 320|#define WS(stride, i) (stride * i) ------------------ 591| 320| TX = ii[WS(rs, 6)]; ------------------ | | 825| 320|#define WS(stride, i) (stride * i) ------------------ 592| 320| TY = FMA(TU, TV, TW * TX); ------------------ | | 1131| 320|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 593| 320| T1Z = FNMS(TW, TV, TU * TX); ------------------ | | 1134| 320|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 594| 320| } 595| 320| TZ = TT + TY; 596| 320| T2N = T1Y + T1Z; 597| 320| T1X = TT - TY; 598| 320| T20 = T1Y - T1Z; 599| 320| T21 = T1X + T20; 600| 320| T2x = T1X - T20; 601| 320| } 602| 320| { 603| 320| E T1r, T2k, T1J, T2h, T1A, T2l, T1E, T2g; 604| 320| { 605| 320| E T1p, T1q, T1G, T1I; 606| 320| T1p = ri[WS(rs, 15)]; ------------------ | | 825| 320|#define WS(stride, i) (stride * i) ------------------ 607| 320| T1q = ii[WS(rs, 15)]; ------------------ | | 825| 320|#define WS(stride, i) (stride * i) ------------------ 608| 320| T1r = FMA(TN, T1p, TO * T1q); ------------------ | | 1131| 320|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 609| 320| T2k = FNMS(TO, T1p, TN * T1q); ------------------ | | 1134| 320|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 610| 320| T1G = ri[WS(rs, 11)]; ------------------ | | 825| 320|#define WS(stride, i) (stride * i) ------------------ 611| 320| T1I = ii[WS(rs, 11)]; ------------------ | | 825| 320|#define WS(stride, i) (stride * i) ------------------ 612| 320| T1J = FMA(T1F, T1G, T1H * T1I); ------------------ | | 1131| 320|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 613| 320| T2h = FNMS(T1H, T1G, T1F * T1I); ------------------ | | 1134| 320|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 614| 320| } 615| 320| { 616| 320| E T1v, T1z, T1C, T1D; 617| 320| T1v = ri[WS(rs, 7)]; ------------------ | | 825| 320|#define WS(stride, i) (stride * i) ------------------ 618| 320| T1z = ii[WS(rs, 7)]; ------------------ | | 825| 320|#define WS(stride, i) (stride * i) ------------------ 619| 320| T1A = FMA(T1u, T1v, T1y * T1z); ------------------ | | 1131| 320|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 620| 320| T2l = FNMS(T1y, T1v, T1u * T1z); ------------------ | | 1134| 320|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 621| 320| T1C = ri[WS(rs, 3)]; ------------------ | | 825| 320|#define WS(stride, i) (stride * i) ------------------ 622| 320| T1D = ii[WS(rs, 3)]; ------------------ | | 825| 320|#define WS(stride, i) (stride * i) ------------------ 623| 320| T1E = FMA(Tg, T1C, Ti * T1D); ------------------ | | 1131| 320|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 624| 320| T2g = FNMS(Ti, T1C, Tg * T1D); ------------------ | | 1134| 320|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 625| 320| } 626| 320| T1B = T1r + T1A; 627| 320| T1K = T1E + T1J; 628| 320| T2V = T1B - T1K; 629| 320| T2W = T2k + T2l; 630| 320| T2X = T2g + T2h; 631| 320| T2Y = T2W - T2X; 632| 320| { 633| 320| E T2f, T2i, T2m, T2n; 634| 320| T2f = T1r - T1A; 635| 320| T2i = T2g - T2h; 636| 320| T2j = T2f - T2i; 637| 320| T2D = T2f + T2i; 638| 320| T2m = T2k - T2l; 639| 320| T2n = T1E - T1J; 640| 320| T2o = T2m + T2n; 641| 320| T2E = T2m - T2n; 642| 320| } 643| 320| } 644| 320| { 645| 320| E T14, T24, T1m, T2b, T17, T25, T1h, T2a; 646| 320| { 647| 320| E T12, T13, T1j, T1l; 648| 320| T12 = ri[WS(rs, 1)]; ------------------ | | 825| 320|#define WS(stride, i) (stride * i) ------------------ 649| 320| T13 = ii[WS(rs, 1)]; ------------------ | | 825| 320|#define WS(stride, i) (stride * i) ------------------ 650| 320| T14 = FMA(T2, T12, T5 * T13); ------------------ | | 1131| 320|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 651| 320| T24 = FNMS(T5, T12, T2 * T13); ------------------ | | 1134| 320|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 652| 320| T1j = ri[WS(rs, 13)]; ------------------ | | 825| 320|#define WS(stride, i) (stride * i) ------------------ 653| 320| T1l = ii[WS(rs, 13)]; ------------------ | | 825| 320|#define WS(stride, i) (stride * i) ------------------ 654| 320| T1m = FMA(T1i, T1j, T1k * T1l); ------------------ | | 1131| 320|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 655| 320| T2b = FNMS(T1k, T1j, T1i * T1l); ------------------ | | 1134| 320|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 656| 320| } 657| 320| { 658| 320| E T15, T16, T1c, T1g; 659| 320| T15 = ri[WS(rs, 9)]; ------------------ | | 825| 320|#define WS(stride, i) (stride * i) ------------------ 660| 320| T16 = ii[WS(rs, 9)]; ------------------ | | 825| 320|#define WS(stride, i) (stride * i) ------------------ 661| 320| T17 = FMA(T3, T15, T6 * T16); ------------------ | | 1131| 320|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 662| 320| T25 = FNMS(T6, T15, T3 * T16); ------------------ | | 1134| 320|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 663| 320| T1c = ri[WS(rs, 5)]; ------------------ | | 825| 320|#define WS(stride, i) (stride * i) ------------------ 664| 320| T1g = ii[WS(rs, 5)]; ------------------ | | 825| 320|#define WS(stride, i) (stride * i) ------------------ 665| 320| T1h = FMA(T1b, T1c, T1f * T1g); ------------------ | | 1131| 320|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 666| 320| T2a = FNMS(T1f, T1c, T1b * T1g); ------------------ | | 1134| 320|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 667| 320| } 668| 320| T18 = T14 + T17; 669| 320| T1n = T1h + T1m; 670| 320| T2Q = T18 - T1n; 671| 320| T2R = T24 + T25; 672| 320| T2S = T2a + T2b; 673| 320| T2T = T2R - T2S; 674| 320| { 675| 320| E T26, T27, T29, T2c; 676| 320| T26 = T24 - T25; 677| 320| T27 = T1h - T1m; 678| 320| T28 = T26 + T27; 679| 320| T2A = T26 - T27; 680| 320| T29 = T14 - T17; 681| 320| T2c = T2a - T2b; 682| 320| T2d = T29 - T2c; 683| 320| T2B = T29 + T2c; 684| 320| } 685| 320| } 686| 320| { 687| 320| E T23, T2r, T3A, T3C, T2q, T3B, T2u, T3x; 688| 320| { 689| 320| E T1R, T22, T3y, T3z; 690| 320| T1R = T1N - T1Q; 691| 320| T22 = KP707106781 * (T1W - T21); 692| 320| T23 = T1R + T22; 693| 320| T2r = T1R - T22; 694| 320| T3y = KP707106781 * (T2x - T2w); 695| 320| T3z = T3s + T3r; 696| 320| T3A = T3y + T3z; 697| 320| T3C = T3z - T3y; 698| 320| } 699| 320| { 700| 320| E T2e, T2p, T2s, T2t; 701| 320| T2e = FMA(KP923879532, T28, KP382683432 * T2d); ------------------ | | 1131| 320|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 702| 320| T2p = FNMS(KP923879532, T2o, KP382683432 * T2j); ------------------ | | 1134| 320|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 703| 320| T2q = T2e + T2p; 704| 320| T3B = T2p - T2e; 705| 320| T2s = FNMS(KP923879532, T2d, KP382683432 * T28); ------------------ | | 1134| 320|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 706| 320| T2t = FMA(KP382683432, T2o, KP923879532 * T2j); ------------------ | | 1131| 320|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 707| 320| T2u = T2s - T2t; 708| 320| T3x = T2s + T2t; 709| 320| } 710| 320| ri[WS(rs, 11)] = T23 - T2q; ------------------ | | 825| 320|#define WS(stride, i) (stride * i) ------------------ 711| 320| ii[WS(rs, 11)] = T3A - T3x; ------------------ | | 825| 320|#define WS(stride, i) (stride * i) ------------------ 712| 320| ri[WS(rs, 3)] = T23 + T2q; ------------------ | | 825| 320|#define WS(stride, i) (stride * i) ------------------ 713| 320| ii[WS(rs, 3)] = T3x + T3A; ------------------ | | 825| 320|#define WS(stride, i) (stride * i) ------------------ 714| 320| ri[WS(rs, 15)] = T2r - T2u; ------------------ | | 825| 320|#define WS(stride, i) (stride * i) ------------------ 715| 320| ii[WS(rs, 15)] = T3C - T3B; ------------------ | | 825| 320|#define WS(stride, i) (stride * i) ------------------ 716| 320| ri[WS(rs, 7)] = T2r + T2u; ------------------ | | 825| 320|#define WS(stride, i) (stride * i) ------------------ 717| 320| ii[WS(rs, 7)] = T3B + T3C; ------------------ | | 825| 320|#define WS(stride, i) (stride * i) ------------------ 718| 320| } 719| 320| { 720| 320| E T2P, T31, T3m, T3o, T30, T3n, T34, T3j; 721| 320| { 722| 320| E T2L, T2O, T3k, T3l; 723| 320| T2L = Tf - TA; 724| 320| T2O = T2M - T2N; 725| 320| T2P = T2L + T2O; 726| 320| T31 = T2L - T2O; 727| 320| T3k = TZ - TM; 728| 320| T3l = T3e - T3b; 729| 320| T3m = T3k + T3l; 730| 320| T3o = T3l - T3k; 731| 320| } 732| 320| { 733| 320| E T2U, T2Z, T32, T33; 734| 320| T2U = T2Q + T2T; 735| 320| T2Z = T2V - T2Y; 736| 320| T30 = KP707106781 * (T2U + T2Z); 737| 320| T3n = KP707106781 * (T2Z - T2U); 738| 320| T32 = T2T - T2Q; 739| 320| T33 = T2V + T2Y; 740| 320| T34 = KP707106781 * (T32 - T33); 741| 320| T3j = KP707106781 * (T32 + T33); 742| 320| } 743| 320| ri[WS(rs, 10)] = T2P - T30; ------------------ | | 825| 320|#define WS(stride, i) (stride * i) ------------------ 744| 320| ii[WS(rs, 10)] = T3m - T3j; ------------------ | | 825| 320|#define WS(stride, i) (stride * i) ------------------ 745| 320| ri[WS(rs, 2)] = T2P + T30; ------------------ | | 825| 320|#define WS(stride, i) (stride * i) ------------------ 746| 320| ii[WS(rs, 2)] = T3j + T3m; ------------------ | | 825| 320|#define WS(stride, i) (stride * i) ------------------ 747| 320| ri[WS(rs, 14)] = T31 - T34; ------------------ | | 825| 320|#define WS(stride, i) (stride * i) ------------------ 748| 320| ii[WS(rs, 14)] = T3o - T3n; ------------------ | | 825| 320|#define WS(stride, i) (stride * i) ------------------ 749| 320| ri[WS(rs, 6)] = T31 + T34; ------------------ | | 825| 320|#define WS(stride, i) (stride * i) ------------------ 750| 320| ii[WS(rs, 6)] = T3n + T3o; ------------------ | | 825| 320|#define WS(stride, i) (stride * i) ------------------ 751| 320| } 752| 320| { 753| 320| E T2z, T2H, T3u, T3w, T2G, T3v, T2K, T3p; 754| 320| { 755| 320| E T2v, T2y, T3q, T3t; 756| 320| T2v = T1N + T1Q; 757| 320| T2y = KP707106781 * (T2w + T2x); 758| 320| T2z = T2v + T2y; 759| 320| T2H = T2v - T2y; 760| 320| T3q = KP707106781 * (T1W + T21); 761| 320| T3t = T3r - T3s; 762| 320| T3u = T3q + T3t; 763| 320| T3w = T3t - T3q; 764| 320| } 765| 320| { 766| 320| E T2C, T2F, T2I, T2J; 767| 320| T2C = FMA(KP382683432, T2A, KP923879532 * T2B); ------------------ | | 1131| 320|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 768| 320| T2F = FNMS(KP382683432, T2E, KP923879532 * T2D); ------------------ | | 1134| 320|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 769| 320| T2G = T2C + T2F; 770| 320| T3v = T2F - T2C; 771| 320| T2I = FNMS(KP382683432, T2B, KP923879532 * T2A); ------------------ | | 1134| 320|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 772| 320| T2J = FMA(KP923879532, T2E, KP382683432 * T2D); ------------------ | | 1131| 320|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 773| 320| T2K = T2I - T2J; 774| 320| T3p = T2I + T2J; 775| 320| } 776| 320| ri[WS(rs, 9)] = T2z - T2G; ------------------ | | 825| 320|#define WS(stride, i) (stride * i) ------------------ 777| 320| ii[WS(rs, 9)] = T3u - T3p; ------------------ | | 825| 320|#define WS(stride, i) (stride * i) ------------------ 778| 320| ri[WS(rs, 1)] = T2z + T2G; ------------------ | | 825| 320|#define WS(stride, i) (stride * i) ------------------ 779| 320| ii[WS(rs, 1)] = T3p + T3u; ------------------ | | 825| 320|#define WS(stride, i) (stride * i) ------------------ 780| 320| ri[WS(rs, 13)] = T2H - T2K; ------------------ | | 825| 320|#define WS(stride, i) (stride * i) ------------------ 781| 320| ii[WS(rs, 13)] = T3w - T3v; ------------------ | | 825| 320|#define WS(stride, i) (stride * i) ------------------ 782| 320| ri[WS(rs, 5)] = T2H + T2K; ------------------ | | 825| 320|#define WS(stride, i) (stride * i) ------------------ 783| 320| ii[WS(rs, 5)] = T3v + T3w; ------------------ | | 825| 320|#define WS(stride, i) (stride * i) ------------------ 784| 320| } 785| 320| { 786| 320| E T11, T35, T3g, T3i, T1M, T3h, T38, T39; 787| 320| { 788| 320| E TB, T10, T3a, T3f; 789| 320| TB = Tf + TA; 790| 320| T10 = TM + TZ; 791| 320| T11 = TB + T10; 792| 320| T35 = TB - T10; 793| 320| T3a = T2M + T2N; 794| 320| T3f = T3b + T3e; 795| 320| T3g = T3a + T3f; 796| 320| T3i = T3f - T3a; 797| 320| } 798| 320| { 799| 320| E T1o, T1L, T36, T37; 800| 320| T1o = T18 + T1n; 801| 320| T1L = T1B + T1K; 802| 320| T1M = T1o + T1L; 803| 320| T3h = T1L - T1o; 804| 320| T36 = T2R + T2S; 805| 320| T37 = T2W + T2X; 806| 320| T38 = T36 - T37; 807| 320| T39 = T36 + T37; 808| 320| } 809| 320| ri[WS(rs, 8)] = T11 - T1M; ------------------ | | 825| 320|#define WS(stride, i) (stride * i) ------------------ 810| 320| ii[WS(rs, 8)] = T3g - T39; ------------------ | | 825| 320|#define WS(stride, i) (stride * i) ------------------ 811| 320| ri[0] = T11 + T1M; 812| 320| ii[0] = T39 + T3g; 813| 320| ri[WS(rs, 12)] = T35 - T38; ------------------ | | 825| 320|#define WS(stride, i) (stride * i) ------------------ 814| 320| ii[WS(rs, 12)] = T3i - T3h; ------------------ | | 825| 320|#define WS(stride, i) (stride * i) ------------------ 815| 320| ri[WS(rs, 4)] = T35 + T38; ------------------ | | 825| 320|#define WS(stride, i) (stride * i) ------------------ 816| 320| ii[WS(rs, 4)] = T3h + T3i; ------------------ | | 825| 320|#define WS(stride, i) (stride * i) ------------------ 817| 320| } 818| 320| } 819| 320| } 820| 19| } 821| 19|} fftw_codelet_t2_20: 1094| 1|void X(codelet_t2_20) (planner *p) { 1095| 1| X(kdft_dit_register) (p, t2_20, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 1096| 1|} t2_20.c:t2_20: 615| 39|{ 616| 39| DK(KP587785252, +0.587785252292473129168705954639072768597652438); ------------------ | | 1056| 39|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 39|# define K(x) ((E) x) | | ------------------ ------------------ 617| 39| DK(KP951056516, +0.951056516295153572116439333379382143405698634); ------------------ | | 1056| 39|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 39|# define K(x) ((E) x) | | ------------------ ------------------ 618| 39| DK(KP250000000, +0.250000000000000000000000000000000000000000000); ------------------ | | 1056| 39|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 39|# define K(x) ((E) x) | | ------------------ ------------------ 619| 39| DK(KP559016994, +0.559016994374947424102293417182819058860154590); ------------------ | | 1056| 39|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 39|# define K(x) ((E) x) | | ------------------ ------------------ 620| 39| { 621| 39| INT m; 622| 670| for (m = mb, W = W + (mb * 8); m < me; m = m + 1, ri = ri + ms, ii = ii + ms, W = W + 8, MAKE_VOLATILE_STRIDE(40, rs)) { ------------------ | | 851| 631| (nptr <= ESTIMATED_AVAILABLE_INDEX_REGISTERS ? \ | | ------------------ | | | | 849| 1.26k|#define ESTIMATED_AVAILABLE_INDEX_REGISTERS 16 | | ------------------ | | | Branch (851:7): [Folded, False: 631] | | ------------------ | | 852| 631| 0 : \ | | 853| 631| ((x) = (x) ^ X(an_INT_guaranteed_to_be_zero))) | | ------------------ | | | | 78| 631|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 631|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ | Branch (622:35): [True: 631, False: 39] ------------------ 623| 631| E T2, T5, Tg, Ti, Tk, To, T1h, T1f, T6, T3, T8, T14, T1Q, Tc, T1O; 624| 631| E T1v, T18, T1t, T1n, T24, T1j, T22, Tq, Tu, T1E, T1G, Tx, Ty, Tz, TJ; 625| 631| E T1Z, TB, T1X, T1A, TZ, TL, T1y, TX; 626| 631| { 627| 631| E T7, T16, Ta, T13, T4, T17, Tb, T12; 628| 631| { 629| 631| E Th, Tn, Tj, Tm; 630| 631| T2 = W[0]; 631| 631| T5 = W[1]; 632| 631| Tg = W[2]; 633| 631| Ti = W[3]; 634| 631| Th = T2 * Tg; 635| 631| Tn = T5 * Tg; 636| 631| Tj = T5 * Ti; 637| 631| Tm = T2 * Ti; 638| 631| Tk = Th - Tj; 639| 631| To = Tm + Tn; 640| 631| T1h = Tm - Tn; 641| 631| T1f = Th + Tj; 642| 631| T6 = W[5]; 643| 631| T7 = T5 * T6; 644| 631| T16 = Tg * T6; 645| 631| Ta = T2 * T6; 646| 631| T13 = Ti * T6; 647| 631| T3 = W[4]; 648| 631| T4 = T2 * T3; 649| 631| T17 = Ti * T3; 650| 631| Tb = T5 * T3; 651| 631| T12 = Tg * T3; 652| 631| } 653| 631| T8 = T4 - T7; 654| 631| T14 = T12 + T13; 655| 631| T1Q = T16 + T17; 656| 631| Tc = Ta + Tb; 657| 631| T1O = T12 - T13; 658| 631| T1v = Ta - Tb; 659| 631| T18 = T16 - T17; 660| 631| T1t = T4 + T7; 661| 631| { 662| 631| E T1l, T1m, T1g, T1i; 663| 631| T1l = T1f * T6; 664| 631| T1m = T1h * T3; 665| 631| T1n = T1l + T1m; 666| 631| T24 = T1l - T1m; 667| 631| T1g = T1f * T3; 668| 631| T1i = T1h * T6; 669| 631| T1j = T1g - T1i; 670| 631| T22 = T1g + T1i; 671| 631| { 672| 631| E Tl, Tp, Ts, Tt; 673| 631| Tl = Tk * T3; 674| 631| Tp = To * T6; 675| 631| Tq = Tl + Tp; 676| 631| Ts = Tk * T6; 677| 631| Tt = To * T3; 678| 631| Tu = Ts - Tt; 679| 631| T1E = Tl - Tp; 680| 631| T1G = Ts + Tt; 681| 631| Tx = W[6]; 682| 631| Ty = W[7]; 683| 631| Tz = FMA(Tk, Tx, To * Ty); ------------------ | | 1131| 631|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 684| 631| TJ = FMA(Tq, Tx, Tu * Ty); ------------------ | | 1131| 631|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 685| 631| T1Z = FNMS(T1h, Tx, T1f * Ty); ------------------ | | 1134| 631|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 686| 631| TB = FNMS(To, Tx, Tk * Ty); ------------------ | | 1134| 631|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 687| 631| T1X = FMA(T1f, Tx, T1h * Ty); ------------------ | | 1131| 631|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 688| 631| T1A = FNMS(T5, Tx, T2 * Ty); ------------------ | | 1134| 631|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 689| 631| TZ = FNMS(Ti, Tx, Tg * Ty); ------------------ | | 1134| 631|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 690| 631| TL = FNMS(Tu, Tx, Tq * Ty); ------------------ | | 1134| 631|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 691| 631| T1y = FMA(T2, Tx, T5 * Ty); ------------------ | | 1131| 631|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 692| 631| TX = FMA(Tg, Tx, Ti * Ty); ------------------ | | 1131| 631|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 693| 631| } 694| 631| } 695| 631| } 696| 631| { 697| 631| E TF, T2b, T4A, T4J, T2K, T3r, T4a, T4m, T1N, T28, T29, T3C, T3F, T4o, T3X; 698| 631| E T3Y, T44, T2f, T2g, T2h, T2n, T2s, T4L, T3g, T3h, T4w, T3n, T3o, T3p, T30; 699| 631| E T35, T36, TW, T1r, T1s, T3J, T3M, T4n, T3U, T3V, T43, T2c, T2d, T2e, T2y; 700| 631| E T2D, T4K, T3d, T3e, T4v, T3k, T3l, T3m, T2P, T2U, T2V; 701| 631| { 702| 631| E T1, T48, Te, T47, Tw, T2H, TD, T2I, T9, Td; 703| 631| T1 = ri[0]; 704| 631| T48 = ii[0]; 705| 631| T9 = ri[WS(rs, 10)]; ------------------ | | 825| 631|#define WS(stride, i) (stride * i) ------------------ 706| 631| Td = ii[WS(rs, 10)]; ------------------ | | 825| 631|#define WS(stride, i) (stride * i) ------------------ 707| 631| Te = FMA(T8, T9, Tc * Td); ------------------ | | 1131| 631|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 708| 631| T47 = FNMS(Tc, T9, T8 * Td); ------------------ | | 1134| 631|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 709| 631| { 710| 631| E Tr, Tv, TA, TC; 711| 631| Tr = ri[WS(rs, 5)]; ------------------ | | 825| 631|#define WS(stride, i) (stride * i) ------------------ 712| 631| Tv = ii[WS(rs, 5)]; ------------------ | | 825| 631|#define WS(stride, i) (stride * i) ------------------ 713| 631| Tw = FMA(Tq, Tr, Tu * Tv); ------------------ | | 1131| 631|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 714| 631| T2H = FNMS(Tu, Tr, Tq * Tv); ------------------ | | 1134| 631|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 715| 631| TA = ri[WS(rs, 15)]; ------------------ | | 825| 631|#define WS(stride, i) (stride * i) ------------------ 716| 631| TC = ii[WS(rs, 15)]; ------------------ | | 825| 631|#define WS(stride, i) (stride * i) ------------------ 717| 631| TD = FMA(Tz, TA, TB * TC); ------------------ | | 1131| 631|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 718| 631| T2I = FNMS(TB, TA, Tz * TC); ------------------ | | 1134| 631|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 719| 631| } 720| 631| { 721| 631| E Tf, TE, T4y, T4z; 722| 631| Tf = T1 + Te; 723| 631| TE = Tw + TD; 724| 631| TF = Tf - TE; 725| 631| T2b = Tf + TE; 726| 631| T4y = T48 - T47; 727| 631| T4z = Tw - TD; 728| 631| T4A = T4y - T4z; 729| 631| T4J = T4z + T4y; 730| 631| } 731| 631| { 732| 631| E T2G, T2J, T46, T49; 733| 631| T2G = T1 - Te; 734| 631| T2J = T2H - T2I; 735| 631| T2K = T2G - T2J; 736| 631| T3r = T2G + T2J; 737| 631| T46 = T2H + T2I; 738| 631| T49 = T47 + T48; 739| 631| T4a = T46 + T49; 740| 631| T4m = T49 - T46; 741| 631| } 742| 631| } 743| 631| { 744| 631| E T1D, T3A, T2l, T2W, T27, T3E, T2r, T34, T1M, T3B, T2m, T2Z, T1W, T3D, T2q; 745| 631| E T31; 746| 631| { 747| 631| E T1x, T2j, T1C, T2k; 748| 631| { 749| 631| E T1u, T1w, T1z, T1B; 750| 631| T1u = ri[WS(rs, 8)]; ------------------ | | 825| 631|#define WS(stride, i) (stride * i) ------------------ 751| 631| T1w = ii[WS(rs, 8)]; ------------------ | | 825| 631|#define WS(stride, i) (stride * i) ------------------ 752| 631| T1x = FMA(T1t, T1u, T1v * T1w); ------------------ | | 1131| 631|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 753| 631| T2j = FNMS(T1v, T1u, T1t * T1w); ------------------ | | 1134| 631|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 754| 631| T1z = ri[WS(rs, 18)]; ------------------ | | 825| 631|#define WS(stride, i) (stride * i) ------------------ 755| 631| T1B = ii[WS(rs, 18)]; ------------------ | | 825| 631|#define WS(stride, i) (stride * i) ------------------ 756| 631| T1C = FMA(T1y, T1z, T1A * T1B); ------------------ | | 1131| 631|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 757| 631| T2k = FNMS(T1A, T1z, T1y * T1B); ------------------ | | 1134| 631|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 758| 631| } 759| 631| T1D = T1x + T1C; 760| 631| T3A = T2j + T2k; 761| 631| T2l = T2j - T2k; 762| 631| T2W = T1x - T1C; 763| 631| } 764| 631| { 765| 631| E T21, T32, T26, T33; 766| 631| { 767| 631| E T1Y, T20, T23, T25; 768| 631| T1Y = ri[WS(rs, 17)]; ------------------ | | 825| 631|#define WS(stride, i) (stride * i) ------------------ 769| 631| T20 = ii[WS(rs, 17)]; ------------------ | | 825| 631|#define WS(stride, i) (stride * i) ------------------ 770| 631| T21 = FMA(T1X, T1Y, T1Z * T20); ------------------ | | 1131| 631|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 771| 631| T32 = FNMS(T1Z, T1Y, T1X * T20); ------------------ | | 1134| 631|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 772| 631| T23 = ri[WS(rs, 7)]; ------------------ | | 825| 631|#define WS(stride, i) (stride * i) ------------------ 773| 631| T25 = ii[WS(rs, 7)]; ------------------ | | 825| 631|#define WS(stride, i) (stride * i) ------------------ 774| 631| T26 = FMA(T22, T23, T24 * T25); ------------------ | | 1131| 631|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 775| 631| T33 = FNMS(T24, T23, T22 * T25); ------------------ | | 1134| 631|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 776| 631| } 777| 631| T27 = T21 + T26; 778| 631| T3E = T32 + T33; 779| 631| T2r = T21 - T26; 780| 631| T34 = T32 - T33; 781| 631| } 782| 631| { 783| 631| E T1I, T2X, T1L, T2Y; 784| 631| { 785| 631| E T1F, T1H, T1J, T1K; 786| 631| T1F = ri[WS(rs, 13)]; ------------------ | | 825| 631|#define WS(stride, i) (stride * i) ------------------ 787| 631| T1H = ii[WS(rs, 13)]; ------------------ | | 825| 631|#define WS(stride, i) (stride * i) ------------------ 788| 631| T1I = FMA(T1E, T1F, T1G * T1H); ------------------ | | 1131| 631|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 789| 631| T2X = FNMS(T1G, T1F, T1E * T1H); ------------------ | | 1134| 631|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 790| 631| T1J = ri[WS(rs, 3)]; ------------------ | | 825| 631|#define WS(stride, i) (stride * i) ------------------ 791| 631| T1K = ii[WS(rs, 3)]; ------------------ | | 825| 631|#define WS(stride, i) (stride * i) ------------------ 792| 631| T1L = FMA(Tg, T1J, Ti * T1K); ------------------ | | 1131| 631|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 793| 631| T2Y = FNMS(Ti, T1J, Tg * T1K); ------------------ | | 1134| 631|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 794| 631| } 795| 631| T1M = T1I + T1L; 796| 631| T3B = T2X + T2Y; 797| 631| T2m = T1I - T1L; 798| 631| T2Z = T2X - T2Y; 799| 631| } 800| 631| { 801| 631| E T1S, T2o, T1V, T2p; 802| 631| { 803| 631| E T1P, T1R, T1T, T1U; 804| 631| T1P = ri[WS(rs, 12)]; ------------------ | | 825| 631|#define WS(stride, i) (stride * i) ------------------ 805| 631| T1R = ii[WS(rs, 12)]; ------------------ | | 825| 631|#define WS(stride, i) (stride * i) ------------------ 806| 631| T1S = FMA(T1O, T1P, T1Q * T1R); ------------------ | | 1131| 631|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 807| 631| T2o = FNMS(T1Q, T1P, T1O * T1R); ------------------ | | 1134| 631|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 808| 631| T1T = ri[WS(rs, 2)]; ------------------ | | 825| 631|#define WS(stride, i) (stride * i) ------------------ 809| 631| T1U = ii[WS(rs, 2)]; ------------------ | | 825| 631|#define WS(stride, i) (stride * i) ------------------ 810| 631| T1V = FMA(T1f, T1T, T1h * T1U); ------------------ | | 1131| 631|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 811| 631| T2p = FNMS(T1h, T1T, T1f * T1U); ------------------ | | 1134| 631|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 812| 631| } 813| 631| T1W = T1S + T1V; 814| 631| T3D = T2o + T2p; 815| 631| T2q = T2o - T2p; 816| 631| T31 = T1S - T1V; 817| 631| } 818| 631| T1N = T1D - T1M; 819| 631| T28 = T1W - T27; 820| 631| T29 = T1N + T28; 821| 631| T3C = T3A - T3B; 822| 631| T3F = T3D - T3E; 823| 631| T4o = T3C + T3F; 824| 631| T3X = T3A + T3B; 825| 631| T3Y = T3D + T3E; 826| 631| T44 = T3X + T3Y; 827| 631| T2f = T1D + T1M; 828| 631| T2g = T1W + T27; 829| 631| T2h = T2f + T2g; 830| 631| T2n = T2l + T2m; 831| 631| T2s = T2q + T2r; 832| 631| T4L = T2n + T2s; 833| 631| T3g = T2l - T2m; 834| 631| T3h = T2q - T2r; 835| 631| T4w = T3g + T3h; 836| 631| T3n = T2W + T2Z; 837| 631| T3o = T31 + T34; 838| 631| T3p = T3n + T3o; 839| 631| T30 = T2W - T2Z; 840| 631| T35 = T31 - T34; 841| 631| T36 = T30 + T35; 842| 631| } 843| 631| { 844| 631| E TO, T3H, T2w, T2L, T1q, T3L, T2C, T2T, TV, T3I, T2x, T2O, T1b, T3K, T2B; 845| 631| E T2Q; 846| 631| { 847| 631| E TI, T2u, TN, T2v; 848| 631| { 849| 631| E TG, TH, TK, TM; 850| 631| TG = ri[WS(rs, 4)]; ------------------ | | 825| 631|#define WS(stride, i) (stride * i) ------------------ 851| 631| TH = ii[WS(rs, 4)]; ------------------ | | 825| 631|#define WS(stride, i) (stride * i) ------------------ 852| 631| TI = FMA(Tk, TG, To * TH); ------------------ | | 1131| 631|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 853| 631| T2u = FNMS(To, TG, Tk * TH); ------------------ | | 1134| 631|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 854| 631| TK = ri[WS(rs, 14)]; ------------------ | | 825| 631|#define WS(stride, i) (stride * i) ------------------ 855| 631| TM = ii[WS(rs, 14)]; ------------------ | | 825| 631|#define WS(stride, i) (stride * i) ------------------ 856| 631| TN = FMA(TJ, TK, TL * TM); ------------------ | | 1131| 631|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 857| 631| T2v = FNMS(TL, TK, TJ * TM); ------------------ | | 1134| 631|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 858| 631| } 859| 631| TO = TI + TN; 860| 631| T3H = T2u + T2v; 861| 631| T2w = T2u - T2v; 862| 631| T2L = TI - TN; 863| 631| } 864| 631| { 865| 631| E T1e, T2R, T1p, T2S; 866| 631| { 867| 631| E T1c, T1d, T1k, T1o; 868| 631| T1c = ri[WS(rs, 1)]; ------------------ | | 825| 631|#define WS(stride, i) (stride * i) ------------------ 869| 631| T1d = ii[WS(rs, 1)]; ------------------ | | 825| 631|#define WS(stride, i) (stride * i) ------------------ 870| 631| T1e = FMA(T2, T1c, T5 * T1d); ------------------ | | 1131| 631|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 871| 631| T2R = FNMS(T5, T1c, T2 * T1d); ------------------ | | 1134| 631|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 872| 631| T1k = ri[WS(rs, 11)]; ------------------ | | 825| 631|#define WS(stride, i) (stride * i) ------------------ 873| 631| T1o = ii[WS(rs, 11)]; ------------------ | | 825| 631|#define WS(stride, i) (stride * i) ------------------ 874| 631| T1p = FMA(T1j, T1k, T1n * T1o); ------------------ | | 1131| 631|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 875| 631| T2S = FNMS(T1n, T1k, T1j * T1o); ------------------ | | 1134| 631|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 876| 631| } 877| 631| T1q = T1e + T1p; 878| 631| T3L = T2R + T2S; 879| 631| T2C = T1e - T1p; 880| 631| T2T = T2R - T2S; 881| 631| } 882| 631| { 883| 631| E TR, T2M, TU, T2N; 884| 631| { 885| 631| E TP, TQ, TS, TT; 886| 631| TP = ri[WS(rs, 9)]; ------------------ | | 825| 631|#define WS(stride, i) (stride * i) ------------------ 887| 631| TQ = ii[WS(rs, 9)]; ------------------ | | 825| 631|#define WS(stride, i) (stride * i) ------------------ 888| 631| TR = FMA(T3, TP, T6 * TQ); ------------------ | | 1131| 631|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 889| 631| T2M = FNMS(T6, TP, T3 * TQ); ------------------ | | 1134| 631|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 890| 631| TS = ri[WS(rs, 19)]; ------------------ | | 825| 631|#define WS(stride, i) (stride * i) ------------------ 891| 631| TT = ii[WS(rs, 19)]; ------------------ | | 825| 631|#define WS(stride, i) (stride * i) ------------------ 892| 631| TU = FMA(Tx, TS, Ty * TT); ------------------ | | 1131| 631|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 893| 631| T2N = FNMS(Ty, TS, Tx * TT); ------------------ | | 1134| 631|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 894| 631| } 895| 631| TV = TR + TU; 896| 631| T3I = T2M + T2N; 897| 631| T2x = TR - TU; 898| 631| T2O = T2M - T2N; 899| 631| } 900| 631| { 901| 631| E T11, T2z, T1a, T2A; 902| 631| { 903| 631| E TY, T10, T15, T19; 904| 631| TY = ri[WS(rs, 16)]; ------------------ | | 825| 631|#define WS(stride, i) (stride * i) ------------------ 905| 631| T10 = ii[WS(rs, 16)]; ------------------ | | 825| 631|#define WS(stride, i) (stride * i) ------------------ 906| 631| T11 = FMA(TX, TY, TZ * T10); ------------------ | | 1131| 631|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 907| 631| T2z = FNMS(TZ, TY, TX * T10); ------------------ | | 1134| 631|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 908| 631| T15 = ri[WS(rs, 6)]; ------------------ | | 825| 631|#define WS(stride, i) (stride * i) ------------------ 909| 631| T19 = ii[WS(rs, 6)]; ------------------ | | 825| 631|#define WS(stride, i) (stride * i) ------------------ 910| 631| T1a = FMA(T14, T15, T18 * T19); ------------------ | | 1131| 631|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 911| 631| T2A = FNMS(T18, T15, T14 * T19); ------------------ | | 1134| 631|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 912| 631| } 913| 631| T1b = T11 + T1a; 914| 631| T3K = T2z + T2A; 915| 631| T2B = T2z - T2A; 916| 631| T2Q = T11 - T1a; 917| 631| } 918| 631| TW = TO - TV; 919| 631| T1r = T1b - T1q; 920| 631| T1s = TW + T1r; 921| 631| T3J = T3H - T3I; 922| 631| T3M = T3K - T3L; 923| 631| T4n = T3J + T3M; 924| 631| T3U = T3H + T3I; 925| 631| T3V = T3K + T3L; 926| 631| T43 = T3U + T3V; 927| 631| T2c = TO + TV; 928| 631| T2d = T1b + T1q; 929| 631| T2e = T2c + T2d; 930| 631| T2y = T2w + T2x; 931| 631| T2D = T2B + T2C; 932| 631| T4K = T2y + T2D; 933| 631| T3d = T2w - T2x; 934| 631| T3e = T2B - T2C; 935| 631| T4v = T3d + T3e; 936| 631| T3k = T2L + T2O; 937| 631| T3l = T2Q + T2T; 938| 631| T3m = T3k + T3l; 939| 631| T2P = T2L - T2O; 940| 631| T2U = T2Q - T2T; 941| 631| T2V = T2P + T2U; 942| 631| } 943| 631| { 944| 631| E T3y, T2a, T3x, T3O, T3Q, T3G, T3N, T3P, T3z; 945| 631| T3y = KP559016994 * (T1s - T29); 946| 631| T2a = T1s + T29; 947| 631| T3x = FNMS(KP250000000, T2a, TF); ------------------ | | 1134| 631|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 948| 631| T3G = T3C - T3F; 949| 631| T3N = T3J - T3M; 950| 631| T3O = FNMS(KP587785252, T3N, KP951056516 * T3G); ------------------ | | 1134| 631|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 951| 631| T3Q = FMA(KP951056516, T3N, KP587785252 * T3G); ------------------ | | 1131| 631|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 952| 631| ri[WS(rs, 10)] = TF + T2a; ------------------ | | 825| 631|#define WS(stride, i) (stride * i) ------------------ 953| 631| T3P = T3y + T3x; 954| 631| ri[WS(rs, 14)] = T3P - T3Q; ------------------ | | 825| 631|#define WS(stride, i) (stride * i) ------------------ 955| 631| ri[WS(rs, 6)] = T3P + T3Q; ------------------ | | 825| 631|#define WS(stride, i) (stride * i) ------------------ 956| 631| T3z = T3x - T3y; 957| 631| ri[WS(rs, 2)] = T3z - T3O; ------------------ | | 825| 631|#define WS(stride, i) (stride * i) ------------------ 958| 631| ri[WS(rs, 18)] = T3z + T3O; ------------------ | | 825| 631|#define WS(stride, i) (stride * i) ------------------ 959| 631| } 960| 631| { 961| 631| E T4r, T4p, T4q, T4l, T4u, T4j, T4k, T4t, T4s; 962| 631| T4r = KP559016994 * (T4n - T4o); 963| 631| T4p = T4n + T4o; 964| 631| T4q = FNMS(KP250000000, T4p, T4m); ------------------ | | 1134| 631|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 965| 631| T4j = T1N - T28; 966| 631| T4k = TW - T1r; 967| 631| T4l = FNMS(KP587785252, T4k, KP951056516 * T4j); ------------------ | | 1134| 631|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 968| 631| T4u = FMA(KP951056516, T4k, KP587785252 * T4j); ------------------ | | 1131| 631|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 969| 631| ii[WS(rs, 10)] = T4p + T4m; ------------------ | | 825| 631|#define WS(stride, i) (stride * i) ------------------ 970| 631| T4t = T4r + T4q; 971| 631| ii[WS(rs, 6)] = T4t - T4u; ------------------ | | 825| 631|#define WS(stride, i) (stride * i) ------------------ 972| 631| ii[WS(rs, 14)] = T4u + T4t; ------------------ | | 825| 631|#define WS(stride, i) (stride * i) ------------------ 973| 631| T4s = T4q - T4r; 974| 631| ii[WS(rs, 2)] = T4l + T4s; ------------------ | | 825| 631|#define WS(stride, i) (stride * i) ------------------ 975| 631| ii[WS(rs, 18)] = T4s - T4l; ------------------ | | 825| 631|#define WS(stride, i) (stride * i) ------------------ 976| 631| } 977| 631| { 978| 631| E T3R, T2i, T3S, T40, T42, T3W, T3Z, T41, T3T; 979| 631| T3R = KP559016994 * (T2e - T2h); 980| 631| T2i = T2e + T2h; 981| 631| T3S = FNMS(KP250000000, T2i, T2b); ------------------ | | 1134| 631|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 982| 631| T3W = T3U - T3V; 983| 631| T3Z = T3X - T3Y; 984| 631| T40 = FMA(KP951056516, T3W, KP587785252 * T3Z); ------------------ | | 1131| 631|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 985| 631| T42 = FNMS(KP587785252, T3W, KP951056516 * T3Z); ------------------ | | 1134| 631|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 986| 631| ri[0] = T2b + T2i; 987| 631| T41 = T3S - T3R; 988| 631| ri[WS(rs, 12)] = T41 - T42; ------------------ | | 825| 631|#define WS(stride, i) (stride * i) ------------------ 989| 631| ri[WS(rs, 8)] = T41 + T42; ------------------ | | 825| 631|#define WS(stride, i) (stride * i) ------------------ 990| 631| T3T = T3R + T3S; 991| 631| ri[WS(rs, 4)] = T3T - T40; ------------------ | | 825| 631|#define WS(stride, i) (stride * i) ------------------ 992| 631| ri[WS(rs, 16)] = T3T + T40; ------------------ | | 825| 631|#define WS(stride, i) (stride * i) ------------------ 993| 631| } 994| 631| { 995| 631| E T4e, T45, T4f, T4d, T4i, T4b, T4c, T4h, T4g; 996| 631| T4e = KP559016994 * (T43 - T44); 997| 631| T45 = T43 + T44; 998| 631| T4f = FNMS(KP250000000, T45, T4a); ------------------ | | 1134| 631|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 999| 631| T4b = T2c - T2d; 1000| 631| T4c = T2f - T2g; 1001| 631| T4d = FMA(KP951056516, T4b, KP587785252 * T4c); ------------------ | | 1131| 631|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1002| 631| T4i = FNMS(KP587785252, T4b, KP951056516 * T4c); ------------------ | | 1134| 631|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1003| 631| ii[0] = T45 + T4a; 1004| 631| T4h = T4f - T4e; 1005| 631| ii[WS(rs, 8)] = T4h - T4i; ------------------ | | 825| 631|#define WS(stride, i) (stride * i) ------------------ 1006| 631| ii[WS(rs, 12)] = T4i + T4h; ------------------ | | 825| 631|#define WS(stride, i) (stride * i) ------------------ 1007| 631| T4g = T4e + T4f; 1008| 631| ii[WS(rs, 4)] = T4d + T4g; ------------------ | | 825| 631|#define WS(stride, i) (stride * i) ------------------ 1009| 631| ii[WS(rs, 16)] = T4g - T4d; ------------------ | | 825| 631|#define WS(stride, i) (stride * i) ------------------ 1010| 631| } 1011| 631| { 1012| 631| E T39, T37, T38, T2F, T3b, T2t, T2E, T3c, T3a; 1013| 631| T39 = KP559016994 * (T2V - T36); 1014| 631| T37 = T2V + T36; 1015| 631| T38 = FNMS(KP250000000, T37, T2K); ------------------ | | 1134| 631|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1016| 631| T2t = T2n - T2s; 1017| 631| T2E = T2y - T2D; 1018| 631| T2F = FNMS(KP587785252, T2E, KP951056516 * T2t); ------------------ | | 1134| 631|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1019| 631| T3b = FMA(KP951056516, T2E, KP587785252 * T2t); ------------------ | | 1131| 631|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1020| 631| ri[WS(rs, 15)] = T2K + T37; ------------------ | | 825| 631|#define WS(stride, i) (stride * i) ------------------ 1021| 631| T3c = T39 + T38; 1022| 631| ri[WS(rs, 11)] = T3b + T3c; ------------------ | | 825| 631|#define WS(stride, i) (stride * i) ------------------ 1023| 631| ri[WS(rs, 19)] = T3c - T3b; ------------------ | | 825| 631|#define WS(stride, i) (stride * i) ------------------ 1024| 631| T3a = T38 - T39; 1025| 631| ri[WS(rs, 3)] = T2F + T3a; ------------------ | | 825| 631|#define WS(stride, i) (stride * i) ------------------ 1026| 631| ri[WS(rs, 7)] = T3a - T2F; ------------------ | | 825| 631|#define WS(stride, i) (stride * i) ------------------ 1027| 631| } 1028| 631| { 1029| 631| E T4O, T4M, T4N, T4S, T4U, T4Q, T4R, T4T, T4P; 1030| 631| T4O = KP559016994 * (T4K - T4L); 1031| 631| T4M = T4K + T4L; 1032| 631| T4N = FNMS(KP250000000, T4M, T4J); ------------------ | | 1134| 631|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1033| 631| T4Q = T30 - T35; 1034| 631| T4R = T2P - T2U; 1035| 631| T4S = FNMS(KP587785252, T4R, KP951056516 * T4Q); ------------------ | | 1134| 631|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1036| 631| T4U = FMA(KP951056516, T4R, KP587785252 * T4Q); ------------------ | | 1131| 631|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1037| 631| ii[WS(rs, 15)] = T4M + T4J; ------------------ | | 825| 631|#define WS(stride, i) (stride * i) ------------------ 1038| 631| T4T = T4O + T4N; 1039| 631| ii[WS(rs, 11)] = T4T - T4U; ------------------ | | 825| 631|#define WS(stride, i) (stride * i) ------------------ 1040| 631| ii[WS(rs, 19)] = T4U + T4T; ------------------ | | 825| 631|#define WS(stride, i) (stride * i) ------------------ 1041| 631| T4P = T4N - T4O; 1042| 631| ii[WS(rs, 3)] = T4P - T4S; ------------------ | | 825| 631|#define WS(stride, i) (stride * i) ------------------ 1043| 631| ii[WS(rs, 7)] = T4S + T4P; ------------------ | | 825| 631|#define WS(stride, i) (stride * i) ------------------ 1044| 631| } 1045| 631| { 1046| 631| E T3q, T3s, T3t, T3j, T3v, T3f, T3i, T3w, T3u; 1047| 631| T3q = KP559016994 * (T3m - T3p); 1048| 631| T3s = T3m + T3p; 1049| 631| T3t = FNMS(KP250000000, T3s, T3r); ------------------ | | 1134| 631|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1050| 631| T3f = T3d - T3e; 1051| 631| T3i = T3g - T3h; 1052| 631| T3j = FMA(KP951056516, T3f, KP587785252 * T3i); ------------------ | | 1131| 631|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1053| 631| T3v = FNMS(KP587785252, T3f, KP951056516 * T3i); ------------------ | | 1134| 631|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1054| 631| ri[WS(rs, 5)] = T3r + T3s; ------------------ | | 825| 631|#define WS(stride, i) (stride * i) ------------------ 1055| 631| T3w = T3t - T3q; 1056| 631| ri[WS(rs, 13)] = T3v + T3w; ------------------ | | 825| 631|#define WS(stride, i) (stride * i) ------------------ 1057| 631| ri[WS(rs, 17)] = T3w - T3v; ------------------ | | 825| 631|#define WS(stride, i) (stride * i) ------------------ 1058| 631| T3u = T3q + T3t; 1059| 631| ri[WS(rs, 1)] = T3j + T3u; ------------------ | | 825| 631|#define WS(stride, i) (stride * i) ------------------ 1060| 631| ri[WS(rs, 9)] = T3u - T3j; ------------------ | | 825| 631|#define WS(stride, i) (stride * i) ------------------ 1061| 631| } 1062| 631| { 1063| 631| E T4x, T4B, T4C, T4G, T4I, T4E, T4F, T4H, T4D; 1064| 631| T4x = KP559016994 * (T4v - T4w); 1065| 631| T4B = T4v + T4w; 1066| 631| T4C = FNMS(KP250000000, T4B, T4A); ------------------ | | 1134| 631|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1067| 631| T4E = T3k - T3l; 1068| 631| T4F = T3n - T3o; 1069| 631| T4G = FMA(KP951056516, T4E, KP587785252 * T4F); ------------------ | | 1131| 631|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1070| 631| T4I = FNMS(KP587785252, T4E, KP951056516 * T4F); ------------------ | | 1134| 631|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1071| 631| ii[WS(rs, 5)] = T4B + T4A; ------------------ | | 825| 631|#define WS(stride, i) (stride * i) ------------------ 1072| 631| T4H = T4C - T4x; 1073| 631| ii[WS(rs, 13)] = T4H - T4I; ------------------ | | 825| 631|#define WS(stride, i) (stride * i) ------------------ 1074| 631| ii[WS(rs, 17)] = T4I + T4H; ------------------ | | 825| 631|#define WS(stride, i) (stride * i) ------------------ 1075| 631| T4D = T4x + T4C; 1076| 631| ii[WS(rs, 1)] = T4D - T4G; ------------------ | | 825| 631|#define WS(stride, i) (stride * i) ------------------ 1077| 631| ii[WS(rs, 9)] = T4G + T4D; ------------------ | | 825| 631|#define WS(stride, i) (stride * i) ------------------ 1078| 631| } 1079| 631| } 1080| 631| } 1081| 39| } 1082| 39|} fftw_codelet_t2_25: 1617| 1|void X(codelet_t2_25) (planner *p) { 1618| 1| X(kdft_dit_register) (p, t2_25, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 1619| 1|} t2_25.c:t2_25: 890| 3|{ 891| 3| DK(KP998026728, +0.998026728428271561952336806863450553336905220); ------------------ | | 1056| 3|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 3|# define K(x) ((E) x) | | ------------------ ------------------ 892| 3| DK(KP062790519, +0.062790519529313376076178224565631133122484832); ------------------ | | 1056| 3|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 3|# define K(x) ((E) x) | | ------------------ ------------------ 893| 3| DK(KP425779291, +0.425779291565072648862502445744251703979973042); ------------------ | | 1056| 3|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 3|# define K(x) ((E) x) | | ------------------ ------------------ 894| 3| DK(KP904827052, +0.904827052466019527713668647932697593970413911); ------------------ | | 1056| 3|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 3|# define K(x) ((E) x) | | ------------------ ------------------ 895| 3| DK(KP992114701, +0.992114701314477831049793042785778521453036709); ------------------ | | 1056| 3|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 3|# define K(x) ((E) x) | | ------------------ ------------------ 896| 3| DK(KP125333233, +0.125333233564304245373118759816508793942918247); ------------------ | | 1056| 3|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 3|# define K(x) ((E) x) | | ------------------ ------------------ 897| 3| DK(KP637423989, +0.637423989748689710176712811676016195434917298); ------------------ | | 1056| 3|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 3|# define K(x) ((E) x) | | ------------------ ------------------ 898| 3| DK(KP770513242, +0.770513242775789230803009636396177847271667672); ------------------ | | 1056| 3|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 3|# define K(x) ((E) x) | | ------------------ ------------------ 899| 3| DK(KP684547105, +0.684547105928688673732283357621209269889519233); ------------------ | | 1056| 3|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 3|# define K(x) ((E) x) | | ------------------ ------------------ 900| 3| DK(KP728968627, +0.728968627421411523146730319055259111372571664); ------------------ | | 1056| 3|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 3|# define K(x) ((E) x) | | ------------------ ------------------ 901| 3| DK(KP481753674, +0.481753674101715274987191502872129653528542010); ------------------ | | 1056| 3|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 3|# define K(x) ((E) x) | | ------------------ ------------------ 902| 3| DK(KP876306680, +0.876306680043863587308115903922062583399064238); ------------------ | | 1056| 3|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 3|# define K(x) ((E) x) | | ------------------ ------------------ 903| 3| DK(KP844327925, +0.844327925502015078548558063966681505381659241); ------------------ | | 1056| 3|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 3|# define K(x) ((E) x) | | ------------------ ------------------ 904| 3| DK(KP535826794, +0.535826794978996618271308767867639978063575346); ------------------ | | 1056| 3|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 3|# define K(x) ((E) x) | | ------------------ ------------------ 905| 3| DK(KP248689887, +0.248689887164854788242283746006447968417567406); ------------------ | | 1056| 3|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 3|# define K(x) ((E) x) | | ------------------ ------------------ 906| 3| DK(KP968583161, +0.968583161128631119490168375464735813836012403); ------------------ | | 1056| 3|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 3|# define K(x) ((E) x) | | ------------------ ------------------ 907| 3| DK(KP587785252, +0.587785252292473129168705954639072768597652438); ------------------ | | 1056| 3|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 3|# define K(x) ((E) x) | | ------------------ ------------------ 908| 3| DK(KP951056516, +0.951056516295153572116439333379382143405698634); ------------------ | | 1056| 3|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 3|# define K(x) ((E) x) | | ------------------ ------------------ 909| 3| DK(KP250000000, +0.250000000000000000000000000000000000000000000); ------------------ | | 1056| 3|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 3|# define K(x) ((E) x) | | ------------------ ------------------ 910| 3| DK(KP559016994, +0.559016994374947424102293417182819058860154590); ------------------ | | 1056| 3|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 3|# define K(x) ((E) x) | | ------------------ ------------------ 911| 3| { 912| 3| INT m; 913| 57| for (m = mb, W = W + (mb * 8); m < me; m = m + 1, ri = ri + ms, ii = ii + ms, W = W + 8, MAKE_VOLATILE_STRIDE(50, rs)) { ------------------ | | 851| 54| (nptr <= ESTIMATED_AVAILABLE_INDEX_REGISTERS ? \ | | ------------------ | | | | 849| 108|#define ESTIMATED_AVAILABLE_INDEX_REGISTERS 16 | | ------------------ | | | Branch (851:7): [Folded, False: 54] | | ------------------ | | 852| 54| 0 : \ | | 853| 54| ((x) = (x) ^ X(an_INT_guaranteed_to_be_zero))) | | ------------------ | | | | 78| 54|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 54|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ | Branch (913:35): [True: 54, False: 3] ------------------ 914| 54| E T2, T5, T3, T6, T8, Td, T16, T14, Te, T9, T21, T23, Tx, TR, T1g; 915| 54| E TB, T1f, TV, T1Q, Tg, T1S, Tk, T18, T2s, T1c, T2q, Tn, To, Tp, Tr; 916| 54| E T28, T2x, TY, T2k, T2m, T2v, TG, TE, T10, T1h, T1E, T26, T1B, T1G, T1V; 917| 54| E T1X, T1z, T1j; 918| 54| { 919| 54| E Tw, TT, Tz, TQ, Tv, TU, TA, TP; 920| 54| { 921| 54| E T4, Tc, T7, Tb; 922| 54| T2 = W[0]; 923| 54| T5 = W[1]; 924| 54| T3 = W[2]; 925| 54| T6 = W[3]; 926| 54| T4 = T2 * T3; 927| 54| Tc = T5 * T3; 928| 54| T7 = T5 * T6; 929| 54| Tb = T2 * T6; 930| 54| T8 = T4 - T7; 931| 54| Td = Tb + Tc; 932| 54| T16 = Tb - Tc; 933| 54| T14 = T4 + T7; 934| 54| Te = W[5]; 935| 54| Tw = T5 * Te; 936| 54| TT = T3 * Te; 937| 54| Tz = T2 * Te; 938| 54| TQ = T6 * Te; 939| 54| T9 = W[4]; 940| 54| Tv = T2 * T9; 941| 54| TU = T6 * T9; 942| 54| TA = T5 * T9; 943| 54| TP = T3 * T9; 944| 54| } 945| 54| T21 = TP - TQ; 946| 54| T23 = TT + TU; 947| 54| { 948| 54| E T15, T17, Ta, Tf, T1a, T1b, Ti, Tj; 949| 54| Tx = Tv - Tw; 950| 54| TR = TP + TQ; 951| 54| T1g = Tz - TA; 952| 54| TB = Tz + TA; 953| 54| T1f = Tv + Tw; 954| 54| TV = TT - TU; 955| 54| T15 = T14 * T9; 956| 54| T17 = T16 * Te; 957| 54| T1Q = T15 + T17; 958| 54| Ta = T8 * T9; 959| 54| Tf = Td * Te; 960| 54| Tg = Ta + Tf; 961| 54| T1a = T14 * Te; 962| 54| T1b = T16 * T9; 963| 54| T1S = T1a - T1b; 964| 54| Ti = T8 * Te; 965| 54| Tj = Td * T9; 966| 54| Tk = Ti - Tj; 967| 54| T18 = T15 - T17; 968| 54| T2s = Ti + Tj; 969| 54| T1c = T1a + T1b; 970| 54| T2q = Ta - Tf; 971| 54| Tn = W[6]; 972| 54| To = W[7]; 973| 54| Tp = FMA(T8, Tn, Td * To); ------------------ | | 1131| 54|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 974| 54| Tr = FNMS(Td, Tn, T8 * To); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 975| 54| T28 = FNMS(T1S, Tn, T1Q * To); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 976| 54| T2x = FNMS(TV, Tn, TR * To); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 977| 54| TY = FMA(T3, Tn, T6 * To); ------------------ | | 1131| 54|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 978| 54| T2k = FMA(T2, Tn, T5 * To); ------------------ | | 1131| 54|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 979| 54| T2m = FNMS(T5, Tn, T2 * To); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 980| 54| T2v = FMA(TR, Tn, TV * To); ------------------ | | 1131| 54|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 981| 54| TG = FNMS(Te, Tn, T9 * To); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 982| 54| TE = FMA(T9, Tn, Te * To); ------------------ | | 1131| 54|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 983| 54| T10 = FNMS(T6, Tn, T3 * To); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 984| 54| T1h = FMA(T1f, Tn, T1g * To); ------------------ | | 1131| 54|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 985| 54| T1E = FMA(Tg, Tn, Tk * To); ------------------ | | 1131| 54|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 986| 54| T26 = FMA(T1Q, Tn, T1S * To); ------------------ | | 1131| 54|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 987| 54| T1B = FNMS(TB, Tn, Tx * To); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 988| 54| T1G = FNMS(Tk, Tn, Tg * To); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 989| 54| T1V = FMA(T14, Tn, T16 * To); ------------------ | | 1131| 54|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 990| 54| T1X = FNMS(T16, Tn, T14 * To); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 991| 54| T1z = FMA(Tx, Tn, TB * To); ------------------ | | 1131| 54|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 992| 54| T1j = FNMS(T1g, Tn, T1f * To); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 993| 54| } 994| 54| } 995| 54| { 996| 54| E T1, T6v, T2F, T6I, TK, T2G, T6u, T6J, T6N, T7c, T2O, T52, T2C, T6k, T48; 997| 54| E T5X, T4L, T5s, T4j, T5W, T4K, T5v, T1o, T6g, T30, T5M, T4A, T56, T3b, T5N; 998| 54| E T4B, T59, T1L, T6h, T3n, T5Q, T4D, T5g, T3y, T5P, T4E, T5d, T2d, T6j, T3L; 999| 54| E T5T, T4I, T5l, T3W, T5U, T4H, T5o; 1000| 54| { 1001| 54| E Tm, T2I, Tt, T2J, Tu, T6s, TD, T2L, TI, T2M, TJ, T6t; 1002| 54| T1 = ri[0]; 1003| 54| T6v = ii[0]; 1004| 54| { 1005| 54| E Th, Tl, Tq, Ts; 1006| 54| Th = ri[WS(rs, 5)]; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1007| 54| Tl = ii[WS(rs, 5)]; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1008| 54| Tm = FMA(Tg, Th, Tk * Tl); ------------------ | | 1131| 54|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1009| 54| T2I = FNMS(Tk, Th, Tg * Tl); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1010| 54| Tq = ri[WS(rs, 20)]; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1011| 54| Ts = ii[WS(rs, 20)]; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1012| 54| Tt = FMA(Tp, Tq, Tr * Ts); ------------------ | | 1131| 54|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1013| 54| T2J = FNMS(Tr, Tq, Tp * Ts); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1014| 54| } 1015| 54| Tu = Tm + Tt; 1016| 54| T6s = T2I + T2J; 1017| 54| { 1018| 54| E Ty, TC, TF, TH; 1019| 54| Ty = ri[WS(rs, 10)]; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1020| 54| TC = ii[WS(rs, 10)]; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1021| 54| TD = FMA(Tx, Ty, TB * TC); ------------------ | | 1131| 54|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1022| 54| T2L = FNMS(TB, Ty, Tx * TC); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1023| 54| TF = ri[WS(rs, 15)]; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1024| 54| TH = ii[WS(rs, 15)]; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1025| 54| TI = FMA(TE, TF, TG * TH); ------------------ | | 1131| 54|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1026| 54| T2M = FNMS(TG, TF, TE * TH); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1027| 54| } 1028| 54| TJ = TD + TI; 1029| 54| T6t = T2L + T2M; 1030| 54| T2F = KP559016994 * (Tu - TJ); 1031| 54| T6I = KP559016994 * (T6s - T6t); 1032| 54| TK = Tu + TJ; 1033| 54| T2G = FNMS(KP250000000, TK, T1); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1034| 54| T6u = T6s + T6t; 1035| 54| T6J = FNMS(KP250000000, T6u, T6v); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1036| 54| { 1037| 54| E T6L, T6M, T2K, T2N; 1038| 54| T6L = Tm - Tt; 1039| 54| T6M = TD - TI; 1040| 54| T6N = FMA(KP951056516, T6L, KP587785252 * T6M); ------------------ | | 1131| 54|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1041| 54| T7c = FNMS(KP587785252, T6L, KP951056516 * T6M); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1042| 54| T2K = T2I - T2J; 1043| 54| T2N = T2L - T2M; 1044| 54| T2O = FMA(KP951056516, T2K, KP587785252 * T2N); ------------------ | | 1131| 54|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1045| 54| T52 = FNMS(KP587785252, T2K, KP951056516 * T2N); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1046| 54| } 1047| 54| } 1048| 54| { 1049| 54| E T2g, T4c, T43, T46, T4h, T4g, T49, T4a, T4d, T2p, T2A, T2B, T2e, T2f; 1050| 54| T2e = ri[WS(rs, 3)]; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1051| 54| T2f = ii[WS(rs, 3)]; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1052| 54| T2g = FMA(T3, T2e, T6 * T2f); ------------------ | | 1131| 54|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1053| 54| T4c = FNMS(T6, T2e, T3 * T2f); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1054| 54| { 1055| 54| E T2j, T41, T2z, T45, T2o, T42, T2u, T44; 1056| 54| { 1057| 54| E T2h, T2i, T2w, T2y; 1058| 54| T2h = ri[WS(rs, 8)]; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1059| 54| T2i = ii[WS(rs, 8)]; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1060| 54| T2j = FMA(T1f, T2h, T1g * T2i); ------------------ | | 1131| 54|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1061| 54| T41 = FNMS(T1g, T2h, T1f * T2i); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1062| 54| T2w = ri[WS(rs, 18)]; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1063| 54| T2y = ii[WS(rs, 18)]; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1064| 54| T2z = FMA(T2v, T2w, T2x * T2y); ------------------ | | 1131| 54|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1065| 54| T45 = FNMS(T2x, T2w, T2v * T2y); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1066| 54| } 1067| 54| { 1068| 54| E T2l, T2n, T2r, T2t; 1069| 54| T2l = ri[WS(rs, 23)]; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1070| 54| T2n = ii[WS(rs, 23)]; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1071| 54| T2o = FMA(T2k, T2l, T2m * T2n); ------------------ | | 1131| 54|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1072| 54| T42 = FNMS(T2m, T2l, T2k * T2n); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1073| 54| T2r = ri[WS(rs, 13)]; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1074| 54| T2t = ii[WS(rs, 13)]; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1075| 54| T2u = FMA(T2q, T2r, T2s * T2t); ------------------ | | 1131| 54|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1076| 54| T44 = FNMS(T2s, T2r, T2q * T2t); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1077| 54| } 1078| 54| T43 = T41 - T42; 1079| 54| T46 = T44 - T45; 1080| 54| T4h = T2u - T2z; 1081| 54| T4g = T2j - T2o; 1082| 54| T49 = T41 + T42; 1083| 54| T4a = T44 + T45; 1084| 54| T4d = T49 + T4a; 1085| 54| T2p = T2j + T2o; 1086| 54| T2A = T2u + T2z; 1087| 54| T2B = T2p + T2A; 1088| 54| } 1089| 54| T2C = T2g + T2B; 1090| 54| T6k = T4c + T4d; 1091| 54| { 1092| 54| E T47, T5r, T40, T5q, T3Y, T3Z; 1093| 54| T47 = FMA(KP951056516, T43, KP587785252 * T46); ------------------ | | 1131| 54|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1094| 54| T5r = FNMS(KP587785252, T43, KP951056516 * T46); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1095| 54| T3Y = KP559016994 * (T2p - T2A); 1096| 54| T3Z = FNMS(KP250000000, T2B, T2g); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1097| 54| T40 = T3Y + T3Z; 1098| 54| T5q = T3Z - T3Y; 1099| 54| T48 = T40 + T47; 1100| 54| T5X = T5q + T5r; 1101| 54| T4L = T40 - T47; 1102| 54| T5s = T5q - T5r; 1103| 54| } 1104| 54| { 1105| 54| E T4i, T5t, T4f, T5u, T4b, T4e; 1106| 54| T4i = FMA(KP951056516, T4g, KP587785252 * T4h); ------------------ | | 1131| 54|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1107| 54| T5t = FNMS(KP587785252, T4g, KP951056516 * T4h); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1108| 54| T4b = KP559016994 * (T49 - T4a); 1109| 54| T4e = FNMS(KP250000000, T4d, T4c); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1110| 54| T4f = T4b + T4e; 1111| 54| T5u = T4e - T4b; 1112| 54| T4j = T4f - T4i; 1113| 54| T5W = T5u - T5t; 1114| 54| T4K = T4i + T4f; 1115| 54| T5v = T5t + T5u; 1116| 54| } 1117| 54| } 1118| 54| { 1119| 54| E TO, T34, T2V, T2Y, T39, T38, T31, T32, T35, T13, T1m, T1n, TM, TN; 1120| 54| TM = ri[WS(rs, 1)]; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1121| 54| TN = ii[WS(rs, 1)]; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1122| 54| TO = FMA(T2, TM, T5 * TN); ------------------ | | 1131| 54|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1123| 54| T34 = FNMS(T5, TM, T2 * TN); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1124| 54| { 1125| 54| E TX, T2T, T1l, T2X, T12, T2U, T1e, T2W; 1126| 54| { 1127| 54| E TS, TW, T1i, T1k; 1128| 54| TS = ri[WS(rs, 6)]; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1129| 54| TW = ii[WS(rs, 6)]; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1130| 54| TX = FMA(TR, TS, TV * TW); ------------------ | | 1131| 54|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1131| 54| T2T = FNMS(TV, TS, TR * TW); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1132| 54| T1i = ri[WS(rs, 16)]; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1133| 54| T1k = ii[WS(rs, 16)]; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1134| 54| T1l = FMA(T1h, T1i, T1j * T1k); ------------------ | | 1131| 54|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1135| 54| T2X = FNMS(T1j, T1i, T1h * T1k); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1136| 54| } 1137| 54| { 1138| 54| E TZ, T11, T19, T1d; 1139| 54| TZ = ri[WS(rs, 21)]; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1140| 54| T11 = ii[WS(rs, 21)]; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1141| 54| T12 = FMA(TY, TZ, T10 * T11); ------------------ | | 1131| 54|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1142| 54| T2U = FNMS(T10, TZ, TY * T11); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1143| 54| T19 = ri[WS(rs, 11)]; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1144| 54| T1d = ii[WS(rs, 11)]; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1145| 54| T1e = FMA(T18, T19, T1c * T1d); ------------------ | | 1131| 54|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1146| 54| T2W = FNMS(T1c, T19, T18 * T1d); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1147| 54| } 1148| 54| T2V = T2T - T2U; 1149| 54| T2Y = T2W - T2X; 1150| 54| T39 = T1e - T1l; 1151| 54| T38 = TX - T12; 1152| 54| T31 = T2T + T2U; 1153| 54| T32 = T2W + T2X; 1154| 54| T35 = T31 + T32; 1155| 54| T13 = TX + T12; 1156| 54| T1m = T1e + T1l; 1157| 54| T1n = T13 + T1m; 1158| 54| } 1159| 54| T1o = TO + T1n; 1160| 54| T6g = T34 + T35; 1161| 54| { 1162| 54| E T2Z, T55, T2S, T54, T2Q, T2R; 1163| 54| T2Z = FMA(KP951056516, T2V, KP587785252 * T2Y); ------------------ | | 1131| 54|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1164| 54| T55 = FNMS(KP587785252, T2V, KP951056516 * T2Y); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1165| 54| T2Q = KP559016994 * (T13 - T1m); 1166| 54| T2R = FNMS(KP250000000, T1n, TO); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1167| 54| T2S = T2Q + T2R; 1168| 54| T54 = T2R - T2Q; 1169| 54| T30 = T2S + T2Z; 1170| 54| T5M = T54 + T55; 1171| 54| T4A = T2S - T2Z; 1172| 54| T56 = T54 - T55; 1173| 54| } 1174| 54| { 1175| 54| E T3a, T57, T37, T58, T33, T36; 1176| 54| T3a = FMA(KP951056516, T38, KP587785252 * T39); ------------------ | | 1131| 54|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1177| 54| T57 = FNMS(KP587785252, T38, KP951056516 * T39); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1178| 54| T33 = KP559016994 * (T31 - T32); 1179| 54| T36 = FNMS(KP250000000, T35, T34); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1180| 54| T37 = T33 + T36; 1181| 54| T58 = T36 - T33; 1182| 54| T3b = T37 - T3a; 1183| 54| T5N = T58 - T57; 1184| 54| T4B = T3a + T37; 1185| 54| T59 = T57 + T58; 1186| 54| } 1187| 54| } 1188| 54| { 1189| 54| E T1r, T3r, T3i, T3l, T3w, T3v, T3o, T3p, T3s, T1y, T1J, T1K, T1p, T1q; 1190| 54| T1p = ri[WS(rs, 4)]; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1191| 54| T1q = ii[WS(rs, 4)]; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1192| 54| T1r = FMA(T8, T1p, Td * T1q); ------------------ | | 1131| 54|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1193| 54| T3r = FNMS(Td, T1p, T8 * T1q); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1194| 54| { 1195| 54| E T1u, T3g, T1I, T3k, T1x, T3h, T1D, T3j; 1196| 54| { 1197| 54| E T1s, T1t, T1F, T1H; 1198| 54| T1s = ri[WS(rs, 9)]; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1199| 54| T1t = ii[WS(rs, 9)]; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1200| 54| T1u = FMA(T9, T1s, Te * T1t); ------------------ | | 1131| 54|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1201| 54| T3g = FNMS(Te, T1s, T9 * T1t); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1202| 54| T1F = ri[WS(rs, 19)]; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1203| 54| T1H = ii[WS(rs, 19)]; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1204| 54| T1I = FMA(T1E, T1F, T1G * T1H); ------------------ | | 1131| 54|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1205| 54| T3k = FNMS(T1G, T1F, T1E * T1H); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1206| 54| } 1207| 54| { 1208| 54| E T1v, T1w, T1A, T1C; 1209| 54| T1v = ri[WS(rs, 24)]; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1210| 54| T1w = ii[WS(rs, 24)]; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1211| 54| T1x = FMA(Tn, T1v, To * T1w); ------------------ | | 1131| 54|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1212| 54| T3h = FNMS(To, T1v, Tn * T1w); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1213| 54| T1A = ri[WS(rs, 14)]; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1214| 54| T1C = ii[WS(rs, 14)]; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1215| 54| T1D = FMA(T1z, T1A, T1B * T1C); ------------------ | | 1131| 54|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1216| 54| T3j = FNMS(T1B, T1A, T1z * T1C); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1217| 54| } 1218| 54| T3i = T3g - T3h; 1219| 54| T3l = T3j - T3k; 1220| 54| T3w = T1D - T1I; 1221| 54| T3v = T1u - T1x; 1222| 54| T3o = T3g + T3h; 1223| 54| T3p = T3j + T3k; 1224| 54| T3s = T3o + T3p; 1225| 54| T1y = T1u + T1x; 1226| 54| T1J = T1D + T1I; 1227| 54| T1K = T1y + T1J; 1228| 54| } 1229| 54| T1L = T1r + T1K; 1230| 54| T6h = T3r + T3s; 1231| 54| { 1232| 54| E T3m, T5f, T3f, T5e, T3d, T3e; 1233| 54| T3m = FMA(KP951056516, T3i, KP587785252 * T3l); ------------------ | | 1131| 54|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1234| 54| T5f = FNMS(KP587785252, T3i, KP951056516 * T3l); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1235| 54| T3d = KP559016994 * (T1y - T1J); 1236| 54| T3e = FNMS(KP250000000, T1K, T1r); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1237| 54| T3f = T3d + T3e; 1238| 54| T5e = T3e - T3d; 1239| 54| T3n = T3f + T3m; 1240| 54| T5Q = T5e + T5f; 1241| 54| T4D = T3f - T3m; 1242| 54| T5g = T5e - T5f; 1243| 54| } 1244| 54| { 1245| 54| E T3x, T5b, T3u, T5c, T3q, T3t; 1246| 54| T3x = FMA(KP951056516, T3v, KP587785252 * T3w); ------------------ | | 1131| 54|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1247| 54| T5b = FNMS(KP587785252, T3v, KP951056516 * T3w); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1248| 54| T3q = KP559016994 * (T3o - T3p); 1249| 54| T3t = FNMS(KP250000000, T3s, T3r); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1250| 54| T3u = T3q + T3t; 1251| 54| T5c = T3t - T3q; 1252| 54| T3y = T3u - T3x; 1253| 54| T5P = T5c - T5b; 1254| 54| T4E = T3x + T3u; 1255| 54| T5d = T5b + T5c; 1256| 54| } 1257| 54| } 1258| 54| { 1259| 54| E T1P, T3P, T3G, T3J, T3U, T3T, T3M, T3N, T3Q, T20, T2b, T2c, T1N, T1O; 1260| 54| T1N = ri[WS(rs, 2)]; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1261| 54| T1O = ii[WS(rs, 2)]; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1262| 54| T1P = FMA(T14, T1N, T16 * T1O); ------------------ | | 1131| 54|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1263| 54| T3P = FNMS(T16, T1N, T14 * T1O); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1264| 54| { 1265| 54| E T1U, T3E, T2a, T3I, T1Z, T3F, T25, T3H; 1266| 54| { 1267| 54| E T1R, T1T, T27, T29; 1268| 54| T1R = ri[WS(rs, 7)]; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1269| 54| T1T = ii[WS(rs, 7)]; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1270| 54| T1U = FMA(T1Q, T1R, T1S * T1T); ------------------ | | 1131| 54|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1271| 54| T3E = FNMS(T1S, T1R, T1Q * T1T); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1272| 54| T27 = ri[WS(rs, 17)]; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1273| 54| T29 = ii[WS(rs, 17)]; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1274| 54| T2a = FMA(T26, T27, T28 * T29); ------------------ | | 1131| 54|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1275| 54| T3I = FNMS(T28, T27, T26 * T29); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1276| 54| } 1277| 54| { 1278| 54| E T1W, T1Y, T22, T24; 1279| 54| T1W = ri[WS(rs, 22)]; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1280| 54| T1Y = ii[WS(rs, 22)]; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1281| 54| T1Z = FMA(T1V, T1W, T1X * T1Y); ------------------ | | 1131| 54|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1282| 54| T3F = FNMS(T1X, T1W, T1V * T1Y); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1283| 54| T22 = ri[WS(rs, 12)]; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1284| 54| T24 = ii[WS(rs, 12)]; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1285| 54| T25 = FMA(T21, T22, T23 * T24); ------------------ | | 1131| 54|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1286| 54| T3H = FNMS(T23, T22, T21 * T24); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1287| 54| } 1288| 54| T3G = T3E - T3F; 1289| 54| T3J = T3H - T3I; 1290| 54| T3U = T25 - T2a; 1291| 54| T3T = T1U - T1Z; 1292| 54| T3M = T3E + T3F; 1293| 54| T3N = T3H + T3I; 1294| 54| T3Q = T3M + T3N; 1295| 54| T20 = T1U + T1Z; 1296| 54| T2b = T25 + T2a; 1297| 54| T2c = T20 + T2b; 1298| 54| } 1299| 54| T2d = T1P + T2c; 1300| 54| T6j = T3P + T3Q; 1301| 54| { 1302| 54| E T3K, T5k, T3D, T5j, T3B, T3C; 1303| 54| T3K = FMA(KP951056516, T3G, KP587785252 * T3J); ------------------ | | 1131| 54|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1304| 54| T5k = FNMS(KP587785252, T3G, KP951056516 * T3J); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1305| 54| T3B = KP559016994 * (T20 - T2b); 1306| 54| T3C = FNMS(KP250000000, T2c, T1P); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1307| 54| T3D = T3B + T3C; 1308| 54| T5j = T3C - T3B; 1309| 54| T3L = T3D + T3K; 1310| 54| T5T = T5j + T5k; 1311| 54| T4I = T3D - T3K; 1312| 54| T5l = T5j - T5k; 1313| 54| } 1314| 54| { 1315| 54| E T3V, T5m, T3S, T5n, T3O, T3R; 1316| 54| T3V = FMA(KP951056516, T3T, KP587785252 * T3U); ------------------ | | 1131| 54|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1317| 54| T5m = FNMS(KP587785252, T3T, KP951056516 * T3U); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1318| 54| T3O = KP559016994 * (T3M - T3N); 1319| 54| T3R = FNMS(KP250000000, T3Q, T3P); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1320| 54| T3S = T3O + T3R; 1321| 54| T5n = T3R - T3O; 1322| 54| T3W = T3S - T3V; 1323| 54| T5U = T5n - T5m; 1324| 54| T4H = T3V + T3S; 1325| 54| T5o = T5m + T5n; 1326| 54| } 1327| 54| } 1328| 54| { 1329| 54| E T6m, T6o, TL, T2E, T6d, T6e, T6n, T6f; 1330| 54| { 1331| 54| E T6i, T6l, T1M, T2D; 1332| 54| T6i = T6g - T6h; 1333| 54| T6l = T6j - T6k; 1334| 54| T6m = FMA(KP951056516, T6i, KP587785252 * T6l); ------------------ | | 1131| 54|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1335| 54| T6o = FNMS(KP587785252, T6i, KP951056516 * T6l); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1336| 54| TL = T1 + TK; 1337| 54| T1M = T1o + T1L; 1338| 54| T2D = T2d + T2C; 1339| 54| T2E = T1M + T2D; 1340| 54| T6d = KP559016994 * (T1M - T2D); 1341| 54| T6e = FNMS(KP250000000, T2E, TL); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1342| 54| } 1343| 54| ri[0] = TL + T2E; 1344| 54| T6n = T6e - T6d; 1345| 54| ri[WS(rs, 10)] = T6n - T6o; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1346| 54| ri[WS(rs, 15)] = T6n + T6o; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1347| 54| T6f = T6d + T6e; 1348| 54| ri[WS(rs, 20)] = T6f - T6m; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1349| 54| ri[WS(rs, 5)] = T6f + T6m; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1350| 54| } 1351| 54| { 1352| 54| E T6C, T6D, T6w, T6r, T6x, T6y, T6E, T6z; 1353| 54| { 1354| 54| E T6A, T6B, T6p, T6q; 1355| 54| T6A = T1o - T1L; 1356| 54| T6B = T2d - T2C; 1357| 54| T6C = FMA(KP951056516, T6A, KP587785252 * T6B); ------------------ | | 1131| 54|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1358| 54| T6D = FNMS(KP587785252, T6A, KP951056516 * T6B); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1359| 54| T6w = T6u + T6v; 1360| 54| T6p = T6g + T6h; 1361| 54| T6q = T6j + T6k; 1362| 54| T6r = T6p + T6q; 1363| 54| T6x = KP559016994 * (T6p - T6q); 1364| 54| T6y = FNMS(KP250000000, T6r, T6w); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1365| 54| } 1366| 54| ii[0] = T6r + T6w; 1367| 54| T6E = T6y - T6x; 1368| 54| ii[WS(rs, 10)] = T6D + T6E; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1369| 54| ii[WS(rs, 15)] = T6E - T6D; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1370| 54| T6z = T6x + T6y; 1371| 54| ii[WS(rs, 5)] = T6z - T6C; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1372| 54| ii[WS(rs, 20)] = T6C + T6z; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1373| 54| } 1374| 54| { 1375| 54| E T2P, T4z, T6O, T70, T4m, T6T, T4n, T6S, T4U, T71, T4X, T6Z, T4O, T75, T4P; 1376| 54| E T74, T4s, T6P, T4v, T6H, T2H, T6K; 1377| 54| T2H = T2F + T2G; 1378| 54| T2P = T2H + T2O; 1379| 54| T4z = T2H - T2O; 1380| 54| T6K = T6I + T6J; 1381| 54| T6O = T6K - T6N; 1382| 54| T70 = T6N + T6K; 1383| 54| { 1384| 54| E T3c, T3z, T3A, T3X, T4k, T4l; 1385| 54| T3c = FMA(KP968583161, T30, KP248689887 * T3b); ------------------ | | 1131| 54|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1386| 54| T3z = FMA(KP535826794, T3n, KP844327925 * T3y); ------------------ | | 1131| 54|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1387| 54| T3A = T3c + T3z; 1388| 54| T3X = FMA(KP876306680, T3L, KP481753674 * T3W); ------------------ | | 1131| 54|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1389| 54| T4k = FMA(KP728968627, T48, KP684547105 * T4j); ------------------ | | 1131| 54|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1390| 54| T4l = T3X + T4k; 1391| 54| T4m = T3A + T4l; 1392| 54| T6T = T3X - T4k; 1393| 54| T4n = KP559016994 * (T3A - T4l); 1394| 54| T6S = T3c - T3z; 1395| 54| } 1396| 54| { 1397| 54| E T4S, T4T, T6X, T4V, T4W, T6Y; 1398| 54| T4S = FNMS(KP844327925, T4A, KP535826794 * T4B); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1399| 54| T4T = FNMS(KP637423989, T4E, KP770513242 * T4D); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1400| 54| T6X = T4S + T4T; 1401| 54| T4V = FMA(KP125333233, T4L, KP992114701 * T4K); ------------------ | | 1131| 54|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1402| 54| T4W = FMA(KP904827052, T4I, KP425779291 * T4H); ------------------ | | 1131| 54|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1403| 54| T6Y = T4W + T4V; 1404| 54| T4U = T4S - T4T; 1405| 54| T71 = KP559016994 * (T6X + T6Y); 1406| 54| T4X = T4V - T4W; 1407| 54| T6Z = T6X - T6Y; 1408| 54| } 1409| 54| { 1410| 54| E T4C, T4F, T4G, T4J, T4M, T4N; 1411| 54| T4C = FMA(KP535826794, T4A, KP844327925 * T4B); ------------------ | | 1131| 54|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1412| 54| T4F = FMA(KP637423989, T4D, KP770513242 * T4E); ------------------ | | 1131| 54|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1413| 54| T4G = T4C - T4F; 1414| 54| T4J = FNMS(KP425779291, T4I, KP904827052 * T4H); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1415| 54| T4M = FNMS(KP992114701, T4L, KP125333233 * T4K); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1416| 54| T4N = T4J + T4M; 1417| 54| T4O = T4G + T4N; 1418| 54| T75 = T4J - T4M; 1419| 54| T4P = KP559016994 * (T4G - T4N); 1420| 54| T74 = T4C + T4F; 1421| 54| } 1422| 54| { 1423| 54| E T4q, T4r, T6F, T4t, T4u, T6G; 1424| 54| T4q = FNMS(KP248689887, T30, KP968583161 * T3b); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1425| 54| T4r = FNMS(KP844327925, T3n, KP535826794 * T3y); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1426| 54| T6F = T4q + T4r; 1427| 54| T4t = FNMS(KP481753674, T3L, KP876306680 * T3W); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1428| 54| T4u = FNMS(KP684547105, T48, KP728968627 * T4j); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1429| 54| T6G = T4t + T4u; 1430| 54| T4s = T4q - T4r; 1431| 54| T6P = KP559016994 * (T6F - T6G); 1432| 54| T4v = T4t - T4u; 1433| 54| T6H = T6F + T6G; 1434| 54| } 1435| 54| ri[WS(rs, 1)] = T2P + T4m; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1436| 54| ii[WS(rs, 1)] = T6H + T6O; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1437| 54| ri[WS(rs, 4)] = T4z + T4O; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1438| 54| ii[WS(rs, 4)] = T6Z + T70; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1439| 54| { 1440| 54| E T4w, T4y, T4p, T4x, T4o; 1441| 54| T4w = FMA(KP951056516, T4s, KP587785252 * T4v); ------------------ | | 1131| 54|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1442| 54| T4y = FNMS(KP587785252, T4s, KP951056516 * T4v); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1443| 54| T4o = FNMS(KP250000000, T4m, T2P); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1444| 54| T4p = T4n + T4o; 1445| 54| T4x = T4o - T4n; 1446| 54| ri[WS(rs, 21)] = T4p - T4w; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1447| 54| ri[WS(rs, 16)] = T4x + T4y; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1448| 54| ri[WS(rs, 6)] = T4p + T4w; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1449| 54| ri[WS(rs, 11)] = T4x - T4y; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1450| 54| } 1451| 54| { 1452| 54| E T6U, T6V, T6R, T6W, T6Q; 1453| 54| T6U = FMA(KP951056516, T6S, KP587785252 * T6T); ------------------ | | 1131| 54|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1454| 54| T6V = FNMS(KP587785252, T6S, KP951056516 * T6T); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1455| 54| T6Q = FNMS(KP250000000, T6H, T6O); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1456| 54| T6R = T6P + T6Q; 1457| 54| T6W = T6Q - T6P; 1458| 54| ii[WS(rs, 6)] = T6R - T6U; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1459| 54| ii[WS(rs, 16)] = T6W - T6V; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1460| 54| ii[WS(rs, 21)] = T6U + T6R; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1461| 54| ii[WS(rs, 11)] = T6V + T6W; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1462| 54| } 1463| 54| { 1464| 54| E T4Y, T50, T4R, T4Z, T4Q; 1465| 54| T4Y = FMA(KP951056516, T4U, KP587785252 * T4X); ------------------ | | 1131| 54|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1466| 54| T50 = FNMS(KP587785252, T4U, KP951056516 * T4X); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1467| 54| T4Q = FNMS(KP250000000, T4O, T4z); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1468| 54| T4R = T4P + T4Q; 1469| 54| T4Z = T4Q - T4P; 1470| 54| ri[WS(rs, 24)] = T4R - T4Y; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1471| 54| ri[WS(rs, 19)] = T4Z + T50; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1472| 54| ri[WS(rs, 9)] = T4R + T4Y; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1473| 54| ri[WS(rs, 14)] = T4Z - T50; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1474| 54| } 1475| 54| { 1476| 54| E T76, T77, T73, T78, T72; 1477| 54| T76 = FMA(KP951056516, T74, KP587785252 * T75); ------------------ | | 1131| 54|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1478| 54| T77 = FNMS(KP587785252, T74, KP951056516 * T75); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1479| 54| T72 = FNMS(KP250000000, T6Z, T70); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1480| 54| T73 = T71 + T72; 1481| 54| T78 = T72 - T71; 1482| 54| ii[WS(rs, 9)] = T73 - T76; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1483| 54| ii[WS(rs, 19)] = T78 - T77; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1484| 54| ii[WS(rs, 24)] = T76 + T73; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1485| 54| ii[WS(rs, 14)] = T77 + T78; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1486| 54| } 1487| 54| } 1488| 54| { 1489| 54| E T53, T5L, T7e, T7q, T5y, T7j, T5z, T7i, T66, T7r, T69, T7p, T60, T7v, T61; 1490| 54| E T7u, T5E, T7f, T5H, T7b, T51, T7d; 1491| 54| T51 = T2G - T2F; 1492| 54| T53 = T51 - T52; 1493| 54| T5L = T51 + T52; 1494| 54| T7d = T6J - T6I; 1495| 54| T7e = T7c + T7d; 1496| 54| T7q = T7d - T7c; 1497| 54| { 1498| 54| E T5a, T5h, T5i, T5p, T5w, T5x; 1499| 54| T5a = FMA(KP876306680, T56, KP481753674 * T59); ------------------ | | 1131| 54|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1500| 54| T5h = FNMS(KP425779291, T5g, KP904827052 * T5d); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1501| 54| T5i = T5a + T5h; 1502| 54| T5p = FMA(KP535826794, T5l, KP844327925 * T5o); ------------------ | | 1131| 54|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1503| 54| T5w = FMA(KP062790519, T5s, KP998026728 * T5v); ------------------ | | 1131| 54|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1504| 54| T5x = T5p + T5w; 1505| 54| T5y = T5i + T5x; 1506| 54| T7j = T5p - T5w; 1507| 54| T5z = KP559016994 * (T5i - T5x); 1508| 54| T7i = T5a - T5h; 1509| 54| } 1510| 54| { 1511| 54| E T64, T65, T7n, T67, T68, T7o; 1512| 54| T64 = FNMS(KP684547105, T5M, KP728968627 * T5N); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1513| 54| T65 = FMA(KP125333233, T5Q, KP992114701 * T5P); ------------------ | | 1131| 54|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1514| 54| T7n = T64 - T65; 1515| 54| T67 = FNMS(KP998026728, T5T, KP062790519 * T5U); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1516| 54| T68 = FMA(KP770513242, T5X, KP637423989 * T5W); ------------------ | | 1131| 54|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1517| 54| T7o = T67 - T68; 1518| 54| T66 = T64 + T65; 1519| 54| T7r = KP559016994 * (T7n - T7o); 1520| 54| T69 = T67 + T68; 1521| 54| T7p = T7n + T7o; 1522| 54| } 1523| 54| { 1524| 54| E T5O, T5R, T5S, T5V, T5Y, T5Z; 1525| 54| T5O = FMA(KP728968627, T5M, KP684547105 * T5N); ------------------ | | 1131| 54|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1526| 54| T5R = FNMS(KP992114701, T5Q, KP125333233 * T5P); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1527| 54| T5S = T5O + T5R; 1528| 54| T5V = FMA(KP062790519, T5T, KP998026728 * T5U); ------------------ | | 1131| 54|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1529| 54| T5Y = FNMS(KP637423989, T5X, KP770513242 * T5W); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1530| 54| T5Z = T5V + T5Y; 1531| 54| T60 = T5S + T5Z; 1532| 54| T7v = T5V - T5Y; 1533| 54| T61 = KP559016994 * (T5S - T5Z); 1534| 54| T7u = T5O - T5R; 1535| 54| } 1536| 54| { 1537| 54| E T5C, T5D, T79, T5F, T5G, T7a; 1538| 54| T5C = FNMS(KP481753674, T56, KP876306680 * T59); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1539| 54| T5D = FMA(KP904827052, T5g, KP425779291 * T5d); ------------------ | | 1131| 54|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1540| 54| T79 = T5C - T5D; 1541| 54| T5F = FNMS(KP844327925, T5l, KP535826794 * T5o); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1542| 54| T5G = FNMS(KP998026728, T5s, KP062790519 * T5v); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1543| 54| T7a = T5F + T5G; 1544| 54| T5E = T5C + T5D; 1545| 54| T7f = KP559016994 * (T79 - T7a); 1546| 54| T5H = T5F - T5G; 1547| 54| T7b = T79 + T7a; 1548| 54| } 1549| 54| ri[WS(rs, 2)] = T53 + T5y; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1550| 54| ii[WS(rs, 2)] = T7b + T7e; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1551| 54| ri[WS(rs, 3)] = T5L + T60; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1552| 54| ii[WS(rs, 3)] = T7p + T7q; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1553| 54| { 1554| 54| E T5I, T5K, T5B, T5J, T5A; 1555| 54| T5I = FMA(KP951056516, T5E, KP587785252 * T5H); ------------------ | | 1131| 54|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1556| 54| T5K = FNMS(KP587785252, T5E, KP951056516 * T5H); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1557| 54| T5A = FNMS(KP250000000, T5y, T53); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1558| 54| T5B = T5z + T5A; 1559| 54| T5J = T5A - T5z; 1560| 54| ri[WS(rs, 22)] = T5B - T5I; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1561| 54| ri[WS(rs, 17)] = T5J + T5K; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1562| 54| ri[WS(rs, 7)] = T5B + T5I; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1563| 54| ri[WS(rs, 12)] = T5J - T5K; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1564| 54| } 1565| 54| { 1566| 54| E T7k, T7l, T7h, T7m, T7g; 1567| 54| T7k = FMA(KP951056516, T7i, KP587785252 * T7j); ------------------ | | 1131| 54|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1568| 54| T7l = FNMS(KP587785252, T7i, KP951056516 * T7j); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1569| 54| T7g = FNMS(KP250000000, T7b, T7e); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1570| 54| T7h = T7f + T7g; 1571| 54| T7m = T7g - T7f; 1572| 54| ii[WS(rs, 7)] = T7h - T7k; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1573| 54| ii[WS(rs, 17)] = T7m - T7l; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1574| 54| ii[WS(rs, 22)] = T7k + T7h; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1575| 54| ii[WS(rs, 12)] = T7l + T7m; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1576| 54| } 1577| 54| { 1578| 54| E T6a, T6c, T63, T6b, T62; 1579| 54| T6a = FMA(KP951056516, T66, KP587785252 * T69); ------------------ | | 1131| 54|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1580| 54| T6c = FNMS(KP587785252, T66, KP951056516 * T69); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1581| 54| T62 = FNMS(KP250000000, T60, T5L); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1582| 54| T63 = T61 + T62; 1583| 54| T6b = T62 - T61; 1584| 54| ri[WS(rs, 23)] = T63 - T6a; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1585| 54| ri[WS(rs, 18)] = T6b + T6c; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1586| 54| ri[WS(rs, 8)] = T63 + T6a; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1587| 54| ri[WS(rs, 13)] = T6b - T6c; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1588| 54| } 1589| 54| { 1590| 54| E T7w, T7x, T7t, T7y, T7s; 1591| 54| T7w = FMA(KP951056516, T7u, KP587785252 * T7v); ------------------ | | 1131| 54|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1592| 54| T7x = FNMS(KP587785252, T7u, KP951056516 * T7v); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1593| 54| T7s = FNMS(KP250000000, T7p, T7q); ------------------ | | 1134| 54|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1594| 54| T7t = T7r + T7s; 1595| 54| T7y = T7s - T7r; 1596| 54| ii[WS(rs, 8)] = T7t - T7w; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1597| 54| ii[WS(rs, 18)] = T7y - T7x; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1598| 54| ii[WS(rs, 23)] = T7w + T7t; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1599| 54| ii[WS(rs, 13)] = T7x + T7y; ------------------ | | 825| 54|#define WS(stride, i) (stride * i) ------------------ 1600| 54| } 1601| 54| } 1602| 54| } 1603| 54| } 1604| 3| } 1605| 3|} fftw_codelet_t2_32: 1890| 1|void X(codelet_t2_32) (planner *p) { 1891| 1| X(kdft_dit_register) (p, t2_32, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 1892| 1|} t2_32.c:t2_32: 1025| 6|{ 1026| 6| DK(KP195090322, +0.195090322016128267848284868477022240927691618); ------------------ | | 1056| 6|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 6|# define K(x) ((E) x) | | ------------------ ------------------ 1027| 6| DK(KP980785280, +0.980785280403230449126182236134239036973933731); ------------------ | | 1056| 6|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 6|# define K(x) ((E) x) | | ------------------ ------------------ 1028| 6| DK(KP555570233, +0.555570233019602224742830813948532874374937191); ------------------ | | 1056| 6|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 6|# define K(x) ((E) x) | | ------------------ ------------------ 1029| 6| DK(KP831469612, +0.831469612302545237078788377617905756738560812); ------------------ | | 1056| 6|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 6|# define K(x) ((E) x) | | ------------------ ------------------ 1030| 6| DK(KP382683432, +0.382683432365089771728459984030398866761344562); ------------------ | | 1056| 6|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 6|# define K(x) ((E) x) | | ------------------ ------------------ 1031| 6| DK(KP923879532, +0.923879532511286756128183189396788286822416626); ------------------ | | 1056| 6|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 6|# define K(x) ((E) x) | | ------------------ ------------------ 1032| 6| DK(KP707106781, +0.707106781186547524400844362104849039284835938); ------------------ | | 1056| 6|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 6|# define K(x) ((E) x) | | ------------------ ------------------ 1033| 6| { 1034| 6| INT m; 1035| 78| for (m = mb, W = W + (mb * 8); m < me; m = m + 1, ri = ri + ms, ii = ii + ms, W = W + 8, MAKE_VOLATILE_STRIDE(64, rs)) { ------------------ | | 851| 72| (nptr <= ESTIMATED_AVAILABLE_INDEX_REGISTERS ? \ | | ------------------ | | | | 849| 144|#define ESTIMATED_AVAILABLE_INDEX_REGISTERS 16 | | ------------------ | | | Branch (851:7): [Folded, False: 72] | | ------------------ | | 852| 72| 0 : \ | | 853| 72| ((x) = (x) ^ X(an_INT_guaranteed_to_be_zero))) | | ------------------ | | | | 78| 72|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 72|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ | Branch (1035:35): [True: 72, False: 6] ------------------ 1036| 72| E T2, T5, T3, T6, T8, TM, TO, Td, T9, Te, Th, Tl, TD, TH, T1y; 1037| 72| E T1H, T15, T1A, T11, T1F, T1n, T1p, T2q, T2I, T2u, T2K, T2V, T3b, T2Z, T3d; 1038| 72| E Tu, Ty, T3l, T3n, T1t, T1v, T2f, T2h, T1a, T1e, T32, T34, T1W, T1Y, T2C; 1039| 72| E T2E, Tg, TR, Tk, TS, Tm, TV, To, TT, T1M, T21, T1P, T22, T1Q, T25; 1040| 72| E T1S, T23; 1041| 72| { 1042| 72| E Ts, T1d, Tx, T18, Tt, T1c, Tw, T19, TB, T14, TG, TZ, TC, T13, TF; 1043| 72| E T10; 1044| 72| { 1045| 72| E T4, Tc, T7, Tb; 1046| 72| T2 = W[0]; 1047| 72| T5 = W[1]; 1048| 72| T3 = W[2]; 1049| 72| T6 = W[3]; 1050| 72| T4 = T2 * T3; 1051| 72| Tc = T5 * T3; 1052| 72| T7 = T5 * T6; 1053| 72| Tb = T2 * T6; 1054| 72| T8 = T4 + T7; 1055| 72| TM = T4 - T7; 1056| 72| TO = Tb + Tc; 1057| 72| Td = Tb - Tc; 1058| 72| T9 = W[4]; 1059| 72| Ts = T2 * T9; 1060| 72| T1d = T6 * T9; 1061| 72| Tx = T5 * T9; 1062| 72| T18 = T3 * T9; 1063| 72| Te = W[5]; 1064| 72| Tt = T5 * Te; 1065| 72| T1c = T3 * Te; 1066| 72| Tw = T2 * Te; 1067| 72| T19 = T6 * Te; 1068| 72| Th = W[6]; 1069| 72| TB = T3 * Th; 1070| 72| T14 = T5 * Th; 1071| 72| TG = T6 * Th; 1072| 72| TZ = T2 * Th; 1073| 72| Tl = W[7]; 1074| 72| TC = T6 * Tl; 1075| 72| T13 = T2 * Tl; 1076| 72| TF = T3 * Tl; 1077| 72| T10 = T5 * Tl; 1078| 72| } 1079| 72| TD = TB + TC; 1080| 72| TH = TF - TG; 1081| 72| T1y = TZ + T10; 1082| 72| T1H = TF + TG; 1083| 72| T15 = T13 + T14; 1084| 72| T1A = T13 - T14; 1085| 72| T11 = TZ - T10; 1086| 72| T1F = TB - TC; 1087| 72| T1n = FMA(T9, Th, Te * Tl); ------------------ | | 1131| 72|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1088| 72| T1p = FNMS(Te, Th, T9 * Tl); ------------------ | | 1134| 72|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1089| 72| { 1090| 72| E T2o, T2p, T2s, T2t; 1091| 72| T2o = T8 * Th; 1092| 72| T2p = Td * Tl; 1093| 72| T2q = T2o + T2p; 1094| 72| T2I = T2o - T2p; 1095| 72| T2s = T8 * Tl; 1096| 72| T2t = Td * Th; 1097| 72| T2u = T2s - T2t; 1098| 72| T2K = T2s + T2t; 1099| 72| } 1100| 72| { 1101| 72| E T2T, T2U, T2X, T2Y; 1102| 72| T2T = TM * Th; 1103| 72| T2U = TO * Tl; 1104| 72| T2V = T2T - T2U; 1105| 72| T3b = T2T + T2U; 1106| 72| T2X = TM * Tl; 1107| 72| T2Y = TO * Th; 1108| 72| T2Z = T2X + T2Y; 1109| 72| T3d = T2X - T2Y; 1110| 72| Tu = Ts + Tt; 1111| 72| Ty = Tw - Tx; 1112| 72| T3l = FMA(Tu, Th, Ty * Tl); ------------------ | | 1131| 72|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1113| 72| T3n = FNMS(Ty, Th, Tu * Tl); ------------------ | | 1134| 72|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1114| 72| } 1115| 72| T1t = Ts - Tt; 1116| 72| T1v = Tw + Tx; 1117| 72| T2f = FMA(T1t, Th, T1v * Tl); ------------------ | | 1131| 72|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1118| 72| T2h = FNMS(T1v, Th, T1t * Tl); ------------------ | | 1134| 72|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1119| 72| T1a = T18 - T19; 1120| 72| T1e = T1c + T1d; 1121| 72| T32 = FMA(T1a, Th, T1e * Tl); ------------------ | | 1131| 72|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1122| 72| T34 = FNMS(T1e, Th, T1a * Tl); ------------------ | | 1134| 72|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1123| 72| T1W = T18 + T19; 1124| 72| T1Y = T1c - T1d; 1125| 72| T2C = FMA(T1W, Th, T1Y * Tl); ------------------ | | 1131| 72|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1126| 72| T2E = FNMS(T1Y, Th, T1W * Tl); ------------------ | | 1134| 72|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1127| 72| { 1128| 72| E Ta, Tf, Ti, Tj; 1129| 72| Ta = T8 * T9; 1130| 72| Tf = Td * Te; 1131| 72| Tg = Ta - Tf; 1132| 72| TR = Ta + Tf; 1133| 72| Ti = T8 * Te; 1134| 72| Tj = Td * T9; 1135| 72| Tk = Ti + Tj; 1136| 72| TS = Ti - Tj; 1137| 72| } 1138| 72| Tm = FMA(Tg, Th, Tk * Tl); ------------------ | | 1131| 72|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1139| 72| TV = FNMS(TS, Th, TR * Tl); ------------------ | | 1134| 72|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1140| 72| To = FNMS(Tk, Th, Tg * Tl); ------------------ | | 1134| 72|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1141| 72| TT = FMA(TR, Th, TS * Tl); ------------------ | | 1131| 72|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1142| 72| { 1143| 72| E T1K, T1L, T1N, T1O; 1144| 72| T1K = TM * T9; 1145| 72| T1L = TO * Te; 1146| 72| T1M = T1K - T1L; 1147| 72| T21 = T1K + T1L; 1148| 72| T1N = TM * Te; 1149| 72| T1O = TO * T9; 1150| 72| T1P = T1N + T1O; 1151| 72| T22 = T1N - T1O; 1152| 72| } 1153| 72| T1Q = FMA(T1M, Th, T1P * Tl); ------------------ | | 1131| 72|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1154| 72| T25 = FNMS(T22, Th, T21 * Tl); ------------------ | | 1134| 72|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1155| 72| T1S = FNMS(T1P, Th, T1M * Tl); ------------------ | | 1134| 72|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1156| 72| T23 = FMA(T21, Th, T22 * Tl); ------------------ | | 1131| 72|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1157| 72| } 1158| 72| { 1159| 72| E TL, T6f, T8c, T8q, T3F, T5t, T7I, T7W, T2y, T6B, T6y, T7j, T4k, T5J, T4B; 1160| 72| E T5G, T3h, T6H, T6O, T7o, T4L, T5N, T52, T5Q, T1i, T7V, T6i, T7D, T3K, T5u; 1161| 72| E T3P, T5v, T1E, T6n, T6m, T7e, T3W, T5y, T41, T5z, T29, T6p, T6s, T7f, T47; 1162| 72| E T5B, T4c, T5C, T2R, T6z, T6E, T7k, T4v, T5H, T4E, T5K, T3y, T6P, T6K, T7p; 1163| 72| E T4W, T5R, T55, T5O; 1164| 72| { 1165| 72| E T1, T7G, Tq, T7F, TA, T3C, TJ, T3D, Tn, Tp; 1166| 72| T1 = ri[0]; 1167| 72| T7G = ii[0]; 1168| 72| Tn = ri[WS(rs, 16)]; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1169| 72| Tp = ii[WS(rs, 16)]; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1170| 72| Tq = FMA(Tm, Tn, To * Tp); ------------------ | | 1131| 72|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1171| 72| T7F = FNMS(To, Tn, Tm * Tp); ------------------ | | 1134| 72|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1172| 72| { 1173| 72| E Tv, Tz, TE, TI; 1174| 72| Tv = ri[WS(rs, 8)]; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1175| 72| Tz = ii[WS(rs, 8)]; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1176| 72| TA = FMA(Tu, Tv, Ty * Tz); ------------------ | | 1131| 72|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1177| 72| T3C = FNMS(Ty, Tv, Tu * Tz); ------------------ | | 1134| 72|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1178| 72| TE = ri[WS(rs, 24)]; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1179| 72| TI = ii[WS(rs, 24)]; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1180| 72| TJ = FMA(TD, TE, TH * TI); ------------------ | | 1131| 72|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1181| 72| T3D = FNMS(TH, TE, TD * TI); ------------------ | | 1134| 72|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1182| 72| } 1183| 72| { 1184| 72| E Tr, TK, T8a, T8b; 1185| 72| Tr = T1 + Tq; 1186| 72| TK = TA + TJ; 1187| 72| TL = Tr + TK; 1188| 72| T6f = Tr - TK; 1189| 72| T8a = T7G - T7F; 1190| 72| T8b = TA - TJ; 1191| 72| T8c = T8a - T8b; 1192| 72| T8q = T8b + T8a; 1193| 72| } 1194| 72| { 1195| 72| E T3B, T3E, T7E, T7H; 1196| 72| T3B = T1 - Tq; 1197| 72| T3E = T3C - T3D; 1198| 72| T3F = T3B - T3E; 1199| 72| T5t = T3B + T3E; 1200| 72| T7E = T3C + T3D; 1201| 72| T7H = T7F + T7G; 1202| 72| T7I = T7E + T7H; 1203| 72| T7W = T7H - T7E; 1204| 72| } 1205| 72| } 1206| 72| { 1207| 72| E T2e, T4g, T2w, T4z, T2j, T4h, T2n, T4y; 1208| 72| { 1209| 72| E T2c, T2d, T2r, T2v; 1210| 72| T2c = ri[WS(rs, 1)]; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1211| 72| T2d = ii[WS(rs, 1)]; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1212| 72| T2e = FMA(T2, T2c, T5 * T2d); ------------------ | | 1131| 72|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1213| 72| T4g = FNMS(T5, T2c, T2 * T2d); ------------------ | | 1134| 72|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1214| 72| T2r = ri[WS(rs, 25)]; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1215| 72| T2v = ii[WS(rs, 25)]; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1216| 72| T2w = FMA(T2q, T2r, T2u * T2v); ------------------ | | 1131| 72|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1217| 72| T4z = FNMS(T2u, T2r, T2q * T2v); ------------------ | | 1134| 72|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1218| 72| } 1219| 72| { 1220| 72| E T2g, T2i, T2l, T2m; 1221| 72| T2g = ri[WS(rs, 17)]; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1222| 72| T2i = ii[WS(rs, 17)]; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1223| 72| T2j = FMA(T2f, T2g, T2h * T2i); ------------------ | | 1131| 72|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1224| 72| T4h = FNMS(T2h, T2g, T2f * T2i); ------------------ | | 1134| 72|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1225| 72| T2l = ri[WS(rs, 9)]; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1226| 72| T2m = ii[WS(rs, 9)]; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1227| 72| T2n = FMA(T9, T2l, Te * T2m); ------------------ | | 1131| 72|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1228| 72| T4y = FNMS(Te, T2l, T9 * T2m); ------------------ | | 1134| 72|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1229| 72| } 1230| 72| { 1231| 72| E T2k, T2x, T6w, T6x; 1232| 72| T2k = T2e + T2j; 1233| 72| T2x = T2n + T2w; 1234| 72| T2y = T2k + T2x; 1235| 72| T6B = T2k - T2x; 1236| 72| T6w = T4g + T4h; 1237| 72| T6x = T4y + T4z; 1238| 72| T6y = T6w - T6x; 1239| 72| T7j = T6w + T6x; 1240| 72| } 1241| 72| { 1242| 72| E T4i, T4j, T4x, T4A; 1243| 72| T4i = T4g - T4h; 1244| 72| T4j = T2n - T2w; 1245| 72| T4k = T4i + T4j; 1246| 72| T5J = T4i - T4j; 1247| 72| T4x = T2e - T2j; 1248| 72| T4A = T4y - T4z; 1249| 72| T4B = T4x - T4A; 1250| 72| T5G = T4x + T4A; 1251| 72| } 1252| 72| } 1253| 72| { 1254| 72| E T31, T4Y, T3f, T4J, T36, T4Z, T3a, T4I; 1255| 72| { 1256| 72| E T2W, T30, T3c, T3e; 1257| 72| T2W = ri[WS(rs, 31)]; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1258| 72| T30 = ii[WS(rs, 31)]; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1259| 72| T31 = FMA(T2V, T2W, T2Z * T30); ------------------ | | 1131| 72|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1260| 72| T4Y = FNMS(T2Z, T2W, T2V * T30); ------------------ | | 1134| 72|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1261| 72| T3c = ri[WS(rs, 23)]; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1262| 72| T3e = ii[WS(rs, 23)]; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1263| 72| T3f = FMA(T3b, T3c, T3d * T3e); ------------------ | | 1131| 72|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1264| 72| T4J = FNMS(T3d, T3c, T3b * T3e); ------------------ | | 1134| 72|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1265| 72| } 1266| 72| { 1267| 72| E T33, T35, T38, T39; 1268| 72| T33 = ri[WS(rs, 15)]; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1269| 72| T35 = ii[WS(rs, 15)]; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1270| 72| T36 = FMA(T32, T33, T34 * T35); ------------------ | | 1131| 72|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1271| 72| T4Z = FNMS(T34, T33, T32 * T35); ------------------ | | 1134| 72|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1272| 72| T38 = ri[WS(rs, 7)]; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1273| 72| T39 = ii[WS(rs, 7)]; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1274| 72| T3a = FMA(TR, T38, TS * T39); ------------------ | | 1131| 72|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1275| 72| T4I = FNMS(TS, T38, TR * T39); ------------------ | | 1134| 72|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1276| 72| } 1277| 72| { 1278| 72| E T37, T3g, T6M, T6N; 1279| 72| T37 = T31 + T36; 1280| 72| T3g = T3a + T3f; 1281| 72| T3h = T37 + T3g; 1282| 72| T6H = T37 - T3g; 1283| 72| T6M = T4Y + T4Z; 1284| 72| T6N = T4I + T4J; 1285| 72| T6O = T6M - T6N; 1286| 72| T7o = T6M + T6N; 1287| 72| } 1288| 72| { 1289| 72| E T4H, T4K, T50, T51; 1290| 72| T4H = T31 - T36; 1291| 72| T4K = T4I - T4J; 1292| 72| T4L = T4H - T4K; 1293| 72| T5N = T4H + T4K; 1294| 72| T50 = T4Y - T4Z; 1295| 72| T51 = T3a - T3f; 1296| 72| T52 = T50 + T51; 1297| 72| T5Q = T50 - T51; 1298| 72| } 1299| 72| } 1300| 72| { 1301| 72| E TQ, T3G, T1g, T3N, TX, T3H, T17, T3M; 1302| 72| { 1303| 72| E TN, TP, T1b, T1f; 1304| 72| TN = ri[WS(rs, 4)]; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1305| 72| TP = ii[WS(rs, 4)]; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1306| 72| TQ = FMA(TM, TN, TO * TP); ------------------ | | 1131| 72|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1307| 72| T3G = FNMS(TO, TN, TM * TP); ------------------ | | 1134| 72|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1308| 72| T1b = ri[WS(rs, 12)]; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1309| 72| T1f = ii[WS(rs, 12)]; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1310| 72| T1g = FMA(T1a, T1b, T1e * T1f); ------------------ | | 1131| 72|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1311| 72| T3N = FNMS(T1e, T1b, T1a * T1f); ------------------ | | 1134| 72|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1312| 72| } 1313| 72| { 1314| 72| E TU, TW, T12, T16; 1315| 72| TU = ri[WS(rs, 20)]; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1316| 72| TW = ii[WS(rs, 20)]; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1317| 72| TX = FMA(TT, TU, TV * TW); ------------------ | | 1131| 72|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1318| 72| T3H = FNMS(TV, TU, TT * TW); ------------------ | | 1134| 72|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1319| 72| T12 = ri[WS(rs, 28)]; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1320| 72| T16 = ii[WS(rs, 28)]; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1321| 72| T17 = FMA(T11, T12, T15 * T16); ------------------ | | 1131| 72|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1322| 72| T3M = FNMS(T15, T12, T11 * T16); ------------------ | | 1134| 72|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1323| 72| } 1324| 72| { 1325| 72| E TY, T1h, T6g, T6h; 1326| 72| TY = TQ + TX; 1327| 72| T1h = T17 + T1g; 1328| 72| T1i = TY + T1h; 1329| 72| T7V = T1h - TY; 1330| 72| T6g = T3G + T3H; 1331| 72| T6h = T3M + T3N; 1332| 72| T6i = T6g - T6h; 1333| 72| T7D = T6g + T6h; 1334| 72| } 1335| 72| { 1336| 72| E T3I, T3J, T3L, T3O; 1337| 72| T3I = T3G - T3H; 1338| 72| T3J = TQ - TX; 1339| 72| T3K = T3I - T3J; 1340| 72| T5u = T3J + T3I; 1341| 72| T3L = T17 - T1g; 1342| 72| T3O = T3M - T3N; 1343| 72| T3P = T3L + T3O; 1344| 72| T5v = T3L - T3O; 1345| 72| } 1346| 72| } 1347| 72| { 1348| 72| E T1m, T3S, T1C, T3Z, T1r, T3T, T1x, T3Y; 1349| 72| { 1350| 72| E T1k, T1l, T1z, T1B; 1351| 72| T1k = ri[WS(rs, 2)]; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1352| 72| T1l = ii[WS(rs, 2)]; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1353| 72| T1m = FMA(T8, T1k, Td * T1l); ------------------ | | 1131| 72|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1354| 72| T3S = FNMS(Td, T1k, T8 * T1l); ------------------ | | 1134| 72|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1355| 72| T1z = ri[WS(rs, 26)]; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1356| 72| T1B = ii[WS(rs, 26)]; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1357| 72| T1C = FMA(T1y, T1z, T1A * T1B); ------------------ | | 1131| 72|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1358| 72| T3Z = FNMS(T1A, T1z, T1y * T1B); ------------------ | | 1134| 72|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1359| 72| } 1360| 72| { 1361| 72| E T1o, T1q, T1u, T1w; 1362| 72| T1o = ri[WS(rs, 18)]; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1363| 72| T1q = ii[WS(rs, 18)]; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1364| 72| T1r = FMA(T1n, T1o, T1p * T1q); ------------------ | | 1131| 72|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1365| 72| T3T = FNMS(T1p, T1o, T1n * T1q); ------------------ | | 1134| 72|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1366| 72| T1u = ri[WS(rs, 10)]; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1367| 72| T1w = ii[WS(rs, 10)]; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1368| 72| T1x = FMA(T1t, T1u, T1v * T1w); ------------------ | | 1131| 72|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1369| 72| T3Y = FNMS(T1v, T1u, T1t * T1w); ------------------ | | 1134| 72|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1370| 72| } 1371| 72| { 1372| 72| E T1s, T1D, T6k, T6l; 1373| 72| T1s = T1m + T1r; 1374| 72| T1D = T1x + T1C; 1375| 72| T1E = T1s + T1D; 1376| 72| T6n = T1s - T1D; 1377| 72| T6k = T3S + T3T; 1378| 72| T6l = T3Y + T3Z; 1379| 72| T6m = T6k - T6l; 1380| 72| T7e = T6k + T6l; 1381| 72| } 1382| 72| { 1383| 72| E T3U, T3V, T3X, T40; 1384| 72| T3U = T3S - T3T; 1385| 72| T3V = T1x - T1C; 1386| 72| T3W = T3U + T3V; 1387| 72| T5y = T3U - T3V; 1388| 72| T3X = T1m - T1r; 1389| 72| T40 = T3Y - T3Z; 1390| 72| T41 = T3X - T40; 1391| 72| T5z = T3X + T40; 1392| 72| } 1393| 72| } 1394| 72| { 1395| 72| E T1J, T43, T27, T4a, T1U, T44, T20, T49; 1396| 72| { 1397| 72| E T1G, T1I, T24, T26; 1398| 72| T1G = ri[WS(rs, 30)]; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1399| 72| T1I = ii[WS(rs, 30)]; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1400| 72| T1J = FMA(T1F, T1G, T1H * T1I); ------------------ | | 1131| 72|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1401| 72| T43 = FNMS(T1H, T1G, T1F * T1I); ------------------ | | 1134| 72|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1402| 72| T24 = ri[WS(rs, 22)]; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1403| 72| T26 = ii[WS(rs, 22)]; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1404| 72| T27 = FMA(T23, T24, T25 * T26); ------------------ | | 1131| 72|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1405| 72| T4a = FNMS(T25, T24, T23 * T26); ------------------ | | 1134| 72|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1406| 72| } 1407| 72| { 1408| 72| E T1R, T1T, T1X, T1Z; 1409| 72| T1R = ri[WS(rs, 14)]; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1410| 72| T1T = ii[WS(rs, 14)]; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1411| 72| T1U = FMA(T1Q, T1R, T1S * T1T); ------------------ | | 1131| 72|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1412| 72| T44 = FNMS(T1S, T1R, T1Q * T1T); ------------------ | | 1134| 72|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1413| 72| T1X = ri[WS(rs, 6)]; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1414| 72| T1Z = ii[WS(rs, 6)]; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1415| 72| T20 = FMA(T1W, T1X, T1Y * T1Z); ------------------ | | 1131| 72|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1416| 72| T49 = FNMS(T1Y, T1X, T1W * T1Z); ------------------ | | 1134| 72|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1417| 72| } 1418| 72| { 1419| 72| E T1V, T28, T6q, T6r; 1420| 72| T1V = T1J + T1U; 1421| 72| T28 = T20 + T27; 1422| 72| T29 = T1V + T28; 1423| 72| T6p = T1V - T28; 1424| 72| T6q = T43 + T44; 1425| 72| T6r = T49 + T4a; 1426| 72| T6s = T6q - T6r; 1427| 72| T7f = T6q + T6r; 1428| 72| } 1429| 72| { 1430| 72| E T45, T46, T48, T4b; 1431| 72| T45 = T43 - T44; 1432| 72| T46 = T20 - T27; 1433| 72| T47 = T45 + T46; 1434| 72| T5B = T45 - T46; 1435| 72| T48 = T1J - T1U; 1436| 72| T4b = T49 - T4a; 1437| 72| T4c = T48 - T4b; 1438| 72| T5C = T48 + T4b; 1439| 72| } 1440| 72| } 1441| 72| { 1442| 72| E T2B, T4r, T2G, T4s, T4q, T4t, T2M, T4m, T2P, T4n, T4l, T4o; 1443| 72| { 1444| 72| E T2z, T2A, T2D, T2F; 1445| 72| T2z = ri[WS(rs, 5)]; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1446| 72| T2A = ii[WS(rs, 5)]; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1447| 72| T2B = FMA(T21, T2z, T22 * T2A); ------------------ | | 1131| 72|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1448| 72| T4r = FNMS(T22, T2z, T21 * T2A); ------------------ | | 1134| 72|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1449| 72| T2D = ri[WS(rs, 21)]; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1450| 72| T2F = ii[WS(rs, 21)]; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1451| 72| T2G = FMA(T2C, T2D, T2E * T2F); ------------------ | | 1131| 72|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1452| 72| T4s = FNMS(T2E, T2D, T2C * T2F); ------------------ | | 1134| 72|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1453| 72| } 1454| 72| T4q = T2B - T2G; 1455| 72| T4t = T4r - T4s; 1456| 72| { 1457| 72| E T2J, T2L, T2N, T2O; 1458| 72| T2J = ri[WS(rs, 29)]; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1459| 72| T2L = ii[WS(rs, 29)]; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1460| 72| T2M = FMA(T2I, T2J, T2K * T2L); ------------------ | | 1131| 72|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1461| 72| T4m = FNMS(T2K, T2J, T2I * T2L); ------------------ | | 1134| 72|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1462| 72| T2N = ri[WS(rs, 13)]; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1463| 72| T2O = ii[WS(rs, 13)]; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1464| 72| T2P = FMA(T1M, T2N, T1P * T2O); ------------------ | | 1131| 72|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1465| 72| T4n = FNMS(T1P, T2N, T1M * T2O); ------------------ | | 1134| 72|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1466| 72| } 1467| 72| T4l = T2M - T2P; 1468| 72| T4o = T4m - T4n; 1469| 72| { 1470| 72| E T2H, T2Q, T6C, T6D; 1471| 72| T2H = T2B + T2G; 1472| 72| T2Q = T2M + T2P; 1473| 72| T2R = T2H + T2Q; 1474| 72| T6z = T2Q - T2H; 1475| 72| T6C = T4r + T4s; 1476| 72| T6D = T4m + T4n; 1477| 72| T6E = T6C - T6D; 1478| 72| T7k = T6C + T6D; 1479| 72| } 1480| 72| { 1481| 72| E T4p, T4u, T4C, T4D; 1482| 72| T4p = T4l - T4o; 1483| 72| T4u = T4q + T4t; 1484| 72| T4v = KP707106781 * (T4p - T4u); 1485| 72| T5H = KP707106781 * (T4u + T4p); 1486| 72| T4C = T4t - T4q; 1487| 72| T4D = T4l + T4o; 1488| 72| T4E = KP707106781 * (T4C - T4D); 1489| 72| T5K = KP707106781 * (T4C + T4D); 1490| 72| } 1491| 72| } 1492| 72| { 1493| 72| E T3k, T4M, T3p, T4N, T4O, T4P, T3t, T4S, T3w, T4T, T4R, T4U; 1494| 72| { 1495| 72| E T3i, T3j, T3m, T3o; 1496| 72| T3i = ri[WS(rs, 3)]; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1497| 72| T3j = ii[WS(rs, 3)]; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1498| 72| T3k = FMA(T3, T3i, T6 * T3j); ------------------ | | 1131| 72|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1499| 72| T4M = FNMS(T6, T3i, T3 * T3j); ------------------ | | 1134| 72|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1500| 72| T3m = ri[WS(rs, 19)]; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1501| 72| T3o = ii[WS(rs, 19)]; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1502| 72| T3p = FMA(T3l, T3m, T3n * T3o); ------------------ | | 1131| 72|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1503| 72| T4N = FNMS(T3n, T3m, T3l * T3o); ------------------ | | 1134| 72|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1504| 72| } 1505| 72| T4O = T4M - T4N; 1506| 72| T4P = T3k - T3p; 1507| 72| { 1508| 72| E T3r, T3s, T3u, T3v; 1509| 72| T3r = ri[WS(rs, 27)]; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1510| 72| T3s = ii[WS(rs, 27)]; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1511| 72| T3t = FMA(Th, T3r, Tl * T3s); ------------------ | | 1131| 72|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1512| 72| T4S = FNMS(Tl, T3r, Th * T3s); ------------------ | | 1134| 72|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1513| 72| T3u = ri[WS(rs, 11)]; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1514| 72| T3v = ii[WS(rs, 11)]; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1515| 72| T3w = FMA(Tg, T3u, Tk * T3v); ------------------ | | 1131| 72|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1516| 72| T4T = FNMS(Tk, T3u, Tg * T3v); ------------------ | | 1134| 72|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1517| 72| } 1518| 72| T4R = T3t - T3w; 1519| 72| T4U = T4S - T4T; 1520| 72| { 1521| 72| E T3q, T3x, T6I, T6J; 1522| 72| T3q = T3k + T3p; 1523| 72| T3x = T3t + T3w; 1524| 72| T3y = T3q + T3x; 1525| 72| T6P = T3x - T3q; 1526| 72| T6I = T4M + T4N; 1527| 72| T6J = T4S + T4T; 1528| 72| T6K = T6I - T6J; 1529| 72| T7p = T6I + T6J; 1530| 72| } 1531| 72| { 1532| 72| E T4Q, T4V, T53, T54; 1533| 72| T4Q = T4O - T4P; 1534| 72| T4V = T4R + T4U; 1535| 72| T4W = KP707106781 * (T4Q - T4V); 1536| 72| T5R = KP707106781 * (T4Q + T4V); 1537| 72| T53 = T4R - T4U; 1538| 72| T54 = T4P + T4O; 1539| 72| T55 = KP707106781 * (T53 - T54); 1540| 72| T5O = KP707106781 * (T54 + T53); 1541| 72| } 1542| 72| } 1543| 72| { 1544| 72| E T2b, T7x, T7K, T7M, T3A, T7L, T7A, T7B; 1545| 72| { 1546| 72| E T1j, T2a, T7C, T7J; 1547| 72| T1j = TL + T1i; 1548| 72| T2a = T1E + T29; 1549| 72| T2b = T1j + T2a; 1550| 72| T7x = T1j - T2a; 1551| 72| T7C = T7e + T7f; 1552| 72| T7J = T7D + T7I; 1553| 72| T7K = T7C + T7J; 1554| 72| T7M = T7J - T7C; 1555| 72| } 1556| 72| { 1557| 72| E T2S, T3z, T7y, T7z; 1558| 72| T2S = T2y + T2R; 1559| 72| T3z = T3h + T3y; 1560| 72| T3A = T2S + T3z; 1561| 72| T7L = T3z - T2S; 1562| 72| T7y = T7j + T7k; 1563| 72| T7z = T7o + T7p; 1564| 72| T7A = T7y - T7z; 1565| 72| T7B = T7y + T7z; 1566| 72| } 1567| 72| ri[WS(rs, 16)] = T2b - T3A; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1568| 72| ii[WS(rs, 16)] = T7K - T7B; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1569| 72| ri[0] = T2b + T3A; 1570| 72| ii[0] = T7B + T7K; 1571| 72| ri[WS(rs, 24)] = T7x - T7A; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1572| 72| ii[WS(rs, 24)] = T7M - T7L; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1573| 72| ri[WS(rs, 8)] = T7x + T7A; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1574| 72| ii[WS(rs, 8)] = T7L + T7M; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1575| 72| } 1576| 72| { 1577| 72| E T7h, T7t, T7Q, T7S, T7m, T7u, T7r, T7v; 1578| 72| { 1579| 72| E T7d, T7g, T7O, T7P; 1580| 72| T7d = TL - T1i; 1581| 72| T7g = T7e - T7f; 1582| 72| T7h = T7d + T7g; 1583| 72| T7t = T7d - T7g; 1584| 72| T7O = T29 - T1E; 1585| 72| T7P = T7I - T7D; 1586| 72| T7Q = T7O + T7P; 1587| 72| T7S = T7P - T7O; 1588| 72| } 1589| 72| { 1590| 72| E T7i, T7l, T7n, T7q; 1591| 72| T7i = T2y - T2R; 1592| 72| T7l = T7j - T7k; 1593| 72| T7m = T7i + T7l; 1594| 72| T7u = T7l - T7i; 1595| 72| T7n = T3h - T3y; 1596| 72| T7q = T7o - T7p; 1597| 72| T7r = T7n - T7q; 1598| 72| T7v = T7n + T7q; 1599| 72| } 1600| 72| { 1601| 72| E T7s, T7N, T7w, T7R; 1602| 72| T7s = KP707106781 * (T7m + T7r); 1603| 72| ri[WS(rs, 20)] = T7h - T7s; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1604| 72| ri[WS(rs, 4)] = T7h + T7s; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1605| 72| T7N = KP707106781 * (T7u + T7v); 1606| 72| ii[WS(rs, 4)] = T7N + T7Q; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1607| 72| ii[WS(rs, 20)] = T7Q - T7N; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1608| 72| T7w = KP707106781 * (T7u - T7v); 1609| 72| ri[WS(rs, 28)] = T7t - T7w; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1610| 72| ri[WS(rs, 12)] = T7t + T7w; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1611| 72| T7R = KP707106781 * (T7r - T7m); 1612| 72| ii[WS(rs, 12)] = T7R + T7S; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1613| 72| ii[WS(rs, 28)] = T7S - T7R; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1614| 72| } 1615| 72| } 1616| 72| { 1617| 72| E T6j, T7X, T83, T6X, T6u, T7U, T77, T7b, T70, T82, T6G, T6U, T74, T7a, T6R; 1618| 72| E T6V; 1619| 72| { 1620| 72| E T6o, T6t, T6A, T6F; 1621| 72| T6j = T6f - T6i; 1622| 72| T7X = T7V + T7W; 1623| 72| T83 = T7W - T7V; 1624| 72| T6X = T6f + T6i; 1625| 72| T6o = T6m - T6n; 1626| 72| T6t = T6p + T6s; 1627| 72| T6u = KP707106781 * (T6o - T6t); 1628| 72| T7U = KP707106781 * (T6o + T6t); 1629| 72| { 1630| 72| E T75, T76, T6Y, T6Z; 1631| 72| T75 = T6H + T6K; 1632| 72| T76 = T6O + T6P; 1633| 72| T77 = FNMS(KP382683432, T76, KP923879532 * T75); ------------------ | | 1134| 72|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1634| 72| T7b = FMA(KP923879532, T76, KP382683432 * T75); ------------------ | | 1131| 72|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1635| 72| T6Y = T6n + T6m; 1636| 72| T6Z = T6p - T6s; 1637| 72| T70 = KP707106781 * (T6Y + T6Z); 1638| 72| T82 = KP707106781 * (T6Z - T6Y); 1639| 72| } 1640| 72| T6A = T6y - T6z; 1641| 72| T6F = T6B - T6E; 1642| 72| T6G = FMA(KP923879532, T6A, KP382683432 * T6F); ------------------ | | 1131| 72|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1643| 72| T6U = FNMS(KP923879532, T6F, KP382683432 * T6A); ------------------ | | 1134| 72|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1644| 72| { 1645| 72| E T72, T73, T6L, T6Q; 1646| 72| T72 = T6y + T6z; 1647| 72| T73 = T6B + T6E; 1648| 72| T74 = FMA(KP382683432, T72, KP923879532 * T73); ------------------ | | 1131| 72|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1649| 72| T7a = FNMS(KP382683432, T73, KP923879532 * T72); ------------------ | | 1134| 72|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1650| 72| T6L = T6H - T6K; 1651| 72| T6Q = T6O - T6P; 1652| 72| T6R = FNMS(KP923879532, T6Q, KP382683432 * T6L); ------------------ | | 1134| 72|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1653| 72| T6V = FMA(KP382683432, T6Q, KP923879532 * T6L); ------------------ | | 1131| 72|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1654| 72| } 1655| 72| } 1656| 72| { 1657| 72| E T6v, T6S, T81, T84; 1658| 72| T6v = T6j + T6u; 1659| 72| T6S = T6G + T6R; 1660| 72| ri[WS(rs, 22)] = T6v - T6S; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1661| 72| ri[WS(rs, 6)] = T6v + T6S; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1662| 72| T81 = T6U + T6V; 1663| 72| T84 = T82 + T83; 1664| 72| ii[WS(rs, 6)] = T81 + T84; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1665| 72| ii[WS(rs, 22)] = T84 - T81; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1666| 72| } 1667| 72| { 1668| 72| E T6T, T6W, T85, T86; 1669| 72| T6T = T6j - T6u; 1670| 72| T6W = T6U - T6V; 1671| 72| ri[WS(rs, 30)] = T6T - T6W; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1672| 72| ri[WS(rs, 14)] = T6T + T6W; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1673| 72| T85 = T6R - T6G; 1674| 72| T86 = T83 - T82; 1675| 72| ii[WS(rs, 14)] = T85 + T86; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1676| 72| ii[WS(rs, 30)] = T86 - T85; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1677| 72| } 1678| 72| { 1679| 72| E T71, T78, T7T, T7Y; 1680| 72| T71 = T6X + T70; 1681| 72| T78 = T74 + T77; 1682| 72| ri[WS(rs, 18)] = T71 - T78; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1683| 72| ri[WS(rs, 2)] = T71 + T78; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1684| 72| T7T = T7a + T7b; 1685| 72| T7Y = T7U + T7X; 1686| 72| ii[WS(rs, 2)] = T7T + T7Y; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1687| 72| ii[WS(rs, 18)] = T7Y - T7T; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1688| 72| } 1689| 72| { 1690| 72| E T79, T7c, T7Z, T80; 1691| 72| T79 = T6X - T70; 1692| 72| T7c = T7a - T7b; 1693| 72| ri[WS(rs, 26)] = T79 - T7c; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1694| 72| ri[WS(rs, 10)] = T79 + T7c; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1695| 72| T7Z = T77 - T74; 1696| 72| T80 = T7X - T7U; 1697| 72| ii[WS(rs, 10)] = T7Z + T80; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1698| 72| ii[WS(rs, 26)] = T80 - T7Z; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1699| 72| } 1700| 72| } 1701| 72| { 1702| 72| E T3R, T5d, T8r, T8x, T4e, T8o, T5n, T5r, T4G, T5a, T5g, T8w, T5k, T5q, T57; 1703| 72| E T5b, T3Q, T8p; 1704| 72| T3Q = KP707106781 * (T3K - T3P); 1705| 72| T3R = T3F - T3Q; 1706| 72| T5d = T3F + T3Q; 1707| 72| T8p = KP707106781 * (T5v - T5u); 1708| 72| T8r = T8p + T8q; 1709| 72| T8x = T8q - T8p; 1710| 72| { 1711| 72| E T42, T4d, T5l, T5m; 1712| 72| T42 = FNMS(KP923879532, T41, KP382683432 * T3W); ------------------ | | 1134| 72|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1713| 72| T4d = FMA(KP382683432, T47, KP923879532 * T4c); ------------------ | | 1131| 72|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1714| 72| T4e = T42 - T4d; 1715| 72| T8o = T42 + T4d; 1716| 72| T5l = T4L + T4W; 1717| 72| T5m = T52 + T55; 1718| 72| T5n = FNMS(KP555570233, T5m, KP831469612 * T5l); ------------------ | | 1134| 72|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1719| 72| T5r = FMA(KP831469612, T5m, KP555570233 * T5l); ------------------ | | 1131| 72|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1720| 72| } 1721| 72| { 1722| 72| E T4w, T4F, T5e, T5f; 1723| 72| T4w = T4k - T4v; 1724| 72| T4F = T4B - T4E; 1725| 72| T4G = FMA(KP980785280, T4w, KP195090322 * T4F); ------------------ | | 1131| 72|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1726| 72| T5a = FNMS(KP980785280, T4F, KP195090322 * T4w); ------------------ | | 1134| 72|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1727| 72| T5e = FMA(KP923879532, T3W, KP382683432 * T41); ------------------ | | 1131| 72|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1728| 72| T5f = FNMS(KP923879532, T47, KP382683432 * T4c); ------------------ | | 1134| 72|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1729| 72| T5g = T5e + T5f; 1730| 72| T8w = T5f - T5e; 1731| 72| } 1732| 72| { 1733| 72| E T5i, T5j, T4X, T56; 1734| 72| T5i = T4k + T4v; 1735| 72| T5j = T4B + T4E; 1736| 72| T5k = FMA(KP555570233, T5i, KP831469612 * T5j); ------------------ | | 1131| 72|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1737| 72| T5q = FNMS(KP555570233, T5j, KP831469612 * T5i); ------------------ | | 1134| 72|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1738| 72| T4X = T4L - T4W; 1739| 72| T56 = T52 - T55; 1740| 72| T57 = FNMS(KP980785280, T56, KP195090322 * T4X); ------------------ | | 1134| 72|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1741| 72| T5b = FMA(KP195090322, T56, KP980785280 * T4X); ------------------ | | 1131| 72|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1742| 72| } 1743| 72| { 1744| 72| E T4f, T58, T8v, T8y; 1745| 72| T4f = T3R + T4e; 1746| 72| T58 = T4G + T57; 1747| 72| ri[WS(rs, 23)] = T4f - T58; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1748| 72| ri[WS(rs, 7)] = T4f + T58; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1749| 72| T8v = T5a + T5b; 1750| 72| T8y = T8w + T8x; 1751| 72| ii[WS(rs, 7)] = T8v + T8y; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1752| 72| ii[WS(rs, 23)] = T8y - T8v; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1753| 72| } 1754| 72| { 1755| 72| E T59, T5c, T8z, T8A; 1756| 72| T59 = T3R - T4e; 1757| 72| T5c = T5a - T5b; 1758| 72| ri[WS(rs, 31)] = T59 - T5c; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1759| 72| ri[WS(rs, 15)] = T59 + T5c; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1760| 72| T8z = T57 - T4G; 1761| 72| T8A = T8x - T8w; 1762| 72| ii[WS(rs, 15)] = T8z + T8A; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1763| 72| ii[WS(rs, 31)] = T8A - T8z; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1764| 72| } 1765| 72| { 1766| 72| E T5h, T5o, T8n, T8s; 1767| 72| T5h = T5d + T5g; 1768| 72| T5o = T5k + T5n; 1769| 72| ri[WS(rs, 19)] = T5h - T5o; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1770| 72| ri[WS(rs, 3)] = T5h + T5o; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1771| 72| T8n = T5q + T5r; 1772| 72| T8s = T8o + T8r; 1773| 72| ii[WS(rs, 3)] = T8n + T8s; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1774| 72| ii[WS(rs, 19)] = T8s - T8n; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1775| 72| } 1776| 72| { 1777| 72| E T5p, T5s, T8t, T8u; 1778| 72| T5p = T5d - T5g; 1779| 72| T5s = T5q - T5r; 1780| 72| ri[WS(rs, 27)] = T5p - T5s; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1781| 72| ri[WS(rs, 11)] = T5p + T5s; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1782| 72| T8t = T5n - T5k; 1783| 72| T8u = T8r - T8o; 1784| 72| ii[WS(rs, 11)] = T8t + T8u; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1785| 72| ii[WS(rs, 27)] = T8u - T8t; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1786| 72| } 1787| 72| } 1788| 72| { 1789| 72| E T5x, T5Z, T8d, T8j, T5E, T88, T69, T6d, T5M, T5W, T62, T8i, T66, T6c, T5T; 1790| 72| E T5X, T5w, T89; 1791| 72| T5w = KP707106781 * (T5u + T5v); 1792| 72| T5x = T5t - T5w; 1793| 72| T5Z = T5t + T5w; 1794| 72| T89 = KP707106781 * (T3K + T3P); 1795| 72| T8d = T89 + T8c; 1796| 72| T8j = T8c - T89; 1797| 72| { 1798| 72| E T5A, T5D, T67, T68; 1799| 72| T5A = FNMS(KP382683432, T5z, KP923879532 * T5y); ------------------ | | 1134| 72|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1800| 72| T5D = FMA(KP923879532, T5B, KP382683432 * T5C); ------------------ | | 1131| 72|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1801| 72| T5E = T5A - T5D; 1802| 72| T88 = T5A + T5D; 1803| 72| T67 = T5N + T5O; 1804| 72| T68 = T5Q + T5R; 1805| 72| T69 = FNMS(KP195090322, T68, KP980785280 * T67); ------------------ | | 1134| 72|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1806| 72| T6d = FMA(KP195090322, T67, KP980785280 * T68); ------------------ | | 1131| 72|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1807| 72| } 1808| 72| { 1809| 72| E T5I, T5L, T60, T61; 1810| 72| T5I = T5G - T5H; 1811| 72| T5L = T5J - T5K; 1812| 72| T5M = FMA(KP555570233, T5I, KP831469612 * T5L); ------------------ | | 1131| 72|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1813| 72| T5W = FNMS(KP831469612, T5I, KP555570233 * T5L); ------------------ | | 1134| 72|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1814| 72| T60 = FMA(KP382683432, T5y, KP923879532 * T5z); ------------------ | | 1131| 72|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1815| 72| T61 = FNMS(KP382683432, T5B, KP923879532 * T5C); ------------------ | | 1134| 72|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1816| 72| T62 = T60 + T61; 1817| 72| T8i = T61 - T60; 1818| 72| } 1819| 72| { 1820| 72| E T64, T65, T5P, T5S; 1821| 72| T64 = T5G + T5H; 1822| 72| T65 = T5J + T5K; 1823| 72| T66 = FMA(KP980785280, T64, KP195090322 * T65); ------------------ | | 1131| 72|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1824| 72| T6c = FNMS(KP195090322, T64, KP980785280 * T65); ------------------ | | 1134| 72|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1825| 72| T5P = T5N - T5O; 1826| 72| T5S = T5Q - T5R; 1827| 72| T5T = FNMS(KP831469612, T5S, KP555570233 * T5P); ------------------ | | 1134| 72|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 1828| 72| T5X = FMA(KP831469612, T5P, KP555570233 * T5S); ------------------ | | 1131| 72|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 1829| 72| } 1830| 72| { 1831| 72| E T5F, T5U, T8h, T8k; 1832| 72| T5F = T5x + T5E; 1833| 72| T5U = T5M + T5T; 1834| 72| ri[WS(rs, 21)] = T5F - T5U; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1835| 72| ri[WS(rs, 5)] = T5F + T5U; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1836| 72| T8h = T5W + T5X; 1837| 72| T8k = T8i + T8j; 1838| 72| ii[WS(rs, 5)] = T8h + T8k; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1839| 72| ii[WS(rs, 21)] = T8k - T8h; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1840| 72| } 1841| 72| { 1842| 72| E T5V, T5Y, T8l, T8m; 1843| 72| T5V = T5x - T5E; 1844| 72| T5Y = T5W - T5X; 1845| 72| ri[WS(rs, 29)] = T5V - T5Y; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1846| 72| ri[WS(rs, 13)] = T5V + T5Y; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1847| 72| T8l = T5T - T5M; 1848| 72| T8m = T8j - T8i; 1849| 72| ii[WS(rs, 13)] = T8l + T8m; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1850| 72| ii[WS(rs, 29)] = T8m - T8l; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1851| 72| } 1852| 72| { 1853| 72| E T63, T6a, T87, T8e; 1854| 72| T63 = T5Z + T62; 1855| 72| T6a = T66 + T69; 1856| 72| ri[WS(rs, 17)] = T63 - T6a; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1857| 72| ri[WS(rs, 1)] = T63 + T6a; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1858| 72| T87 = T6c + T6d; 1859| 72| T8e = T88 + T8d; 1860| 72| ii[WS(rs, 1)] = T87 + T8e; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1861| 72| ii[WS(rs, 17)] = T8e - T87; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1862| 72| } 1863| 72| { 1864| 72| E T6b, T6e, T8f, T8g; 1865| 72| T6b = T5Z - T62; 1866| 72| T6e = T6c - T6d; 1867| 72| ri[WS(rs, 25)] = T6b - T6e; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1868| 72| ri[WS(rs, 9)] = T6b + T6e; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1869| 72| T8f = T69 - T66; 1870| 72| T8g = T8d - T88; 1871| 72| ii[WS(rs, 9)] = T8f + T8g; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1872| 72| ii[WS(rs, 25)] = T8g - T8f; ------------------ | | 825| 72|#define WS(stride, i) (stride * i) ------------------ 1873| 72| } 1874| 72| } 1875| 72| } 1876| 72| } 1877| 6| } 1878| 6|} fftw_codelet_t2_4: 197| 1|void X(codelet_t2_4) (planner *p) { 198| 1| X(kdft_dit_register) (p, t2_4, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 199| 1|} t2_4.c:t2_4: 132| 2|{ 133| 2| { 134| 2| INT m; 135| 30| for (m = mb, W = W + (mb * 4); m < me; m = m + 1, ri = ri + ms, ii = ii + ms, W = W + 4, MAKE_VOLATILE_STRIDE(8, rs)) { ------------------ | | 851| 28| (nptr <= ESTIMATED_AVAILABLE_INDEX_REGISTERS ? \ | | ------------------ | | | | 849| 56|#define ESTIMATED_AVAILABLE_INDEX_REGISTERS 16 | | ------------------ | | | Branch (851:7): [True: 28, Folded] | | ------------------ | | 852| 28| 0 : \ | | 853| 28| ((x) = (x) ^ X(an_INT_guaranteed_to_be_zero))) | | ------------------ | | | | 78| 0|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ | Branch (135:35): [True: 28, False: 2] ------------------ 136| 28| E T2, T4, T3, T5, T6, T8; 137| 28| T2 = W[0]; 138| 28| T4 = W[1]; 139| 28| T3 = W[2]; 140| 28| T5 = W[3]; 141| 28| T6 = FMA(T2, T3, T4 * T5); ------------------ | | 1131| 28|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 142| 28| T8 = FNMS(T4, T3, T2 * T5); ------------------ | | 1134| 28|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 143| 28| { 144| 28| E T1, Tp, Ta, To, Te, Tk, Th, Tl, T7, T9; 145| 28| T1 = ri[0]; 146| 28| Tp = ii[0]; 147| 28| T7 = ri[WS(rs, 2)]; ------------------ | | 825| 28|#define WS(stride, i) (stride * i) ------------------ 148| 28| T9 = ii[WS(rs, 2)]; ------------------ | | 825| 28|#define WS(stride, i) (stride * i) ------------------ 149| 28| Ta = FMA(T6, T7, T8 * T9); ------------------ | | 1131| 28|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 150| 28| To = FNMS(T8, T7, T6 * T9); ------------------ | | 1134| 28|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 151| 28| { 152| 28| E Tc, Td, Tf, Tg; 153| 28| Tc = ri[WS(rs, 1)]; ------------------ | | 825| 28|#define WS(stride, i) (stride * i) ------------------ 154| 28| Td = ii[WS(rs, 1)]; ------------------ | | 825| 28|#define WS(stride, i) (stride * i) ------------------ 155| 28| Te = FMA(T2, Tc, T4 * Td); ------------------ | | 1131| 28|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 156| 28| Tk = FNMS(T4, Tc, T2 * Td); ------------------ | | 1134| 28|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 157| 28| Tf = ri[WS(rs, 3)]; ------------------ | | 825| 28|#define WS(stride, i) (stride * i) ------------------ 158| 28| Tg = ii[WS(rs, 3)]; ------------------ | | 825| 28|#define WS(stride, i) (stride * i) ------------------ 159| 28| Th = FMA(T3, Tf, T5 * Tg); ------------------ | | 1131| 28|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 160| 28| Tl = FNMS(T5, Tf, T3 * Tg); ------------------ | | 1134| 28|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 161| 28| } 162| 28| { 163| 28| E Tb, Ti, Tn, Tq; 164| 28| Tb = T1 + Ta; 165| 28| Ti = Te + Th; 166| 28| ri[WS(rs, 2)] = Tb - Ti; ------------------ | | 825| 28|#define WS(stride, i) (stride * i) ------------------ 167| 28| ri[0] = Tb + Ti; 168| 28| Tn = Tk + Tl; 169| 28| Tq = To + Tp; 170| 28| ii[0] = Tn + Tq; 171| 28| ii[WS(rs, 2)] = Tq - Tn; ------------------ | | 825| 28|#define WS(stride, i) (stride * i) ------------------ 172| 28| } 173| 28| { 174| 28| E Tj, Tm, Tr, Ts; 175| 28| Tj = T1 - Ta; 176| 28| Tm = Tk - Tl; 177| 28| ri[WS(rs, 3)] = Tj - Tm; ------------------ | | 825| 28|#define WS(stride, i) (stride * i) ------------------ 178| 28| ri[WS(rs, 1)] = Tj + Tm; ------------------ | | 825| 28|#define WS(stride, i) (stride * i) ------------------ 179| 28| Tr = Tp - To; 180| 28| Ts = Te - Th; 181| 28| ii[WS(rs, 1)] = Tr - Ts; ------------------ | | 825| 28|#define WS(stride, i) (stride * i) ------------------ 182| 28| ii[WS(rs, 3)] = Ts + Tr; ------------------ | | 825| 28|#define WS(stride, i) (stride * i) ------------------ 183| 28| } 184| 28| } 185| 28| } 186| 2| } 187| 2|} fftw_codelet_t2_5: 261| 1|void X(codelet_t2_5) (planner *p) { 262| 1| X(kdft_dit_register) (p, t2_5, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 263| 1|} t2_5.c:t2_5: 157| 106|{ 158| 106| DK(KP250000000, +0.250000000000000000000000000000000000000000000); ------------------ | | 1056| 106|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 106|# define K(x) ((E) x) | | ------------------ ------------------ 159| 106| DK(KP559016994, +0.559016994374947424102293417182819058860154590); ------------------ | | 1056| 106|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 106|# define K(x) ((E) x) | | ------------------ ------------------ 160| 106| DK(KP587785252, +0.587785252292473129168705954639072768597652438); ------------------ | | 1056| 106|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 106|# define K(x) ((E) x) | | ------------------ ------------------ 161| 106| DK(KP951056516, +0.951056516295153572116439333379382143405698634); ------------------ | | 1056| 106|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 106|# define K(x) ((E) x) | | ------------------ ------------------ 162| 106| { 163| 106| INT m; 164| 1.72k| for (m = mb, W = W + (mb * 4); m < me; m = m + 1, ri = ri + ms, ii = ii + ms, W = W + 4, MAKE_VOLATILE_STRIDE(10, rs)) { ------------------ | | 851| 1.62k| (nptr <= ESTIMATED_AVAILABLE_INDEX_REGISTERS ? \ | | ------------------ | | | | 849| 3.24k|#define ESTIMATED_AVAILABLE_INDEX_REGISTERS 16 | | ------------------ | | | Branch (851:7): [True: 1.62k, Folded] | | ------------------ | | 852| 1.62k| 0 : \ | | 853| 1.62k| ((x) = (x) ^ X(an_INT_guaranteed_to_be_zero))) | | ------------------ | | | | 78| 0|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ | Branch (164:35): [True: 1.62k, False: 106] ------------------ 165| 1.62k| E T2, T4, T7, T9, Tb, Tl, Tf, Tj; 166| 1.62k| { 167| 1.62k| E T8, Te, Ta, Td; 168| 1.62k| T2 = W[0]; 169| 1.62k| T4 = W[1]; 170| 1.62k| T7 = W[2]; 171| 1.62k| T9 = W[3]; 172| 1.62k| T8 = T2 * T7; 173| 1.62k| Te = T4 * T7; 174| 1.62k| Ta = T4 * T9; 175| 1.62k| Td = T2 * T9; 176| 1.62k| Tb = T8 - Ta; 177| 1.62k| Tl = Td - Te; 178| 1.62k| Tf = Td + Te; 179| 1.62k| Tj = T8 + Ta; 180| 1.62k| } 181| 1.62k| { 182| 1.62k| E T1, TI, Ty, TB, TN, TM, TF, TG, TH, Ti, Tr, Ts; 183| 1.62k| T1 = ri[0]; 184| 1.62k| TI = ii[0]; 185| 1.62k| { 186| 1.62k| E T6, Tw, Tq, TA, Th, Tx, Tn, Tz; 187| 1.62k| { 188| 1.62k| E T3, T5, To, Tp; 189| 1.62k| T3 = ri[WS(rs, 1)]; ------------------ | | 825| 1.62k|#define WS(stride, i) (stride * i) ------------------ 190| 1.62k| T5 = ii[WS(rs, 1)]; ------------------ | | 825| 1.62k|#define WS(stride, i) (stride * i) ------------------ 191| 1.62k| T6 = FMA(T2, T3, T4 * T5); ------------------ | | 1131| 1.62k|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 192| 1.62k| Tw = FNMS(T4, T3, T2 * T5); ------------------ | | 1134| 1.62k|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 193| 1.62k| To = ri[WS(rs, 3)]; ------------------ | | 825| 1.62k|#define WS(stride, i) (stride * i) ------------------ 194| 1.62k| Tp = ii[WS(rs, 3)]; ------------------ | | 825| 1.62k|#define WS(stride, i) (stride * i) ------------------ 195| 1.62k| Tq = FMA(T7, To, T9 * Tp); ------------------ | | 1131| 1.62k|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 196| 1.62k| TA = FNMS(T9, To, T7 * Tp); ------------------ | | 1134| 1.62k|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 197| 1.62k| } 198| 1.62k| { 199| 1.62k| E Tc, Tg, Tk, Tm; 200| 1.62k| Tc = ri[WS(rs, 4)]; ------------------ | | 825| 1.62k|#define WS(stride, i) (stride * i) ------------------ 201| 1.62k| Tg = ii[WS(rs, 4)]; ------------------ | | 825| 1.62k|#define WS(stride, i) (stride * i) ------------------ 202| 1.62k| Th = FMA(Tb, Tc, Tf * Tg); ------------------ | | 1131| 1.62k|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 203| 1.62k| Tx = FNMS(Tf, Tc, Tb * Tg); ------------------ | | 1134| 1.62k|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 204| 1.62k| Tk = ri[WS(rs, 2)]; ------------------ | | 825| 1.62k|#define WS(stride, i) (stride * i) ------------------ 205| 1.62k| Tm = ii[WS(rs, 2)]; ------------------ | | 825| 1.62k|#define WS(stride, i) (stride * i) ------------------ 206| 1.62k| Tn = FMA(Tj, Tk, Tl * Tm); ------------------ | | 1131| 1.62k|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 207| 1.62k| Tz = FNMS(Tl, Tk, Tj * Tm); ------------------ | | 1134| 1.62k|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 208| 1.62k| } 209| 1.62k| Ty = Tw - Tx; 210| 1.62k| TB = Tz - TA; 211| 1.62k| TN = Tn - Tq; 212| 1.62k| TM = T6 - Th; 213| 1.62k| TF = Tw + Tx; 214| 1.62k| TG = Tz + TA; 215| 1.62k| TH = TF + TG; 216| 1.62k| Ti = T6 + Th; 217| 1.62k| Tr = Tn + Tq; 218| 1.62k| Ts = Ti + Tr; 219| 1.62k| } 220| 1.62k| ri[0] = T1 + Ts; 221| 1.62k| ii[0] = TH + TI; 222| 1.62k| { 223| 1.62k| E TC, TE, Tv, TD, Tt, Tu; 224| 1.62k| TC = FMA(KP951056516, Ty, KP587785252 * TB); ------------------ | | 1131| 1.62k|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 225| 1.62k| TE = FNMS(KP587785252, Ty, KP951056516 * TB); ------------------ | | 1134| 1.62k|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 226| 1.62k| Tt = KP559016994 * (Ti - Tr); 227| 1.62k| Tu = FNMS(KP250000000, Ts, T1); ------------------ | | 1134| 1.62k|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 228| 1.62k| Tv = Tt + Tu; 229| 1.62k| TD = Tu - Tt; 230| 1.62k| ri[WS(rs, 4)] = Tv - TC; ------------------ | | 825| 1.62k|#define WS(stride, i) (stride * i) ------------------ 231| 1.62k| ri[WS(rs, 3)] = TD + TE; ------------------ | | 825| 1.62k|#define WS(stride, i) (stride * i) ------------------ 232| 1.62k| ri[WS(rs, 1)] = Tv + TC; ------------------ | | 825| 1.62k|#define WS(stride, i) (stride * i) ------------------ 233| 1.62k| ri[WS(rs, 2)] = TD - TE; ------------------ | | 825| 1.62k|#define WS(stride, i) (stride * i) ------------------ 234| 1.62k| } 235| 1.62k| { 236| 1.62k| E TO, TP, TL, TQ, TJ, TK; 237| 1.62k| TO = FMA(KP951056516, TM, KP587785252 * TN); ------------------ | | 1131| 1.62k|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 238| 1.62k| TP = FNMS(KP587785252, TM, KP951056516 * TN); ------------------ | | 1134| 1.62k|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 239| 1.62k| TJ = KP559016994 * (TF - TG); 240| 1.62k| TK = FNMS(KP250000000, TH, TI); ------------------ | | 1134| 1.62k|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 241| 1.62k| TL = TJ + TK; 242| 1.62k| TQ = TK - TJ; 243| 1.62k| ii[WS(rs, 1)] = TL - TO; ------------------ | | 825| 1.62k|#define WS(stride, i) (stride * i) ------------------ 244| 1.62k| ii[WS(rs, 3)] = TQ - TP; ------------------ | | 825| 1.62k|#define WS(stride, i) (stride * i) ------------------ 245| 1.62k| ii[WS(rs, 4)] = TO + TL; ------------------ | | 825| 1.62k|#define WS(stride, i) (stride * i) ------------------ 246| 1.62k| ii[WS(rs, 2)] = TP + TQ; ------------------ | | 825| 1.62k|#define WS(stride, i) (stride * i) ------------------ 247| 1.62k| } 248| 1.62k| } 249| 1.62k| } 250| 106| } 251| 106|} fftw_codelet_t2_64: 4240| 1|void X(codelet_t2_64) (planner *p) { 4241| 1| X(kdft_dit_register) (p, t2_64, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 4242| 1|} fftw_codelet_t2_8: 387| 1|void X(codelet_t2_8) (planner *p) { 388| 1| X(kdft_dit_register) (p, t2_8, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 389| 1|} t2_8.c:t2_8: 228| 51|{ 229| 51| DK(KP707106781, +0.707106781186547524400844362104849039284835938); ------------------ | | 1056| 51|#define DK(name, value) const E name = K(value) | | ------------------ | | | | 1054| 51|# define K(x) ((E) x) | | ------------------ ------------------ 230| 51| { 231| 51| INT m; 232| 968| for (m = mb, W = W + (mb * 6); m < me; m = m + 1, ri = ri + ms, ii = ii + ms, W = W + 6, MAKE_VOLATILE_STRIDE(16, rs)) { ------------------ | | 851| 917| (nptr <= ESTIMATED_AVAILABLE_INDEX_REGISTERS ? \ | | ------------------ | | | | 849| 1.83k|#define ESTIMATED_AVAILABLE_INDEX_REGISTERS 16 | | ------------------ | | | Branch (851:7): [True: 917, Folded] | | ------------------ | | 852| 917| 0 : \ | | 853| 917| ((x) = (x) ^ X(an_INT_guaranteed_to_be_zero))) | | ------------------ | | | | 78| 0|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ | Branch (232:35): [True: 917, False: 51] ------------------ 233| 917| E T2, T5, T3, T6, T8, Tc, Tg, Ti, Tl, Tm, Tn, Tz, Tp, Tx; 234| 917| { 235| 917| E T4, Tb, T7, Ta; 236| 917| T2 = W[0]; 237| 917| T5 = W[1]; 238| 917| T3 = W[2]; 239| 917| T6 = W[3]; 240| 917| T4 = T2 * T3; 241| 917| Tb = T5 * T3; 242| 917| T7 = T5 * T6; 243| 917| Ta = T2 * T6; 244| 917| T8 = T4 - T7; 245| 917| Tc = Ta + Tb; 246| 917| Tg = T4 + T7; 247| 917| Ti = Ta - Tb; 248| 917| Tl = W[4]; 249| 917| Tm = W[5]; 250| 917| Tn = FMA(T2, Tl, T5 * Tm); ------------------ | | 1131| 917|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 251| 917| Tz = FNMS(Ti, Tl, Tg * Tm); ------------------ | | 1134| 917|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 252| 917| Tp = FNMS(T5, Tl, T2 * Tm); ------------------ | | 1134| 917|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 253| 917| Tx = FMA(Tg, Tl, Ti * Tm); ------------------ | | 1131| 917|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 254| 917| } 255| 917| { 256| 917| E Tf, T1i, TL, T1d, TJ, T17, TV, TY, Ts, T1j, TO, T1a, TC, T16, TQ; 257| 917| E TT; 258| 917| { 259| 917| E T1, T1c, Te, T1b, T9, Td; 260| 917| T1 = ri[0]; 261| 917| T1c = ii[0]; 262| 917| T9 = ri[WS(rs, 4)]; ------------------ | | 825| 917|#define WS(stride, i) (stride * i) ------------------ 263| 917| Td = ii[WS(rs, 4)]; ------------------ | | 825| 917|#define WS(stride, i) (stride * i) ------------------ 264| 917| Te = FMA(T8, T9, Tc * Td); ------------------ | | 1131| 917|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 265| 917| T1b = FNMS(Tc, T9, T8 * Td); ------------------ | | 1134| 917|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 266| 917| Tf = T1 + Te; 267| 917| T1i = T1c - T1b; 268| 917| TL = T1 - Te; 269| 917| T1d = T1b + T1c; 270| 917| } 271| 917| { 272| 917| E TF, TW, TI, TX; 273| 917| { 274| 917| E TD, TE, TG, TH; 275| 917| TD = ri[WS(rs, 7)]; ------------------ | | 825| 917|#define WS(stride, i) (stride * i) ------------------ 276| 917| TE = ii[WS(rs, 7)]; ------------------ | | 825| 917|#define WS(stride, i) (stride * i) ------------------ 277| 917| TF = FMA(Tl, TD, Tm * TE); ------------------ | | 1131| 917|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 278| 917| TW = FNMS(Tm, TD, Tl * TE); ------------------ | | 1134| 917|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 279| 917| TG = ri[WS(rs, 3)]; ------------------ | | 825| 917|#define WS(stride, i) (stride * i) ------------------ 280| 917| TH = ii[WS(rs, 3)]; ------------------ | | 825| 917|#define WS(stride, i) (stride * i) ------------------ 281| 917| TI = FMA(T3, TG, T6 * TH); ------------------ | | 1131| 917|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 282| 917| TX = FNMS(T6, TG, T3 * TH); ------------------ | | 1134| 917|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 283| 917| } 284| 917| TJ = TF + TI; 285| 917| T17 = TW + TX; 286| 917| TV = TF - TI; 287| 917| TY = TW - TX; 288| 917| } 289| 917| { 290| 917| E Tk, TM, Tr, TN; 291| 917| { 292| 917| E Th, Tj, To, Tq; 293| 917| Th = ri[WS(rs, 2)]; ------------------ | | 825| 917|#define WS(stride, i) (stride * i) ------------------ 294| 917| Tj = ii[WS(rs, 2)]; ------------------ | | 825| 917|#define WS(stride, i) (stride * i) ------------------ 295| 917| Tk = FMA(Tg, Th, Ti * Tj); ------------------ | | 1131| 917|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 296| 917| TM = FNMS(Ti, Th, Tg * Tj); ------------------ | | 1134| 917|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 297| 917| To = ri[WS(rs, 6)]; ------------------ | | 825| 917|#define WS(stride, i) (stride * i) ------------------ 298| 917| Tq = ii[WS(rs, 6)]; ------------------ | | 825| 917|#define WS(stride, i) (stride * i) ------------------ 299| 917| Tr = FMA(Tn, To, Tp * Tq); ------------------ | | 1131| 917|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 300| 917| TN = FNMS(Tp, To, Tn * Tq); ------------------ | | 1134| 917|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 301| 917| } 302| 917| Ts = Tk + Tr; 303| 917| T1j = Tk - Tr; 304| 917| TO = TM - TN; 305| 917| T1a = TM + TN; 306| 917| } 307| 917| { 308| 917| E Tw, TR, TB, TS; 309| 917| { 310| 917| E Tu, Tv, Ty, TA; 311| 917| Tu = ri[WS(rs, 1)]; ------------------ | | 825| 917|#define WS(stride, i) (stride * i) ------------------ 312| 917| Tv = ii[WS(rs, 1)]; ------------------ | | 825| 917|#define WS(stride, i) (stride * i) ------------------ 313| 917| Tw = FMA(T2, Tu, T5 * Tv); ------------------ | | 1131| 917|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 314| 917| TR = FNMS(T5, Tu, T2 * Tv); ------------------ | | 1134| 917|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 315| 917| Ty = ri[WS(rs, 5)]; ------------------ | | 825| 917|#define WS(stride, i) (stride * i) ------------------ 316| 917| TA = ii[WS(rs, 5)]; ------------------ | | 825| 917|#define WS(stride, i) (stride * i) ------------------ 317| 917| TB = FMA(Tx, Ty, Tz * TA); ------------------ | | 1131| 917|#define FMA(a, b, c) (((a) * (b)) + (c)) ------------------ 318| 917| TS = FNMS(Tz, Ty, Tx * TA); ------------------ | | 1134| 917|#define FNMS(a, b, c) ((c) - ((a) * (b))) ------------------ 319| 917| } 320| 917| TC = Tw + TB; 321| 917| T16 = TR + TS; 322| 917| TQ = Tw - TB; 323| 917| TT = TR - TS; 324| 917| } 325| 917| { 326| 917| E Tt, TK, T1f, T1g; 327| 917| Tt = Tf + Ts; 328| 917| TK = TC + TJ; 329| 917| ri[WS(rs, 4)] = Tt - TK; ------------------ | | 825| 917|#define WS(stride, i) (stride * i) ------------------ 330| 917| ri[0] = Tt + TK; 331| 917| { 332| 917| E T19, T1e, T15, T18; 333| 917| T19 = T16 + T17; 334| 917| T1e = T1a + T1d; 335| 917| ii[0] = T19 + T1e; 336| 917| ii[WS(rs, 4)] = T1e - T19; ------------------ | | 825| 917|#define WS(stride, i) (stride * i) ------------------ 337| 917| T15 = Tf - Ts; 338| 917| T18 = T16 - T17; 339| 917| ri[WS(rs, 6)] = T15 - T18; ------------------ | | 825| 917|#define WS(stride, i) (stride * i) ------------------ 340| 917| ri[WS(rs, 2)] = T15 + T18; ------------------ | | 825| 917|#define WS(stride, i) (stride * i) ------------------ 341| 917| } 342| 917| T1f = TJ - TC; 343| 917| T1g = T1d - T1a; 344| 917| ii[WS(rs, 2)] = T1f + T1g; ------------------ | | 825| 917|#define WS(stride, i) (stride * i) ------------------ 345| 917| ii[WS(rs, 6)] = T1g - T1f; ------------------ | | 825| 917|#define WS(stride, i) (stride * i) ------------------ 346| 917| { 347| 917| E T11, T1k, T14, T1h, T12, T13; 348| 917| T11 = TL - TO; 349| 917| T1k = T1i - T1j; 350| 917| T12 = TT - TQ; 351| 917| T13 = TV + TY; 352| 917| T14 = KP707106781 * (T12 - T13); 353| 917| T1h = KP707106781 * (T12 + T13); 354| 917| ri[WS(rs, 7)] = T11 - T14; ------------------ | | 825| 917|#define WS(stride, i) (stride * i) ------------------ 355| 917| ii[WS(rs, 5)] = T1k - T1h; ------------------ | | 825| 917|#define WS(stride, i) (stride * i) ------------------ 356| 917| ri[WS(rs, 3)] = T11 + T14; ------------------ | | 825| 917|#define WS(stride, i) (stride * i) ------------------ 357| 917| ii[WS(rs, 1)] = T1h + T1k; ------------------ | | 825| 917|#define WS(stride, i) (stride * i) ------------------ 358| 917| } 359| 917| { 360| 917| E TP, T1m, T10, T1l, TU, TZ; 361| 917| TP = TL + TO; 362| 917| T1m = T1j + T1i; 363| 917| TU = TQ + TT; 364| 917| TZ = TV - TY; 365| 917| T10 = KP707106781 * (TU + TZ); 366| 917| T1l = KP707106781 * (TZ - TU); 367| 917| ri[WS(rs, 5)] = TP - T10; ------------------ | | 825| 917|#define WS(stride, i) (stride * i) ------------------ 368| 917| ii[WS(rs, 7)] = T1m - T1l; ------------------ | | 825| 917|#define WS(stride, i) (stride * i) ------------------ 369| 917| ri[WS(rs, 1)] = TP + T10; ------------------ | | 825| 917|#define WS(stride, i) (stride * i) ------------------ 370| 917| ii[WS(rs, 3)] = T1l + T1m; ------------------ | | 825| 917|#define WS(stride, i) (stride * i) ------------------ 371| 917| } 372| 917| } 373| 917| } 374| 917| } 375| 51| } 376| 51|} n.c:okp: 29| 2.35k|{ 30| 2.35k| UNUSED(ri); UNUSED(ii); UNUSED(ro); UNUSED(io); UNUSED(vl); UNUSED(plnr); ------------------ | | 88| 2.35k|#define UNUSED(x) (void)x ------------------ UNUSED(ri); UNUSED(ii); UNUSED(ro); UNUSED(io); UNUSED(vl); UNUSED(plnr); ------------------ | | 88| 2.35k|#define UNUSED(x) (void)x ------------------ UNUSED(ri); UNUSED(ii); UNUSED(ro); UNUSED(io); UNUSED(vl); UNUSED(plnr); ------------------ | | 88| 2.35k|#define UNUSED(x) (void)x ------------------ UNUSED(ri); UNUSED(ii); UNUSED(ro); UNUSED(io); UNUSED(vl); UNUSED(plnr); ------------------ | | 88| 2.35k|#define UNUSED(x) (void)x ------------------ UNUSED(ri); UNUSED(ii); UNUSED(ro); UNUSED(io); UNUSED(vl); UNUSED(plnr); ------------------ | | 88| 2.35k|#define UNUSED(x) (void)x ------------------ UNUSED(ri); UNUSED(ii); UNUSED(ro); UNUSED(io); UNUSED(vl); UNUSED(plnr); ------------------ | | 88| 2.35k|#define UNUSED(x) (void)x ------------------ 31| 2.35k| return (1 ------------------ | Branch (31:14): [True: 2.35k, Folded] ------------------ 32| 2.35k| && (!d->is || (d->is == is)) ------------------ | Branch (32:11): [True: 2.35k, False: 0] | Branch (32:21): [True: 0, False: 0] ------------------ 33| 2.35k| && (!d->os || (d->os == os)) ------------------ | Branch (33:11): [True: 2.35k, False: 0] | Branch (33:21): [True: 0, False: 0] ------------------ 34| 2.35k| && (!d->ivs || (d->ivs == ivs)) ------------------ | Branch (34:11): [True: 2.35k, False: 0] | Branch (34:22): [True: 0, False: 0] ------------------ 35| 2.35k| && (!d->ovs || (d->ovs == ovs)) ------------------ | Branch (35:11): [True: 2.35k, False: 0] | Branch (35:22): [True: 0, False: 0] ------------------ 36| 2.35k| ); 37| 2.35k|} t.c:okp: 28| 3.70k|{ 29| 3.70k| UNUSED(rio); UNUSED(iio); UNUSED(m); UNUSED(mb); UNUSED(me); UNUSED(plnr); ------------------ | | 88| 3.70k|#define UNUSED(x) (void)x ------------------ UNUSED(rio); UNUSED(iio); UNUSED(m); UNUSED(mb); UNUSED(me); UNUSED(plnr); ------------------ | | 88| 3.70k|#define UNUSED(x) (void)x ------------------ UNUSED(rio); UNUSED(iio); UNUSED(m); UNUSED(mb); UNUSED(me); UNUSED(plnr); ------------------ | | 88| 3.70k|#define UNUSED(x) (void)x ------------------ UNUSED(rio); UNUSED(iio); UNUSED(m); UNUSED(mb); UNUSED(me); UNUSED(plnr); ------------------ | | 88| 3.70k|#define UNUSED(x) (void)x ------------------ UNUSED(rio); UNUSED(iio); UNUSED(m); UNUSED(mb); UNUSED(me); UNUSED(plnr); ------------------ | | 88| 3.70k|#define UNUSED(x) (void)x ------------------ UNUSED(rio); UNUSED(iio); UNUSED(m); UNUSED(mb); UNUSED(me); UNUSED(plnr); ------------------ | | 88| 3.70k|#define UNUSED(x) (void)x ------------------ 30| 3.70k| return (1 ------------------ | Branch (30:14): [True: 3.70k, Folded] ------------------ 31| 3.70k| && (!d->rs || (d->rs == rs)) ------------------ | Branch (31:11): [True: 3.70k, False: 0] | Branch (31:21): [True: 0, False: 0] ------------------ 32| 3.70k| && (!d->vs || (d->vs == vs)) ------------------ | Branch (32:11): [True: 3.70k, False: 0] | Branch (32:21): [True: 0, False: 0] ------------------ 33| 3.70k| && (!d->ms || (d->ms == ms)) ------------------ | Branch (33:11): [True: 3.70k, False: 0] | Branch (33:21): [True: 0, False: 0] ------------------ 34| 3.70k| ); 35| 3.70k|} fftw_dft_solve: 26| 285|{ 27| 285| const plan_dft *ego = (const plan_dft *) ego_; 28| 285| const problem_dft *p = (const problem_dft *) p_; 29| 285| ego->apply(ego_, 30| 285| UNTAINT(p->ri), UNTAINT(p->ii), ------------------ | | 1037| 285|#define UNTAINT(p) (p) ------------------ UNTAINT(p->ri), UNTAINT(p->ii), ------------------ | | 1037| 285|#define UNTAINT(p) (p) ------------------ 31| 285| UNTAINT(p->ro), UNTAINT(p->io)); ------------------ | | 1037| 285|#define UNTAINT(p) (p) ------------------ UNTAINT(p->ro), UNTAINT(p->io)); ------------------ | | 1037| 285|#define UNTAINT(p) (p) ------------------ 32| 285|} fftw_dft_vrank_geq1_register: 202| 1|{ 203| | /* FIXME: Should we try other vecloop_dim values? */ 204| 1| static const int buddies[] = { 1, -1 }; 205| 1| size_t i; 206| | 207| 3| for (i = 0; i < NELEM(buddies); ++i) ------------------ | | 90| 3|#define NELEM(array) ((sizeof(array) / sizeof((array)[0]))) ------------------ | Branch (207:18): [True: 2, False: 1] ------------------ 208| 2| REGISTER_SOLVER(p, mksolver(buddies[i], buddies, NELEM(buddies))); ------------------ | | 95| 2|#define REGISTER_SOLVER(p, s) X(solver_register)(p, s) | | ------------------ | | | | 78| 2|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 2|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 209| 1|} vrank-geq1.c:mksolver: 192| 4|{ 193| 4| static const solver_adt sadt = { PROBLEM_DFT, mkplan, 0 }; 194| 4| S *slv = MKSOLVER(S, &sadt); ------------------ | | 604| 4|#define MKSOLVER(type, adt) (type *)X(mksolver)(sizeof(type), adt) | | ------------------ | | | | 78| 4|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 4|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 195| 4| slv->vecloop_dim = vecloop_dim; 196| 4| slv->buddies = buddies; 197| 4| slv->nbuddies = nbuddies; 198| 4| return &(slv->super); 199| 4|} vrank-geq1.c:mkplan: 146| 3.06k|{ 147| 3.06k| const S *ego = (const S *) ego_; 148| 3.06k| const problem_dft *p; 149| 3.06k| P *pln; 150| 3.06k| plan *cld; 151| 3.06k| int vdim; 152| 3.06k| iodim *d; 153| | 154| 3.06k| static const plan_adt padt = { 155| 3.06k| X(dft_solve), awake, print, destroy ------------------ | | 78| 3.06k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 3.06k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 156| 3.06k| }; 157| | 158| 3.06k| if (!applicable(ego_, p_, plnr, &vdim)) ------------------ | Branch (158:10): [True: 2.70k, False: 361] ------------------ 159| 2.70k| return (plan *) 0; 160| 361| p = (const problem_dft *) p_; 161| | 162| 361| d = p->vecsz->dims + vdim; 163| | 164| 361| A(d->n > 1); 165| 361| cld = X(mkplan_d)(plnr, ------------------ | | 78| 361|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 361|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 166| 361| X(mkproblem_dft_d)( ------------------ | | 78| 361|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 361|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 167| 361| X(tensor_copy)(p->sz), ------------------ | | 78| 361|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 361|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 168| 361| X(tensor_copy_except)(p->vecsz, vdim), ------------------ | | 78| 361|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 361|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 169| 361| TAINT(p->ri, d->is), TAINT(p->ii, d->is), ------------------ | | 1036| 361|#define TAINT(p, s) (p) ------------------ TAINT(p->ri, d->is), TAINT(p->ii, d->is), ------------------ | | 1036| 361|#define TAINT(p, s) (p) ------------------ 170| 361| TAINT(p->ro, d->os), TAINT(p->io, d->os))); ------------------ | | 1036| 361|#define TAINT(p, s) (p) ------------------ TAINT(p->ro, d->os), TAINT(p->io, d->os))); ------------------ | | 1036| 361|#define TAINT(p, s) (p) ------------------ 171| 361| if (!cld) return (plan *) 0; ------------------ | Branch (171:10): [True: 9, False: 352] ------------------ 172| | 173| 352| pln = MKPLAN_DFT(P, &padt, apply); ------------------ | | 60| 352| (type *)X(mkplan_dft)(sizeof(type), adt, apply) | | ------------------ | | | | 78| 352|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 352|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 174| | 175| 352| pln->cld = cld; 176| 352| pln->vl = d->n; 177| 352| pln->ivs = d->is; 178| 352| pln->ovs = d->os; 179| | 180| 352| pln->solver = ego; 181| 352| X(ops_zero)(&pln->super.super.ops); ------------------ | | 78| 352|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 352|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 182| 352| pln->super.super.ops.other = 3.14159; /* magic to prefer codelet loops */ 183| 352| X(ops_madd2)(pln->vl, &cld->ops, &pln->super.super.ops); ------------------ | | 78| 352|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 352|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 184| | 185| 352| if (p->sz->rnk != 1 || (p->sz->dims[0].n > 64)) ------------------ | Branch (185:10): [True: 0, False: 352] | Branch (185:29): [True: 36, False: 316] ------------------ 186| 36| pln->super.super.pcost = pln->vl * cld->pcost; 187| | 188| 352| return &(pln->super.super); 189| 361|} vrank-geq1.c:awake: 68| 242|{ 69| 242| P *ego = (P *) ego_; 70| 242| X(plan_awake)(ego->cld, wakefulness); ------------------ | | 78| 242|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 242|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 71| 242|} vrank-geq1.c:destroy: 74| 352|{ 75| 352| P *ego = (P *) ego_; 76| 352| X(plan_destroy_internal)(ego->cld); ------------------ | | 78| 352|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 352|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 77| 352|} vrank-geq1.c:applicable: 112| 2.39k|{ 113| 2.39k| const S *ego = (const S *)ego_; 114| 2.39k| const problem_dft *p; 115| | 116| 2.39k| if (!applicable0(ego_, p_, dp)) return 0; ------------------ | Branch (116:10): [True: 2.01k, False: 384] ------------------ 117| | 118| | /* fftw2 behavior */ 119| 384| if (NO_VRANK_SPLITSP(plnr) && (ego->vecloop_dim != ego->buddies[0])) ------------------ | | 685| 768|#define NO_VRANK_SPLITSP(plnr) (PLNR_L(plnr) & NO_VRANK_SPLITS) | | ------------------ | | | | 674| 384|#define PLNR_L(plnr) ((plnr)->flags.l) | | ------------------ | | | Branch (685:32): [True: 384, False: 0] | | ------------------ ------------------ | Branch (119:36): [True: 23, False: 361] ------------------ 120| 23| return 0; 121| | 122| 361| p = (const problem_dft *) p_; 123| | 124| 361| if (NO_UGLYP(plnr)) { ------------------ | | 689| 361|#define NO_UGLYP(plnr) (PLNR_L(plnr) & NO_UGLY) | | ------------------ | | | | 674| 361|#define PLNR_L(plnr) ((plnr)->flags.l) | | ------------------ | | | Branch (689:24): [True: 361, False: 0] | | ------------------ ------------------ 125| | /* Heuristic: if the transform is multi-dimensional, and the 126| | vector stride is less than the transform size, then we 127| | probably want to use a rank>=2 plan first in order to combine 128| | this vector with the transform-dimension vectors. */ 129| 361| { 130| 361| iodim *d = p->vecsz->dims + *dp; 131| 361| if (1 ------------------ | Branch (131:13): [True: 361, Folded] ------------------ 132| 361| && p->sz->rnk > 1 ------------------ | Branch (132:9): [True: 0, False: 361] ------------------ 133| 0| && X(imin)(X(iabs)(d->is), X(iabs)(d->os)) ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ && X(imin)(X(iabs)(d->is), X(iabs)(d->os)) ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ && X(imin)(X(iabs)(d->is), X(iabs)(d->os)) ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ | Branch (133:9): [True: 0, False: 0] ------------------ 134| 0| < X(tensor_max_index)(p->sz) ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 135| 361| ) 136| 0| return 0; 137| 361| } 138| | 139| 361| if (NO_NONTHREADEDP(plnr)) return 0; /* prefer threaded version */ ------------------ | | 693| 361| ((PLNR_L(plnr) & NO_NONTHREADED) && (plnr)->nthr > 1) | | ------------------ | | | | 674| 361|#define PLNR_L(plnr) ((plnr)->flags.l) | | ------------------ | | | Branch (693:4): [True: 361, False: 0] | | | Branch (693:39): [True: 0, False: 361] | | ------------------ ------------------ 140| 361| } 141| | 142| 361| return 1; 143| 361|} vrank-geq1.c:applicable0: 94| 2.39k|{ 95| 2.39k| const S *ego = (const S *) ego_; 96| 2.39k| const problem_dft *p = (const problem_dft *) p_; 97| | 98| 2.39k| return (1 ------------------ | Branch (98:14): [True: 2.39k, Folded] ------------------ 99| 2.39k| && FINITE_RNK(p->vecsz->rnk) ------------------ | | 442| 4.79k|#define FINITE_RNK(rnk) ((rnk) != RNK_MINFTY) | | ------------------ | | | | 441| 2.39k|#define RNK_MINFTY INT_MAX | | ------------------ | | | Branch (442:25): [True: 2.06k, False: 332] | | ------------------ ------------------ 100| 2.06k| && p->vecsz->rnk > 0 ------------------ | Branch (100:10): [True: 1.54k, False: 522] ------------------ 101| | 102| | /* do not bother looping over rank-0 problems, 103| | since they are handled via rdft */ 104| 1.54k| && p->sz->rnk > 0 ------------------ | Branch (104:10): [True: 877, False: 668] ------------------ 105| | 106| 877| && pickdim(ego, p->vecsz, p->ri != p->ro, dp) ------------------ | Branch (106:10): [True: 384, False: 493] ------------------ 107| 2.39k| ); 108| 2.39k|} vrank-geq1.c:pickdim: 88| 1.54k|{ 89| 1.54k| return X(pickdim)(ego->vecloop_dim, ego->buddies, ego->nbuddies, ------------------ | | 78| 1.54k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1.54k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 90| 1.54k| vecsz, oop, dp); 91| 1.54k|} vrank-geq1.c:apply: 55| 169|{ 56| 169| const P *ego = (const P *) ego_; 57| 169| INT i, vl = ego->vl; 58| 169| INT ivs = ego->ivs, ovs = ego->ovs; 59| 169| dftapply cldapply = ((plan_dft *) ego->cld)->apply; 60| | 61| 1.50k| for (i = 0; i < vl; ++i) { ------------------ | Branch (61:18): [True: 1.33k, False: 169] ------------------ 62| 1.33k| cldapply(ego->cld, 63| 1.33k| ri + i * ivs, ii + i * ivs, ro + i * ovs, io + i * ovs); 64| 1.33k| } 65| 169|} fftw_ialignment_of: 34| 14.4k|{ 35| 14.4k|#if ALGN == 0 36| 14.4k| UNUSED(p); ------------------ | | 88| 14.4k|#define UNUSED(x) (void)x ------------------ 37| 14.4k| return 0; 38| |#else 39| | return (int)(((uintptr_t) p) % ALGN); 40| |#endif 41| 14.4k|} fftw_malloc_plain: 24| 35.1k|{ 25| 35.1k| void *p; 26| 35.1k| if (n == 0) ------------------ | Branch (26:10): [True: 0, False: 35.1k] ------------------ 27| 0| n = 1; 28| 35.1k| p = X(kernel_malloc)(n); ------------------ | | 78| 35.1k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 35.1k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 29| 35.1k| CK(p); ------------------ | | 264| 35.1k| (void)((ex) || (X(assertion_failed)(#ex, __LINE__, __FILE__), 0)) | | ------------------ | | | | 78| 0|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ | | | Branch (264:14): [True: 35.1k, False: 0] | | | Branch (264:22): [True: 0, False: 0] | | ------------------ ------------------ 30| | 31| |#ifdef MIN_ALIGNMENT 32| | A((((uintptr_t)p) % MIN_ALIGNMENT) == 0); 33| |#endif 34| | 35| 35.1k| return p; 36| 35.1k|} fftw_ifree: 39| 34.5k|{ 40| 34.5k| X(kernel_free)(p); ------------------ | | 78| 34.5k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 34.5k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 41| 34.5k|} fftw_ifree0: 44| 5.59k|{ 45| | /* common pattern */ 46| 5.59k| if (p) X(ifree)(p); ------------------ | | 78| 1.06k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1.06k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ | Branch (46:10): [True: 1.06k, False: 4.52k] ------------------ 47| 5.59k|} fftw_null_awake: 25| 740|{ 26| 740| UNUSED(ego); ------------------ | | 88| 740|#define UNUSED(x) (void)x ------------------ 27| 740| UNUSED(wakefulness); ------------------ | | 88| 740|#define UNUSED(x) (void)x ------------------ 28| | /* do nothing */ 29| 740|} fftw_nbuf: 31| 1.74k|{ 32| 1.74k| INT i, nbuf, lb; 33| | 34| 1.74k| if (!maxnbuf) ------------------ | Branch (34:10): [True: 0, False: 1.74k] ------------------ 35| 0| maxnbuf = DEFAULT_MAXNBUF; ------------------ | | 25| 0|#define DEFAULT_MAXNBUF ((INT)256) ------------------ 36| | 37| 1.74k| nbuf = X(imin)(maxnbuf, ------------------ | | 78| 1.74k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1.74k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 38| 1.74k| X(imin)(vl, X(imax)((INT)1, MAXBUFSZ / n))); ------------------ | | 78| 1.74k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1.74k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ X(imin)(vl, X(imax)((INT)1, MAXBUFSZ / n))); ------------------ | | 78| 1.74k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1.74k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ X(imin)(vl, X(imax)((INT)1, MAXBUFSZ / n))); ------------------ | | 28| 1.74k|#define MAXBUFSZ (256 * 1024 / (INT)(sizeof(R))) ------------------ 39| | 40| | /* 41| | * Look for a buffer number (not too small) that divides the 42| | * vector length, in order that we only need one child plan: 43| | */ 44| 1.74k| lb = X(imax)(1, nbuf / 4); ------------------ | | 78| 1.74k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1.74k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 45| 2.52k| for (i = nbuf; i >= lb; --i) ------------------ | Branch (45:21): [True: 2.52k, False: 5] ------------------ 46| 2.52k| if (vl % i == 0) ------------------ | Branch (46:15): [True: 1.74k, False: 780] ------------------ 47| 1.74k| return i; 48| | 49| | /* whatever... */ 50| 5| return nbuf; 51| 1.74k|} fftw_bufdist: 57| 204|{ 58| 204| if (vl == 1) ------------------ | Branch (58:10): [True: 0, False: 204] ------------------ 59| 0| return n; 60| 204| else 61| | /* return smallest X such that X >= N and X == SKEW (mod SKEWMOD) */ 62| 204| return n + X(modulo)(SKEW - n, SKEWMOD); ------------------ | | 78| 204|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 204|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ return n + X(modulo)(SKEW - n, SKEWMOD); ------------------ | | 53| 204|#define SKEW 6 /* need to be even for SIMD */ ------------------ return n + X(modulo)(SKEW - n, SKEWMOD); ------------------ | | 54| 204|#define SKEWMOD 8 ------------------ 63| 204|} fftw_toobig: 66| 1.40k|{ 67| 1.40k| return n > MAXBUFSZ; ------------------ | | 28| 1.40k|#define MAXBUFSZ (256 * 1024 / (INT)(sizeof(R))) ------------------ 68| 1.40k|} fftw_nbuf_redundant: 75| 1.20k|{ 76| 1.20k| size_t i; 77| 1.20k| (void)nmaxnbuf; /* UNUSED */ 78| 1.37k| for (i = 0; i < which; ++i) ------------------ | Branch (78:18): [True: 615, False: 762] ------------------ 79| 615| if (X(nbuf)(n, vl, maxnbuf[i]) == X(nbuf)(n, vl, maxnbuf[which])) ------------------ | | 78| 615|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 615|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ if (X(nbuf)(n, vl, maxnbuf[i]) == X(nbuf)(n, vl, maxnbuf[which])) ------------------ | | 78| 615|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 615|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ | Branch (79:8): [True: 441, False: 174] ------------------ 80| 441| return 1; 81| 762| return 0; 82| 1.20k|} fftw_cpy1d: 25| 83|{ 26| 83| INT i0, v; 27| | 28| 83| A(I != O); 29| 83| switch (vl) { 30| 14| case 1: ------------------ | Branch (30:3): [True: 14, False: 69] ------------------ 31| 14| if ((n0 & 1) || is0 != 1 || os0 != 1) { ------------------ | Branch (31:12): [True: 0, False: 14] | Branch (31:24): [True: 0, False: 14] | Branch (31:36): [True: 0, False: 14] ------------------ 32| 0| for (; n0 > 0; --n0, I += is0, O += os0) ------------------ | Branch (32:13): [True: 0, False: 0] ------------------ 33| 0| *O = *I; 34| 0| break; 35| 0| } 36| 14| n0 /= 2; is0 = 2; os0 = 2; 37| | /* fall through */ 38| 14| case 2: ------------------ | Branch (38:3): [True: 0, False: 83] ------------------ 39| 14| if ((n0 & 1) || is0 != 2 || os0 != 2) { ------------------ | Branch (39:12): [True: 14, False: 0] | Branch (39:24): [True: 0, False: 0] | Branch (39:36): [True: 0, False: 0] ------------------ 40| 28| for (; n0 > 0; --n0, I += is0, O += os0) { ------------------ | Branch (40:13): [True: 14, False: 14] ------------------ 41| 14| R x0 = I[0]; 42| 14| R x1 = I[1]; 43| 14| O[0] = x0; 44| 14| O[1] = x1; 45| 14| } 46| 14| break; 47| 14| } 48| 0| n0 /= 2; is0 = 4; os0 = 4; 49| | /* fall through */ 50| 0| case 4: ------------------ | Branch (50:3): [True: 0, False: 83] ------------------ 51| 0| for (; n0 > 0; --n0, I += is0, O += os0) { ------------------ | Branch (51:15): [True: 0, False: 0] ------------------ 52| 0| R x0 = I[0]; 53| 0| R x1 = I[1]; 54| 0| R x2 = I[2]; 55| 0| R x3 = I[3]; 56| 0| O[0] = x0; 57| 0| O[1] = x1; 58| 0| O[2] = x2; 59| 0| O[3] = x3; 60| 0| } 61| 0| break; 62| 69| default: ------------------ | Branch (62:3): [True: 69, False: 14] ------------------ 63| 1.23k| for (i0 = 0; i0 < n0; ++i0) ------------------ | Branch (63:21): [True: 1.16k, False: 69] ------------------ 64| 44.1k| for (v = 0; v < vl; ++v) { ------------------ | Branch (64:18): [True: 42.9k, False: 1.16k] ------------------ 65| 42.9k| R x0 = I[i0 * is0 + v]; 66| 42.9k| O[i0 * os0 + v] = x0; 67| 42.9k| } 68| 69| break; 69| 83| } 70| 83|} fftw_cpy2d_pair: 27| 43|{ 28| 43| INT i0, i1; 29| | 30| 478| for (i1 = 0; i1 < n1; ++i1) ------------------ | Branch (30:19): [True: 435, False: 43] ------------------ 31| 4.62k| for (i0 = 0; i0 < n0; ++i0) { ------------------ | Branch (31:17): [True: 4.19k, False: 435] ------------------ 32| 4.19k| R x0 = I0[i0 * is0 + i1 * is1]; 33| 4.19k| R x1 = I1[i0 * is0 + i1 * is1]; 34| 4.19k| O0[i0 * os0 + i1 * os1] = x0; 35| 4.19k| O1[i0 * os0 + i1 * os1] = x1; 36| 4.19k| } 37| 43|} fftw_cpy2d_pair_ci: 52| 43|{ 53| 43| if (IABS(is0) < IABS(is1)) /* inner loop is for n0 */ ------------------ | | 381| 43|#define IABS(x) (((x) < 0) ? (0 - (x)) : (x)) | | ------------------ | | | Branch (381:18): [True: 0, False: 43] | | ------------------ ------------------ if (IABS(is0) < IABS(is1)) /* inner loop is for n0 */ ------------------ | | 381| 43|#define IABS(x) (((x) < 0) ? (0 - (x)) : (x)) | | ------------------ | | | Branch (381:18): [True: 0, False: 43] | | ------------------ ------------------ | Branch (53:10): [True: 0, False: 43] ------------------ 54| 0| X(cpy2d_pair) (I0, I1, O0, O1, n0, is0, os0, n1, is1, os1); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 55| 43| else 56| 43| X(cpy2d_pair) (I0, I1, O0, O1, n1, is1, os1, n0, is0, os0); ------------------ | | 78| 43|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 43|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 57| 43|} fftw_ct_uglyp: 29| 1.84k|{ 30| 1.84k| return (n <= min_n) || (POW2P(n) && (v * (n / r)) <= 4); ------------------ | | 25| 2.36k|#define POW2P(n) (((n) > 0) && (((n) & ((n) - 1)) == 0)) | | ------------------ | | | Branch (25:19): [True: 1.18k, False: 0] | | | Branch (25:32): [True: 35, False: 1.14k] | | ------------------ ------------------ | Branch (30:13): [True: 660, False: 1.18k] | Branch (30:42): [True: 19, False: 16] ------------------ 31| 1.84k|} fftw_extract_reim: 28| 570|{ 29| 570| if (sign == FFT_SIGN) { ------------------ | | 92| 570|#define FFT_SIGN (-1) /* sign convention for forward transforms */ ------------------ | Branch (29:10): [True: 570, False: 0] ------------------ 30| 570| *r = c + 0; 31| 570| *i = c + 1; 32| 570| } else { 33| 0| *r = c + 1; 34| 0| *i = c + 0; 35| 0| } 36| 570|} fftw_hash: 24| 613|{ 25| 613| unsigned h = 0xDEADBEEFu; 26| 13.5k| do { 27| 13.5k| h = h * 17 + (unsigned)(*s & 0xFF); 28| 13.5k| } while (*s++); ------------------ | Branch (28:15): [True: 12.9k, False: 613] ------------------ 29| 613| return h; 30| 613|} fftw_iabs: 25| 17.3k|{ 26| 17.3k| return a < 0 ? (0 - a) : a; ------------------ | Branch (26:13): [True: 0, False: 17.3k] ------------------ 27| 17.3k|} fftw_kernel_malloc: 72| 35.1k|{ 73| 35.1k| void *p; 74| | 75| |#if defined(MIN_ALIGNMENT) 76| | 77| |# if defined(WITH_OUR_MALLOC) 78| | p = our_malloc(n); 79| |# undef real_free 80| |# define real_free our_free 81| | 82| |# elif defined(__FreeBSD__) && (MIN_ALIGNMENT <= 16) 83| | /* FreeBSD does not have memalign, but its malloc is 16-byte aligned. */ 84| | p = malloc(n); 85| | 86| |# elif (defined(__MACOSX__) || defined(__APPLE__)) && (MIN_ALIGNMENT <= 16) 87| | /* MacOS X malloc is already 16-byte aligned */ 88| | p = malloc(n); 89| | 90| |# elif defined(HAVE_MEMALIGN) 91| | p = memalign(MIN_ALIGNMENT, n); 92| | 93| |# elif defined(HAVE_POSIX_MEMALIGN) 94| | /* note: posix_memalign is broken in glibc 2.2.5: it constrains 95| | the size, not the alignment, to be (power of two) * sizeof(void*). 96| | The bug seems to have been fixed as of glibc 2.3.1. */ 97| | if (posix_memalign(&p, MIN_ALIGNMENT, n)) 98| | p = (void*) 0; 99| | 100| |# elif defined(__ICC) || defined(__INTEL_COMPILER) || defined(HAVE__MM_MALLOC) 101| | /* Intel's C compiler defines _mm_malloc and _mm_free intrinsics */ 102| | p = (void *) _mm_malloc(n, MIN_ALIGNMENT); 103| |# undef real_free 104| |# define real_free _mm_free 105| | 106| |# elif defined(_MSC_VER) 107| | /* MS Visual C++ 6.0 with a "Processor Pack" supports SIMD 108| | and _aligned_malloc/free (uses malloc.h) */ 109| | p = (void *) _aligned_malloc(n, MIN_ALIGNMENT); 110| |# undef real_free 111| |# define real_free _aligned_free 112| | 113| |# elif defined(macintosh) /* MacOS 9 */ 114| | p = (void *) MPAllocateAligned(n, 115| |# if MIN_ALIGNMENT == 8 116| | kMPAllocate8ByteAligned, 117| |# elif MIN_ALIGNMENT == 16 118| | kMPAllocate16ByteAligned, 119| |# elif MIN_ALIGNMENT == 32 120| | kMPAllocate32ByteAligned, 121| |# else 122| |# error "Unknown alignment for MPAllocateAligned" 123| |# endif 124| | 0); 125| |# undef real_free 126| |# define real_free MPFree 127| | 128| |# else 129| | /* Add your machine here and send a patch to fftw@fftw.org 130| | or (e.g. for Windows) configure --with-our-malloc */ 131| |# error "Don't know how to malloc() aligned memory ... try configuring --with-our-malloc" 132| |# endif 133| | 134| |#else /* !defined(MIN_ALIGNMENT) */ 135| 35.1k| p = malloc(n); 136| 35.1k|#endif 137| | 138| 35.1k| return p; 139| 35.1k|} fftw_kernel_free: 142| 34.5k|{ 143| 34.5k| real_free(p); ------------------ | | 42| 34.5k|#define real_free free /* memalign and malloc use ordinary free */ ------------------ 144| 34.5k|} fftw_md5putb: 25| 59.5k|{ 26| 59.5k| size_t i; 27| 59.5k| const unsigned char *d = (const unsigned char *)d_; 28| 401k| for (i = 0; i < len; ++i) ------------------ | Branch (28:18): [True: 341k, False: 59.5k] ------------------ 29| 341k| X(md5putc)(p, d[i]); ------------------ | | 78| 341k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 341k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 30| 59.5k|} fftw_md5puts: 33| 3.82k|{ 34| | /* also hash final '\0' */ 35| 15.7k| do { 36| 15.7k| X(md5putc)(p, (unsigned)(*s & 0xFF)); ------------------ | | 78| 15.7k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 15.7k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 37| 15.7k| } while(*s++); ------------------ | Branch (37:14): [True: 11.9k, False: 3.82k] ------------------ 38| 3.82k|} fftw_md5int: 41| 29.7k|{ 42| 29.7k| X(md5putb)(p, &i, sizeof(i)); ------------------ | | 78| 29.7k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 29.7k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 43| 29.7k|} fftw_md5INT: 46| 25.8k|{ 47| 25.8k| X(md5putb)(p, &i, sizeof(i)); ------------------ | | 78| 25.8k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 25.8k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 48| 25.8k|} fftw_md5unsigned: 51| 3.82k|{ 52| 3.82k| X(md5putb)(p, &i, sizeof(i)); ------------------ | | 78| 3.82k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 3.82k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 53| 3.82k|} fftw_md5begin: 111| 3.82k|{ 112| 3.82k| p->s[0] = 0x67452301; 113| 3.82k| p->s[1] = 0xefcdab89; 114| 3.82k| p->s[2] = 0x98badcfe; 115| 3.82k| p->s[3] = 0x10325476; 116| 3.82k| p->l = 0; 117| 3.82k|} fftw_md5putc: 120| 495k|{ 121| 495k| p->c[p->l % 64] = c; 122| 495k| if (((++p->l) % 64) == 0) doblock(p->s, p->c); ------------------ | Branch (122:10): [True: 7.74k, False: 488k] ------------------ 123| 495k|} fftw_md5end: 126| 3.82k|{ 127| 3.82k| unsigned l, i; 128| | 129| 3.82k| l = 8 * p->l; /* length before padding, in bits */ 130| | 131| | /* rfc 1321 section 3.1: padding */ 132| 3.82k| X(md5putc)(p, 0x80); ------------------ | | 78| 3.82k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 3.82k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 133| 107k| while ((p->l % 64) != 56) X(md5putc)(p, 0x00); ------------------ | | 78| 103k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 103k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ | Branch (133:13): [True: 103k, False: 3.82k] ------------------ 134| | 135| | /* rfc 1321 section 3.2: length (little endian) */ 136| 34.4k| for (i = 0; i < 8; ++i) { ------------------ | Branch (136:18): [True: 30.6k, False: 3.82k] ------------------ 137| 30.6k| X(md5putc)(p, (unsigned char)(l & 0xFF)); ------------------ | | 78| 30.6k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 30.6k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 138| 30.6k| l = l >> 8; 139| 30.6k| } 140| | 141| | /* Now p->l % 64 == 0 and signature is in p->s */ 142| 3.82k|} md5.c:doblock: 77| 7.74k|{ 78| 7.74k| md5uint a, b, c, d, t, x[16]; 79| 7.74k| const md5uint msk = (md5uint)0xffffffffUL; 80| 7.74k| int i; 81| | 82| | /* encode input bytes into md5uint */ 83| 131k| for (i = 0; i < 16; ++i) { ------------------ | Branch (83:18): [True: 123k, False: 7.74k] ------------------ 84| 123k| const unsigned char *p = data + 4 * i; 85| 123k| x[i] = (unsigned)p[0] | ((unsigned)p[1] << 8) | ((unsigned)p[2] << 16) | ((unsigned)p[3] << 24); 86| 123k| } 87| | 88| 7.74k| a = state[0]; b = state[1]; c = state[2]; d = state[3]; 89| 503k| for (i = 0; i < 64; ++i) { ------------------ | Branch (89:18): [True: 495k, False: 7.74k] ------------------ 90| 495k| const struct roundtab *p = roundtab + i; 91| 495k| switch (i >> 4) { ------------------ | Branch (91:12): [True: 495k, False: 0] ------------------ 92| 123k| case 0: a += (b & c) | (~b & d); break; ------------------ | Branch (92:8): [True: 123k, False: 371k] ------------------ 93| 123k| case 1: a += (b & d) | (c & ~d); break; ------------------ | Branch (93:8): [True: 123k, False: 371k] ------------------ 94| 123k| case 2: a += b ^ c ^ d; break; ------------------ | Branch (94:8): [True: 123k, False: 371k] ------------------ 95| 123k| case 3: a += c ^ (b | ~d); break; ------------------ | Branch (95:8): [True: 123k, False: 371k] ------------------ 96| 495k| } 97| 495k| a += sintab[i]; 98| 495k| a += x[(int)(p->k)]; 99| 495k| a &= msk; 100| 495k| t = b + rol(a, p->s); ------------------ | | 74| 495k|#define rol(a, s) ((a << (int)(s)) | (a >> (32 - (int)(s)))) ------------------ 101| 495k| a = d; d = c; c = b; b = t; 102| 495k| } 103| 7.74k| state[0] = (state[0] + a) & msk; 104| 7.74k| state[1] = (state[1] + b) & msk; 105| 7.74k| state[2] = (state[2] + c) & msk; 106| 7.74k| state[3] = (state[3] + d) & msk; 107| 7.74k|} fftw_imax: 25| 3.97k|{ 26| 3.97k| return (a > b) ? a : b; ------------------ | Branch (26:13): [True: 1.16k, False: 2.81k] ------------------ 27| 3.97k|} fftw_imin: 30| 5.93k|{ 31| 5.93k| return (a < b) ? a : b; ------------------ | Branch (31:13): [True: 2.64k, False: 3.29k] ------------------ 32| 5.93k|} fftw_ops_zero: 25| 9.80k|{ 26| 9.80k| dst->add = dst->mul = dst->fma = dst->other = 0; 27| 9.80k|} fftw_ops_cpy: 30| 12|{ 31| 12| *dst = *src; 32| 12|} fftw_ops_other: 35| 546|{ 36| 546| X(ops_zero)(dst); ------------------ | | 78| 546|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 546|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 37| 546| dst->other = o; 38| 546|} fftw_ops_madd: 41| 4.94k|{ 42| 4.94k| dst->add = m * a->add + b->add; 43| 4.94k| dst->mul = m * a->mul + b->mul; 44| 4.94k| dst->fma = m * a->fma + b->fma; 45| 4.94k| dst->other = m * a->other + b->other; 46| 4.94k|} fftw_ops_add: 49| 1.64k|{ 50| 1.64k| X(ops_madd)(1, a, b, dst); ------------------ | | 78| 1.64k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1.64k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 51| 1.64k|} fftw_ops_madd2: 59| 3.09k|{ 60| 3.09k| X(ops_madd)(m, a, dst, dst); ------------------ | | 78| 3.09k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 3.09k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 61| 3.09k|} fftw_pickdim: 65| 1.54k|{ 66| 1.54k| size_t i; 67| 1.54k| int d1; 68| | 69| 1.54k| if (!really_pickdim(which_dim, sz, oop, dp)) ------------------ | Branch (69:10): [True: 322, False: 1.22k] ------------------ 70| 322| return 0; 71| | 72| | /* check whether some buddy solver would produce the same dim. 73| | If so, consider this solver unapplicable and let the buddy 74| | take care of it. The smallest-indexed buddy is applicable. */ 75| 1.40k| for (i = 0; i < nbuddies; ++i) { ------------------ | Branch (75:18): [True: 1.40k, False: 0] ------------------ 76| 1.40k| if (buddies[i] == which_dim) ------------------ | Branch (76:15): [True: 875, False: 529] ------------------ 77| 875| break; /* found self */ 78| 529| if (really_pickdim(buddies[i], sz, oop, &d1) && *dp == d1) ------------------ | Branch (78:15): [True: 529, False: 0] | Branch (78:59): [True: 350, False: 179] ------------------ 79| 350| return 0; /* found equivalent buddy */ 80| 529| } 81| 875| return 1; 82| 1.22k|} pickdim.c:really_pickdim: 30| 2.07k|{ 31| 2.07k| int i; 32| 2.07k| int count_ok = 0; 33| 2.07k| if (which_dim > 0) { ------------------ | Branch (33:10): [True: 1.38k, False: 690] ------------------ 34| 2.19k| for (i = 0; i < sz->rnk; ++i) { ------------------ | Branch (34:23): [True: 2.03k, False: 161] ------------------ 35| 2.03k| if (oop || sz->dims[i].is == sz->dims[i].os) ------------------ | Branch (35:20): [True: 805, False: 1.22k] | Branch (35:27): [True: 420, False: 805] ------------------ 36| 1.22k| if (++count_ok == which_dim) { ------------------ | Branch (36:25): [True: 1.22k, False: 0] ------------------ 37| 1.22k| *dp = i; 38| 1.22k| return 1; 39| 1.22k| } 40| 2.03k| } 41| 1.38k| } 42| 690| else if (which_dim < 0) { ------------------ | Branch (42:15): [True: 690, False: 0] ------------------ 43| 863| for (i = sz->rnk - 1; i >= 0; --i) { ------------------ | Branch (43:33): [True: 702, False: 161] ------------------ 44| 702| if (oop || sz->dims[i].is == sz->dims[i].os) ------------------ | Branch (44:20): [True: 322, False: 380] | Branch (44:27): [True: 207, False: 173] ------------------ 45| 529| if (++count_ok == -which_dim) { ------------------ | Branch (45:25): [True: 529, False: 0] ------------------ 46| 529| *dp = i; 47| 529| return 1; 48| 529| } 49| 702| } 50| 690| } 51| 0| else { /* zero: pick the middle, if valid */ 52| 0| i = (sz->rnk - 1) / 2; 53| 0| if (i >= 0 && (oop || sz->dims[i].is == sz->dims[i].os)) { ------------------ | Branch (53:8): [True: 0, False: 0] | Branch (53:19): [True: 0, False: 0] | Branch (53:26): [True: 0, False: 0] ------------------ 54| 0| *dp = i; 55| 0| return 1; 56| 0| } 57| 0| } 58| 322| return 0; 59| 2.07k|} fftw_mkplan: 29| 5.97k|{ 30| 5.97k| plan *p = (plan *)MALLOC(size, PLANS); ------------------ | | 306| 5.97k|#define MALLOC(n, what) X(malloc_plain)(n) | | ------------------ | | | | 78| 5.97k|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 5.97k|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 31| | 32| 5.97k| A(adt->destroy); 33| 5.97k| p->adt = adt; 34| 5.97k| X(ops_zero)(&p->ops); ------------------ | | 78| 5.97k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 5.97k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 35| 5.97k| p->pcost = 0.0; 36| 5.97k| p->wakefulness = SLEEPY; 37| 5.97k| p->could_prune_now_p = 0; 38| | 39| 5.97k| return p; 40| 5.97k|} fftw_plan_destroy_internal: 46| 19.1k|{ 47| 19.1k| if (ego) { ------------------ | Branch (47:10): [True: 5.97k, False: 13.1k] ------------------ 48| 5.97k| A(ego->wakefulness == SLEEPY); 49| 5.97k| ego->adt->destroy(ego); 50| 5.97k| X(ifree)(ego); ------------------ | | 78| 5.97k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 5.97k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 51| 5.97k| } 52| 19.1k|} fftw_plan_null_destroy: 56| 941|{ 57| 941| UNUSED(ego); ------------------ | | 88| 941|#define UNUSED(x) (void)x ------------------ 58| | /* nothing */ 59| 941|} fftw_plan_awake: 62| 2.88k|{ 63| 2.88k| if (ego) { ------------------ | Branch (63:10): [True: 2.88k, False: 0] ------------------ 64| 2.88k| A(((wakefulness == SLEEPY) ^ (ego->wakefulness == SLEEPY))); 65| | 66| 2.88k| ego->adt->awake(ego, wakefulness); 67| 2.88k| ego->wakefulness = wakefulness; 68| 2.88k| } 69| 2.88k|} fftw_iestimate_cost: 427| 2.19k|{ 428| 2.19k| double cost = 429| 2.19k| + pln->ops.add 430| 2.19k| + pln->ops.mul 431| | 432| |#if HAVE_FMA 433| | + pln->ops.fma 434| |#else 435| 2.19k| + 2 * pln->ops.fma 436| 2.19k|#endif 437| | 438| 2.19k| + pln->ops.other; 439| 2.19k| if (ego->cost_hook) ------------------ | Branch (439:10): [True: 0, False: 2.19k] ------------------ 440| 0| cost = ego->cost_hook(p, cost, COST_MAX); 441| 2.19k| return cost; 442| 2.19k|} fftw_mkplanner: 912| 1|{ 913| 1| int i; 914| | 915| 1| static const planner_adt padt = { 916| 1| register_solver, mkplan, forget, exprt, imprt 917| 1| }; 918| | 919| 1| planner *p = (planner *) MALLOC(sizeof(planner), PLANNERS); ------------------ | | 306| 1|#define MALLOC(n, what) X(malloc_plain)(n) | | ------------------ | | | | 78| 1|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 920| | 921| 1| p->adt = &padt; 922| 1| p->nplan = p->nprob = 0; 923| 1| p->pcost = p->epcost = 0.0; 924| 1| p->hook = 0; 925| 1| p->cost_hook = 0; 926| 1| p->wisdom_ok_hook = 0; 927| 1| p->nowisdom_hook = 0; 928| 1| p->bogosity_hook = 0; 929| 1| p->cur_reg_nam = 0; 930| 1| p->wisdom_state = WISDOM_NORMAL; 931| | 932| 1| p->slvdescs = 0; 933| 1| p->nslvdesc = p->slvdescsiz = 0; 934| | 935| 1| p->flags.l = 0; 936| 1| p->flags.u = 0; 937| 1| p->flags.timelimit_impatience = 0; 938| 1| p->flags.hash_info = 0; 939| 1| p->nthr = 1; 940| 1| p->need_timeout_check = 1; 941| 1| p->timelimit = -1; 942| | 943| 1| mkhashtab(&p->htab_blessed); 944| 1| mkhashtab(&p->htab_unblessed); 945| | 946| 9| for (i = 0; i < PROBLEM_LAST; ++i) ------------------ | Branch (946:18): [True: 8, False: 1] ------------------ 947| 8| p->slvdescs_for_problem_kind[i] = -1; 948| | 949| 1| return p; 950| 1|} fftw_mkplan_d: 969| 3.25k|{ 970| 3.25k| plan *pln = ego->adt->mkplan(ego, p); 971| 3.25k| X(problem_destroy)(p); ------------------ | | 78| 3.25k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 3.25k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 972| 3.25k| return pln; 973| 3.25k|} fftw_mkplan_f_d: 978| 640|{ 979| 640| flags_t oflags = ego->flags; 980| 640| plan *pln; 981| | 982| 640| PLNR_U(ego) &= ~u_reset; ------------------ | | 675| 640|#define PLNR_U(plnr) ((plnr)->flags.u) ------------------ 983| 640| PLNR_L(ego) &= ~u_reset; ------------------ | | 674| 640|#define PLNR_L(plnr) ((plnr)->flags.l) ------------------ 984| 640| PLNR_L(ego) |= l_set; ------------------ | | 674| 640|#define PLNR_L(plnr) ((plnr)->flags.l) ------------------ 985| 640| PLNR_U(ego) |= u_set | l_set; ------------------ | | 675| 640|#define PLNR_U(plnr) ((plnr)->flags.u) ------------------ 986| 640| pln = X(mkplan_d)(ego, p); ------------------ | | 78| 640|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 640|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 987| 640| ego->flags = oflags; 988| 640| return pln; 989| 640|} planner.c:register_solver: 95| 613|{ 96| 613| slvdesc *n; 97| 613| int kind; 98| | 99| 613| if (s) { /* add s to solver list */ ------------------ | Branch (99:10): [True: 613, False: 0] ------------------ 100| 613| X(solver_use)(s); ------------------ | | 78| 613|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 613|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 101| | 102| 613| A(ego->nslvdesc < INFEASIBLE_SLVNDX); 103| 613| if (ego->nslvdesc >= ego->slvdescsiz) ------------------ | Branch (103:8): [True: 25, False: 588] ------------------ 104| 25| sgrow(ego); 105| | 106| 613| n = ego->slvdescs + ego->nslvdesc; 107| | 108| 613| n->slv = s; 109| 613| n->reg_nam = ego->cur_reg_nam; 110| 613| n->reg_id = ego->cur_reg_id++; 111| | 112| 613| A(strlen(n->reg_nam) < MAXNAM); 113| 613| n->nam_hash = X(hash)(n->reg_nam); ------------------ | | 78| 613|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 613|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 114| | 115| 613| kind = s->adt->problem_kind; 116| 613| n->next_for_same_problem_kind = ego->slvdescs_for_problem_kind[kind]; 117| 613| ego->slvdescs_for_problem_kind[kind] = (int)/*from unsigned*/ego->nslvdesc; 118| | 119| 613| ego->nslvdesc++; 120| 613| } 121| 613|} planner.c:sgrow: 81| 25|{ 82| 25| unsigned osiz = ego->slvdescsiz, nsiz = 1 + osiz + osiz / 4; 83| 25| slvdesc *ntab = (slvdesc *)MALLOC(nsiz * sizeof(slvdesc), SLVDESCS); ------------------ | | 306| 25|#define MALLOC(n, what) X(malloc_plain)(n) | | ------------------ | | | | 78| 25|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 25|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 84| 25| slvdesc *otab = ego->slvdescs; 85| 25| unsigned i; 86| | 87| 25| ego->slvdescs = ntab; 88| 25| ego->slvdescsiz = nsiz; 89| 2.80k| for (i = 0; i < osiz; ++i) ------------------ | Branch (89:18): [True: 2.77k, False: 25] ------------------ 90| 2.77k| ntab[i] = otab[i]; 91| 25| X(ifree0)(otab); ------------------ | | 78| 25|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 25|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 92| 25|} planner.c:mkplan: 624| 3.82k|{ 625| 3.82k| plan *pln; 626| 3.82k| md5 m; 627| 3.82k| unsigned slvndx; 628| 3.82k| flags_t flags_of_solution; 629| 3.82k| solution *sol; 630| 3.82k| solver *s; 631| | 632| 3.82k| ASSERT_ALIGNED_DOUBLE; 633| 3.82k| A(LEQ(PLNR_L(ego), PLNR_U(ego))); 634| | 635| 3.82k| if (ESTIMATEP(ego)) ------------------ | | 678| 3.82k|#define ESTIMATEP(plnr) (PLNR_U(plnr) & ESTIMATE) | | ------------------ | | | | 675| 3.82k|#define PLNR_U(plnr) ((plnr)->flags.u) | | ------------------ | | | Branch (678:25): [True: 3.82k, False: 0] | | ------------------ ------------------ 636| 3.82k| PLNR_TIMELIMIT_IMPATIENCE(ego) = 0; /* canonical form */ ------------------ | | 676| 3.82k|#define PLNR_TIMELIMIT_IMPATIENCE(plnr) ((plnr)->flags.timelimit_impatience) ------------------ 637| | 638| | 639| |#ifdef FFTW_DEBUG 640| | check(&ego->htab_blessed); 641| | check(&ego->htab_unblessed); 642| |#endif 643| | 644| 3.82k| pln = 0; 645| | 646| 3.82k| CHECK_FOR_BOGOSITY; ------------------ | | 618| 3.82k| if ((ego->bogosity_hook ? \ | | ------------------ | | | Branch (618:10): [True: 0, False: 3.82k] | | | Branch (618:11): [True: 0, False: 3.82k] | | ------------------ | | 619| 3.82k| (ego->wisdom_state = ego->bogosity_hook(ego->wisdom_state, p)) \ | | 620| 3.82k| : ego->wisdom_state) == WISDOM_IS_BOGUS) \ | | 621| 3.82k| goto wisdom_is_bogus; ------------------ 647| | 648| 3.82k| ego->timed_out = 0; 649| | 650| 3.82k| ++ego->nprob; 651| 3.82k| md5hash(&m, p, ego); 652| | 653| 3.82k| flags_of_solution = ego->flags; 654| | 655| 3.82k| if (ego->wisdom_state != WISDOM_IGNORE_ALL) { ------------------ | Branch (655:10): [True: 3.82k, False: 0] ------------------ 656| 3.82k| if ((sol = hlookup(ego, m.s, &flags_of_solution))) { ------------------ | Branch (656:8): [True: 1.73k, False: 2.09k] ------------------ 657| | /* wisdom is acceptable */ 658| 1.73k| wisdom_state_t owisdom_state = ego->wisdom_state; 659| | 660| | /* this hook is mainly for MPI, to make sure that 661| | wisdom is in sync across all processes for MPI problems */ 662| 1.73k| if (ego->wisdom_ok_hook && !ego->wisdom_ok_hook(p, sol->flags)) ------------------ | Branch (662:13): [True: 0, False: 1.73k] | Branch (662:36): [True: 0, False: 0] ------------------ 663| 0| goto do_search; /* ignore not-ok wisdom */ 664| | 665| 1.73k| slvndx = SLVNDX(sol); ------------------ | | 35| 1.73k|#define SLVNDX(solution) ((solution)->flags.slvndx) ------------------ 666| | 667| 1.73k| if (slvndx == INFEASIBLE_SLVNDX) { ------------------ | | 37| 1.73k|#define INFEASIBLE_SLVNDX ((1U<wisdom_state == WISDOM_IGNORE_INFEASIBLE) ------------------ | Branch (668:11): [True: 0, False: 6] ------------------ 669| 0| goto do_search; 670| 6| else 671| 6| return 0; /* known to be infeasible */ 672| 6| } 673| | 674| 1.73k| flags_of_solution = sol->flags; 675| | 676| | /* inherit blessing either from wisdom 677| | or from the planner */ 678| 1.73k| flags_of_solution.hash_info |= BLISS(ego->flags); ------------------ | | 36| 1.73k|#define BLISS(flags) (((flags).hash_info) & BLESSING) ------------------ 679| | 680| 1.73k| ego->wisdom_state = WISDOM_ONLY; 681| | 682| 1.73k| s = ego->slvdescs[slvndx].slv; 683| 1.73k| if (p->adt->problem_kind != s->adt->problem_kind) ------------------ | Branch (683:13): [True: 0, False: 1.73k] ------------------ 684| 0| goto wisdom_is_bogus; 685| | 686| 1.73k| pln = invoke_solver(ego, p, s, &flags_of_solution); 687| | 688| 1.73k| CHECK_FOR_BOGOSITY; /* catch error in child solvers */ ------------------ | | 618| 1.73k| if ((ego->bogosity_hook ? \ | | ------------------ | | | Branch (618:10): [True: 0, False: 1.73k] | | | Branch (618:11): [True: 0, False: 1.73k] | | ------------------ | | 619| 1.73k| (ego->wisdom_state = ego->bogosity_hook(ego->wisdom_state, p)) \ | | 620| 1.73k| : ego->wisdom_state) == WISDOM_IS_BOGUS) \ | | 621| 1.73k| goto wisdom_is_bogus; ------------------ 689| | 690| 1.73k| sol = 0; /* Paranoia: SOL may be dangling after 691| | invoke_solver(); make sure we don't accidentally 692| | reuse it. */ 693| | 694| 1.73k| if (!pln) ------------------ | Branch (694:13): [True: 0, False: 1.73k] ------------------ 695| 0| goto wisdom_is_bogus; 696| | 697| 1.73k| ego->wisdom_state = owisdom_state; 698| | 699| 1.73k| goto skip_search; 700| 1.73k| } 701| 2.09k| else if (ego->nowisdom_hook) /* for MPI, make sure lack of wisdom */ ------------------ | Branch (701:13): [True: 0, False: 2.09k] ------------------ 702| 0| ego->nowisdom_hook(p); /* is in sync across all processes */ 703| 3.82k| } 704| | 705| 2.09k| do_search: 706| | /* cannot search in WISDOM_ONLY mode */ 707| 2.09k| if (ego->wisdom_state == WISDOM_ONLY) ------------------ | Branch (707:10): [True: 0, False: 2.09k] ------------------ 708| 0| goto wisdom_is_bogus; 709| | 710| 2.09k| flags_of_solution = ego->flags; 711| 2.09k| pln = search(ego, p, &slvndx, &flags_of_solution); 712| 2.09k| CHECK_FOR_BOGOSITY; /* catch error in child solvers */ ------------------ | | 618| 2.09k| if ((ego->bogosity_hook ? \ | | ------------------ | | | Branch (618:10): [True: 0, False: 2.09k] | | | Branch (618:11): [True: 0, False: 2.09k] | | ------------------ | | 619| 2.09k| (ego->wisdom_state = ego->bogosity_hook(ego->wisdom_state, p)) \ | | 620| 2.09k| : ego->wisdom_state) == WISDOM_IS_BOGUS) \ | | 621| 2.09k| goto wisdom_is_bogus; ------------------ 713| | 714| 2.09k| if (ego->timed_out) { ------------------ | Branch (714:10): [True: 0, False: 2.09k] ------------------ 715| 0| A(!pln); 716| 0| if (PLNR_TIMELIMIT_IMPATIENCE(ego) != 0) { ------------------ | | 676| 0|#define PLNR_TIMELIMIT_IMPATIENCE(plnr) ((plnr)->flags.timelimit_impatience) ------------------ | Branch (716:8): [True: 0, False: 0] ------------------ 717| | /* record (below) that this plan has failed because of 718| | timeout */ 719| 0| flags_of_solution.hash_info |= BLESSING; 720| 0| } else { 721| | /* this is not the top-level problem or timeout is not 722| | active: record no wisdom. */ 723| 0| return 0; 724| 0| } 725| 2.09k| } else { 726| | /* canonicalize to infinite timeout */ 727| 2.09k| flags_of_solution.timelimit_impatience = 0; 728| 2.09k| } 729| | 730| 3.82k| skip_search: 731| 3.82k| if (ego->wisdom_state == WISDOM_NORMAL || ------------------ | Branch (731:10): [True: 2.60k, False: 1.21k] ------------------ 732| 3.82k| ego->wisdom_state == WISDOM_ONLY) { ------------------ | Branch (732:3): [True: 1.21k, False: 0] ------------------ 733| 3.82k| if (pln) { ------------------ | Branch (733:8): [True: 3.54k, False: 274] ------------------ 734| 3.54k| hinsert(ego, m.s, &flags_of_solution, slvndx); 735| 3.54k| invoke_hook(ego, pln, p, 1); 736| 3.54k| } else { 737| 274| hinsert(ego, m.s, &flags_of_solution, INFEASIBLE_SLVNDX); ------------------ | | 37| 274|#define INFEASIBLE_SLVNDX ((1U<wisdom_state = WISDOM_IS_BOGUS; 746| 0| return 0; 747| 2.09k|} planner.c:md5hash: 171| 3.82k|{ 172| 3.82k| X(md5begin)(m); ------------------ | | 78| 3.82k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 3.82k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 173| 3.82k| X(md5unsigned)(m, sizeof(R)); /* so we don't mix different precisions */ ------------------ | | 78| 3.82k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 3.82k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 174| 3.82k| X(md5int)(m, plnr->nthr); ------------------ | | 78| 3.82k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 3.82k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 175| 3.82k| p->adt->hash(p, m); 176| 3.82k| X(md5end)(m); ------------------ | | 78| 3.82k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 3.82k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 177| 3.82k|} planner.c:hlookup: 241| 3.82k|{ 242| 3.82k| solution *sol = htab_lookup(&ego->htab_blessed, s, flagsp); 243| 3.82k| if (!sol) sol = htab_lookup(&ego->htab_unblessed, s, flagsp); ------------------ | Branch (243:10): [True: 2.75k, False: 1.07k] ------------------ 244| 3.82k| return sol; 245| 3.82k|} planner.c:htab_lookup: 205| 6.58k|{ 206| 6.58k| unsigned g, h = h1(ht, s), d = h2(ht, s); 207| 6.58k| solution *best = 0; 208| | 209| 6.58k| ++ht->lookup; 210| | 211| | /* search all entries that match; select the one with 212| | the lowest flags.u */ 213| | /* This loop may potentially traverse the whole table, since at 214| | least one element is guaranteed to be !LIVEP, but all elements 215| | may be VALIDP. Hence, we stop after at the first invalid 216| | element or after traversing the whole table. */ 217| 6.58k| g = h; 218| 36.0k| do { 219| 36.0k| solution *l = ht->solutions + g; 220| 36.0k| ++ht->lookup_iter; 221| 36.0k| if (VALIDP(l)) { ------------------ | | 33| 36.0k|#define VALIDP(solution) ((solution)->flags.hash_info & H_VALID) | | ------------------ | | | Branch (33:26): [True: 29.4k, False: 6.58k] | | ------------------ ------------------ 222| 29.4k| if (LIVEP(l) ------------------ | | 34| 58.9k|#define LIVEP(solution) ((solution)->flags.hash_info & H_LIVE) | | ------------------ | | | Branch (34:25): [True: 29.4k, False: 0] | | ------------------ ------------------ 223| 29.4k| && md5eq(s, l->s) ------------------ | Branch (223:9): [True: 1.82k, False: 27.6k] ------------------ 224| 1.82k| && subsumes(&l->flags, SLVNDX(l), flagsp) ) { ------------------ | | 35| 1.82k|#define SLVNDX(solution) ((solution)->flags.slvndx) ------------------ | Branch (224:9): [True: 1.73k, False: 83] ------------------ 225| 1.73k| if (!best || LEQ(l->flags.u, best->flags.u)) ------------------ | | 50| 0|#define LEQ(x, y) (((x) & (y)) == (x)) | | ------------------ | | | Branch (50:19): [True: 0, False: 0] | | ------------------ ------------------ | Branch (225:11): [True: 1.73k, False: 0] ------------------ 226| 1.73k| best = l; 227| 1.73k| } 228| 29.4k| } else 229| 6.58k| break; 230| | 231| 29.4k| g = addmod(g, d, ht->hashsiz); 232| 29.4k| } while (g != h); ------------------ | Branch (232:15): [True: 29.4k, False: 0] ------------------ 233| | 234| 6.58k| if (best) ------------------ | Branch (234:10): [True: 1.73k, False: 4.84k] ------------------ 235| 1.73k| ++ht->succ_lookup; 236| 6.58k| return best; 237| 6.58k|} planner.c:h1: 156| 24.7k|{ 157| 24.7k| unsigned h = s[0] % ht->hashsiz; 158| 24.7k| A(h == (s[0] % ht->hashsiz)); 159| 24.7k| return h; 160| 24.7k|} planner.c:h2: 164| 24.7k|{ 165| 24.7k| unsigned h = 1U + s[1] % (ht->hashsiz - 1); 166| 24.7k| A(h == (1U + s[1] % (ht->hashsiz - 1))); 167| 24.7k| return h; 168| 24.7k|} planner.c:md5eq: 180| 43.3k|{ 181| 43.3k| return a[0] == b[0] && a[1] == b[1] && a[2] == b[2] && a[3] == b[3]; ------------------ | Branch (181:13): [True: 2.60k, False: 40.7k] | Branch (181:29): [True: 2.60k, False: 0] | Branch (181:45): [True: 2.60k, False: 0] | Branch (181:61): [True: 2.60k, False: 0] ------------------ 182| 43.3k|} planner.c:subsumes: 54| 2.60k|{ 55| 2.60k| if (slvndx_a != INFEASIBLE_SLVNDX) { ------------------ | | 37| 2.60k|#define INFEASIBLE_SLVNDX ((1U<timelimit_impatience == 0); 57| 2.58k| return (LEQ(a->u, b->u) && LEQ(b->l, a->l)); ------------------ | | 50| 5.16k|#define LEQ(x, y) (((x) & (y)) == (x)) | | ------------------ | | | Branch (50:19): [True: 2.52k, False: 59] | | ------------------ ------------------ return (LEQ(a->u, b->u) && LEQ(b->l, a->l)); ------------------ | | 50| 2.52k|#define LEQ(x, y) (((x) & (y)) == (x)) | | ------------------ | | | Branch (50:19): [True: 2.45k, False: 68] | | ------------------ ------------------ 58| 2.58k| } else { 59| 20| return (LEQ(a->l, b->l) ------------------ | | 50| 40|#define LEQ(x, y) (((x) & (y)) == (x)) | | ------------------ | | | Branch (50:19): [True: 10, False: 10] | | ------------------ ------------------ 60| 10| && a->timelimit_impatience <= b->timelimit_impatience); ------------------ | Branch (60:8): [True: 10, False: 0] ------------------ 61| 20| } 62| 2.60k|} planner.c:addmod: 65| 58.0k|{ 66| | /* gcc-2.95/sparc produces incorrect code for the fast version below. */ 67| |#if defined(__sparc__) && defined(__GNUC__) 68| | /* slow version */ 69| | return (a + b) % p; 70| |#else 71| | /* faster version */ 72| 58.0k| unsigned c = a + b; 73| 58.0k| return c >= p ? c - p : c; ------------------ | Branch (73:13): [True: 29.4k, False: 28.5k] ------------------ 74| 58.0k|#endif 75| 58.0k|} planner.c:invoke_solver: 479| 302k|{ 480| 302k| flags_t flags = ego->flags; 481| 302k| int nthr = ego->nthr; 482| 302k| plan *pln; 483| 302k| ego->flags = *nflags; 484| 302k| PLNR_TIMELIMIT_IMPATIENCE(ego) = 0; ------------------ | | 676| 302k|#define PLNR_TIMELIMIT_IMPATIENCE(plnr) ((plnr)->flags.timelimit_impatience) ------------------ 485| 302k| A(p->adt->problem_kind == s->adt->problem_kind); 486| 302k| pln = s->adt->mkplan(s, p, ego); 487| 302k| ego->nthr = nthr; 488| 302k| ego->flags = flags; 489| 302k| return pln; 490| 302k|} planner.c:search: 574| 2.09k|{ 575| 2.09k| plan *pln = 0; 576| 2.09k| unsigned i; 577| | 578| | /* relax impatience in this order: */ 579| 2.09k| static const unsigned relax_tab[] = { 580| 2.09k| 0, /* relax nothing */ 581| 2.09k| NO_VRECURSE, 582| 2.09k| NO_FIXED_RADIX_LARGE_N, 583| 2.09k| NO_SLOW, 584| 2.09k| NO_UGLY 585| 2.09k| }; 586| | 587| 2.09k| unsigned l_orig = flagsp->l; 588| 2.09k| unsigned x = flagsp->u; 589| | 590| | /* guaranteed to be different from X */ 591| 2.09k| unsigned last_x = ~x; 592| | 593| 3.48k| for (i = 0; i < sizeof(relax_tab) / sizeof(relax_tab[0]); ++i) { ------------------ | Branch (593:18): [True: 3.21k, False: 274] ------------------ 594| 3.21k| if (LEQ(l_orig, x & ~relax_tab[i])) ------------------ | | 50| 3.21k|#define LEQ(x, y) (((x) & (y)) == (x)) | | ------------------ | | | Branch (50:19): [True: 2.12k, False: 1.09k] | | ------------------ ------------------ 595| 2.12k| x = x & ~relax_tab[i]; 596| | 597| 3.21k| if (x != last_x) { ------------------ | Branch (597:8): [True: 2.12k, False: 1.09k] ------------------ 598| 2.12k| last_x = x; 599| 2.12k| flagsp->l = x; 600| 2.12k| pln = search0(ego, p, slvndx, flagsp); 601| 2.12k| if (pln) break; ------------------ | Branch (601:13): [True: 1.81k, False: 307] ------------------ 602| 2.12k| } 603| 3.21k| } 604| | 605| 2.09k| if (!pln) { ------------------ | Branch (605:10): [True: 274, False: 1.81k] ------------------ 606| | /* search [L_ORIG, U] */ 607| 274| if (l_orig != last_x) { ------------------ | Branch (607:8): [True: 0, False: 274] ------------------ 608| 0| last_x = l_orig; 609| 0| flagsp->l = l_orig; 610| 0| pln = search0(ego, p, slvndx, flagsp); 611| 0| } 612| 274| } 613| | 614| 2.09k| return pln; 615| 2.09k|} planner.c:search0: 520| 2.12k|{ 521| 2.12k| plan *best = 0; 522| 2.12k| int best_not_yet_timed = 1; 523| | 524| | /* Do not start a search if the planner timed out. This check is 525| | necessary, lest the relaxation mechanism kick in */ 526| 2.12k| if (timeout_p(ego, p)) ------------------ | Branch (526:10): [True: 0, False: 2.12k] ------------------ 527| 0| return 0; 528| | 529| 2.12k| FORALL_SOLVERS_OF_KIND(p->adt->problem_kind, ego, s, sp, { ------------------ | | 802| 2.12k|#define FORALL_SOLVERS_OF_KIND(kind, ego, s, p, what) \ | | 803| 2.12k|{ \ | | 804| 2.12k| int _cnt = ego->slvdescs_for_problem_kind[kind]; \ | | 805| 302k| while (_cnt >= 0) { \ | | ------------------ | | | Branch (805:13): [True: 301k, False: 1.45k] | | ------------------ | | 806| 301k| slvdesc *p = ego->slvdescs + _cnt; \ | | 807| 301k| solver *s = p->slv; \ | | 808| 625k| what; \ | | ------------------ | | | Branch (808:4): [True: 0, False: 0] | | | Branch (808:4): [True: 0, False: 301k] | | | Branch (808:4): [True: 965, False: 260] | | | Branch (808:4): [True: 744, False: 481] | | | Branch (808:4): [True: 1.22k, False: 1.81k] | | | Branch (808:4): [True: 3.04k, False: 0] | | | Branch (808:4): [True: 672, False: 2.36k] | | | Branch (808:4): [True: 3.04k, False: 297k] | | ------------------ | | 809| 300k| _cnt = p->next_for_same_problem_kind; \ | | 810| 300k| } \ | | 811| 2.12k|} ------------------ 530| 2.12k| plan *pln; 531| | 532| 2.12k| pln = invoke_solver(ego, p, s, flagsp); 533| | 534| 2.12k| if (ego->need_timeout_check) 535| 2.12k| if (timeout_p(ego, p)) { 536| 2.12k| X(plan_destroy_internal)(pln); 537| 2.12k| X(plan_destroy_internal)(best); 538| 2.12k| return 0; 539| 2.12k| } 540| | 541| 2.12k| if (pln) { 542| | /* read COULD_PRUNE_NOW_P because PLN may be destroyed 543| | before we use COULD_PRUNE_NOW_P */ 544| 2.12k| int could_prune_now_p = pln->could_prune_now_p; 545| | 546| 2.12k| if (best) { 547| 2.12k| if (best_not_yet_timed) { 548| 2.12k| evaluate_plan(ego, best, p); 549| 2.12k| best_not_yet_timed = 0; 550| 2.12k| } 551| 2.12k| evaluate_plan(ego, pln, p); 552| 2.12k| if (pln->pcost < best->pcost) { 553| 2.12k| X(plan_destroy_internal)(best); 554| 2.12k| best = pln; 555| 2.12k| *slvndx = (unsigned)/*from ptrdiff_t*/(sp - ego->slvdescs); 556| 2.12k| } else { 557| 2.12k| X(plan_destroy_internal)(pln); 558| 2.12k| } 559| 2.12k| } else { 560| 2.12k| best = pln; 561| 2.12k| *slvndx = (unsigned)/*from ptrdiff_t*/(sp - ego->slvdescs); 562| 2.12k| } 563| | 564| 2.12k| if (ALLOW_PRUNINGP(ego) && could_prune_now_p) 565| 2.12k| break; 566| 2.12k| } 567| 2.12k| }); 568| | 569| 2.12k| return best; 570| 2.12k|} planner.c:timeout_p: 494| 2.12k|{ 495| | /* do not timeout when estimating. First, the estimator is the 496| | planner of last resort. Second, calling X(elapsed_since)() is 497| | slower than estimating */ 498| 2.12k| if (!ESTIMATEP(ego)) { ------------------ | | 678| 2.12k|#define ESTIMATEP(plnr) (PLNR_U(plnr) & ESTIMATE) | | ------------------ | | | | 675| 2.12k|#define PLNR_U(plnr) ((plnr)->flags.u) | | ------------------ ------------------ | Branch (498:10): [True: 0, False: 2.12k] ------------------ 499| | /* do not assume that X(elapsed_since)() is monotonic */ 500| 0| if (ego->timed_out) { ------------------ | Branch (500:8): [True: 0, False: 0] ------------------ 501| 0| A(ego->need_timeout_check); 502| 0| return 1; 503| 0| } 504| | 505| 0| if (ego->timelimit >= 0 && ------------------ | Branch (505:8): [True: 0, False: 0] ------------------ 506| 0| X(elapsed_since)(ego, p, ego->start_time) >= ego->timelimit) { ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ | Branch (506:8): [True: 0, False: 0] ------------------ 507| 0| ego->timed_out = 1; 508| 0| ego->need_timeout_check = 1; 509| 0| return 1; 510| 0| } 511| 0| } 512| | 513| 2.12k| A(!ego->timed_out); 514| 2.12k| ego->need_timeout_check = 0; 515| 2.12k| return 0; 516| 2.12k|} planner.c:evaluate_plan: 445| 2.19k|{ 446| 2.19k| if (ESTIMATEP(ego) || !BELIEVE_PCOSTP(ego) || pln->pcost == 0.0) { ------------------ | | 678| 4.38k|#define ESTIMATEP(plnr) (PLNR_U(plnr) & ESTIMATE) | | ------------------ | | | | 675| 2.19k|#define PLNR_U(plnr) ((plnr)->flags.u) | | ------------------ | | | Branch (678:25): [True: 2.19k, False: 0] | | ------------------ ------------------ if (ESTIMATEP(ego) || !BELIEVE_PCOSTP(ego) || pln->pcost == 0.0) { ------------------ | | 679| 2.19k|#define BELIEVE_PCOSTP(plnr) (PLNR_U(plnr) & BELIEVE_PCOST) | | ------------------ | | | | 675| 0|#define PLNR_U(plnr) ((plnr)->flags.u) | | ------------------ ------------------ | Branch (446:28): [True: 0, False: 0] | Branch (446:52): [True: 0, False: 0] ------------------ 447| 2.19k| ego->nplan++; 448| | 449| 2.19k| if (ESTIMATEP(ego)) { ------------------ | | 678| 2.19k|#define ESTIMATEP(plnr) (PLNR_U(plnr) & ESTIMATE) | | ------------------ | | | | 675| 2.19k|#define PLNR_U(plnr) ((plnr)->flags.u) | | ------------------ | | | Branch (678:25): [True: 2.19k, False: 0] | | ------------------ ------------------ 450| 2.19k| estimate: 451| | /* heuristic */ 452| |#ifdef FFTW_RANDOM_ESTIMATOR 453| | pln->pcost = random_estimate(ego, pln, p); 454| | ego->epcost += X(iestimate_cost)(ego, pln, p); 455| |#else 456| 2.19k| pln->pcost = X(iestimate_cost)(ego, pln, p); ------------------ | | 78| 2.19k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 2.19k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 457| 2.19k| ego->epcost += pln->pcost; 458| 2.19k|#endif 459| 2.19k| } else { 460| 0| double t = X(measure_execution_time)(ego, pln, p); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 461| | 462| 0| if (t < 0) { /* unavailable cycle counter */ ------------------ | Branch (462:13): [True: 0, False: 0] ------------------ 463| | /* Real programmers can write FORTRAN in any language */ 464| 0| goto estimate; 465| 0| } 466| | 467| 0| pln->pcost = t; 468| 0| ego->pcost += t; 469| 0| ego->need_timeout_check = 1; 470| 0| } 471| 2.19k| } 472| | 473| 2.19k| invoke_hook(ego, pln, p, 0); 474| 2.19k|} planner.c:hinsert: 391| 3.82k|{ 392| 3.82k| htab_insert(BLISS(*flagsp) ? &ego->htab_blessed : &ego->htab_unblessed, ------------------ | | 36| 3.82k|#define BLISS(flags) (((flags).hash_info) & BLESSING) | | ------------------ | | | Branch (36:22): [True: 1.07k, False: 2.75k] | | ------------------ ------------------ 393| 3.82k| s, flagsp, slvndx ); 394| 3.82k|} planner.c:htab_insert: 349| 3.82k|{ 350| 3.82k| unsigned g, h = h1(ht, s), d = h2(ht, s); 351| 3.82k| solution *first = 0; 352| | 353| | /* Remove all entries that are subsumed by the new one. */ 354| | /* This loop may potentially traverse the whole table, since at 355| | least one element is guaranteed to be !LIVEP, but all elements 356| | may be VALIDP. Hence, we stop after at the first invalid 357| | element or after traversing the whole table. */ 358| 3.82k| g = h; 359| 17.7k| do { 360| 17.7k| solution *l = ht->solutions + g; 361| 17.7k| ++ht->insert_iter; 362| 17.7k| if (VALIDP(l)) { ------------------ | | 33| 17.7k|#define VALIDP(solution) ((solution)->flags.hash_info & H_VALID) | | ------------------ | | | Branch (33:26): [True: 13.9k, False: 3.82k] | | ------------------ ------------------ 363| 13.9k| if (LIVEP(l) && md5eq(s, l->s)) { ------------------ | | 34| 27.8k|#define LIVEP(solution) ((solution)->flags.hash_info & H_LIVE) | | ------------------ | | | Branch (34:25): [True: 13.9k, False: 0] | | ------------------ ------------------ | Branch (363:25): [True: 781, False: 13.1k] ------------------ 364| 781| if (subsumes(flagsp, slvndx, &l->flags)) { ------------------ | Branch (364:11): [True: 727, False: 54] ------------------ 365| 727| if (!first) first = l; ------------------ | Branch (365:9): [True: 727, False: 0] ------------------ 366| 727| kill_slot(ht, l); 367| 727| } else { 368| | /* It is an error to insert an element that 369| | is subsumed by an existing entry. */ 370| 54| A(!subsumes(&l->flags, SLVNDX(l), flagsp)); 371| 54| } 372| 781| } 373| 13.9k| } else 374| 3.82k| break; 375| | 376| 13.9k| g = addmod(g, d, ht->hashsiz); 377| 13.9k| } while (g != h); ------------------ | Branch (377:15): [True: 13.9k, False: 0] ------------------ 378| | 379| 3.82k| if (first) { ------------------ | Branch (379:10): [True: 727, False: 3.09k] ------------------ 380| | /* overwrite FIRST */ 381| 727| fill_slot(ht, s, flagsp, slvndx, first); 382| 3.09k| } else { 383| | /* create a new entry */ 384| 3.09k| hgrow(ht); 385| 3.09k| hinsert0(ht, s, flagsp, slvndx); 386| 3.09k| } 387| 3.82k|} planner.c:kill_slot: 266| 727|{ 267| 727| A(LIVEP(slot)); /* ==> */ A(VALIDP(slot)); 268| | 269| 727| --ht->nelem; 270| 727| slot->flags.hash_info = H_VALID; 271| 727|} planner.c:fill_slot: 249| 15.0k|{ 250| 15.0k| ++ht->insert; 251| 15.0k| ++ht->nelem; 252| 15.0k| A(!LIVEP(slot)); 253| 15.0k| slot->flags.u = flagsp->u; 254| 15.0k| slot->flags.l = flagsp->l; 255| 15.0k| slot->flags.timelimit_impatience = flagsp->timelimit_impatience; 256| 15.0k| slot->flags.hash_info |= H_VALID | H_LIVE; 257| 15.0k| SLVNDX(slot) = slvndx; ------------------ | | 35| 15.0k|#define SLVNDX(solution) ((solution)->flags.slvndx) ------------------ 258| | 259| | /* keep this check enabled in case we add so many solvers 260| | that the bitfield overflows */ 261| 15.0k| CK(SLVNDX(slot) == slvndx); ------------------ | | 264| 15.0k| (void)((ex) || (X(assertion_failed)(#ex, __LINE__, __FILE__), 0)) | | ------------------ | | | | 78| 0|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ | | | Branch (264:14): [True: 15.0k, False: 0] | | | Branch (264:22): [True: 0, False: 0] | | ------------------ ------------------ 262| 15.0k| sigcpy(s, slot->s); 263| 15.0k|} planner.c:sigcpy: 185| 15.0k|{ 186| 15.0k| b[0] = a[0]; b[1] = a[1]; b[2] = a[2]; b[3] = a[3]; 187| 15.0k|} planner.c:hgrow: 331| 3.38k|{ 332| 3.38k| unsigned nelem = ht->nelem; 333| 3.38k| if (minsz(nelem) >= ht->hashsiz) ------------------ | Branch (333:10): [True: 1.27k, False: 2.10k] ------------------ 334| 1.27k| rehash(ht, nextsz(nelem)); 335| 3.38k|} planner.c:minsz: 321| 5.93k|{ 322| 5.93k| return 1U + nelem + nelem / 8U; 323| 5.93k|} planner.c:rehash: 293| 1.27k|{ 294| 1.27k| unsigned osiz = ht->hashsiz, h; 295| 1.27k| solution *osol = ht->solutions, *nsol; 296| | 297| 1.27k| nsiz = (unsigned)X(next_prime)((INT)nsiz); ------------------ | | 78| 1.27k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1.27k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 298| 1.27k| nsol = (solution *)MALLOC(nsiz * sizeof(solution), HASHT); ------------------ | | 306| 1.27k|#define MALLOC(n, what) X(malloc_plain)(n) | | ------------------ | | | | 78| 1.27k|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 1.27k|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 299| 1.27k| ++ht->nrehash; 300| | 301| | /* init new table */ 302| 18.6k| for (h = 0; h < nsiz; ++h) ------------------ | Branch (302:18): [True: 17.3k, False: 1.27k] ------------------ 303| 17.3k| nsol[h].flags.hash_info = 0; 304| | 305| | /* install new table */ 306| 1.27k| ht->hashsiz = nsiz; 307| 1.27k| ht->solutions = nsol; 308| 1.27k| ht->nelem = 0; 309| | 310| | /* copy table */ 311| 14.5k| for (h = 0; h < osiz; ++h) { ------------------ | Branch (311:18): [True: 13.3k, False: 1.27k] ------------------ 312| 13.3k| solution *l = osol + h; 313| 13.3k| if (LIVEP(l)) ------------------ | | 34| 13.3k|#define LIVEP(solution) ((solution)->flags.hash_info & H_LIVE) | | ------------------ | | | Branch (34:25): [True: 11.2k, False: 2.04k] | | ------------------ ------------------ 314| 11.2k| hinsert0(ht, l->s, &l->flags, SLVNDX(l)); ------------------ | | 35| 11.2k|#define SLVNDX(solution) ((solution)->flags.slvndx) ------------------ 315| 13.3k| } 316| | 317| 1.27k| X(ifree0)(osol); ------------------ | | 78| 1.27k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1.27k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 318| 1.27k|} planner.c:nextsz: 326| 1.27k|{ 327| 1.27k| return minsz(minsz(nelem)); 328| 1.27k|} planner.c:hinsert0: 275| 14.3k|{ 276| 14.3k| solution *l; 277| 14.3k| unsigned g, h = h1(ht, s), d = h2(ht, s); 278| | 279| 14.3k| ++ht->insert_unknown; 280| | 281| | /* search for nonfull slot */ 282| 29.0k| for (g = h; ; g = addmod(g, d, ht->hashsiz)) { 283| 29.0k| ++ht->insert_iter; 284| 29.0k| l = ht->solutions + g; 285| 29.0k| if (!LIVEP(l)) break; ------------------ | | 34| 29.0k|#define LIVEP(solution) ((solution)->flags.hash_info & H_LIVE) ------------------ | Branch (285:8): [True: 14.3k, False: 14.6k] ------------------ 286| 14.6k| A((g + d) % ht->hashsiz != h); 287| 14.6k| } 288| | 289| 14.3k| fill_slot(ht, s, flagsp, slvndx, l); 290| 14.3k|} planner.c:invoke_hook: 399| 5.73k|{ 400| 5.73k| if (ego->hook) ------------------ | Branch (400:10): [True: 0, False: 5.73k] ------------------ 401| 0| ego->hook(ego, pln, p, optimalp); 402| 5.73k|} planner.c:forget: 770| 285|{ 771| 285| switch (a) { 772| 0| case FORGET_EVERYTHING: ------------------ | Branch (772:3): [True: 0, False: 285] ------------------ 773| 0| htab_destroy(&ego->htab_blessed); 774| 0| mkhashtab(&ego->htab_blessed); 775| | /* fall through */ 776| 285| case FORGET_ACCURSED: ------------------ | Branch (776:3): [True: 285, False: 0] ------------------ 777| 285| htab_destroy(&ego->htab_unblessed); 778| 285| mkhashtab(&ego->htab_unblessed); 779| 285| break; 780| 0| default: ------------------ | Branch (780:3): [True: 0, False: 285] ------------------ 781| 0| break; 782| 285| } 783| 285|} planner.c:mkhashtab: 757| 287|{ 758| 287| ht->nrehash = 0; 759| 287| ht->succ_lookup = ht->lookup = ht->lookup_iter = 0; 760| 287| ht->insert = ht->insert_iter = ht->insert_unknown = 0; 761| | 762| 287| ht->solutions = 0; 763| 287| ht->hashsiz = ht->nelem = 0U; 764| 287| hgrow(ht); /* so that hashsiz > 0 */ 765| 287|} planner.c:htab_destroy: 750| 285|{ 751| 285| X(ifree)(ht->solutions); ------------------ | | 78| 285|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 285|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 752| 285| ht->solutions = 0; 753| 285| ht->nelem = 0U; 754| 285|} fftw_power_mod: 65| 7.69k|{ 66| 7.69k| A(p > 0); 67| 7.69k| if (m == 0) ------------------ | Branch (67:10): [True: 2.27k, False: 5.42k] ------------------ 68| 2.27k| return 1; 69| 5.42k| else if (m % 2 == 0) { ------------------ | Branch (69:15): [True: 3.37k, False: 2.04k] ------------------ 70| 3.37k| INT x = X(power_mod)(n, m / 2, p); ------------------ | | 78| 3.37k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 3.37k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 71| 3.37k| return MULMOD(x, x, p); ------------------ | | 925| 3.37k| (((x) <= 92681 - (y)) ? ((x) * (y)) % (p) : X(safe_mulmod)(x, y, p)) | | ------------------ | | | | 78| 0|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ | | | Branch (925:5): [True: 3.37k, False: 0] | | ------------------ ------------------ 72| 3.37k| } 73| 2.04k| else 74| 2.04k| return MULMOD(n, X(power_mod)(n, m - 1, p), p); ------------------ | | 925| 2.04k| (((x) <= 92681 - (y)) ? ((x) * (y)) % (p) : X(safe_mulmod)(x, y, p)) | | ------------------ | | | | 78| 0|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ | | | Branch (925:5): [True: 2.04k, False: 0] | | ------------------ ------------------ 75| 7.69k|} fftw_find_generator: 106| 45|{ 107| 45| INT n, i, size; 108| 45| INT primef[16]; /* smallest number = 32589158477190044730 > 2^64 */ 109| 45| INT pm1 = p - 1; 110| | 111| 45| if (p == 2) ------------------ | Branch (111:9): [True: 0, False: 45] ------------------ 112| 0| return 1; 113| | 114| 45| size = get_prime_factors(pm1, primef); 115| 45| n = 2; 116| 221| for (i = 0; i < size; i++) ------------------ | Branch (116:17): [True: 176, False: 45] ------------------ 117| 176| if (X(power_mod)(n, pm1 / primef[i], p) == 1) { ------------------ | | 78| 176|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 176|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ | Branch (117:13): [True: 66, False: 110] ------------------ 118| 66| i = -1; 119| 66| n++; 120| 66| } 121| 45| return n; 122| 45|} fftw_first_divisor: 127| 4.35k|{ 128| 4.35k| INT i; 129| 4.35k| if (n <= 1) ------------------ | Branch (129:10): [True: 0, False: 4.35k] ------------------ 130| 0| return n; 131| 4.35k| if (n % 2 == 0) ------------------ | Branch (131:10): [True: 1.79k, False: 2.55k] ------------------ 132| 1.79k| return 2; 133| 5.97k| for (i = 3; i*i <= n; i += 2) ------------------ | Branch (133:18): [True: 4.29k, False: 1.67k] ------------------ 134| 4.29k| if (n % i == 0) ------------------ | Branch (134:8): [True: 880, False: 3.41k] ------------------ 135| 880| return i; 136| 1.67k| return n; 137| 2.55k|} fftw_is_prime: 140| 3.33k|{ 141| 3.33k| return(n > 1 && X(first_divisor)(n) == n); ------------------ | | 78| 3.33k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 3.33k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ | Branch (141:13): [True: 3.33k, False: 0] | Branch (141:22): [True: 1.73k, False: 1.59k] ------------------ 142| 3.33k|} fftw_next_prime: 145| 1.27k|{ 146| 2.56k| while (!X(is_prime)(n)) ++n; ------------------ | | 78| 2.56k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 2.56k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ | Branch (146:13): [True: 1.28k, False: 1.27k] ------------------ 147| 1.27k| return n; 148| 1.27k|} fftw_factors_into: 151| 933|{ 152| 3.73k| for (; *primes != 0; ++primes) ------------------ | Branch (152:13): [True: 2.79k, False: 933] ------------------ 153| 4.77k| while ((n % *primes) == 0) ------------------ | Branch (153:11): [True: 1.98k, False: 2.79k] ------------------ 154| 1.98k| n /= *primes; 155| 933| return (n == 1); 156| 933|} fftw_isqrt: 160| 4.83k|{ 161| 4.83k| INT guess, iguess; 162| | 163| 4.83k| A(n >= 0); 164| 4.83k| if (n == 0) return 0; ------------------ | Branch (164:10): [True: 0, False: 4.83k] ------------------ 165| | 166| 4.83k| guess = n; iguess = 1; 167| | 168| 15.4k| do { 169| 15.4k| guess = (guess + iguess) / 2; 170| 15.4k| iguess = n / guess; 171| 15.4k| } while (guess > iguess); ------------------ | Branch (171:15): [True: 10.6k, False: 4.83k] ------------------ 172| | 173| 4.83k| return guess; 174| 4.83k|} fftw_choose_radix: 184| 86.1k|{ 185| 86.1k| if (r > 0) { ------------------ | Branch (185:10): [True: 64.6k, False: 21.5k] ------------------ 186| 64.6k| if (divides(r, n)) return r; ------------------ | | 182| 64.6k|#define divides(a, b) (((b) % (a)) == 0) | | ------------------ | | | Branch (182:23): [True: 10.3k, False: 54.3k] | | ------------------ ------------------ 187| 54.3k| return 0; 188| 64.6k| } else if (r == 0) { ------------------ | Branch (188:17): [True: 1.02k, False: 20.5k] ------------------ 189| 1.02k| return X(first_divisor)(n); ------------------ | | 78| 1.02k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1.02k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 190| 20.5k| } else { 191| | /* r is negative. If n = (-r) * q^2, take q as the radix */ 192| 20.5k| r = 0 - r; 193| 20.5k| return (n > r && divides(r, n)) ? isqrt_maybe(n / r) : 0; ------------------ | | 182| 15.7k|#define divides(a, b) (((b) % (a)) == 0) | | ------------------ | | | Branch (182:23): [True: 4.72k, False: 11.0k] | | ------------------ ------------------ | Branch (193:12): [True: 15.7k, False: 4.80k] ------------------ 194| 20.5k| } 195| 86.1k|} fftw_modulo: 199| 204|{ 200| 204| A(n > 0); 201| 204| if (a >= 0) ------------------ | Branch (201:10): [True: 48, False: 156] ------------------ 202| 48| return a % n; 203| 156| else 204| 156| return (n - 1) - ((-(a + (INT)1)) % n); 205| 204|} fftw_factors_into_small_primes: 209| 933|{ 210| 933| static const INT primes[] = { 2, 3, 5, 0 }; 211| 933| return X(factors_into)(n, primes); ------------------ | | 78| 933|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 933|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 212| 933|} primes.c:get_prime_factors: 79| 45|{ 80| 45| INT i; 81| 45| INT size = 0; 82| | 83| 45| A(n % 2 == 0); /* this routine is designed only for even n */ 84| 45| primef[size++] = (INT)2; 85| 110| do { 86| 110| n >>= 1; 87| 110| } while ((n & 1) == 0); ------------------ | Branch (87:15): [True: 65, False: 45] ------------------ 88| | 89| 45| if (n == 1) ------------------ | Branch (89:10): [True: 0, False: 45] ------------------ 90| 0| return size; 91| | 92| 85| for (i = 3; i * i <= n; i += 2) ------------------ | Branch (92:18): [True: 40, False: 45] ------------------ 93| 40| if (!(n % i)) { ------------------ | Branch (93:8): [True: 36, False: 4] ------------------ 94| 36| primef[size++] = i; 95| 72| do { 96| 72| n /= i; 97| 72| } while (!(n % i)); ------------------ | Branch (97:18): [True: 36, False: 36] ------------------ 98| 36| } 99| 45| if (n == 1) ------------------ | Branch (99:10): [True: 26, False: 19] ------------------ 100| 26| return size; 101| 19| primef[size++] = n; 102| 19| return size; 103| 45|} primes.c:isqrt_maybe: 177| 4.72k|{ 178| 4.72k| INT guess = X(isqrt)(n); ------------------ | | 78| 4.72k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 4.72k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 179| 4.72k| return guess * guess == n ? guess : 0; ------------------ | Branch (179:13): [True: 995, False: 3.73k] ------------------ 180| 4.72k|} fftw_mkproblem: 26| 3.54k|{ 27| 3.54k| problem *p = (problem *)MALLOC(sz, PROBLEMS); ------------------ | | 306| 3.54k|#define MALLOC(n, what) X(malloc_plain)(n) | | ------------------ | | | | 78| 3.54k|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 3.54k|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 28| | 29| 3.54k| p->adt = adt; 30| 3.54k| return p; 31| 3.54k|} fftw_problem_destroy: 35| 3.54k|{ 36| 3.54k| if (ego) ------------------ | Branch (36:10): [True: 3.54k, False: 0] ------------------ 37| 3.54k| ego->adt->destroy(ego); 38| 3.54k|} fftw_rader_tl_insert: 37| 45|{ 38| 45| rader_tl *t = (rader_tl *) MALLOC(sizeof(rader_tl), TWIDDLES); ------------------ | | 306| 45|#define MALLOC(n, what) X(malloc_plain)(n) | | ------------------ | | | | 78| 45|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 45|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 39| 45| t->k1 = k1; t->k2 = k2; t->k3 = k3; t->W = W; 40| 45| t->refcnt = 1; t->cdr = *tl; *tl = t; 41| 45|} fftw_rader_tl_find: 44| 45|{ 45| 45| while (t && (t->k1 != k1 || t->k2 != k2 || t->k3 != k3)) ------------------ | Branch (45:13): [True: 0, False: 45] | Branch (45:19): [True: 0, False: 0] | Branch (45:34): [True: 0, False: 0] | Branch (45:49): [True: 0, False: 0] ------------------ 46| 0| t = t->cdr; 47| 45| if (t) { ------------------ | Branch (47:10): [True: 0, False: 45] ------------------ 48| 0| ++t->refcnt; 49| 0| return t->W; 50| 0| } else 51| 45| return 0; 52| 45|} fftw_rader_tl_delete: 55| 45|{ 56| 45| if (W) { ------------------ | Branch (56:10): [True: 45, False: 0] ------------------ 57| 45| rader_tl **tp, *t; 58| | 59| 45| for (tp = tl; (t = *tp) && t->W != W; tp = &t->cdr) ------------------ | Branch (59:18): [True: 45, False: 0] | Branch (59:31): [True: 0, False: 45] ------------------ 60| 0| ; 61| | 62| 45| if (t && --t->refcnt <= 0) { ------------------ | Branch (62:8): [True: 45, False: 0] | Branch (62:13): [True: 45, False: 0] ------------------ 63| 45| *tp = t->cdr; 64| 45| X(ifree)(t->W); ------------------ | | 78| 45|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 45|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 65| 45| X(ifree)(t); ------------------ | | 78| 45|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 45|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 66| 45| } 67| 45| } 68| 45|} fftw_mksolver: 25| 613|{ 26| 613| solver *s = (solver *)MALLOC(size, SOLVERS); ------------------ | | 306| 613|#define MALLOC(n, what) X(malloc_plain)(n) | | ------------------ | | | | 78| 613|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 613|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 27| | 28| 613| s->adt = adt; 29| 613| s->refcnt = 0; 30| 613| return s; 31| 613|} fftw_solver_use: 34| 613|{ 35| 613| ++ego->refcnt; 36| 613|} fftw_solver_register: 48| 613|{ 49| 613| plnr->adt->register_solver(plnr, s); 50| 613|} fftw_solvtab_exec: 25| 7|{ 26| 269| for (; tbl->reg_nam; ++tbl) { ------------------ | Branch (26:13): [True: 262, False: 7] ------------------ 27| 262| p->cur_reg_nam = tbl->reg_nam; 28| 262| p->cur_reg_id = 0; 29| 262| tbl->reg(p); 30| 262| } 31| 7| p->cur_reg_nam = 0; 32| 7|} fftw_mktensor: 25| 21.5k|{ 26| 21.5k| tensor *x; 27| | 28| 21.5k| A(rnk >= 0); 29| | 30| 21.5k|#if defined(STRUCT_HACK_KR) 31| 21.5k| if (FINITE_RNK(rnk) && rnk > 1) ------------------ | | 442| 43.1k|#define FINITE_RNK(rnk) ((rnk) != RNK_MINFTY) | | ------------------ | | | | 441| 21.5k|#define RNK_MINFTY INT_MAX | | ------------------ | | | Branch (442:25): [True: 21.3k, False: 219] | | ------------------ ------------------ | Branch (31:29): [True: 8.39k, False: 12.9k] ------------------ 32| 8.39k| x = (tensor *)MALLOC(sizeof(tensor) + (unsigned)(rnk - 1) * sizeof(iodim), ------------------ | | 306| 8.39k|#define MALLOC(n, what) X(malloc_plain)(n) | | ------------------ | | | | 78| 8.39k|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 8.39k|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 33| 21.5k| TENSORS); 34| 13.1k| else 35| 13.1k| x = (tensor *)MALLOC(sizeof(tensor), TENSORS); ------------------ | | 306| 13.1k|#define MALLOC(n, what) X(malloc_plain)(n) | | ------------------ | | | | 78| 13.1k|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 13.1k|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 36| |#elif defined(STRUCT_HACK_C99) 37| | if (FINITE_RNK(rnk)) 38| | x = (tensor *)MALLOC(sizeof(tensor) + (unsigned)rnk * sizeof(iodim), 39| | TENSORS); 40| | else 41| | x = (tensor *)MALLOC(sizeof(tensor), TENSORS); 42| |#else 43| | x = (tensor *)MALLOC(sizeof(tensor), TENSORS); 44| | if (FINITE_RNK(rnk) && rnk > 0) 45| | x->dims = (iodim *)MALLOC(sizeof(iodim) * (unsigned)rnk, TENSORS); 46| | else 47| | x->dims = 0; 48| |#endif 49| | 50| 21.5k| x->rnk = rnk; 51| 21.5k| return x; 52| 21.5k|} fftw_tensor_destroy: 55| 21.5k|{ 56| |#if !defined(STRUCT_HACK_C99) && !defined(STRUCT_HACK_KR) 57| | X(ifree0)(sz->dims); 58| |#endif 59| 21.5k| X(ifree)(sz); ------------------ | | 78| 21.5k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 21.5k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 60| 21.5k|} fftw_tensor_sz: 63| 6.28k|{ 64| 6.28k| int i; 65| 6.28k| INT n = 1; 66| | 67| 6.28k| if (!FINITE_RNK(sz->rnk)) ------------------ | | 442| 6.28k|#define FINITE_RNK(rnk) ((rnk) != RNK_MINFTY) | | ------------------ | | | | 441| 6.28k|#define RNK_MINFTY INT_MAX | | ------------------ ------------------ | Branch (67:10): [True: 0, False: 6.28k] ------------------ 68| 0| return 0; 69| | 70| 18.5k| for (i = 0; i < sz->rnk; ++i) ------------------ | Branch (70:18): [True: 12.3k, False: 6.28k] ------------------ 71| 12.3k| n *= sz->dims[i].n; 72| 6.28k| return n; 73| 6.28k|} fftw_tensor_md5: 76| 7.65k|{ 77| 7.65k| int i; 78| 7.65k| X(md5int)(p, t->rnk); ------------------ | | 78| 7.65k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 7.65k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 79| 7.65k| if (FINITE_RNK(t->rnk)) { ------------------ | | 442| 7.65k|#define FINITE_RNK(rnk) ((rnk) != RNK_MINFTY) | | ------------------ | | | | 441| 7.65k|#define RNK_MINFTY INT_MAX | | ------------------ | | | Branch (442:25): [True: 7.44k, False: 209] | | ------------------ ------------------ 80| 13.8k| for (i = 0; i < t->rnk; ++i) { ------------------ | Branch (80:16): [True: 6.35k, False: 7.44k] ------------------ 81| 6.35k| const iodim *q = t->dims + i; 82| 6.35k| X(md5INT)(p, q->n); ------------------ | | 78| 6.35k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 6.35k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 83| 6.35k| X(md5INT)(p, q->is); ------------------ | | 78| 6.35k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 6.35k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 84| 6.35k| X(md5INT)(p, q->os); ------------------ | | 78| 6.35k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 6.35k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 85| 6.35k| } 86| 7.44k| } 87| 7.65k|} fftw_tensor_tornk1: 92| 8.13k|{ 93| 8.13k| A(t->rnk <= 1); 94| 8.13k| if (t->rnk == 1) { ------------------ | Branch (94:10): [True: 4.82k, False: 3.30k] ------------------ 95| 4.82k| const iodim *vd = t->dims; 96| 4.82k| *n = vd[0].n; 97| 4.82k| *is = vd[0].is; 98| 4.82k| *os = vd[0].os; 99| 4.82k| } else { 100| 3.30k| *n = 1; 101| 3.30k| *is = *os = 0; 102| 3.30k| } 103| 8.13k| return 1; 104| 8.13k|} fftw_mktensor_0d: 25| 389|{ 26| 389| return X(mktensor(0)); ------------------ | | 78| 389|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 389|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 27| 389|} fftw_mktensor_1d: 30| 3.33k|{ 31| 3.33k| tensor *x = X(mktensor)(1); ------------------ | | 78| 3.33k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 3.33k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 32| 3.33k| x->dims[0].n = n; 33| 3.33k| x->dims[0].is = is; 34| 3.33k| x->dims[0].os = os; 35| 3.33k| return x; 36| 3.33k|} fftw_mktensor_2d: 26| 1.42k|{ 27| 1.42k| tensor *x = X(mktensor)(2); ------------------ | | 78| 1.42k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1.42k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 28| 1.42k| x->dims[0].n = n0; 29| 1.42k| x->dims[0].is = is0; 30| 1.42k| x->dims[0].os = os0; 31| 1.42k| x->dims[1].n = n1; 32| 1.42k| x->dims[1].is = is1; 33| 1.42k| x->dims[1].os = os1; 34| 1.42k| return x; 35| 1.42k|} fftw_tensor_min_istride: 38| 484|#define tensor_min_xstride(sz, xs) { \ 39| 484| A(FINITE_RNK(sz->rnk)); \ 40| 484| if (sz->rnk == 0) return 0; \ ------------------ | Branch (40:10): [True: 0, False: 484] ------------------ 41| 484| else { \ 42| 484| int i; \ 43| 484| INT s = X(iabs)(sz->dims[0].xs); \ ------------------ | | 78| 484|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 484|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 44| 484| for (i = 1; i < sz->rnk; ++i) \ ------------------ | Branch (44:23): [True: 0, False: 484] ------------------ 45| 484| s = X(imin)(s, X(iabs)(sz->dims[i].xs)); \ ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ s = X(imin)(s, X(iabs)(sz->dims[i].xs)); \ ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 46| 484| return s; \ 47| 484| } \ 48| 484|} fftw_tensor_min_ostride: 38| 420|#define tensor_min_xstride(sz, xs) { \ 39| 420| A(FINITE_RNK(sz->rnk)); \ 40| 420| if (sz->rnk == 0) return 0; \ ------------------ | Branch (40:10): [True: 0, False: 420] ------------------ 41| 420| else { \ 42| 420| int i; \ 43| 420| INT s = X(iabs)(sz->dims[0].xs); \ ------------------ | | 78| 420|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 420|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 44| 420| for (i = 1; i < sz->rnk; ++i) \ ------------------ | Branch (44:23): [True: 0, False: 420] ------------------ 45| 420| s = X(imin)(s, X(iabs)(sz->dims[i].xs)); \ ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ s = X(imin)(s, X(iabs)(sz->dims[i].xs)); \ ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 46| 420| return s; \ 47| 420| } \ 48| 420|} fftw_tensor_inplace_strides: 59| 3.06k|{ 60| 3.06k| int i; 61| 3.06k| A(FINITE_RNK(sz->rnk)); 62| 3.67k| for (i = 0; i < sz->rnk; ++i) { ------------------ | Branch (62:18): [True: 2.89k, False: 781] ------------------ 63| 2.89k| const iodim *p = sz->dims + i; 64| 2.89k| if (p->is != p->os) ------------------ | Branch (64:15): [True: 2.28k, False: 616] ------------------ 65| 2.28k| return 0; 66| 2.89k| } 67| 781| return 1; 68| 3.06k|} fftw_tensor_inplace_strides2: 71| 2.08k|{ 72| 2.08k| return X(tensor_inplace_strides(a)) && X(tensor_inplace_strides(b)); ------------------ | | 78| 2.08k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 2.08k|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | | | | Branch (63:30): [True: 459, False: 1.62k] | | | | ------------------ | | ------------------ ------------------ return X(tensor_inplace_strides(a)) && X(tensor_inplace_strides(b)); ------------------ | | 78| 459|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 459|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | | | | Branch (63:30): [True: 322, False: 137] | | | | ------------------ | | ------------------ ------------------ 73| 2.08k|} fftw_tensor_strides_decrease: 100| 346|{ 101| 346| return(tensor_strides_decrease(sz, k) ------------------ | Branch (101:13): [True: 173, False: 173] ------------------ 102| 173| || (X(tensor_inplace_strides)(sz) ------------------ | | 78| 173|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 173|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ | Branch (102:10): [True: 0, False: 173] ------------------ 103| 0| && tensor_strides_decrease(vecsz, k))); ------------------ | Branch (103:6): [True: 0, False: 0] ------------------ 104| 346|} tensor4.c:tensor_strides_decrease: 78| 346|{ 79| 346| if (FINITE_RNK(sz->rnk)) { ------------------ | | 442| 346|#define FINITE_RNK(rnk) ((rnk) != RNK_MINFTY) | | ------------------ | | | | 441| 346|#define RNK_MINFTY INT_MAX | | ------------------ | | | Branch (442:25): [True: 346, False: 0] | | ------------------ ------------------ 80| 346| int i; 81| 519| for (i = 0; i < sz->rnk; ++i) ------------------ | Branch (81:23): [True: 346, False: 173] ------------------ 82| 346| if ((sz->dims[i].os - sz->dims[i].is) ------------------ | Branch (82:20): [True: 173, False: 173] ------------------ 83| 346| * (k == INPLACE_OS ? (INT)1 : (INT)-1) < 0) ------------------ | Branch (83:23): [True: 173, False: 173] ------------------ 84| 173| return 1; 85| 346| } 86| 173| return 0; 87| 346|} fftw_tensor_copy: 33| 2.70k|{ 34| 2.70k| tensor *x = X(mktensor)(sz->rnk); ------------------ | | 78| 2.70k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 2.70k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 35| 2.70k| dimcpy(x->dims, sz->dims, sz->rnk); 36| 2.70k| return x; 37| 2.70k|} fftw_tensor_copy_inplace: 42| 2.09k|{ 43| 2.09k| tensor *x = X(tensor_copy)(sz); ------------------ | | 78| 2.09k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 2.09k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 44| 2.09k| if (FINITE_RNK(x->rnk)) { ------------------ | | 442| 2.09k|#define FINITE_RNK(rnk) ((rnk) != RNK_MINFTY) | | ------------------ | | | | 441| 2.09k|#define RNK_MINFTY INT_MAX | | ------------------ | | | Branch (442:25): [True: 2.09k, False: 0] | | ------------------ ------------------ 45| 2.09k| int i; 46| 2.09k| if (k == INPLACE_OS) ------------------ | Branch (46:8): [True: 1.02k, False: 1.06k] ------------------ 47| 3.63k| for (i = 0; i < x->rnk; ++i) ------------------ | Branch (47:21): [True: 2.61k, False: 1.02k] ------------------ 48| 2.61k| x->dims[i].is = x->dims[i].os; 49| 1.06k| else 50| 3.74k| for (i = 0; i < x->rnk; ++i) ------------------ | Branch (50:21): [True: 2.67k, False: 1.06k] ------------------ 51| 2.67k| x->dims[i].os = x->dims[i].is; 52| 2.09k| } 53| 2.09k| return x; 54| 2.09k|} fftw_tensor_copy_except: 59| 361|{ 60| 361| tensor *x; 61| | 62| 361| A(FINITE_RNK(sz->rnk) && sz->rnk >= 1 && except_dim < sz->rnk); 63| 361| x = X(mktensor)(sz->rnk - 1); ------------------ | | 78| 361|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 361|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 64| 361| dimcpy(x->dims, sz->dims, except_dim); 65| 361| dimcpy(x->dims + except_dim, sz->dims + except_dim + 1, 66| 361| x->rnk - except_dim); 67| 361| return x; 68| 361|} fftw_tensor_append: 83| 1.59k|{ 84| 1.59k| if (!FINITE_RNK(a->rnk) || !FINITE_RNK(b->rnk)) { ------------------ | | 442| 3.19k|#define FINITE_RNK(rnk) ((rnk) != RNK_MINFTY) | | ------------------ | | | | 441| 1.59k|#define RNK_MINFTY INT_MAX | | ------------------ ------------------ if (!FINITE_RNK(a->rnk) || !FINITE_RNK(b->rnk)) { ------------------ | | 442| 1.59k|#define FINITE_RNK(rnk) ((rnk) != RNK_MINFTY) | | ------------------ | | | | 441| 1.59k|#define RNK_MINFTY INT_MAX | | ------------------ ------------------ | Branch (84:10): [True: 0, False: 1.59k] | Branch (84:33): [True: 0, False: 1.59k] ------------------ 85| 0| return X(mktensor)(RNK_MINFTY); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ return X(mktensor)(RNK_MINFTY); ------------------ | | 441| 0|#define RNK_MINFTY INT_MAX ------------------ 86| 1.59k| } else { 87| 1.59k| tensor *x = X(mktensor)(a->rnk + b->rnk); ------------------ | | 78| 1.59k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1.59k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 88| 1.59k| dimcpy(x->dims, a->dims, a->rnk); 89| 1.59k| dimcpy(x->dims + a->rnk, b->dims, b->rnk); 90| 1.59k| return x; 91| 1.59k| } 92| 1.59k|} tensor5.c:dimcpy: 25| 6.62k|{ 26| 6.62k| int i; 27| 6.62k| if (FINITE_RNK(rnk)) ------------------ | | 442| 6.62k|#define FINITE_RNK(rnk) ((rnk) != RNK_MINFTY) | | ------------------ | | | | 441| 6.62k|#define RNK_MINFTY INT_MAX | | ------------------ | | | Branch (442:25): [True: 6.62k, False: 0] | | ------------------ ------------------ 28| 16.7k| for (i = 0; i < rnk; ++i) ------------------ | Branch (28:23): [True: 10.1k, False: 6.62k] ------------------ 29| 10.1k| dst[i] = src[i]; 30| 6.62k|} fftw_dimcmp: 33| 1.22k|{ 34| 1.22k| INT sai = X(iabs)(a->is), sbi = X(iabs)(b->is); ------------------ | | 78| 1.22k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1.22k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ INT sai = X(iabs)(a->is), sbi = X(iabs)(b->is); ------------------ | | 78| 1.22k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1.22k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 35| 1.22k| INT sao = X(iabs)(a->os), sbo = X(iabs)(b->os); ------------------ | | 78| 1.22k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1.22k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ INT sao = X(iabs)(a->os), sbo = X(iabs)(b->os); ------------------ | | 78| 1.22k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1.22k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 36| 1.22k| INT sam = X(imin)(sai, sao), sbm = X(imin)(sbi, sbo); ------------------ | | 78| 1.22k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1.22k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ INT sam = X(imin)(sai, sao), sbm = X(imin)(sbi, sbo); ------------------ | | 78| 1.22k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1.22k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 37| | 38| | /* in descending order of min{istride, ostride} */ 39| 1.22k| if (sam != sbm) ------------------ | Branch (39:10): [True: 828, False: 394] ------------------ 40| 828| return signof(sbm - sam); 41| | 42| | /* in case of a tie, in descending order of istride */ 43| 394| if (sbi != sai) ------------------ | Branch (43:10): [True: 394, False: 0] ------------------ 44| 394| return signof(sbi - sai); 45| | 46| | /* in case of a tie, in descending order of ostride */ 47| 0| if (sbo != sao) ------------------ | Branch (47:10): [True: 0, False: 0] ------------------ 48| 0| return signof(sbo - sao); 49| | 50| | /* in case of a tie, in ascending order of n */ 51| 0| return signof(a->n - b->n); 52| 0|} fftw_tensor_compress: 100| 3.12k|{ 101| 3.12k| tensor *x = really_compress(sz); 102| 3.12k| canonicalize(x); 103| 3.12k| return x; 104| 3.12k|} fftw_tensor_compress_contiguous: 117| 5.53k|{ 118| 5.53k| int i, rnk; 119| 5.53k| tensor *sz2, *x; 120| | 121| 5.53k| if (X(tensor_sz)(sz) == 0) ------------------ | | 78| 5.53k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 5.53k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ | Branch (121:10): [True: 219, False: 5.31k] ------------------ 122| 219| return X(mktensor)(RNK_MINFTY); ------------------ | | 78| 219|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 219|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ return X(mktensor)(RNK_MINFTY); ------------------ | | 441| 219|#define RNK_MINFTY INT_MAX ------------------ 123| | 124| 5.31k| sz2 = really_compress(sz); 125| 5.31k| A(FINITE_RNK(sz2->rnk)); 126| | 127| 5.31k| if (sz2->rnk <= 1) { /* nothing to compress. */ ------------------ | Branch (127:10): [True: 2.91k, False: 2.40k] ------------------ 128| 2.91k| if (0) { ------------------ | Branch (128:8): [Folded, False: 2.91k] ------------------ 129| | /* this call is redundant, because "sz->rnk <= 1" implies 130| | that the tensor is already canonical, but I am writing 131| | it explicitly because "logically" we need to canonicalize 132| | the tensor before returning. */ 133| 0| canonicalize(sz2); 134| 0| } 135| 2.91k| return sz2; 136| 2.91k| } 137| | 138| | /* sort in descending order of |istride|, so that compressible 139| | dimensions appear contigously */ 140| 2.40k| qsort(sz2->dims, (unsigned)sz2->rnk, sizeof(iodim), 141| 2.40k| (int (*)(const void *, const void *))compare_by_istride); 142| | 143| | /* compute what the rank will be after compression */ 144| 5.80k| for (i = rnk = 1; i < sz2->rnk; ++i) ------------------ | Branch (144:24): [True: 3.39k, False: 2.40k] ------------------ 145| 3.39k| if (!strides_contig(sz2->dims + i - 1, sz2->dims + i)) ------------------ | Branch (145:15): [True: 1.19k, False: 2.19k] ------------------ 146| 1.19k| ++rnk; 147| | 148| | /* merge adjacent dimensions whenever possible */ 149| 2.40k| x = X(mktensor)(rnk); ------------------ | | 78| 2.40k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 2.40k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 150| 2.40k| x->dims[0] = sz2->dims[0]; 151| 5.80k| for (i = rnk = 1; i < sz2->rnk; ++i) { ------------------ | Branch (151:24): [True: 3.39k, False: 2.40k] ------------------ 152| 3.39k| if (strides_contig(sz2->dims + i - 1, sz2->dims + i)) { ------------------ | Branch (152:15): [True: 2.19k, False: 1.19k] ------------------ 153| 2.19k| x->dims[rnk - 1].n *= sz2->dims[i].n; 154| 2.19k| x->dims[rnk - 1].is = sz2->dims[i].is; 155| 2.19k| x->dims[rnk - 1].os = sz2->dims[i].os; 156| 2.19k| } else { 157| 1.19k| A(rnk < x->rnk); 158| 1.19k| x->dims[rnk++] = sz2->dims[i]; 159| 1.19k| } 160| 3.39k| } 161| | 162| 2.40k| X(tensor_destroy)(sz2); ------------------ | | 78| 2.40k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 2.40k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 163| | 164| | /* reduce to canonical form */ 165| 2.40k| canonicalize(x); 166| 2.40k| return x; 167| 5.31k|} fftw_tensor_equal: 181| 997|{ 182| 997| if (a->rnk != b->rnk) ------------------ | Branch (182:10): [True: 0, False: 997] ------------------ 183| 0| return 0; 184| | 185| 997| if (FINITE_RNK(a->rnk)) { ------------------ | | 442| 997|#define FINITE_RNK(rnk) ((rnk) != RNK_MINFTY) | | ------------------ | | | | 441| 997|#define RNK_MINFTY INT_MAX | | ------------------ | | | Branch (442:25): [True: 992, False: 5] | | ------------------ ------------------ 186| 992| int i; 187| 2.06k| for (i = 0; i < a->rnk; ++i) ------------------ | Branch (187:16): [True: 1.07k, False: 992] ------------------ 188| 1.07k| if (0 ------------------ | Branch (188:13): [Folded, False: 1.07k] ------------------ 189| 1.07k| || a->dims[i].n != b->dims[i].n ------------------ | Branch (189:9): [True: 0, False: 1.07k] ------------------ 190| 1.07k| || a->dims[i].is != b->dims[i].is ------------------ | Branch (190:9): [True: 0, False: 1.07k] ------------------ 191| 1.07k| || a->dims[i].os != b->dims[i].os ------------------ | Branch (191:9): [True: 0, False: 1.07k] ------------------ 192| 1.07k| ) 193| 0| return 0; 194| 992| } 195| | 196| 997| return 1; 197| 997|} fftw_tensor_inplace_locations: 202| 997|{ 203| 997| tensor *t = X(tensor_append)(sz, vecsz); ------------------ | | 78| 997|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 997|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 204| 997| tensor *ti = X(tensor_copy_inplace)(t, INPLACE_IS); ------------------ | | 78| 997|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 997|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 205| 997| tensor *to = X(tensor_copy_inplace)(t, INPLACE_OS); ------------------ | | 78| 997|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 997|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 206| 997| tensor *tic = X(tensor_compress_contiguous)(ti); ------------------ | | 78| 997|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 997|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 207| 997| tensor *toc = X(tensor_compress_contiguous)(to); ------------------ | | 78| 997|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 997|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 208| | 209| 997| int retval = X(tensor_equal)(tic, toc); ------------------ | | 78| 997|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 997|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 210| | 211| 997| X(tensor_destroy)(t); ------------------ | | 78| 997|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 997|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 212| 997| X(tensor_destroy4)(ti, to, tic, toc); ------------------ | | 78| 997|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 997|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 213| | 214| 997| return retval; 215| 997|} tensor7.c:signof: 25| 5.53k|{ 26| 5.53k| if (x < 0) return -1; ------------------ | Branch (26:10): [True: 2.88k, False: 2.65k] ------------------ 27| 2.65k| if (x == 0) return 0; ------------------ | Branch (27:10): [True: 0, False: 2.65k] ------------------ 28| 2.65k| /* if (x > 0) */ return 1; 29| 2.65k|} tensor7.c:really_compress: 71| 8.44k|{ 72| 8.44k| int i, rnk; 73| 8.44k| tensor *x; 74| | 75| 8.44k| A(FINITE_RNK(sz->rnk)); 76| 21.7k| for (i = rnk = 0; i < sz->rnk; ++i) { ------------------ | Branch (76:24): [True: 13.3k, False: 8.44k] ------------------ 77| 13.3k| A(sz->dims[i].n > 0); 78| 13.3k| if (sz->dims[i].n != 1) ------------------ | Branch (78:15): [True: 10.4k, False: 2.90k] ------------------ 79| 10.4k| ++rnk; 80| 13.3k| } 81| | 82| 8.44k| x = X(mktensor)(rnk); ------------------ | | 78| 8.44k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 8.44k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 83| 21.7k| for (i = rnk = 0; i < sz->rnk; ++i) { ------------------ | Branch (83:24): [True: 13.3k, False: 8.44k] ------------------ 84| 13.3k| if (sz->dims[i].n != 1) ------------------ | Branch (84:15): [True: 10.4k, False: 2.90k] ------------------ 85| 10.4k| x->dims[rnk++] = sz->dims[i]; 86| 13.3k| } 87| 8.44k| return x; 88| 8.44k|} tensor7.c:canonicalize: 55| 5.53k|{ 56| 5.53k| if (x->rnk > 1) { ------------------ | Branch (56:10): [True: 953, False: 4.57k] ------------------ 57| 953| qsort(x->dims, (unsigned)x->rnk, sizeof(iodim), 58| 953| (int (*)(const void *, const void *))X(dimcmp)); ------------------ | | 78| 953|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 953|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 59| 953| } 60| 5.53k|} tensor7.c:compare_by_istride: 63| 4.31k|{ 64| 4.31k| INT sai = X(iabs)(a->is), sbi = X(iabs)(b->is); ------------------ | | 78| 4.31k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 4.31k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ INT sai = X(iabs)(a->is), sbi = X(iabs)(b->is); ------------------ | | 78| 4.31k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 4.31k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 65| | 66| | /* in descending order of istride */ 67| 4.31k| return signof(sbi - sai); 68| 4.31k|} tensor7.c:strides_contig: 109| 6.79k|{ 110| 6.79k| return (a->is == b->is * b->n && a->os == b->os * b->n); ------------------ | Branch (110:14): [True: 5.56k, False: 1.22k] | Branch (110:39): [True: 4.39k, False: 1.17k] ------------------ 111| 6.79k|} fftw_tensor_destroy2: 25| 9.09k|{ 26| 9.09k| X(tensor_destroy)(a); ------------------ | | 78| 9.09k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 9.09k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 27| 9.09k| X(tensor_destroy)(b); ------------------ | | 78| 9.09k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 9.09k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 28| 9.09k|} fftw_tensor_destroy4: 31| 997|{ 32| 997| X(tensor_destroy2)(a, b); ------------------ | | 78| 997|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 997|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 33| 997| X(tensor_destroy2)(c, d); ------------------ | | 78| 997|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 997|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 34| 997|} fftw_compute_tilesz: 50| 114|{ 51| 114| return X(isqrt)(CACHESIZE / ------------------ | | 78| 114|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 114|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ return X(isqrt)(CACHESIZE / ------------------ | | 959| 114|#define CACHESIZE 8192 ------------------ 52| 114| (((INT)sizeof(R)) * vl * (INT)how_many_tiles_in_cache)); 53| 114|} fftw_get_crude_time: 64| 285|{ 65| 285| crude_time tv; 66| 285| gettimeofday(&tv, 0); 67| 285| return tv; 68| 285|} fftw_mktriggen: 170| 415|{ 171| 415| INT i, n0, n1; 172| 415| triggen *p = (triggen *)MALLOC(sizeof(*p), TWIDDLES); ------------------ | | 306| 415|#define MALLOC(n, what) X(malloc_plain)(n) | | ------------------ | | | | 78| 415|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 415|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 173| | 174| 415| p->n = n; 175| 415| p->W0 = p->W1 = 0; 176| 415| p->cexp = 0; 177| 415| p->rotate = 0; 178| | 179| 415| switch (wakefulness) { ------------------ | Branch (179:14): [True: 415, False: 0] ------------------ 180| 0| case SLEEPY: ------------------ | Branch (180:3): [True: 0, False: 415] ------------------ 181| 0| A(0 /* can't happen */); 182| 0| break; 183| | 184| 0| case AWAKE_SQRTN_TABLE: { ------------------ | Branch (184:3): [True: 0, False: 415] ------------------ 185| 0| INT twshft = choose_twshft(n); 186| | 187| 0| p->twshft = twshft; 188| 0| p->twradix = ((INT)1) << twshft; 189| 0| p->twmsk = p->twradix - 1; 190| | 191| 0| n0 = p->twradix; 192| 0| n1 = (n + n0 - 1) / n0; 193| | 194| 0| p->W0 = (trigreal *)MALLOC(n0 * 2 * sizeof(trigreal), TWIDDLES); ------------------ | | 306| 0|#define MALLOC(n, what) X(malloc_plain)(n) | | ------------------ | | | | 78| 0|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 195| 0| p->W1 = (trigreal *)MALLOC(n1 * 2 * sizeof(trigreal), TWIDDLES); ------------------ | | 306| 0|#define MALLOC(n, what) X(malloc_plain)(n) | | ------------------ | | | | 78| 0|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 196| | 197| 0| for (i = 0; i < n0; ++i) ------------------ | Branch (197:20): [True: 0, False: 0] ------------------ 198| 0| real_cexp(i, n, p->W0 + 2 * i); 199| | 200| 0| for (i = 0; i < n1; ++i) ------------------ | Branch (200:20): [True: 0, False: 0] ------------------ 201| 0| real_cexp(i * p->twradix, n, p->W1 + 2 * i); 202| | 203| 0| p->cexpl = cexpl_sqrtn_table; 204| 0| p->rotate = rotate_sqrtn_table; 205| 0| break; 206| 0| } 207| | 208| 415| case AWAKE_SINCOS: ------------------ | Branch (208:3): [True: 415, False: 0] ------------------ 209| 415| p->cexpl = cexpl_sincos; 210| 415| break; 211| | 212| 0| case AWAKE_ZERO: ------------------ | Branch (212:3): [True: 0, False: 415] ------------------ 213| 0| p->cexp = cexp_zero; 214| 0| p->cexpl = cexpl_zero; 215| 0| break; 216| 415| } 217| | 218| 415| if (!p->cexp) { ------------------ | Branch (218:10): [True: 415, False: 0] ------------------ 219| 415| if (sizeof(trigreal) == sizeof(R)) ------------------ | Branch (219:8): [True: 415, Folded] ------------------ 220| 415| p->cexp = (void (*)(triggen *, INT, R *))p->cexpl; 221| 0| else 222| 0| p->cexp = cexp_generic; 223| 415| } 224| 415| if (!p->rotate) ------------------ | Branch (224:10): [True: 415, False: 0] ------------------ 225| 415| p->rotate = rotate_generic; 226| 415| return p; 227| 415|} fftw_triggen_destroy: 230| 415|{ 231| 415| X(ifree0)(p->W0); ------------------ | | 78| 415|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 415|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 232| 415| X(ifree0)(p->W1); ------------------ | | 78| 415|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 415|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 233| 415| X(ifree)(p); ------------------ | | 78| 415|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 415|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 234| 415|} trig.c:real_cexp: 58| 54.3k|{ 59| 54.3k| trigreal theta, c, s, t; 60| 54.3k| unsigned octant = 0; 61| 54.3k| INT quarter_n = n; 62| | 63| 54.3k| n += n; n += n; 64| 54.3k| m += m; m += m; 65| | 66| 54.3k| if (m < 0) m += n; ------------------ | Branch (66:10): [True: 0, False: 54.3k] ------------------ 67| 54.3k| if (m > n - m) { m = n - m; octant |= 4; } ------------------ | Branch (67:10): [True: 19.2k, False: 35.0k] ------------------ 68| 54.3k| if (m - quarter_n > 0) { m = m - quarter_n; octant |= 2; } ------------------ | Branch (68:10): [True: 25.0k, False: 29.2k] ------------------ 69| 54.3k| if (m > quarter_n - m) { m = quarter_n - m; octant |= 1; } ------------------ | Branch (69:10): [True: 25.8k, False: 28.5k] ------------------ 70| | 71| 54.3k| theta = by2pi(m, n); ------------------ | | 50| 54.3k|#define by2pi(m, n) ((K2PI * (m)) / (n)) ------------------ 72| 54.3k| c = COS(theta); s = SIN(theta); ------------------ | | 43| 54.3k|# define COS cos ------------------ c = COS(theta); s = SIN(theta); ------------------ | | 44| 54.3k|# define SIN sin ------------------ 73| | 74| 54.3k| if (octant & 1) { t = c; c = s; s = t; } ------------------ | Branch (74:10): [True: 25.8k, False: 28.5k] ------------------ 75| 54.3k| if (octant & 2) { t = c; c = -s; s = t; } ------------------ | Branch (75:10): [True: 25.0k, False: 29.2k] ------------------ 76| 54.3k| if (octant & 4) { s = -s; } ------------------ | Branch (76:10): [True: 19.2k, False: 35.0k] ------------------ 77| | 78| 54.3k| out[0] = c; 79| 54.3k| out[1] = s; 80| 54.3k|} trig.c:cexpl_sincos: 135| 54.3k|{ 136| 54.3k| real_cexp(m, p->n, res); 137| 54.3k|} fftw_twiddle_awake: 247| 884|{ 248| 884| switch (wakefulness) { 249| 442| case SLEEPY: ------------------ | Branch (249:3): [True: 442, False: 442] ------------------ 250| 442| twiddle_destroy(pp); 251| 442| break; 252| 442| default: ------------------ | Branch (252:3): [True: 442, False: 442] ------------------ 253| 442| mktwiddle(wakefulness, pp, instr, n, r, m); 254| 442| break; 255| 884| } 256| 884|} twiddle.c:twlen0: 91| 346|{ 92| 346| INT ntwiddle = 0; 93| | 94| | /* compute length of bytecode program */ 95| 346| A(r > 0); 96| 887| for ( ; p->op != TW_NEXT; ++p) { ------------------ | Branch (96:14): [True: 541, False: 346] ------------------ 97| 541| switch (p->op) { ------------------ | Branch (97:12): [True: 541, False: 0] ------------------ 98| 170| case TW_FULL: ------------------ | Branch (98:8): [True: 170, False: 371] ------------------ 99| 170| ntwiddle += (r - 1) * 2; 100| 170| break; 101| 68| case TW_HALF: ------------------ | Branch (101:8): [True: 68, False: 473] ------------------ 102| 68| ntwiddle += (r - 1); 103| 68| break; 104| 303| case TW_CEXP: ------------------ | Branch (104:8): [True: 303, False: 238] ------------------ 105| 303| ntwiddle += 2; 106| 303| break; 107| 0| case TW_COS: ------------------ | Branch (107:8): [True: 0, False: 541] ------------------ 108| 0| case TW_SIN: ------------------ | Branch (108:8): [True: 0, False: 541] ------------------ 109| 0| ntwiddle += 1; 110| 0| break; 111| 541| } 112| 541| } 113| | 114| 346| *vl = (INT)p->v; 115| 346| return ntwiddle; 116| 346|} twiddle.c:twiddle_destroy: 225| 442|{ 226| 442| twid *p = *pp; 227| 442| twid **q; 228| | 229| 442| if ((--p->refcnt) == 0) { ------------------ | Branch (229:10): [True: 346, False: 96] ------------------ 230| | /* remove p from twiddle list */ 231| 346| for (q = &twlist[hash(p->n, p->r)]; *q; q = &((*q)->cdr)) { ------------------ | Branch (231:40): [True: 346, False: 0] ------------------ 232| 346| if (*q == p) { ------------------ | Branch (232:13): [True: 346, False: 0] ------------------ 233| 346| *q = p->cdr; 234| 346| X(ifree)(p->W); ------------------ | | 78| 346|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 346|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 235| 346| X(ifree)(p); ------------------ | | 78| 346|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 346|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 236| 346| *pp = 0; 237| 346| return; 238| 346| } 239| 346| } 240| 0| A(0 /* can't happen */ ); 241| 0| } 242| 442|} twiddle.c:hash: 33| 1.13k|{ 34| 1.13k| INT h = n * 17 + r; 35| | 36| 1.13k| if (h < 0) h = -h; ------------------ | Branch (36:10): [True: 0, False: 1.13k] ------------------ 37| | 38| 1.13k| return (h % HASHSZ); ------------------ | | 27| 1.13k|#define HASHSZ 109 ------------------ 39| 1.13k|} twiddle.c:mktwiddle: 199| 442|{ 200| 442| twid *p; 201| 442| INT h; 202| | 203| 442| if ((p = lookup(wakefulness, instr, n, r, m))) { ------------------ | Branch (203:10): [True: 96, False: 346] ------------------ 204| 96| ++p->refcnt; 205| 346| } else { 206| 346| p = (twid *) MALLOC(sizeof(twid), TWIDDLES); ------------------ | | 306| 346|#define MALLOC(n, what) X(malloc_plain)(n) | | ------------------ | | | | 78| 346|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 346|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 207| 346| p->n = n; 208| 346| p->r = r; 209| 346| p->m = m; 210| 346| p->instr = instr; 211| 346| p->refcnt = 1; 212| 346| p->wakefulness = wakefulness; 213| 346| p->W = compute(wakefulness, instr, n, r, m); 214| | 215| | /* cons! onto twlist */ 216| 346| h = hash(n, r); 217| 346| p->cdr = twlist[h]; 218| 346| twlist[h] = p; 219| 346| } 220| | 221| 442| *pp = p; 222| 442|} twiddle.c:lookup: 80| 442|{ 81| 442| twid *p; 82| | 83| 442| for (p = twlist[hash(n,r)]; 84| 442| p && !ok_twid(p, wakefulness, q, n, r, m); ------------------ | Branch (84:4): [True: 96, False: 346] | Branch (84:9): [True: 0, False: 96] ------------------ 85| 442| p = p->cdr) 86| 0| ; 87| 442| return p; 88| 442|} twiddle.c:ok_twid: 70| 96|{ 71| 96| return (wakefulness == t->wakefulness && ------------------ | Branch (71:14): [True: 96, False: 0] ------------------ 72| 96| n == t->n && ------------------ | Branch (72:7): [True: 96, False: 0] ------------------ 73| 96| r == t->r && ------------------ | Branch (73:7): [True: 96, False: 0] ------------------ 74| 96| m <= t->m && ------------------ | Branch (74:7): [True: 96, False: 0] ------------------ 75| 96| equal_instr(t->instr, q)); ------------------ | Branch (75:7): [True: 96, False: 0] ------------------ 76| 96|} twiddle.c:equal_instr: 42| 96|{ 43| 96| if (p == q) ------------------ | Branch (43:10): [True: 96, False: 0] ------------------ 44| 96| return 1; 45| | 46| 0| for (;; ++p, ++q) { 47| 0| if (p->op != q->op) ------------------ | Branch (47:15): [True: 0, False: 0] ------------------ 48| 0| return 0; 49| | 50| 0| switch (p->op) { 51| 0| case TW_NEXT: ------------------ | Branch (51:8): [True: 0, False: 0] ------------------ 52| 0| return (p->v == q->v); /* p->i is ignored */ 53| | 54| 0| case TW_FULL: ------------------ | Branch (54:8): [True: 0, False: 0] ------------------ 55| 0| case TW_HALF: ------------------ | Branch (55:8): [True: 0, False: 0] ------------------ 56| 0| if (p->v != q->v) return 0; /* p->i is ignored */ ------------------ | Branch (56:10): [True: 0, False: 0] ------------------ 57| 0| break; 58| | 59| 0| default: ------------------ | Branch (59:8): [True: 0, False: 0] ------------------ 60| 0| if (p->v != q->v || p->i != q->i) return 0; ------------------ | Branch (60:10): [True: 0, False: 0] | Branch (60:26): [True: 0, False: 0] ------------------ 61| 0| break; 62| 0| } 63| 0| } 64| 0| A(0 /* can't happen */); 65| 0|} twiddle.c:compute: 126| 346|{ 127| 346| INT ntwiddle, j, vl; 128| 346| R *W, *W0; 129| 346| const tw_instr *p; 130| 346| triggen *t = X(mktriggen)(wakefulness, n); ------------------ | | 78| 346|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 346|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 131| | 132| 346| p = instr; 133| 346| ntwiddle = twlen0(r, p, &vl); 134| | 135| 346| A(m % vl == 0); 136| | 137| 346| W0 = W = (R *)MALLOC((ntwiddle * (m / vl)) * sizeof(R), TWIDDLES); ------------------ | | 306| 346|#define MALLOC(n, what) X(malloc_plain)(n) | | ------------------ | | | | 78| 346|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 346|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 138| | 139| 7.53k| for (j = 0; j < m; j += vl) { ------------------ | Branch (139:18): [True: 7.19k, False: 346] ------------------ 140| 17.6k| for (p = instr; p->op != TW_NEXT; ++p) { ------------------ | Branch (140:27): [True: 10.4k, False: 7.19k] ------------------ 141| 10.4k| switch (p->op) { ------------------ | Branch (141:17): [True: 10.4k, False: 0] ------------------ 142| 4.13k| case TW_FULL: { ------------------ | Branch (142:6): [True: 4.13k, False: 6.34k] ------------------ 143| 4.13k| INT i; 144| 15.9k| for (i = 1; i < r; ++i) { ------------------ | Branch (144:16): [True: 11.7k, False: 4.13k] ------------------ 145| 11.7k| A((j + (INT)p->v) * i < n); 146| 11.7k| A((j + (INT)p->v) * i > -n); 147| 11.7k| t->cexp(t, (j + (INT)p->v) * i, W); 148| 11.7k| W += 2; 149| 11.7k| } 150| 4.13k| break; 151| 0| } 152| | 153| 1.22k| case TW_HALF: { ------------------ | Branch (153:6): [True: 1.22k, False: 9.25k] ------------------ 154| 1.22k| INT i; 155| 1.22k| A((r % 2) == 1); 156| 31.1k| for (i = 1; i + i < r; ++i) { ------------------ | Branch (156:16): [True: 29.9k, False: 1.22k] ------------------ 157| 29.9k| t->cexp(t, MULMOD(i, (j + (INT)p->v), n), W); ------------------ | | 925| 29.9k| (((x) <= 92681 - (y)) ? ((x) * (y)) % (p) : X(safe_mulmod)(x, y, p)) | | ------------------ | | | | 78| 0|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ | | | Branch (925:5): [True: 29.9k, False: 0] | | ------------------ ------------------ 158| 29.9k| W += 2; 159| 29.9k| } 160| 1.22k| break; 161| 0| } 162| | 163| 0| case TW_COS: { ------------------ | Branch (163:6): [True: 0, False: 10.4k] ------------------ 164| 0| R d[2]; 165| | 166| 0| A((j + (INT)p->v) * p->i < n); 167| 0| A((j + (INT)p->v) * p->i > -n); 168| 0| t->cexp(t, (j + (INT)p->v) * (INT)p->i, d); 169| 0| *W++ = d[0]; 170| 0| break; 171| 0| } 172| | 173| 0| case TW_SIN: { ------------------ | Branch (173:6): [True: 0, False: 10.4k] ------------------ 174| 0| R d[2]; 175| | 176| 0| A((j + (INT)p->v) * p->i < n); 177| 0| A((j + (INT)p->v) * p->i > -n); 178| 0| t->cexp(t, (j + (INT)p->v) * (INT)p->i, d); 179| 0| *W++ = d[1]; 180| 0| break; 181| 0| } 182| | 183| 5.11k| case TW_CEXP: ------------------ | Branch (183:6): [True: 5.11k, False: 5.35k] ------------------ 184| 5.11k| A((j + (INT)p->v) * p->i < n); 185| 5.11k| A((j + (INT)p->v) * p->i > -n); 186| 5.11k| t->cexp(t, (j + (INT)p->v) * (INT)p->i, W); 187| 5.11k| W += 2; 188| 5.11k| break; 189| 10.4k| } 190| 10.4k| } 191| 7.19k| } 192| | 193| 346| X(triggen_destroy)(t); ------------------ | | 78| 346|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 346|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 194| 346| return W0; 195| 346|} fftw_rdft_buffered_register: 333| 1|{ 334| 1| size_t i; 335| 3| for (i = 0; i < NELEM(maxnbufs); ++i) ------------------ | | 90| 3|#define NELEM(array) ((sizeof(array) / sizeof((array)[0]))) ------------------ | Branch (335:18): [True: 2, False: 1] ------------------ 336| 2| REGISTER_SOLVER(p, mksolver(i)); ------------------ | | 95| 2|#define REGISTER_SOLVER(p, s) X(solver_register)(p, s) | | ------------------ | | | | 78| 2|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 2|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 337| 1|} fftw_rdft2_buffered_register: 371| 1|{ 372| 1| size_t i; 373| 3| for (i = 0; i < NELEM(maxnbufs); ++i) ------------------ | | 90| 3|#define NELEM(array) ((sizeof(array) / sizeof((array)[0]))) ------------------ | Branch (373:18): [True: 2, False: 1] ------------------ 374| 2| REGISTER_SOLVER(p, mksolver(i)); ------------------ | | 95| 2|#define REGISTER_SOLVER(p, s) X(solver_register)(p, s) | | ------------------ | | | | 78| 2|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 2|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 375| 1|} buffered2.c:mksolver: 363| 2|{ 364| 2| static const solver_adt sadt = { PROBLEM_RDFT2, mkplan, 0 }; 365| 2| S *slv = MKSOLVER(S, &sadt); ------------------ | | 604| 2|#define MKSOLVER(type, adt) (type *)X(mksolver)(sizeof(type), adt) | | ------------------ | | | | 78| 2|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 2|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 366| 2| slv->maxnbuf_ndx = maxnbuf_ndx; 367| 2| return &(slv->super); 368| 2|} fftw_rdft_conf_standard: 55| 1|{ 56| 1| X(solvtab_exec)(s, p); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 57| 1| X(solvtab_exec)(X(solvtab_rdft_r2cf), p); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ X(solvtab_exec)(X(solvtab_rdft_r2cf), p); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 58| 1| X(solvtab_exec)(X(solvtab_rdft_r2cb), p); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ X(solvtab_exec)(X(solvtab_rdft_r2cb), p); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 59| 1| X(solvtab_exec)(X(solvtab_rdft_r2r), p); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ X(solvtab_exec)(X(solvtab_rdft_r2r), p); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 60| | 61| |#if HAVE_SSE2 62| | if (X(have_simd_sse2)()) 63| | X(solvtab_exec)(X(solvtab_rdft_sse2), p); 64| |#endif 65| |#if HAVE_AVX 66| | if (X(have_simd_avx)()) 67| | X(solvtab_exec)(X(solvtab_rdft_avx), p); 68| |#endif 69| |#if HAVE_AVX_128_FMA 70| | if (X(have_simd_avx_128_fma)()) 71| | X(solvtab_exec)(X(solvtab_rdft_avx_128_fma), p); 72| |#endif 73| |#if HAVE_AVX2 74| | if (X(have_simd_avx2)()) 75| | X(solvtab_exec)(X(solvtab_rdft_avx2), p); 76| | if (X(have_simd_avx2_128)()) 77| | X(solvtab_exec)(X(solvtab_rdft_avx2_128), p); 78| |#endif 79| |#if HAVE_AVX512 80| | if (X(have_simd_avx512)()) 81| | X(solvtab_exec)(X(solvtab_rdft_avx512), p); 82| |#endif 83| |#if HAVE_KCVI 84| | if (X(have_simd_kcvi)()) 85| | X(solvtab_exec)(X(solvtab_rdft_kcvi), p); 86| |#endif 87| |#if HAVE_ALTIVEC 88| | if (X(have_simd_altivec)()) 89| | X(solvtab_exec)(X(solvtab_rdft_altivec), p); 90| |#endif 91| |#if HAVE_VSX 92| | if (X(have_simd_vsx)()) 93| | X(solvtab_exec)(X(solvtab_rdft_vsx), p); 94| |#endif 95| |#if HAVE_NEON 96| | if (X(have_simd_neon)()) 97| | X(solvtab_exec)(X(solvtab_rdft_neon), p); 98| |#endif 99| |#if HAVE_SVE 100| | if (X(have_simd_sve)(128)) 101| | X(solvtab_exec)(X(solvtab_rdft_sve128), p); 102| | if (X(have_simd_sve)(256)) 103| | X(solvtab_exec)(X(solvtab_rdft_sve256), p); 104| | if (X(have_simd_sve)(512)) 105| | X(solvtab_exec)(X(solvtab_rdft_sve512), p); 106| | if (X(have_simd_sve)(1024)) 107| | X(solvtab_exec)(X(solvtab_rdft_sve1024), p); 108| | if (X(have_simd_sve)(2048)) 109| | X(solvtab_exec)(X(solvtab_rdft_sve2048), p); 110| |#endif 111| |#if HAVE_LSX 112| | if (X(have_simd_lsx)()) 113| | X(solvtab_exec)(X(solvtab_rdft_lsx), p); 114| |#endif 115| |#if HAVE_LASX 116| | if (X(have_simd_lasx)()) 117| | X(solvtab_exec)(X(solvtab_rdft_lasx), p); 118| |#endif 119| |#if HAVE_GENERIC_SIMD128 120| | X(solvtab_exec)(X(solvtab_rdft_generic_simd128), p); 121| |#endif 122| |#if HAVE_GENERIC_SIMD256 123| | X(solvtab_exec)(X(solvtab_rdft_generic_simd256), p); 124| |#endif 125| 1|} fftw_regsolver_hc2c_direct: 401| 56|{ 402| 56| regone(plnr, codelet, desc, hc2ckind, /* bufferedp */0); 403| 56| regone(plnr, codelet, desc, hc2ckind, /* bufferedp */1); 404| 56|} ct-hc2c-direct.c:regone: 390| 112|{ 391| 112| S *slv = (S *)X(mksolver_hc2c)(sizeof(S), desc->radix, hc2ckind, mkcldw); ------------------ | | 78| 112|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 112|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 392| 112| slv->k = codelet; 393| 112| slv->desc = desc; 394| 112| slv->bufferedp = bufferedp; 395| 112| REGISTER_SOLVER(plnr, &(slv->super.super)); ------------------ | | 95| 112|#define REGISTER_SOLVER(p, s) X(solver_register)(p, s) | | ------------------ | | | | 78| 112|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 112|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 396| 112|} fftw_mksolver_hc2c: 279| 112|{ 280| 112| static const solver_adt sadt = { PROBLEM_RDFT2, mkplan, 0 }; 281| 112| hc2c_solver *slv = (hc2c_solver *)X(mksolver)(size, &sadt); ------------------ | | 78| 112|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 112|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 282| 112| slv->r = r; 283| 112| slv->hc2ckind = hc2ckind; 284| 112| slv->mkcldw = mkcldw; 285| 112| return slv; 286| 112|} fftw_dft_r2hc_register: 192| 1|{ 193| 1| REGISTER_SOLVER(p, mksolver()); ------------------ | | 95| 1|#define REGISTER_SOLVER(p, s) X(solver_register)(p, s) | | ------------------ | | | | 78| 1|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 194| 1|} dft-r2hc.c:mksolver: 185| 1|{ 186| 1| static const solver_adt sadt = { PROBLEM_DFT, mkplan, 0 }; 187| 1| S *slv = MKSOLVER(S, &sadt); ------------------ | | 604| 1|#define MKSOLVER(type, adt) (type *)X(mksolver)(sizeof(type), adt) | | ------------------ | | | | 78| 1|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 188| 1| return &(slv->super); 189| 1|} dft-r2hc.c:mkplan: 125| 1.87k|{ 126| 1.87k| P *pln; 127| 1.87k| const problem_dft *p; 128| 1.87k| plan *cld; 129| 1.87k| INT ishift = 0, oshift = 0; 130| | 131| 1.87k| static const plan_adt padt = { 132| 1.87k| X(dft_solve), awake, print, destroy ------------------ | | 78| 1.87k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1.87k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 133| 1.87k| }; 134| | 135| 1.87k| UNUSED(ego_); ------------------ | | 88| 1.87k|#define UNUSED(x) (void)x ------------------ 136| 1.87k| if (!applicable(p_, plnr)) ------------------ | Branch (136:10): [True: 1.45k, False: 417] ------------------ 137| 1.45k| return (plan *)0; 138| | 139| 417| p = (const problem_dft *) p_; 140| | 141| 417| { 142| 417| tensor *ri_vec = X(mktensor_1d)(2, p->ii - p->ri, p->io - p->ro); ------------------ | | 78| 417|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 417|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 143| 417| tensor *cld_vec = X(tensor_append)(ri_vec, p->vecsz); ------------------ | | 78| 417|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 417|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 144| 417| int i; 145| 1.61k| for (i = 0; i < cld_vec->rnk; ++i) { /* make all istrides > 0 */ ------------------ | Branch (145:16): [True: 1.20k, False: 417] ------------------ 146| 1.20k| if (cld_vec->dims[i].is < 0) { ------------------ | Branch (146:13): [True: 0, False: 1.20k] ------------------ 147| 0| INT nm1 = cld_vec->dims[i].n - 1; 148| 0| ishift -= nm1 * (cld_vec->dims[i].is *= -1); 149| 0| oshift -= nm1 * (cld_vec->dims[i].os *= -1); 150| 0| } 151| 1.20k| } 152| 417| cld = X(mkplan_d)(plnr, ------------------ | | 78| 417|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 417|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 153| 417| X(mkproblem_rdft_1)(p->sz, cld_vec, ------------------ | | 78| 417|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 417|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 154| 417| p->ri + ishift, 155| 417| p->ro + oshift, R2HC)); ------------------ | | 48| 417|#define R2HC R2HC00 ------------------ 156| 417| X(tensor_destroy2)(ri_vec, cld_vec); ------------------ | | 78| 417|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 417|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 157| 417| } 158| 417| if (!cld) return (plan *)0; ------------------ | Branch (158:10): [True: 134, False: 283] ------------------ 159| | 160| 283| pln = MKPLAN_DFT(P, &padt, apply); ------------------ | | 60| 283| (type *)X(mkplan_dft)(sizeof(type), adt, apply) | | ------------------ | | | | 78| 283|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 283|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 161| | 162| 283| if (p->sz->rnk == 0) { ------------------ | Branch (162:10): [True: 283, False: 0] ------------------ 163| 283| pln->n = 1; 164| 283| pln->os = 0; 165| 283| } 166| 0| else { 167| 0| pln->n = p->sz->dims[0].n; 168| 0| pln->os = p->sz->dims[0].os; 169| 0| } 170| 283| pln->ishift = ishift; 171| 283| pln->oshift = oshift; 172| | 173| 283| pln->cld = cld; 174| | 175| 283| pln->super.super.ops = cld->ops; 176| 283| pln->super.super.ops.other += 8 * ((pln->n - 1)/2); 177| 283| pln->super.super.ops.add += 4 * ((pln->n - 1)/2); 178| 283| pln->super.super.ops.other += 1; /* estimator hack for nop plans */ 179| | 180| 283| return &(pln->super.super); 181| 417|} dft-r2hc.c:awake: 73| 82|{ 74| 82| P *ego = (P *) ego_; 75| 82| X(plan_awake)(ego->cld, wakefulness); ------------------ | | 78| 82|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 82|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 76| 82|} dft-r2hc.c:destroy: 79| 283|{ 80| 283| P *ego = (P *) ego_; 81| 283| X(plan_destroy_internal)(ego->cld); ------------------ | | 78| 283|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 283|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 82| 283|} dft-r2hc.c:applicable: 105| 1.87k|{ 106| 1.87k| if (!applicable0(p_)) return 0; ------------------ | Branch (106:10): [True: 1.01k, False: 859] ------------------ 107| | 108| 859| { 109| 859| const problem_dft *p = (const problem_dft *) p_; 110| | 111| | /* rank-0 problems are always OK */ 112| 859| if (p->sz->rnk == 0) return 1; ------------------ | Branch (112:8): [True: 417, False: 442] ------------------ 113| | 114| | /* this solver is ok for split arrays */ 115| 442| if (p->sz->rnk == 1 && ------------------ | Branch (115:8): [True: 442, False: 0] ------------------ 116| 442| splitp(p->ri, p->ii, p->sz->dims[0].n, p->sz->dims[0].is) && ------------------ | Branch (116:8): [True: 0, False: 442] ------------------ 117| 0| splitp(p->ro, p->io, p->sz->dims[0].n, p->sz->dims[0].os)) ------------------ | Branch (117:8): [True: 0, False: 0] ------------------ 118| 0| return 1; 119| | 120| 442| return !(NO_DFT_R2HCP(plnr)); ------------------ | | 687| 442|#define NO_DFT_R2HCP(plnr) (PLNR_L(plnr) & NO_DFT_R2HC) | | ------------------ | | | | 674| 442|#define PLNR_L(plnr) ((plnr)->flags.l) | | ------------------ ------------------ 121| 442| } 122| 442|} dft-r2hc.c:applicable0: 92| 1.87k|{ 93| 1.87k| const problem_dft *p = (const problem_dft *) p_; 94| 1.87k| return ((p->sz->rnk == 1 && p->vecsz->rnk == 0) ------------------ | Branch (94:15): [True: 1.45k, False: 417] | Branch (94:34): [True: 442, False: 1.01k] ------------------ 95| 1.42k| || (p->sz->rnk == 0 && FINITE_RNK(p->vecsz->rnk)) ------------------ | | 442| 417|#define FINITE_RNK(rnk) ((rnk) != RNK_MINFTY) | | ------------------ | | | | 441| 417|#define RNK_MINFTY INT_MAX | | ------------------ | | | Branch (442:25): [True: 417, False: 0] | | ------------------ ------------------ | Branch (95:11): [True: 417, False: 1.01k] ------------------ 96| 1.87k| ); 97| 1.87k|} dft-r2hc.c:splitp: 100| 442|{ 101| 442| return ((r > i ? (r - i) : (i - r)) >= n * (s > 0 ? s : 0-s)); ------------------ | Branch (101:15): [True: 0, False: 442] | Branch (101:50): [True: 442, False: 0] ------------------ 102| 442|} dft-r2hc.c:apply: 44| 83|{ 45| 83| const P *ego = (const P *) ego_; 46| 83| INT n; 47| | 48| 83| UNUSED(ii); ------------------ | | 88| 83|#define UNUSED(x) (void)x ------------------ 49| | 50| 83| { /* transform vector of real & imag parts: */ 51| 83| plan_rdft *cld = (plan_rdft *) ego->cld; 52| 83| cld->apply((plan *) cld, ri + ego->ishift, ro + ego->oshift); 53| 83| } 54| | 55| 83| n = ego->n; 56| 83| if (n > 1) { ------------------ | Branch (56:10): [True: 0, False: 83] ------------------ 57| 0| INT i, os = ego->os; 58| 0| for (i = 1; i < (n + 1)/2; ++i) { ------------------ | Branch (58:16): [True: 0, False: 0] ------------------ 59| 0| E rop, iop, iom, rom; 60| 0| rop = ro[os * i]; 61| 0| iop = io[os * i]; 62| 0| rom = ro[os * (n - i)]; 63| 0| iom = io[os * (n - i)]; 64| 0| ro[os * i] = rop - iom; 65| 0| io[os * i] = iop + rom; 66| 0| ro[os * (n - i)] = rop + iom; 67| 0| io[os * (n - i)] = iop - rom; 68| 0| } 69| 0| } 70| 83|} fftw_dht_r2hc_register: 142| 1|{ 143| 1| REGISTER_SOLVER(p, mksolver()); ------------------ | | 95| 1|#define REGISTER_SOLVER(p, s) X(solver_register)(p, s) | | ------------------ | | | | 78| 1|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 144| 1|} dht-r2hc.c:mksolver: 135| 1|{ 136| 1| static const solver_adt sadt = { PROBLEM_RDFT, mkplan, 0 }; 137| 1| S *slv = MKSOLVER(S, &sadt); ------------------ | | 604| 1|#define MKSOLVER(type, adt) (type *)X(mksolver)(sizeof(type), adt) | | ------------------ | | | | 78| 1|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 138| 1| return &(slv->super); 139| 1|} dht-r2hc.c:mkplan: 99| 335|{ 100| 335| P *pln; 101| 335| const problem_rdft *p; 102| 335| plan *cld; 103| | 104| 335| static const plan_adt padt = { 105| 335| X(rdft_solve), awake, print, destroy ------------------ | | 78| 335|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 335|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 106| 335| }; 107| | 108| 335| if (!applicable(ego_, p_, plnr)) ------------------ | Branch (108:10): [True: 335, False: 0] ------------------ 109| 335| return (plan *)0; 110| | 111| 0| p = (const problem_rdft *) p_; 112| | 113| | /* NO_DHT_R2HC stops infinite loops with rdft-dht.c */ 114| 0| cld = X(mkplan_f_d)(plnr, ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 115| 0| X(mkproblem_rdft_1)(p->sz, p->vecsz, ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 116| 0| p->I, p->O, R2HC), ------------------ | | 48| 0|#define R2HC R2HC00 ------------------ 117| 0| NO_DHT_R2HC, 0, 0); 118| 0| if (!cld) return (plan *)0; ------------------ | Branch (118:10): [True: 0, False: 0] ------------------ 119| | 120| 0| pln = MKPLAN_RDFT(P, &padt, apply); ------------------ | | 73| 0| (type *)X(mkplan_rdft)(sizeof(type), adt, apply) | | ------------------ | | | | 78| 0|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 121| | 122| 0| pln->n = p->sz->dims[0].n; 123| 0| pln->os = p->sz->dims[0].os; 124| 0| pln->cld = cld; 125| | 126| 0| pln->super.super.ops = cld->ops; 127| 0| pln->super.super.ops.other += 4 * ((pln->n - 1)/2); 128| 0| pln->super.super.ops.add += 2 * ((pln->n - 1)/2); 129| | 130| 0| return &(pln->super.super); 131| 0|} dht-r2hc.c:applicable: 93| 335|{ 94| 335| UNUSED(ego); ------------------ | | 88| 335|#define UNUSED(x) (void)x ------------------ 95| 335| return (!NO_SLOWP(plnr) && applicable0(p, plnr)); ------------------ | | 688| 670|#define NO_SLOWP(plnr) (PLNR_L(plnr) & NO_SLOW) | | ------------------ | | | | 674| 335|#define PLNR_L(plnr) ((plnr)->flags.l) | | ------------------ ------------------ | Branch (95:14): [True: 0, False: 335] | Branch (95:33): [True: 0, False: 0] ------------------ 96| 335|} fftw_dht_rader_register: 383| 1|{ 384| 1| REGISTER_SOLVER(p, mksolver(0)); ------------------ | | 95| 1|#define REGISTER_SOLVER(p, s) X(solver_register)(p, s) | | ------------------ | | | | 78| 1|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 385| 1| REGISTER_SOLVER(p, mksolver(1)); ------------------ | | 95| 1|#define REGISTER_SOLVER(p, s) X(solver_register)(p, s) | | ------------------ | | | | 78| 1|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 386| 1|} dht-rader.c:mksolver: 375| 2|{ 376| 2| static const solver_adt sadt = { PROBLEM_RDFT, mkplan, 0 }; 377| 2| S *slv = MKSOLVER(S, &sadt); ------------------ | | 604| 2|#define MKSOLVER(type, adt) (type *)X(mksolver)(sizeof(type), adt) | | ------------------ | | | | 78| 2|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 2|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 378| 2| slv->pad = pad; 379| 2| return &(slv->super); 380| 2|} dht-rader.c:mkplan: 277| 670|{ 278| 670| const S *ego = (const S *) ego_; 279| 670| const problem_rdft *p = (const problem_rdft *) p_; 280| 670| P *pln; 281| 670| INT n, npad; 282| 670| INT is, os; 283| 670| plan *cld1 = (plan *) 0; 284| 670| plan *cld2 = (plan *) 0; 285| 670| plan *cld_omega = (plan *) 0; 286| 670| R *buf = (R *) 0; 287| 670| problem *cldp; 288| | 289| 670| static const plan_adt padt = { 290| 670| X(rdft_solve), awake, print, destroy ------------------ | | 78| 670|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 670|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 291| 670| }; 292| | 293| 670| if (!applicable(ego_, p_, plnr)) ------------------ | Branch (293:10): [True: 670, False: 0] ------------------ 294| 670| return (plan *) 0; 295| | 296| 0| n = p->sz->dims[0].n; 297| 0| is = p->sz->dims[0].is; 298| 0| os = p->sz->dims[0].os; 299| | 300| 0| if (ego->pad) ------------------ | Branch (300:10): [True: 0, False: 0] ------------------ 301| 0| npad = choose_transform_size(2 * (n - 1) - 1); 302| 0| else 303| 0| npad = n - 1; 304| | 305| | /* initial allocation for the purpose of planning */ 306| 0| buf = (R *) MALLOC(sizeof(R) * npad, BUFFERS); ------------------ | | 306| 0|#define MALLOC(n, what) X(malloc_plain)(n) | | ------------------ | | | | 78| 0|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 307| | 308| 0| cld1 = X(mkplan_f_d)(plnr, ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 309| 0| X(mkproblem_rdft_1_d)(X(mktensor_1d)(npad, 1, 1), ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ X(mkproblem_rdft_1_d)(X(mktensor_1d)(npad, 1, 1), ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 310| 0| X(mktensor_1d)(1, 0, 0), ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 311| 0| buf, buf, 312| 0| R2HC), ------------------ | | 48| 0|#define R2HC R2HC00 ------------------ 313| 0| NO_SLOW, 0, 0); 314| 0| if (!cld1) goto nada; ------------------ | Branch (314:10): [True: 0, False: 0] ------------------ 315| | 316| 0| cldp = 317| 0| X(mkproblem_rdft_1_d)( ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 318| 0| X(mktensor_1d)(npad, 1, 1), ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 319| 0| X(mktensor_1d)(1, 0, 0), ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 320| 0| buf, buf, 321| 0|#if R2HC_ONLY_CONV 322| 0| R2HC ------------------ | | 48| 0|#define R2HC R2HC00 ------------------ 323| |#else 324| | HC2R 325| |#endif 326| 0| ); 327| 0| if (!(cld2 = X(mkplan_f_d)(plnr, cldp, NO_SLOW, 0, 0))) ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ | Branch (327:10): [True: 0, False: 0] ------------------ 328| 0| goto nada; 329| | 330| | /* plan for omega */ 331| 0| cld_omega = X(mkplan_f_d)(plnr, ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 332| 0| X(mkproblem_rdft_1_d)( ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 333| 0| X(mktensor_1d)(npad, 1, 1), ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 334| 0| X(mktensor_1d)(1, 0, 0), ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 335| 0| buf, buf, R2HC), ------------------ | | 48| 0|#define R2HC R2HC00 ------------------ 336| 0| NO_SLOW, ESTIMATE, 0); 337| 0| if (!cld_omega) goto nada; ------------------ | Branch (337:10): [True: 0, False: 0] ------------------ 338| | 339| | /* deallocate buffers; let awake() or apply() allocate them for real */ 340| 0| X(ifree)(buf); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 341| 0| buf = 0; 342| | 343| 0| pln = MKPLAN_RDFT(P, &padt, apply); ------------------ | | 73| 0| (type *)X(mkplan_rdft)(sizeof(type), adt, apply) | | ------------------ | | | | 78| 0|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 344| 0| pln->cld1 = cld1; 345| 0| pln->cld2 = cld2; 346| 0| pln->cld_omega = cld_omega; 347| 0| pln->omega = 0; 348| 0| pln->n = n; 349| 0| pln->npad = npad; 350| 0| pln->is = is; 351| 0| pln->os = os; 352| | 353| 0| X(ops_add)(&cld1->ops, &cld2->ops, &pln->super.super.ops); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 354| 0| pln->super.super.ops.other += (npad/2-1)*6 + npad + n + (n-1) * ego->pad; 355| 0| pln->super.super.ops.add += (npad/2-1)*2 + 2 + (n-1) * ego->pad; 356| 0| pln->super.super.ops.mul += (npad/2-1)*4 + 2 + ego->pad; 357| 0|#if R2HC_ONLY_CONV 358| 0| pln->super.super.ops.other += n-2 - ego->pad; 359| 0| pln->super.super.ops.add += (npad/2-1)*2 + (n-2) - ego->pad; 360| 0|#endif 361| | 362| 0| return &(pln->super.super); 363| | 364| 0| nada: 365| 0| X(ifree0)(buf); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 366| 0| X(plan_destroy_internal)(cld_omega); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 367| 0| X(plan_destroy_internal)(cld2); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 368| 0| X(plan_destroy_internal)(cld1); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 369| 0| return 0; 370| 0|} dht-rader.c:applicable: 250| 670|{ 251| 670| const problem_rdft *p = (const problem_rdft *) p_; 252| 670| UNUSED(ego); ------------------ | | 88| 670|#define UNUSED(x) (void)x ------------------ 253| 670| return (1 ------------------ | Branch (253:14): [True: 670, Folded] ------------------ 254| 670| && p->sz->rnk == 1 ------------------ | Branch (254:10): [True: 0, False: 670] ------------------ 255| 0| && p->vecsz->rnk == 0 ------------------ | Branch (255:10): [True: 0, False: 0] ------------------ 256| 0| && p->kind[0] == DHT ------------------ | Branch (256:10): [True: 0, False: 0] ------------------ 257| 0| && X(is_prime)(p->sz->dims[0].n) ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ | Branch (257:10): [True: 0, False: 0] ------------------ 258| 0| && p->sz->dims[0].n > 2 ------------------ | Branch (258:10): [True: 0, False: 0] ------------------ 259| 0| && CIMPLIES(NO_SLOWP(plnr), p->sz->dims[0].n > RADER_MAX_SLOW) ------------------ | | 99| 670|#define CIMPLIES(ante, post) (!(ante) || (post)) | | ------------------ | | | Branch (99:31): [True: 0, False: 0] | | | Branch (99:42): [True: 0, False: 0] | | ------------------ ------------------ 260| | /* proclaim the solver SLOW if p-1 is not easily 261| | factorizable. Unlike in the complex case where 262| | Bluestein can solve the problem, in the DHT case we 263| | may have no other choice */ 264| 0| && CIMPLIES(NO_SLOWP(plnr), X(factors_into_small_primes)(p->sz->dims[0].n - 1)) ------------------ | | 99| 0|#define CIMPLIES(ante, post) (!(ante) || (post)) | | ------------------ | | | Branch (99:31): [True: 0, False: 0] | | | Branch (99:42): [True: 0, False: 0] | | ------------------ ------------------ 265| 670| ); 266| 670|} fftw_mksolver_rdft_r2c_direct: 334| 72|{ 335| 72| return mksolver(k, desc, 0); 336| 72|} fftw_mksolver_rdft_r2c_directbuf: 339| 72|{ 340| 72| return mksolver(k, desc, 1); 341| 72|} direct-r2c.c:mksolver: 324| 144|{ 325| 144| static const solver_adt sadt = { PROBLEM_RDFT, mkplan, 0 }; 326| 144| S *slv = MKSOLVER(S, &sadt); ------------------ | | 604| 144|#define MKSOLVER(type, adt) (type *)X(mksolver)(sizeof(type), adt) | | ------------------ | | | | 78| 144|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 144|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 327| 144| slv->k = k; 328| 144| slv->desc = desc; 329| 144| slv->bufferedp = bufferedp; 330| 144| return &(slv->super); 331| 144|} direct-r2c.c:mkplan: 254| 48.2k|{ 255| 48.2k| const S *ego = (const S *) ego_; 256| 48.2k| P *pln; 257| 48.2k| const problem_rdft *p; 258| 48.2k| iodim *d; 259| 48.2k| INT rs, cs, b, n; 260| | 261| 48.2k| static const plan_adt padt = { 262| 48.2k| X(rdft_solve), X(null_awake), print, destroy ------------------ | | 78| 48.2k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 48.2k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ X(rdft_solve), X(null_awake), print, destroy ------------------ | | 78| 48.2k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 48.2k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 263| 48.2k| }; 264| | 265| 48.2k| UNUSED(plnr); ------------------ | | 88| 48.2k|#define UNUSED(x) (void)x ------------------ 266| | 267| 48.2k| if (ego->bufferedp) { ------------------ | Branch (267:10): [True: 24.1k, False: 24.1k] ------------------ 268| 24.1k| if (!applicable_buf(ego_, p_)) ------------------ | Branch (268:8): [True: 24.1k, False: 0] ------------------ 269| 24.1k| return (plan *)0; 270| 24.1k| } else { 271| 24.1k| if (!applicable(ego_, p_)) ------------------ | Branch (271:8): [True: 24.1k, False: 0] ------------------ 272| 24.1k| return (plan *)0; 273| 24.1k| } 274| | 275| 0| p = (const problem_rdft *) p_; 276| | 277| 0| if (R2HC_KINDP(p->kind[0])) { ------------------ | | 57| 0|#define R2HC_KINDP(k) ((k) >= R2HC00 && (k) <= R2HC11) /* uses kr2hc_genus */ | | ------------------ | | | Branch (57:24): [True: 0, False: 0] | | | Branch (57:41): [True: 0, False: 0] | | ------------------ ------------------ 278| 0| rs = p->sz->dims[0].is; cs = p->sz->dims[0].os; 279| 0| pln = MKPLAN_RDFT(P, &padt, ------------------ | | 73| 0| (type *)X(mkplan_rdft)(sizeof(type), adt, apply) | | ------------------ | | | | 78| 0|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ | | | Branch (73:45): [True: 0, False: 0] | | ------------------ ------------------ 280| 0| ego->bufferedp ? apply_buf_r2hc : apply_r2hc); 281| 0| } else { 282| 0| rs = p->sz->dims[0].os; cs = p->sz->dims[0].is; 283| 0| pln = MKPLAN_RDFT(P, &padt, ------------------ | | 73| 0| (type *)X(mkplan_rdft)(sizeof(type), adt, apply) | | ------------------ | | | | 78| 0|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ | | | Branch (73:45): [True: 0, False: 0] | | ------------------ ------------------ 284| 0| ego->bufferedp ? apply_buf_hc2r : apply_hc2r); 285| 0| } 286| | 287| 0| d = p->sz->dims; 288| 0| n = d[0].n; 289| | 290| 0| pln->k = ego->k; 291| 0| pln->n = n; 292| | 293| 0| pln->rs0 = rs; 294| 0| pln->rs = X(mkstride)(n, 2 * rs); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | | | | | 827| 0|#define fftw_mkstride(n, stride) stride | | | | ------------------ | | ------------------ ------------------ 295| 0| pln->csr = X(mkstride)(n, cs); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | | | | | 827| 0|#define fftw_mkstride(n, stride) stride | | | | ------------------ | | ------------------ ------------------ 296| 0| pln->csi = X(mkstride)(n, -cs); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | | | | | 827| 0|#define fftw_mkstride(n, stride) stride | | | | ------------------ | | ------------------ ------------------ 297| 0| pln->ioffset = ioffset(p->kind[0], n, cs); 298| | 299| 0| b = compute_batchsize(n); 300| 0| pln->brs = X(mkstride)(n, 2 * b); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | | | | | 827| 0|#define fftw_mkstride(n, stride) stride | | | | ------------------ | | ------------------ ------------------ 301| 0| pln->bcsr = X(mkstride)(n, b); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | | | | | 827| 0|#define fftw_mkstride(n, stride) stride | | | | ------------------ | | ------------------ ------------------ 302| 0| pln->bcsi = X(mkstride)(n, -b); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | | | | | 827| 0|#define fftw_mkstride(n, stride) stride | | | | ------------------ | | ------------------ ------------------ 303| 0| pln->bioffset = ioffset(p->kind[0], n, b); 304| | 305| 0| X(tensor_tornk1)(p->vecsz, &pln->vl, &pln->ivs, &pln->ovs); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 306| | 307| 0| pln->slv = ego; 308| 0| X(ops_zero)(&pln->super.super.ops); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 309| | 310| 0| X(ops_madd2)(pln->vl / ego->desc->genus->vl, ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 311| 0| &ego->desc->ops, 312| 0| &pln->super.super.ops); 313| | 314| 0| if (ego->bufferedp) ------------------ | Branch (314:10): [True: 0, False: 0] ------------------ 315| 0| pln->super.super.ops.other += 2 * n * pln->vl; 316| | 317| 0| pln->super.super.could_prune_now_p = !ego->bufferedp; 318| | 319| 0| return &(pln->super.super); 320| 48.2k|} direct-r2c.c:applicable_buf: 221| 24.1k|{ 222| 24.1k| const S *ego = (const S *) ego_; 223| 24.1k| const kr2c_desc *desc = ego->desc; 224| 24.1k| const problem_rdft *p = (const problem_rdft *) p_; 225| 24.1k| INT vl, ivs, ovs, batchsz; 226| | 227| 24.1k| return ( 228| 24.1k| 1 ------------------ | Branch (228:4): [True: 24.1k, Folded] ------------------ 229| 24.1k| && p->sz->rnk == 1 ------------------ | Branch (229:7): [True: 0, False: 24.1k] ------------------ 230| 0| && p->vecsz->rnk <= 1 ------------------ | Branch (230:7): [True: 0, False: 0] ------------------ 231| 0| && p->sz->dims[0].n == desc->n ------------------ | Branch (231:7): [True: 0, False: 0] ------------------ 232| 0| && p->kind[0] == desc->genus->kind ------------------ | Branch (232:7): [True: 0, False: 0] ------------------ 233| | 234| | /* check strides etc */ 235| 0| && X(tensor_tornk1)(p->vecsz, &vl, &ivs, &ovs) ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ | Branch (235:7): [True: 0, False: 0] ------------------ 236| | 237| 0| && (batchsz = compute_batchsize(desc->n), 1) ------------------ | Branch (237:7): [True: 0, False: 0] ------------------ 238| | 239| 0| && (0 ------------------ | Branch (239:8): [Folded, False: 0] ------------------ 240| | /* can operate out-of-place */ 241| 0| || p->I != p->O ------------------ | Branch (241:11): [True: 0, False: 0] ------------------ 242| | 243| | /* can operate in-place as long as strides are the same */ 244| 0| || X(tensor_inplace_strides2)(p->sz, p->vecsz) ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ | Branch (244:11): [True: 0, False: 0] ------------------ 245| | 246| | /* can do it if the problem fits in the buffer, no matter 247| | what the strides are */ 248| 0| || vl <= batchsz ------------------ | Branch (248:11): [True: 0, False: 0] ------------------ 249| 0| ) 250| 24.1k| ); 251| 24.1k|} direct-r2c.c:applicable: 191| 24.1k|{ 192| 24.1k| const S *ego = (const S *) ego_; 193| 24.1k| const kr2c_desc *desc = ego->desc; 194| 24.1k| const problem_rdft *p = (const problem_rdft *) p_; 195| 24.1k| INT vl, ivs, ovs; 196| | 197| 24.1k| return ( 198| 24.1k| 1 ------------------ | Branch (198:4): [True: 24.1k, Folded] ------------------ 199| 24.1k| && p->sz->rnk == 1 ------------------ | Branch (199:7): [True: 0, False: 24.1k] ------------------ 200| 0| && p->vecsz->rnk <= 1 ------------------ | Branch (200:7): [True: 0, False: 0] ------------------ 201| 0| && p->sz->dims[0].n == desc->n ------------------ | Branch (201:7): [True: 0, False: 0] ------------------ 202| 0| && p->kind[0] == desc->genus->kind ------------------ | Branch (202:7): [True: 0, False: 0] ------------------ 203| | 204| | /* check strides etc */ 205| 0| && X(tensor_tornk1)(p->vecsz, &vl, &ivs, &ovs) ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ | Branch (205:7): [True: 0, False: 0] ------------------ 206| | 207| 0| && (0 ------------------ | Branch (207:8): [Folded, False: 0] ------------------ 208| | /* can operate out-of-place */ 209| 0| || p->I != p->O ------------------ | Branch (209:11): [True: 0, False: 0] ------------------ 210| | 211| | /* computing one transform */ 212| 0| || vl == 1 ------------------ | Branch (212:11): [True: 0, False: 0] ------------------ 213| | 214| | /* can operate in-place as long as strides are the same */ 215| 0| || X(tensor_inplace_strides2)(p->sz, p->vecsz) ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ | Branch (215:11): [True: 0, False: 0] ------------------ 216| 0| ) 217| 24.1k| ); 218| 24.1k|} fftw_mksolver_rdft_r2r_direct: 138| 2|{ 139| 2| static const solver_adt sadt = { PROBLEM_RDFT, mkplan, 0 }; 140| 2| S *slv = MKSOLVER(S, &sadt); ------------------ | | 604| 2|#define MKSOLVER(type, adt) (type *)X(mksolver)(sizeof(type), adt) | | ------------------ | | | | 78| 2|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 2|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 141| 2| slv->k = k; 142| 2| slv->desc = desc; 143| 2| return &(slv->super); 144| 2|} direct-r2r.c:mkplan: 96| 670|{ 97| 670| const S *ego = (const S *) ego_; 98| 670| P *pln; 99| 670| const problem_rdft *p; 100| 670| iodim *d; 101| | 102| 670| static const plan_adt padt = { 103| 670| X(rdft_solve), X(null_awake), print, destroy ------------------ | | 78| 670|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 670|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ X(rdft_solve), X(null_awake), print, destroy ------------------ | | 78| 670|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 670|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 104| 670| }; 105| | 106| 670| UNUSED(plnr); ------------------ | | 88| 670|#define UNUSED(x) (void)x ------------------ 107| | 108| 670| if (!applicable(ego_, p_)) ------------------ | Branch (108:10): [True: 670, False: 0] ------------------ 109| 670| return (plan *)0; 110| | 111| 0| p = (const problem_rdft *) p_; 112| | 113| | 114| 0| pln = MKPLAN_RDFT(P, &padt, apply); ------------------ | | 73| 0| (type *)X(mkplan_rdft)(sizeof(type), adt, apply) | | ------------------ | | | | 78| 0|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 115| | 116| 0| d = p->sz->dims; 117| | 118| 0| pln->k = ego->k; 119| | 120| 0| pln->is = X(mkstride)(d->n, d->is); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | | | | | 827| 0|#define fftw_mkstride(n, stride) stride | | | | ------------------ | | ------------------ ------------------ 121| 0| pln->os = X(mkstride)(d->n, d->os); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | | | | | 827| 0|#define fftw_mkstride(n, stride) stride | | | | ------------------ | | ------------------ ------------------ 122| | 123| 0| X(tensor_tornk1)(p->vecsz, &pln->vl, &pln->ivs, &pln->ovs); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 124| | 125| 0| pln->slv = ego; 126| 0| X(ops_zero)(&pln->super.super.ops); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 127| 0| X(ops_madd2)(pln->vl / ego->desc->genus->vl, ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 128| 0| &ego->desc->ops, 129| 0| &pln->super.super.ops); 130| | 131| 0| pln->super.super.could_prune_now_p = 1; 132| | 133| 0| return &(pln->super.super); 134| 670|} direct-r2r.c:applicable: 66| 670|{ 67| 670| const S *ego = (const S *) ego_; 68| 670| const problem_rdft *p = (const problem_rdft *) p_; 69| 670| INT vl; 70| 670| INT ivs, ovs; 71| | 72| 670| return ( 73| 670| 1 ------------------ | Branch (73:4): [True: 670, Folded] ------------------ 74| 670| && p->sz->rnk == 1 ------------------ | Branch (74:7): [True: 0, False: 670] ------------------ 75| 0| && p->vecsz->rnk <= 1 ------------------ | Branch (75:7): [True: 0, False: 0] ------------------ 76| 0| && p->sz->dims[0].n == ego->desc->n ------------------ | Branch (76:7): [True: 0, False: 0] ------------------ 77| 0| && p->kind[0] == ego->desc->kind ------------------ | Branch (77:7): [True: 0, False: 0] ------------------ 78| | 79| | /* check strides etc */ 80| 0| && X(tensor_tornk1)(p->vecsz, &vl, &ivs, &ovs) ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ | Branch (80:7): [True: 0, False: 0] ------------------ 81| | 82| 0| && (0 ------------------ | Branch (82:8): [Folded, False: 0] ------------------ 83| | /* can operate out-of-place */ 84| 0| || p->I != p->O ------------------ | Branch (84:11): [True: 0, False: 0] ------------------ 85| | 86| | /* computing one transform */ 87| 0| || vl == 1 ------------------ | Branch (87:11): [True: 0, False: 0] ------------------ 88| | 89| | /* can operate in-place as long as strides are the same */ 90| 0| || X(tensor_inplace_strides2)(p->sz, p->vecsz) ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ | Branch (90:11): [True: 0, False: 0] ------------------ 91| 0| ) 92| 670| ); 93| 670|} fftw_mksolver_rdft2_direct: 165| 72|{ 166| 72| static const solver_adt sadt = { PROBLEM_RDFT2, mkplan, 0 }; 167| 72| S *slv = MKSOLVER(S, &sadt); ------------------ | | 604| 72|#define MKSOLVER(type, adt) (type *)X(mksolver)(sizeof(type), adt) | | ------------------ | | | | 78| 72|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 72|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 168| 72| slv->k = k; 169| 72| slv->desc = desc; 170| 72| return &(slv->super); 171| 72|} fftw_rdft_generic_register: 229| 1|{ 230| 1| REGISTER_SOLVER(p, mksolver(R2HC)); ------------------ | | 95| 1|#define REGISTER_SOLVER(p, s) X(solver_register)(p, s) | | ------------------ | | | | 78| 1|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 231| 1| REGISTER_SOLVER(p, mksolver(HC2R)); ------------------ | | 95| 1|#define REGISTER_SOLVER(p, s) X(solver_register)(p, s) | | ------------------ | | | | 78| 1|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 232| 1|} fftw_regsolver_hc2hc_direct: 276| 46|{ 277| 46| regone(plnr, codelet, desc, /* bufferedp */0); 278| 46| regone(plnr, codelet, desc, /* bufferedp */1); 279| 46|} hc2hc-direct.c:regone: 259| 92|{ 260| 92| S *slv = (S *)X(mksolver_hc2hc)(sizeof(S), desc->radix, mkcldw); ------------------ | | 78| 92|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 92|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 261| 92| slv->k = codelet; 262| 92| slv->desc = desc; 263| 92| slv->bufferedp = bufferedp; 264| 92| REGISTER_SOLVER(plnr, &(slv->super.super)); ------------------ | | 95| 92|#define REGISTER_SOLVER(p, s) X(solver_register)(p, s) | | ------------------ | | | | 78| 92|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 92|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 265| 92| if (X(mksolver_hc2hc_hook)) { ------------------ | | 78| 92|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 92|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | | | | Branch (63:30): [True: 0, False: 92] | | | | ------------------ | | ------------------ ------------------ 266| 0| slv = (S *)X(mksolver_hc2hc_hook)(sizeof(S), desc->radix, mkcldw); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 267| 0| slv->k = codelet; 268| 0| slv->desc = desc; 269| 0| slv->bufferedp = bufferedp; 270| 0| REGISTER_SOLVER(plnr, &(slv->super.super)); ------------------ | | 95| 0|#define REGISTER_SOLVER(p, s) X(solver_register)(p, s) | | ------------------ | | | | 78| 0|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 271| 0| } 272| 92|} fftw_hc2hc_generic_register: 320| 1|{ 321| 1| regsolver(p, 0); 322| 1|} hc2hc-generic.c:regsolver: 310| 1|{ 311| 1| S *slv = (S *)X(mksolver_hc2hc)(sizeof(S), r, mkcldw); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 312| 1| REGISTER_SOLVER(plnr, &(slv->super)); ------------------ | | 95| 1|#define REGISTER_SOLVER(p, s) X(solver_register)(p, s) | | ------------------ | | | | 78| 1|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 313| 1| if (X(mksolver_hc2hc_hook)) { ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | | | | Branch (63:30): [True: 0, False: 1] | | | | ------------------ | | ------------------ ------------------ 314| 0| slv = (S *)X(mksolver_hc2hc_hook)(sizeof(S), r, mkcldw); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 315| 0| REGISTER_SOLVER(plnr, &(slv->super)); ------------------ | | 95| 0|#define REGISTER_SOLVER(p, s) X(solver_register)(p, s) | | ------------------ | | | | 78| 0|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 316| 0| } 317| 1|} fftw_hc2hc_applicable: 102| 31.1k|{ 103| 31.1k| const problem_rdft *p; 104| | 105| 31.1k| if (!applicable0(ego, p_, plnr)) ------------------ | Branch (105:10): [True: 31.1k, False: 0] ------------------ 106| 31.1k| return 0; 107| | 108| 0| p = (const problem_rdft *) p_; 109| | 110| 0| return (0 ------------------ | Branch (110:14): [Folded, False: 0] ------------------ 111| 0| || p->vecsz->rnk == 0 ------------------ | Branch (111:10): [True: 0, False: 0] ------------------ 112| 0| || !NO_VRECURSEP(plnr) ------------------ | | 686| 0|#define NO_VRECURSEP(plnr) (PLNR_L(plnr) & NO_VRECURSE) | | ------------------ | | | | 674| 0|#define PLNR_L(plnr) ((plnr)->flags.l) | | ------------------ ------------------ | Branch (112:10): [True: 0, False: 0] ------------------ 113| 0| ); 114| 31.1k|} fftw_mksolver_hc2hc: 198| 93|{ 199| 93| static const solver_adt sadt = { PROBLEM_RDFT, mkplan, 0 }; 200| 93| hc2hc_solver *slv = (hc2hc_solver *)X(mksolver)(size, &sadt); ------------------ | | 78| 93|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 93|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 201| 93| slv->r = r; 202| 93| slv->mkcldw = mkcldw; 203| 93| return slv; 204| 93|} hc2hc.c:applicable0: 81| 31.1k|{ 82| 31.1k| const problem_rdft *p = (const problem_rdft *) p_; 83| 31.1k| INT r; 84| | 85| 31.1k| return (1 ------------------ | Branch (85:14): [True: 31.1k, Folded] ------------------ 86| 31.1k| && p->sz->rnk == 1 ------------------ | Branch (86:10): [True: 0, False: 31.1k] ------------------ 87| 0| && p->vecsz->rnk <= 1 ------------------ | Branch (87:10): [True: 0, False: 0] ------------------ 88| | 89| 0| && (/* either the problem is R2HC, which is solved by DIT */ 90| 0| (p->kind[0] == R2HC) ------------------ | | 48| 0|#define R2HC R2HC00 ------------------ | Branch (90:5): [True: 0, False: 0] ------------------ 91| 0| || 92| | /* or the problem is HC2R, in which case it is solved 93| | by DIF, which destroys the input */ 94| 0| (p->kind[0] == HC2R && ------------------ | | 49| 0|#define HC2R HC2R00 ------------------ | Branch (94:6): [True: 0, False: 0] ------------------ 95| 0| (p->I == p->O || !NO_DESTROY_INPUTP(plnr)))) ------------------ | | 695| 0|#define NO_DESTROY_INPUTP(plnr) (PLNR_L(plnr) & NO_DESTROY_INPUT) | | ------------------ | | | | 674| 0|#define PLNR_L(plnr) ((plnr)->flags.l) | | ------------------ ------------------ | Branch (95:7): [True: 0, False: 0] | Branch (95:23): [True: 0, False: 0] ------------------ 96| | 97| 0| && ((r = X(choose_radix)(ego->r, p->sz->dims[0].n)) > 0) ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ | Branch (97:10): [True: 0, False: 0] ------------------ 98| 0| && p->sz->dims[0].n > r); ------------------ | Branch (98:10): [True: 0, False: 0] ------------------ 99| 31.1k|} hc2hc.c:mkplan: 117| 31.1k|{ 118| 31.1k| const hc2hc_solver *ego = (const hc2hc_solver *) ego_; 119| 31.1k| const problem_rdft *p; 120| 31.1k| P *pln = 0; 121| 31.1k| plan *cld = 0, *cldw = 0; 122| 31.1k| INT n, r, m, v, ivs, ovs; 123| 31.1k| iodim *d; 124| | 125| 31.1k| static const plan_adt padt = { 126| 31.1k| X(rdft_solve), awake, print, destroy ------------------ | | 78| 31.1k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 31.1k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 127| 31.1k| }; 128| | 129| 31.1k| if (NO_NONTHREADEDP(plnr) || !X(hc2hc_applicable)(ego, p_, plnr)) ------------------ | | 693| 62.3k| ((PLNR_L(plnr) & NO_NONTHREADED) && (plnr)->nthr > 1) | | ------------------ | | | | 674| 31.1k|#define PLNR_L(plnr) ((plnr)->flags.l) | | ------------------ | | | Branch (693:4): [True: 31.1k, False: 0] | | | Branch (693:39): [True: 0, False: 31.1k] | | ------------------ ------------------ if (NO_NONTHREADEDP(plnr) || !X(hc2hc_applicable)(ego, p_, plnr)) ------------------ | | 78| 31.1k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 31.1k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ | Branch (129:35): [True: 31.1k, False: 0] ------------------ 130| 31.1k| return (plan *) 0; 131| | 132| 0| p = (const problem_rdft *) p_; 133| 0| d = p->sz->dims; 134| 0| n = d[0].n; 135| 0| r = X(choose_radix)(ego->r, n); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 136| 0| m = n / r; 137| | 138| 0| X(tensor_tornk1)(p->vecsz, &v, &ivs, &ovs); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 139| | 140| 0| switch (p->kind[0]) { 141| 0| case R2HC: ------------------ | | 48| 0|#define R2HC R2HC00 ------------------ | Branch (141:3): [True: 0, False: 0] ------------------ 142| 0| cldw = ego->mkcldw(ego, 143| 0| R2HC, r, m, d[0].os, v, ovs, 0, (m+2)/2, ------------------ | | 48| 0|#define R2HC R2HC00 ------------------ 144| 0| p->O, plnr); 145| 0| if (!cldw) goto nada; ------------------ | Branch (145:12): [True: 0, False: 0] ------------------ 146| | 147| 0| cld = X(mkplan_d)(plnr, ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 148| 0| X(mkproblem_rdft_d)( ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 149| 0| X(mktensor_1d)(m, r * d[0].is, d[0].os), ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 150| 0| X(mktensor_2d)(r, d[0].is, m * d[0].os, ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 151| 0| v, ivs, ovs), 152| 0| p->I, p->O, p->kind) 153| 0| ); 154| 0| if (!cld) goto nada; ------------------ | Branch (154:12): [True: 0, False: 0] ------------------ 155| | 156| 0| pln = MKPLAN_RDFT(P, &padt, apply_dit); ------------------ | | 73| 0| (type *)X(mkplan_rdft)(sizeof(type), adt, apply) | | ------------------ | | | | 78| 0|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 157| 0| break; 158| | 159| 0| case HC2R: ------------------ | | 49| 0|#define HC2R HC2R00 ------------------ | Branch (159:3): [True: 0, False: 0] ------------------ 160| 0| cldw = ego->mkcldw(ego, 161| 0| HC2R, r, m, d[0].is, v, ivs, 0, (m+2)/2, ------------------ | | 49| 0|#define HC2R HC2R00 ------------------ 162| 0| p->I, plnr); 163| 0| if (!cldw) goto nada; ------------------ | Branch (163:12): [True: 0, False: 0] ------------------ 164| | 165| 0| cld = X(mkplan_d)(plnr, ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 166| 0| X(mkproblem_rdft_d)( ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 167| 0| X(mktensor_1d)(m, d[0].is, r * d[0].os), ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 168| 0| X(mktensor_2d)(r, m * d[0].is, d[0].os, ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 169| 0| v, ivs, ovs), 170| 0| p->I, p->O, p->kind) 171| 0| ); 172| 0| if (!cld) goto nada; ------------------ | Branch (172:12): [True: 0, False: 0] ------------------ 173| | 174| 0| pln = MKPLAN_RDFT(P, &padt, apply_dif); ------------------ | | 73| 0| (type *)X(mkplan_rdft)(sizeof(type), adt, apply) | | ------------------ | | | | 78| 0|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 175| 0| break; 176| | 177| 0| default: ------------------ | Branch (177:3): [True: 0, False: 0] ------------------ 178| 0| A(0); 179| 0| } 180| | 181| 0| pln->cld = cld; 182| 0| pln->cldw = cldw; 183| 0| pln->r = r; 184| 0| X(ops_add)(&cld->ops, &cldw->ops, &pln->super.super.ops); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 185| | 186| | /* inherit could_prune_now_p attribute from cldw */ 187| 0| pln->super.super.could_prune_now_p = cldw->could_prune_now_p; 188| | 189| 0| return &(pln->super.super); 190| | 191| 0| nada: 192| 0| X(plan_destroy_internal)(cldw); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 193| 0| X(plan_destroy_internal)(cld); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 194| 0| return (plan *) 0; 195| 0|} fftw_rdft_indirect_register: 226| 1|{ 227| 1| unsigned i; 228| 1| static const ndrct_adt *const adts[] = { 229| 1| &adt_before, &adt_after 230| 1| }; 231| | 232| 3| for (i = 0; i < sizeof(adts) / sizeof(adts[0]); ++i) ------------------ | Branch (232:18): [True: 2, False: 1] ------------------ 233| 2| REGISTER_SOLVER(p, mksolver(adts[i])); ------------------ | | 95| 2|#define REGISTER_SOLVER(p, s) X(solver_register)(p, s) | | ------------------ | | | | 78| 2|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 2|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 234| 1|} fftw_khc2c_register: 26| 56|{ 27| 56| X(regsolver_hc2c_direct)(p, codelet, desc, hc2ckind); ------------------ | | 78| 56|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 56|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 28| 56|} fftw_khc2hc_register: 25| 46|{ 26| 46| X(regsolver_hc2hc_direct)(p, codelet, desc); ------------------ | | 78| 46|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 46|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 27| 46|} fftw_kr2c_register: 25| 72|{ 26| 72| REGISTER_SOLVER(p, X(mksolver_rdft_r2c_direct)(codelet, desc)); ------------------ | | 95| 72|#define REGISTER_SOLVER(p, s) X(solver_register)(p, s) | | ------------------ | | | | 78| 72|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 72|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 27| 72| REGISTER_SOLVER(p, X(mksolver_rdft_r2c_directbuf)(codelet, desc)); ------------------ | | 95| 72|#define REGISTER_SOLVER(p, s) X(solver_register)(p, s) | | ------------------ | | | | 78| 72|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 72|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 28| 72| REGISTER_SOLVER(p, X(mksolver_rdft2_direct)(codelet, desc)); ------------------ | | 95| 72|#define REGISTER_SOLVER(p, s) X(solver_register)(p, s) | | ------------------ | | | | 78| 72|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 72|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 29| 72|} fftw_kr2r_register: 25| 2|{ 26| 2| REGISTER_SOLVER(p, X(mksolver_rdft_r2r_direct)(codelet, desc)); ------------------ | | 95| 2|#define REGISTER_SOLVER(p, s) X(solver_register)(p, s) | | ------------------ | | | | 78| 2|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 2|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 27| 2|} fftw_rdft_nop_register: 80| 1|{ 81| 1| REGISTER_SOLVER(p, mksolver()); ------------------ | | 95| 1|#define REGISTER_SOLVER(p, s) X(solver_register)(p, s) | | ------------------ | | | | 78| 1|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 82| 1|} fftw_rdft2_nop_register: 88| 1|{ 89| 1| REGISTER_SOLVER(p, mksolver()); ------------------ | | 95| 1|#define REGISTER_SOLVER(p, s) X(solver_register)(p, s) | | ------------------ | | | | 78| 1|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 90| 1|} nop2.c:mksolver: 82| 1|{ 83| 1| static const solver_adt sadt = { PROBLEM_RDFT2, mkplan, 0 }; 84| 1| return MKSOLVER(solver, &sadt); ------------------ | | 604| 1|#define MKSOLVER(type, adt) (type *)X(mksolver)(sizeof(type), adt) | | ------------------ | | | | 78| 1|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 85| 1|} fftw_mkplan_rdft: 25| 546|{ 26| 546| plan_rdft *ego; 27| | 28| 546| ego = (plan_rdft *) X(mkplan)(size, adt); ------------------ | | 78| 546|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 546|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 29| 546| ego->apply = apply; 30| | 31| 546| return &(ego->super); 32| 546|} fftw_mkproblem_rdft: 137| 417|{ 138| 417| problem_rdft *ego; 139| 417| int rnk = sz->rnk; 140| 417| int i; 141| | 142| 417| A(X(tensor_kosherp)(sz)); 143| 417| A(X(tensor_kosherp)(vecsz)); 144| 417| A(FINITE_RNK(sz->rnk)); 145| | 146| 417| if (UNTAINT(I) == UNTAINT(O)) ------------------ | | 1037| 417|#define UNTAINT(p) (p) ------------------ if (UNTAINT(I) == UNTAINT(O)) ------------------ | | 1037| 417|#define UNTAINT(p) (p) ------------------ | Branch (146:10): [True: 185, False: 232] ------------------ 147| 185| I = O = JOIN_TAINT(I, O); ------------------ | | 1039| 185|#define JOIN_TAINT(p1, p2) p1 ------------------ 148| | 149| 417| if (I == O && !X(tensor_inplace_locations)(sz, vecsz)) ------------------ | | 78| 185|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 185|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ | Branch (149:10): [True: 185, False: 232] | Branch (149:20): [True: 0, False: 185] ------------------ 150| 0| return X(mkproblem_unsolvable)(); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 151| | 152| 417| for (i = rnk = 0; i < sz->rnk; ++i) { ------------------ | Branch (152:24): [True: 0, False: 417] ------------------ 153| 0| A(sz->dims[i].n > 0); 154| 0| if (nontrivial(sz->dims + i, kind[i])) ------------------ | Branch (154:15): [True: 0, False: 0] ------------------ 155| 0| ++rnk; 156| 0| } 157| | 158| 417|#if defined(STRUCT_HACK_KR) 159| 417| ego = (problem_rdft *) X(mkproblem)(sizeof(problem_rdft) ------------------ | | 78| 417|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 417|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 160| 417| + sizeof(rdft_kind) 161| 417| * (rnk > 0 ? rnk - 1u : 0u), &padt); ------------------ | Branch (161:10): [True: 0, False: 417] ------------------ 162| |#elif defined(STRUCT_HACK_C99) 163| | ego = (problem_rdft *) X(mkproblem)(sizeof(problem_rdft) 164| | + sizeof(rdft_kind) * (unsigned)rnk, &padt); 165| |#else 166| | ego = (problem_rdft *) X(mkproblem)(sizeof(problem_rdft), &padt); 167| | ego->kind = (rdft_kind *) MALLOC(sizeof(rdft_kind) * (unsigned)rnk, PROBLEMS); 168| |#endif 169| | 170| | /* do compression and sorting as in X(tensor_compress), but take 171| | transform kind into account (sigh) */ 172| 417| ego->sz = X(mktensor)(rnk); ------------------ | | 78| 417|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 417|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 173| 417| for (i = rnk = 0; i < sz->rnk; ++i) { ------------------ | Branch (173:24): [True: 0, False: 417] ------------------ 174| 0| if (nontrivial(sz->dims + i, kind[i])) { ------------------ | Branch (174:15): [True: 0, False: 0] ------------------ 175| 0| ego->kind[rnk] = kind[i]; 176| 0| ego->sz->dims[rnk++] = sz->dims[i]; 177| 0| } 178| 0| } 179| 417| for (i = 0; i + 1 < rnk; ++i) { ------------------ | Branch (179:18): [True: 0, False: 417] ------------------ 180| 0| int j; 181| 0| for (j = i + 1; j < rnk; ++j) ------------------ | Branch (181:20): [True: 0, False: 0] ------------------ 182| 0| if (X(dimcmp)(ego->sz->dims + i, ego->sz->dims + j) > 0) { ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ | Branch (182:13): [True: 0, False: 0] ------------------ 183| 0| iodim dswap; 184| 0| rdft_kind kswap; 185| 0| dswap = ego->sz->dims[i]; 186| 0| ego->sz->dims[i] = ego->sz->dims[j]; 187| 0| ego->sz->dims[j] = dswap; 188| 0| kswap = ego->kind[i]; 189| 0| ego->kind[i] = ego->kind[j]; 190| 0| ego->kind[j] = kswap; 191| 0| } 192| 0| } 193| | 194| 417| for (i = 0; i < rnk; ++i) ------------------ | Branch (194:18): [True: 0, False: 417] ------------------ 195| 0| if (ego->sz->dims[i].n == 2 && (ego->kind[i] == REDFT00 ------------------ | Branch (195:8): [True: 0, False: 0] | Branch (195:36): [True: 0, False: 0] ------------------ 196| 0| || ego->kind[i] == DHT ------------------ | Branch (196:11): [True: 0, False: 0] ------------------ 197| 0| || ego->kind[i] == HC2R)) ------------------ | | 49| 0|#define HC2R HC2R00 ------------------ | Branch (197:11): [True: 0, False: 0] ------------------ 198| 0| ego->kind[i] = R2HC; /* size-2 transforms are equivalent */ ------------------ | | 48| 417|#define R2HC R2HC00 ------------------ 199| | 200| 417| ego->vecsz = X(tensor_compress_contiguous)(vecsz); ------------------ | | 78| 417|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 417|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 201| 417| ego->I = I; 202| 417| ego->O = O; 203| | 204| 417| A(FINITE_RNK(ego->sz->rnk)); 205| | 206| 417| return &(ego->super); 207| 417|} fftw_mkproblem_rdft_1: 221| 417|{ 222| 417| A(sz->rnk <= 1); 223| 417| return X(mkproblem_rdft)(sz, vecsz, I, O, &kind); ------------------ | | 78| 417|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 417|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 224| 417|} problem.c:kind_hash: 36| 417|{ 37| 417| int i; 38| 417| for (i = 0; i < rnk; ++i) ------------------ | Branch (38:18): [True: 0, False: 417] ------------------ 39| 0| X(md5int)(m, kind[i]); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 40| 417|} fftw_rdft2_rank_geq2_register: 230| 1|{ 231| 1| static const int buddies[] = { 1, 0, -2 }; 232| 1| size_t i; 233| | 234| 4| for (i = 0; i < NELEM(buddies); ++i) ------------------ | | 90| 4|#define NELEM(array) ((sizeof(array) / sizeof((array)[0]))) ------------------ | Branch (234:18): [True: 3, False: 1] ------------------ 235| 3| REGISTER_SOLVER(p, mksolver(buddies[i], buddies, NELEM(buddies))); ------------------ | | 95| 3|#define REGISTER_SOLVER(p, s) X(solver_register)(p, s) | | ------------------ | | | | 78| 3|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 3|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 236| | 237| | /* FIXME: Should we try more buddies? See also dft/rank-geq2. */ 238| 1|} rank-geq2-rdft2.c:mksolver: 220| 3|{ 221| 3| static const solver_adt sadt = { PROBLEM_RDFT2, mkplan, 0 }; 222| 3| S *slv = MKSOLVER(S, &sadt); ------------------ | | 604| 3|#define MKSOLVER(type, adt) (type *)X(mksolver)(sizeof(type), adt) | | ------------------ | | | | 78| 3|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 3|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 223| 3| slv->spltrnk = spltrnk; 224| 3| slv->buddies = buddies; 225| 3| slv->nbuddies = nbuddies; 226| 3| return &(slv->super); 227| 3|} fftw_rdft_rank_geq2_register: 199| 1|{ 200| 1| static const int buddies[] = { 1, 0, -2 }; 201| 1| size_t i; 202| | 203| 4| for (i = 0; i < NELEM(buddies); ++i) ------------------ | | 90| 4|#define NELEM(array) ((sizeof(array) / sizeof((array)[0]))) ------------------ | Branch (203:18): [True: 3, False: 1] ------------------ 204| 3| REGISTER_SOLVER(p, mksolver(buddies[i], buddies, NELEM(buddies))); ------------------ | | 95| 3|#define REGISTER_SOLVER(p, s) X(solver_register)(p, s) | | ------------------ | | | | 78| 3|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 3|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 205| | 206| | /* FIXME: Should we try more buddies? See also dft/rank-geq2. */ 207| 1|} fftw_rdft2_rank0_register: 197| 1|{ 198| 1| REGISTER_SOLVER(p, mksolver()); ------------------ | | 95| 1|#define REGISTER_SOLVER(p, s) X(solver_register)(p, s) | | ------------------ | | | | 78| 1|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 199| 1|} rank0-rdft2.c:mksolver: 190| 1|{ 191| 1| static const solver_adt sadt = { PROBLEM_RDFT2, mkplan, 0 }; 192| 1| S *slv = MKSOLVER(S, &sadt); ------------------ | | 604| 1|#define MKSOLVER(type, adt) (type *)X(mksolver)(sizeof(type), adt) | | ------------------ | | | | 78| 1|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 193| 1| return &(slv->super); 194| 1|} fftw_rdft_rank0_register: 346| 1|{ 347| 1| unsigned i; 348| 1| static struct { 349| 1| rdftapply apply; 350| 1| int (*applicable)(const P *, const problem_rdft *); 351| 1| const char *nam; 352| 1| } tab[] = { 353| 1| { apply_memcpy, applicable_memcpy, "rdft-rank0-memcpy" }, 354| 1| { apply_memcpy_loop, applicable_memcpy_loop, 355| 1| "rdft-rank0-memcpy-loop" }, 356| 1| { apply_iter, applicable_iter, "rdft-rank0-iter-ci" }, 357| 1| { apply_cpy2dco, applicable_cpy2dco, "rdft-rank0-iter-co" }, 358| 1| { apply_tiled, applicable_tiled, "rdft-rank0-tiled" }, 359| 1| { apply_tiledbuf, applicable_tiledbuf, "rdft-rank0-tiledbuf" }, ------------------ | | 191| 1|#define applicable_tiledbuf applicable_tiled ------------------ 360| 1| { apply_ip_sq, applicable_ip_sq, "rdft-rank0-ip-sq" }, 361| 1| { 362| 1| apply_ip_sq_tiled, 363| 1| applicable_ip_sq_tiled, 364| 1| "rdft-rank0-ip-sq-tiled" 365| 1| }, 366| 1| { 367| 1| apply_ip_sq_tiledbuf, 368| 1| applicable_ip_sq_tiledbuf, ------------------ | | 289| 1|#define applicable_ip_sq_tiledbuf applicable_ip_sq_tiled ------------------ 369| 1| "rdft-rank0-ip-sq-tiledbuf" 370| 1| }, 371| 1| }; 372| | 373| 10| for (i = 0; i < sizeof(tab) / sizeof(tab[0]); ++i) { ------------------ | Branch (373:18): [True: 9, False: 1] ------------------ 374| 9| static const solver_adt sadt = { PROBLEM_RDFT, mkplan, 0 }; 375| 9| S *slv = MKSOLVER(S, &sadt); ------------------ | | 604| 9|#define MKSOLVER(type, adt) (type *)X(mksolver)(sizeof(type), adt) | | ------------------ | | | | 78| 9|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 9|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 376| 9| slv->apply = tab[i].apply; 377| 9| slv->applicable = tab[i].applicable; 378| 9| slv->nam = tab[i].nam; 379| 9| REGISTER_SOLVER(p, &(slv->super)); ------------------ | | 95| 9|#define REGISTER_SOLVER(p, s) X(solver_register)(p, s) | | ------------------ | | | | 78| 9|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 9|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 380| 9| } 381| 1|} rank0.c:applicable_memcpy: 204| 335|{ 205| 335| return (1 ------------------ | Branch (205:14): [True: 335, Folded] ------------------ 206| 335| && p->I != p->O ------------------ | Branch (206:10): [True: 162, False: 173] ------------------ 207| 162| && pln->rnk == 0 ------------------ | Branch (207:10): [True: 18, False: 144] ------------------ 208| 18| && pln->vl > 2 /* do not bother memcpy-ing complex numbers */ ------------------ | Branch (208:10): [True: 17, False: 1] ------------------ 209| 335| ); 210| 335|} rank0.c:applicable_memcpy_loop: 237| 335|{ 238| 335| return (p->I != p->O ------------------ | Branch (238:14): [True: 162, False: 173] ------------------ 239| 162| && pln->rnk > 0 ------------------ | Branch (239:10): [True: 144, False: 18] ------------------ 240| 144| && pln->vl > 2 /* do not bother memcpy-ing complex numbers */); ------------------ | Branch (240:17): [True: 132, False: 12] ------------------ 241| 335|} rank0.c:apply_iter: 118| 83|{ 119| 83| const P *ego = (const P *) ego_; 120| | 121| 83| switch (ego->rnk) { 122| 14| case 0: ------------------ | Branch (122:3): [True: 14, False: 69] ------------------ 123| 14| X(cpy1d)(I, O, ego->vl, 1, 1, 1); ------------------ | | 78| 14|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 14|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 124| 14| break; 125| 69| case 1: ------------------ | Branch (125:3): [True: 69, False: 14] ------------------ 126| 69| X(cpy1d)(I, O, ------------------ | | 78| 69|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 69|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 127| 69| ego->d[0].n, ego->d[0].is, ego->d[0].os, 128| 69| ego->vl); 129| 69| break; 130| 0| default: ------------------ | Branch (130:3): [True: 0, False: 83] ------------------ 131| 0| copy(ego->d, ego->rnk, ego->vl, I, O, X(cpy2d_ci)); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 132| 0| break; 133| 83| } 134| 83|} rank0.c:applicable_iter: 137| 405|{ 138| 405| UNUSED(pln); ------------------ | | 88| 405|#define UNUSED(x) (void)x ------------------ 139| 405| return (p->I != p->O); 140| 405|} rank0.c:applicable_cpy2dco: 151| 335|{ 152| 335| int rnk = pln->rnk; 153| 335| return (1 ------------------ | Branch (153:14): [True: 335, Folded] ------------------ 154| 335| && p->I != p->O ------------------ | Branch (154:10): [True: 162, False: 173] ------------------ 155| 162| && rnk >= 2 ------------------ | Branch (155:10): [True: 12, False: 150] ------------------ 156| | 157| | /* must not duplicate apply_iter */ 158| 12| && (X(iabs)(pln->d[rnk - 2].is) <= X(iabs)(pln->d[rnk - 1].is) ------------------ | | 78| 12|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 12|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ && (X(iabs)(pln->d[rnk - 2].is) <= X(iabs)(pln->d[rnk - 1].is) ------------------ | | 78| 12|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 12|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ | Branch (158:11): [True: 0, False: 12] ------------------ 159| 12| || 160| 12| X(iabs)(pln->d[rnk - 2].os) <= X(iabs)(pln->d[rnk - 1].os)) ------------------ | | 78| 12|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 12|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ X(iabs)(pln->d[rnk - 2].os) <= X(iabs)(pln->d[rnk - 1].os)) ------------------ | | 78| 12|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 12|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ | Branch (160:4): [True: 12, False: 0] ------------------ 161| 335| ); 162| 335|} rank0.c:applicable_tiled: 173| 670|{ 174| 670| return (1 ------------------ | Branch (174:14): [True: 670, Folded] ------------------ 175| 670| && p->I != p->O ------------------ | Branch (175:10): [True: 324, False: 346] ------------------ 176| 324| && pln->rnk >= 2 ------------------ | Branch (176:10): [True: 24, False: 300] ------------------ 177| | 178| | /* somewhat arbitrary */ 179| 24| && X(compute_tilesz)(pln->vl, 1) > 4 ------------------ | | 78| 24|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 24|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ | Branch (179:10): [True: 24, False: 0] ------------------ 180| 670| ); 181| 670|} rank0.c:applicable_ip_sq: 254| 1.01k|{ 255| 1.01k| return (1 ------------------ | Branch (255:14): [True: 1.01k, Folded] ------------------ 256| 1.01k| && p->I == p->O ------------------ | Branch (256:10): [True: 531, False: 486] ------------------ 257| 531| && pln->rnk >= 2 ------------------ | Branch (257:10): [True: 531, False: 0] ------------------ 258| 531| && transposep(pln)); ------------------ | Branch (258:10): [True: 129, False: 402] ------------------ 259| 1.01k|} rank0.c:transposep: 87| 531|{ 88| 531| int i; 89| | 90| 579| for (i = 0; i < pln->rnk - 2; ++i) ------------------ | Branch (90:18): [True: 48, False: 531] ------------------ 91| 48| if (pln->d[i].is != pln->d[i].os) ------------------ | Branch (91:8): [True: 0, False: 48] ------------------ 92| 0| return 0; 93| | 94| 531| return (pln->d[i].n == pln->d[i+1].n && ------------------ | Branch (94:14): [True: 129, False: 402] ------------------ 95| 129| pln->d[i].is == pln->d[i+1].os && ------------------ | Branch (95:7): [True: 129, False: 0] ------------------ 96| 129| pln->d[i].os == pln->d[i+1].is); ------------------ | Branch (96:7): [True: 129, False: 0] ------------------ 97| 531|} rank0.c:applicable_ip_sq_tiled: 271| 682|{ 272| 682| return (1 ------------------ | Branch (272:14): [True: 682, Folded] ------------------ 273| 682| && applicable_ip_sq(pln, p) ------------------ | Branch (273:10): [True: 90, False: 592] ------------------ 274| | 275| | /* somewhat arbitrary */ 276| 90| && X(compute_tilesz)(pln->vl, 2) > 4 ------------------ | | 78| 90|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 90|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ | Branch (276:10): [True: 90, False: 0] ------------------ 277| 682| ); 278| 682|} rank0.c:mkplan: 315| 3.09k|{ 316| 3.09k| const problem_rdft *p; 317| 3.09k| const S *ego = (const S *) ego_; 318| 3.09k| P *pln; 319| 3.09k| int retval; 320| | 321| 3.09k| static const plan_adt padt = { 322| 3.09k| X(rdft_solve), X(null_awake), print, X(plan_null_destroy) ------------------ | | 78| 3.09k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 3.09k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ X(rdft_solve), X(null_awake), print, X(plan_null_destroy) ------------------ | | 78| 3.09k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 3.09k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ X(rdft_solve), X(null_awake), print, X(plan_null_destroy) ------------------ | | 78| 3.09k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 3.09k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 323| 3.09k| }; 324| | 325| 3.09k| UNUSED(plnr); ------------------ | | 88| 3.09k|#define UNUSED(x) (void)x ------------------ 326| | 327| 3.09k| if (!applicable(ego, p_)) ------------------ | Branch (327:10): [True: 2.55k, False: 546] ------------------ 328| 2.55k| return (plan *) 0; 329| | 330| 546| p = (const problem_rdft *) p_; 331| 546| pln = MKPLAN_RDFT(P, &padt, ego->apply); ------------------ | | 73| 546| (type *)X(mkplan_rdft)(sizeof(type), adt, apply) | | ------------------ | | | | 78| 546|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 546|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 332| | 333| 546| retval = fill_iodim(pln, p); 334| 546| (void)retval; /* UNUSED unless DEBUG */ 335| 546| A(retval); 336| 546| A(pln->vl > 0); /* because FINITE_RNK(p->vecsz->rnk) holds */ 337| 546| pln->nam = ego->nam; 338| | 339| | /* X(tensor_sz)(p->vecsz) loads, X(tensor_sz)(p->vecsz) stores */ 340| 546| X(ops_other)(2 * X(tensor_sz)(p->vecsz), &pln->super.super.ops); ------------------ | | 78| 546|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 546|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ X(ops_other)(2 * X(tensor_sz)(p->vecsz), &pln->super.super.ops); ------------------ | | 78| 546|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 546|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 341| 546| return &(pln->super.super); 342| 3.09k|} rank0.c:applicable: 293| 3.09k|{ 294| 3.09k| const problem_rdft *p = (const problem_rdft *) p_; 295| 3.09k| P pln; 296| 3.09k| return (1 ------------------ | Branch (296:14): [True: 3.09k, Folded] ------------------ 297| 3.09k| && p->sz->rnk == 0 ------------------ | Branch (297:10): [True: 3.09k, False: 0] ------------------ 298| 3.09k| && FINITE_RNK(p->vecsz->rnk) ------------------ | | 442| 6.19k|#define FINITE_RNK(rnk) ((rnk) != RNK_MINFTY) | | ------------------ | | | | 441| 3.09k|#define RNK_MINFTY INT_MAX | | ------------------ | | | Branch (442:25): [True: 3.09k, False: 0] | | ------------------ ------------------ 299| 3.09k| && fill_iodim(&pln, p) ------------------ | Branch (299:10): [True: 3.09k, False: 0] ------------------ 300| 3.09k| && ego->applicable(&pln, p) ------------------ | Branch (300:10): [True: 546, False: 2.55k] ------------------ 301| 3.09k| ); 302| 3.09k|} rank0.c:fill_iodim: 50| 3.64k|{ 51| 3.64k| int i; 52| 3.64k| const tensor *vecsz = p->vecsz; 53| | 54| 3.64k| pln->vl = 1; 55| 3.64k| pln->rnk = 0; 56| 12.7k| for (i = 0; i < vecsz->rnk; ++i) { ------------------ | Branch (56:18): [True: 9.05k, False: 3.64k] ------------------ 57| | /* extract contiguous dimensions */ 58| 9.05k| if (pln->vl == 1 && ------------------ | Branch (58:8): [True: 9.05k, False: 0] ------------------ 59| 9.05k| vecsz->dims[i].is == 1 && vecsz->dims[i].os == 1) ------------------ | Branch (59:8): [True: 3.64k, False: 5.41k] | Branch (59:34): [True: 3.64k, False: 0] ------------------ 60| 3.64k| pln->vl = vecsz->dims[i].n; 61| 5.41k| else if (pln->rnk == MAXRNK) ------------------ | | 30| 5.41k|#define MAXRNK 32 /* FIXME: should malloc() */ ------------------ | Branch (61:13): [True: 0, False: 5.41k] ------------------ 62| 0| return 0; 63| 5.41k| else 64| 5.41k| pln->d[pln->rnk++] = vecsz->dims[i]; 65| 9.05k| } 66| | 67| 3.64k| return 1; 68| 3.64k|} fftw_rdft_dht_register: 218| 1|{ 219| 1| REGISTER_SOLVER(p, mksolver()); ------------------ | | 95| 1|#define REGISTER_SOLVER(p, s) X(solver_register)(p, s) | | ------------------ | | | | 78| 1|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 220| 1|} rdft-dht.c:mksolver: 211| 1|{ 212| 1| static const solver_adt sadt = { PROBLEM_RDFT, mkplan, 0 }; 213| 1| S *slv = MKSOLVER(S, &sadt); ------------------ | | 604| 1|#define MKSOLVER(type, adt) (type *)X(mksolver)(sizeof(type), adt) | | ------------------ | | | | 78| 1|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 214| 1| return &(slv->super); 215| 1|} rdft-dht.c:mkplan: 165| 335|{ 166| 335| P *pln; 167| 335| const problem_rdft *p; 168| 335| problem *cldp; 169| 335| plan *cld; 170| | 171| 335| static const plan_adt padt = { 172| 335| X(rdft_solve), awake, print, destroy ------------------ | | 78| 335|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 335|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 173| 335| }; 174| | 175| 335| if (!applicable(ego_, p_, plnr)) ------------------ | Branch (175:10): [True: 335, False: 0] ------------------ 176| 335| return (plan *)0; 177| | 178| 0| p = (const problem_rdft *) p_; 179| | 180| 0| if (p->kind[0] == R2HC || !NO_DESTROY_INPUTP(plnr)) ------------------ | | 48| 0|#define R2HC R2HC00 ------------------ if (p->kind[0] == R2HC || !NO_DESTROY_INPUTP(plnr)) ------------------ | | 695| 0|#define NO_DESTROY_INPUTP(plnr) (PLNR_L(plnr) & NO_DESTROY_INPUT) | | ------------------ | | | | 674| 0|#define PLNR_L(plnr) ((plnr)->flags.l) | | ------------------ ------------------ | Branch (180:10): [True: 0, False: 0] | Branch (180:32): [True: 0, False: 0] ------------------ 181| 0| cldp = X(mkproblem_rdft_1)(p->sz, p->vecsz, p->I, p->O, DHT); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 182| 0| else { 183| 0| tensor *sz = X(tensor_copy_inplace)(p->sz, INPLACE_OS); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 184| 0| cldp = X(mkproblem_rdft_1)(sz, p->vecsz, p->O, p->O, DHT); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 185| 0| X(tensor_destroy)(sz); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 186| 0| } 187| 0| cld = X(mkplan_d)(plnr, cldp); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 188| 0| if (!cld) return (plan *)0; ------------------ | Branch (188:10): [True: 0, False: 0] ------------------ 189| | 190| 0| pln = MKPLAN_RDFT(P, &padt, p->kind[0] == R2HC ? ------------------ | | 73| 0| (type *)X(mkplan_rdft)(sizeof(type), adt, apply) | | ------------------ | | | | 78| 0|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ | | | Branch (73:45): [True: 0, False: 0] | | | Branch (73:45): [True: 0, False: 0] | | ------------------ ------------------ 191| 0| apply_r2hc : (NO_DESTROY_INPUTP(plnr) ? 192| 0| apply_hc2r_save : apply_hc2r)); 193| 0| pln->n = p->sz->dims[0].n; 194| 0| pln->is = p->sz->dims[0].is; 195| 0| pln->os = p->sz->dims[0].os; 196| 0| pln->cld = cld; 197| | 198| 0| pln->super.super.ops = cld->ops; 199| 0| pln->super.super.ops.other += 4 * ((pln->n - 1)/2); 200| 0| pln->super.super.ops.add += 2 * ((pln->n - 1)/2); 201| 0| if (p->kind[0] == R2HC) ------------------ | | 48| 0|#define R2HC R2HC00 ------------------ | Branch (201:10): [True: 0, False: 0] ------------------ 202| 0| pln->super.super.ops.mul += 2 * ((pln->n - 1)/2); 203| 0| if (pln->super.apply == apply_hc2r_save) ------------------ | Branch (203:10): [True: 0, False: 0] ------------------ 204| 0| pln->super.super.ops.other += 2 + (pln->n % 2 ? 0 : 2); ------------------ | Branch (204:39): [True: 0, False: 0] ------------------ 205| | 206| 0| return &(pln->super.super); 207| 0|} rdft-dht.c:applicable: 160| 335|{ 161| 335| return (!NO_SLOWP(plnr) && applicable0(ego, p_)); ------------------ | | 688| 670|#define NO_SLOWP(plnr) (PLNR_L(plnr) & NO_SLOW) | | ------------------ | | | | 674| 335|#define PLNR_L(plnr) ((plnr)->flags.l) | | ------------------ ------------------ | Branch (161:14): [True: 0, False: 335] | Branch (161:33): [True: 0, False: 0] ------------------ 162| 335|} fftw_rdft2_rdft_register: 326| 1|{ 327| 1| REGISTER_SOLVER(p, mksolver()); ------------------ | | 95| 1|#define REGISTER_SOLVER(p, s) X(solver_register)(p, s) | | ------------------ | | | | 78| 1|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 328| 1|} rdft2-rdft.c:mksolver: 319| 1|{ 320| 1| static const solver_adt sadt = { PROBLEM_RDFT2, mkplan, 0 }; 321| 1| S *slv = MKSOLVER(S, &sadt); ------------------ | | 604| 1|#define MKSOLVER(type, adt) (type *)X(mksolver)(sizeof(type), adt) | | ------------------ | | | | 78| 1|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 322| 1| return &(slv->super); 323| 1|} fftw_codelet_hb2_16: 855| 1|void X(codelet_hb2_16) (planner *p) { 856| 1| X(khc2hc_register) (p, hb2_16, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 857| 1|} fftw_codelet_hb2_20: 1084| 1|void X(codelet_hb2_20) (planner *p) { 1085| 1| X(khc2hc_register) (p, hb2_20, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 1086| 1|} fftw_codelet_hb2_25: 1639| 1|void X(codelet_hb2_25) (planner *p) { 1640| 1| X(khc2hc_register) (p, hb2_25, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 1641| 1|} fftw_codelet_hb2_32: 1879| 1|void X(codelet_hb2_32) (planner *p) { 1880| 1| X(khc2hc_register) (p, hb2_32, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 1881| 1|} fftw_codelet_hb2_4: 191| 1|void X(codelet_hb2_4) (planner *p) { 192| 1| X(khc2hc_register) (p, hb2_4, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 193| 1|} fftw_codelet_hb2_5: 276| 1|void X(codelet_hb2_5) (planner *p) { 277| 1| X(khc2hc_register) (p, hb2_5, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 278| 1|} fftw_codelet_hb2_8: 384| 1|void X(codelet_hb2_8) (planner *p) { 385| 1| X(khc2hc_register) (p, hb2_8, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 386| 1|} fftw_codelet_hb_10: 510| 1|void X(codelet_hb_10) (planner *p) { 511| 1| X(khc2hc_register) (p, hb_10, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 512| 1|} fftw_codelet_hb_12: 594| 1|void X(codelet_hb_12) (planner *p) { 595| 1| X(khc2hc_register) (p, hb_12, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 596| 1|} fftw_codelet_hb_15: 807| 1|void X(codelet_hb_15) (planner *p) { 808| 1| X(khc2hc_register) (p, hb_15, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 809| 1|} fftw_codelet_hb_16: 830| 1|void X(codelet_hb_16) (planner *p) { 831| 1| X(khc2hc_register) (p, hb_16, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 832| 1|} fftw_codelet_hb_2: 114| 1|void X(codelet_hb_2) (planner *p) { 115| 1| X(khc2hc_register) (p, hb_2, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 116| 1|} fftw_codelet_hb_20: 1061| 1|void X(codelet_hb_20) (planner *p) { 1062| 1| X(khc2hc_register) (p, hb_20, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 1063| 1|} fftw_codelet_hb_25: 1606| 1|void X(codelet_hb_25) (planner *p) { 1607| 1| X(khc2hc_register) (p, hb_25, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 1608| 1|} fftw_codelet_hb_3: 163| 1|void X(codelet_hb_3) (planner *p) { 164| 1| X(khc2hc_register) (p, hb_3, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 165| 1|} fftw_codelet_hb_32: 1840| 1|void X(codelet_hb_32) (planner *p) { 1841| 1| X(khc2hc_register) (p, hb_32, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 1842| 1|} fftw_codelet_hb_4: 193| 1|void X(codelet_hb_4) (planner *p) { 194| 1| X(khc2hc_register) (p, hb_4, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 195| 1|} fftw_codelet_hb_5: 271| 1|void X(codelet_hb_5) (planner *p) { 272| 1| X(khc2hc_register) (p, hb_5, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 273| 1|} fftw_codelet_hb_6: 289| 1|void X(codelet_hb_6) (planner *p) { 290| 1| X(khc2hc_register) (p, hb_6, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 291| 1|} fftw_codelet_hb_64: 4022| 1|void X(codelet_hb_64) (planner *p) { 4023| 1| X(khc2hc_register) (p, hb_64, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 4024| 1|} fftw_codelet_hb_7: 353| 1|void X(codelet_hb_7) (planner *p) { 354| 1| X(khc2hc_register) (p, hb_7, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 355| 1|} fftw_codelet_hb_8: 370| 1|void X(codelet_hb_8) (planner *p) { 371| 1| X(khc2hc_register) (p, hb_8, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 372| 1|} fftw_codelet_hb_9: 494| 1|void X(codelet_hb_9) (planner *p) { 495| 1| X(khc2hc_register) (p, hb_9, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 496| 1|} fftw_codelet_hc2cb2_16: 855| 1|void X(codelet_hc2cb2_16) (planner *p) { 856| 1| X(khc2c_register) (p, hc2cb2_16, &desc, HC2C_VIA_RDFT); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 857| 1|} fftw_codelet_hc2cb2_20: 1084| 1|void X(codelet_hc2cb2_20) (planner *p) { 1085| 1| X(khc2c_register) (p, hc2cb2_20, &desc, HC2C_VIA_RDFT); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 1086| 1|} fftw_codelet_hc2cb2_32: 1879| 1|void X(codelet_hc2cb2_32) (planner *p) { 1880| 1| X(khc2c_register) (p, hc2cb2_32, &desc, HC2C_VIA_RDFT); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 1881| 1|} fftw_codelet_hc2cb2_4: 191| 1|void X(codelet_hc2cb2_4) (planner *p) { 192| 1| X(khc2c_register) (p, hc2cb2_4, &desc, HC2C_VIA_RDFT); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 193| 1|} fftw_codelet_hc2cb2_8: 384| 1|void X(codelet_hc2cb2_8) (planner *p) { 385| 1| X(khc2c_register) (p, hc2cb2_8, &desc, HC2C_VIA_RDFT); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 386| 1|} fftw_codelet_hc2cb_10: 510| 1|void X(codelet_hc2cb_10) (planner *p) { 511| 1| X(khc2c_register) (p, hc2cb_10, &desc, HC2C_VIA_RDFT); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 512| 1|} fftw_codelet_hc2cb_12: 594| 1|void X(codelet_hc2cb_12) (planner *p) { 595| 1| X(khc2c_register) (p, hc2cb_12, &desc, HC2C_VIA_RDFT); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 596| 1|} fftw_codelet_hc2cb_16: 830| 1|void X(codelet_hc2cb_16) (planner *p) { 831| 1| X(khc2c_register) (p, hc2cb_16, &desc, HC2C_VIA_RDFT); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 832| 1|} fftw_codelet_hc2cb_2: 114| 1|void X(codelet_hc2cb_2) (planner *p) { 115| 1| X(khc2c_register) (p, hc2cb_2, &desc, HC2C_VIA_RDFT); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 116| 1|} fftw_codelet_hc2cb_20: 1061| 1|void X(codelet_hc2cb_20) (planner *p) { 1062| 1| X(khc2c_register) (p, hc2cb_20, &desc, HC2C_VIA_RDFT); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 1063| 1|} fftw_codelet_hc2cb_32: 1840| 1|void X(codelet_hc2cb_32) (planner *p) { 1841| 1| X(khc2c_register) (p, hc2cb_32, &desc, HC2C_VIA_RDFT); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 1842| 1|} fftw_codelet_hc2cb_4: 193| 1|void X(codelet_hc2cb_4) (planner *p) { 194| 1| X(khc2c_register) (p, hc2cb_4, &desc, HC2C_VIA_RDFT); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 195| 1|} fftw_codelet_hc2cb_6: 289| 1|void X(codelet_hc2cb_6) (planner *p) { 290| 1| X(khc2c_register) (p, hc2cb_6, &desc, HC2C_VIA_RDFT); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 291| 1|} fftw_codelet_hc2cb_8: 370| 1|void X(codelet_hc2cb_8) (planner *p) { 371| 1| X(khc2c_register) (p, hc2cb_8, &desc, HC2C_VIA_RDFT); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 372| 1|} fftw_codelet_hc2cbdft2_16: 889| 1|void X(codelet_hc2cbdft2_16) (planner *p) { 890| 1| X(khc2c_register) (p, hc2cbdft2_16, &desc, HC2C_VIA_DFT); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 891| 1|} fftw_codelet_hc2cbdft2_20: 1146| 1|void X(codelet_hc2cbdft2_20) (planner *p) { 1147| 1| X(khc2c_register) (p, hc2cbdft2_20, &desc, HC2C_VIA_DFT); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 1148| 1|} fftw_codelet_hc2cbdft2_32: 1947| 1|void X(codelet_hc2cbdft2_32) (planner *p) { 1948| 1| X(khc2c_register) (p, hc2cbdft2_32, &desc, HC2C_VIA_DFT); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 1949| 1|} fftw_codelet_hc2cbdft2_4: 215| 1|void X(codelet_hc2cbdft2_4) (planner *p) { 216| 1| X(khc2c_register) (p, hc2cbdft2_4, &desc, HC2C_VIA_DFT); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 217| 1|} fftw_codelet_hc2cbdft2_8: 421| 1|void X(codelet_hc2cbdft2_8) (planner *p) { 422| 1| X(khc2c_register) (p, hc2cbdft2_8, &desc, HC2C_VIA_DFT); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 423| 1|} fftw_codelet_hc2cbdft_10: 542| 1|void X(codelet_hc2cbdft_10) (planner *p) { 543| 1| X(khc2c_register) (p, hc2cbdft_10, &desc, HC2C_VIA_DFT); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 544| 1|} fftw_codelet_hc2cbdft_12: 640| 1|void X(codelet_hc2cbdft_12) (planner *p) { 641| 1| X(khc2c_register) (p, hc2cbdft_12, &desc, HC2C_VIA_DFT); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 642| 1|} fftw_codelet_hc2cbdft_16: 889| 1|void X(codelet_hc2cbdft_16) (planner *p) { 890| 1| X(khc2c_register) (p, hc2cbdft_16, &desc, HC2C_VIA_DFT); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 891| 1|} fftw_codelet_hc2cbdft_2: 128| 1|void X(codelet_hc2cbdft_2) (planner *p) { 129| 1| X(khc2c_register) (p, hc2cbdft_2, &desc, HC2C_VIA_DFT); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 130| 1|} fftw_codelet_hc2cbdft_20: 1146| 1|void X(codelet_hc2cbdft_20) (planner *p) { 1147| 1| X(khc2c_register) (p, hc2cbdft_20, &desc, HC2C_VIA_DFT); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 1148| 1|} fftw_codelet_hc2cbdft_32: 1947| 1|void X(codelet_hc2cbdft_32) (planner *p) { 1948| 1| X(khc2c_register) (p, hc2cbdft_32, &desc, HC2C_VIA_DFT); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 1949| 1|} fftw_codelet_hc2cbdft_4: 215| 1|void X(codelet_hc2cbdft_4) (planner *p) { 216| 1| X(khc2c_register) (p, hc2cbdft_4, &desc, HC2C_VIA_DFT); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 217| 1|} fftw_codelet_hc2cbdft_6: 323| 1|void X(codelet_hc2cbdft_6) (planner *p) { 324| 1| X(khc2c_register) (p, hc2cbdft_6, &desc, HC2C_VIA_DFT); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 325| 1|} fftw_codelet_hc2cbdft_8: 421| 1|void X(codelet_hc2cbdft_8) (planner *p) { 422| 1| X(khc2c_register) (p, hc2cbdft_8, &desc, HC2C_VIA_DFT); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 423| 1|} fftw_codelet_r2cbIII_10: 193| 1|void X(codelet_r2cbIII_10) (planner *p) { X(kr2c_register) (p, r2cbIII_10, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 194| 1|} fftw_codelet_r2cbIII_12: 227| 1|void X(codelet_r2cbIII_12) (planner *p) { X(kr2c_register) (p, r2cbIII_12, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 228| 1|} fftw_codelet_r2cbIII_15: 300| 1|void X(codelet_r2cbIII_15) (planner *p) { X(kr2c_register) (p, r2cbIII_15, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 301| 1|} fftw_codelet_r2cbIII_16: 313| 1|void X(codelet_r2cbIII_16) (planner *p) { X(kr2c_register) (p, r2cbIII_16, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 314| 1|} fftw_codelet_r2cbIII_2: 85| 1|void X(codelet_r2cbIII_2) (planner *p) { X(kr2c_register) (p, r2cbIII_2, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 86| 1|} fftw_codelet_r2cbIII_20: 407| 1|void X(codelet_r2cbIII_20) (planner *p) { X(kr2c_register) (p, r2cbIII_20, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 408| 1|} fftw_codelet_r2cbIII_25: 616| 1|void X(codelet_r2cbIII_25) (planner *p) { X(kr2c_register) (p, r2cbIII_25, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 617| 1|} fftw_codelet_r2cbIII_3: 94| 1|void X(codelet_r2cbIII_3) (planner *p) { X(kr2c_register) (p, r2cbIII_3, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 95| 1|} fftw_codelet_r2cbIII_32: 708| 1|void X(codelet_r2cbIII_32) (planner *p) { X(kr2c_register) (p, r2cbIII_32, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 709| 1|} fftw_codelet_r2cbIII_4: 99| 1|void X(codelet_r2cbIII_4) (planner *p) { X(kr2c_register) (p, r2cbIII_4, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 100| 1|} fftw_codelet_r2cbIII_5: 125| 1|void X(codelet_r2cbIII_5) (planner *p) { X(kr2c_register) (p, r2cbIII_5, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 126| 1|} fftw_codelet_r2cbIII_6: 121| 1|void X(codelet_r2cbIII_6) (planner *p) { X(kr2c_register) (p, r2cbIII_6, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 122| 1|} fftw_codelet_r2cbIII_64: 1582| 1|void X(codelet_r2cbIII_64) (planner *p) { X(kr2c_register) (p, r2cbIII_64, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 1583| 1|} fftw_codelet_r2cbIII_7: 145| 1|void X(codelet_r2cbIII_7) (planner *p) { X(kr2c_register) (p, r2cbIII_7, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 146| 1|} fftw_codelet_r2cbIII_8: 161| 1|void X(codelet_r2cbIII_8) (planner *p) { X(kr2c_register) (p, r2cbIII_8, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 162| 1|} fftw_codelet_r2cbIII_9: 206| 1|void X(codelet_r2cbIII_9) (planner *p) { X(kr2c_register) (p, r2cbIII_9, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 207| 1|} fftw_codelet_r2cb_10: 203| 1|void X(codelet_r2cb_10) (planner *p) { X(kr2c_register) (p, r2cb_10, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 204| 1|} fftw_codelet_r2cb_11: 226| 1|void X(codelet_r2cb_11) (planner *p) { X(kr2c_register) (p, r2cb_11, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 227| 1|} fftw_codelet_r2cb_12: 217| 1|void X(codelet_r2cb_12) (planner *p) { X(kr2c_register) (p, r2cb_12, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 218| 1|} fftw_codelet_r2cb_128: 3243| 1|void X(codelet_r2cb_128) (planner *p) { X(kr2c_register) (p, r2cb_128, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 3244| 1|} fftw_codelet_r2cb_13: 366| 1|void X(codelet_r2cb_13) (planner *p) { X(kr2c_register) (p, r2cb_13, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 367| 1|} fftw_codelet_r2cb_14: 255| 1|void X(codelet_r2cb_14) (planner *p) { X(kr2c_register) (p, r2cb_14, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 256| 1|} fftw_codelet_r2cb_15: 290| 1|void X(codelet_r2cb_15) (planner *p) { X(kr2c_register) (p, r2cb_15, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 291| 1|} fftw_codelet_r2cb_16: 292| 1|void X(codelet_r2cb_16) (planner *p) { X(kr2c_register) (p, r2cb_16, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 293| 1|} fftw_codelet_r2cb_2: 83| 1|void X(codelet_r2cb_2) (planner *p) { X(kr2c_register) (p, r2cb_2, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 84| 1|} fftw_codelet_r2cb_20: 366| 1|void X(codelet_r2cb_20) (planner *p) { X(kr2c_register) (p, r2cb_20, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 367| 1|} fftw_codelet_r2cb_25: 616| 1|void X(codelet_r2cb_25) (planner *p) { X(kr2c_register) (p, r2cb_25, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 617| 1|} fftw_codelet_r2cb_3: 94| 1|void X(codelet_r2cb_3) (planner *p) { X(kr2c_register) (p, r2cb_3, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 95| 1|} fftw_codelet_r2cb_32: 634| 1|void X(codelet_r2cb_32) (planner *p) { X(kr2c_register) (p, r2cb_32, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 635| 1|} fftw_codelet_r2cb_4: 102| 1|void X(codelet_r2cb_4) (planner *p) { X(kr2c_register) (p, r2cb_4, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 103| 1|} fftw_codelet_r2cb_5: 125| 1|void X(codelet_r2cb_5) (planner *p) { X(kr2c_register) (p, r2cb_5, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 126| 1|} fftw_codelet_r2cb_6: 125| 1|void X(codelet_r2cb_6) (planner *p) { X(kr2c_register) (p, r2cb_6, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 126| 1|} fftw_codelet_r2cb_64: 1407| 1|void X(codelet_r2cb_64) (planner *p) { X(kr2c_register) (p, r2cb_64, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 1408| 1|} fftw_codelet_r2cb_7: 145| 1|void X(codelet_r2cb_7) (planner *p) { X(kr2c_register) (p, r2cb_7, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 146| 1|} fftw_codelet_r2cb_8: 152| 1|void X(codelet_r2cb_8) (planner *p) { X(kr2c_register) (p, r2cb_8, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 153| 1|} fftw_codelet_r2cb_9: 206| 1|void X(codelet_r2cb_9) (planner *p) { X(kr2c_register) (p, r2cb_9, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 207| 1|} fftw_codelet_hc2cf2_16: 833| 1|void X(codelet_hc2cf2_16) (planner *p) { 834| 1| X(khc2c_register) (p, hc2cf2_16, &desc, HC2C_VIA_RDFT); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 835| 1|} fftw_codelet_hc2cf2_20: 1094| 1|void X(codelet_hc2cf2_20) (planner *p) { 1095| 1| X(khc2c_register) (p, hc2cf2_20, &desc, HC2C_VIA_RDFT); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 1096| 1|} fftw_codelet_hc2cf2_32: 1890| 1|void X(codelet_hc2cf2_32) (planner *p) { 1891| 1| X(khc2c_register) (p, hc2cf2_32, &desc, HC2C_VIA_RDFT); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 1892| 1|} fftw_codelet_hc2cf2_4: 197| 1|void X(codelet_hc2cf2_4) (planner *p) { 198| 1| X(khc2c_register) (p, hc2cf2_4, &desc, HC2C_VIA_RDFT); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 199| 1|} fftw_codelet_hc2cf2_8: 387| 1|void X(codelet_hc2cf2_8) (planner *p) { 388| 1| X(khc2c_register) (p, hc2cf2_8, &desc, HC2C_VIA_RDFT); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 389| 1|} fftw_codelet_hc2cf_10: 486| 1|void X(codelet_hc2cf_10) (planner *p) { 487| 1| X(khc2c_register) (p, hc2cf_10, &desc, HC2C_VIA_RDFT); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 488| 1|} fftw_codelet_hc2cf_12: 578| 1|void X(codelet_hc2cf_12) (planner *p) { 579| 1| X(khc2c_register) (p, hc2cf_12, &desc, HC2C_VIA_RDFT); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 580| 1|} fftw_codelet_hc2cf_16: 793| 1|void X(codelet_hc2cf_16) (planner *p) { 794| 1| X(khc2c_register) (p, hc2cf_16, &desc, HC2C_VIA_RDFT); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 795| 1|} fftw_codelet_hc2cf_2: 114| 1|void X(codelet_hc2cf_2) (planner *p) { 115| 1| X(khc2c_register) (p, hc2cf_2, &desc, HC2C_VIA_RDFT); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 116| 1|} fftw_codelet_hc2cf_20: 1047| 1|void X(codelet_hc2cf_20) (planner *p) { 1048| 1| X(khc2c_register) (p, hc2cf_20, &desc, HC2C_VIA_RDFT); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 1049| 1|} fftw_codelet_hc2cf_32: 1806| 1|void X(codelet_hc2cf_32) (planner *p) { 1807| 1| X(khc2c_register) (p, hc2cf_32, &desc, HC2C_VIA_RDFT); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 1808| 1|} fftw_codelet_hc2cf_4: 193| 1|void X(codelet_hc2cf_4) (planner *p) { 194| 1| X(khc2c_register) (p, hc2cf_4, &desc, HC2C_VIA_RDFT); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 195| 1|} fftw_codelet_hc2cf_6: 292| 1|void X(codelet_hc2cf_6) (planner *p) { 293| 1| X(khc2c_register) (p, hc2cf_6, &desc, HC2C_VIA_RDFT); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 294| 1|} fftw_codelet_hc2cf_8: 373| 1|void X(codelet_hc2cf_8) (planner *p) { 374| 1| X(khc2c_register) (p, hc2cf_8, &desc, HC2C_VIA_RDFT); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 375| 1|} fftw_codelet_hc2cfdft2_16: 934| 1|void X(codelet_hc2cfdft2_16) (planner *p) { 935| 1| X(khc2c_register) (p, hc2cfdft2_16, &desc, HC2C_VIA_DFT); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 936| 1|} fftw_codelet_hc2cfdft2_20: 1200| 1|void X(codelet_hc2cfdft2_20) (planner *p) { 1201| 1| X(khc2c_register) (p, hc2cfdft2_20, &desc, HC2C_VIA_DFT); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 1202| 1|} fftw_codelet_hc2cfdft2_32: 2054| 1|void X(codelet_hc2cfdft2_32) (planner *p) { 2055| 1| X(khc2c_register) (p, hc2cfdft2_32, &desc, HC2C_VIA_DFT); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 2056| 1|} fftw_codelet_hc2cfdft2_4: 218| 1|void X(codelet_hc2cfdft2_4) (planner *p) { 219| 1| X(khc2c_register) (p, hc2cfdft2_4, &desc, HC2C_VIA_DFT); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 220| 1|} fftw_codelet_hc2cfdft2_8: 439| 1|void X(codelet_hc2cfdft2_8) (planner *p) { 440| 1| X(khc2c_register) (p, hc2cfdft2_8, &desc, HC2C_VIA_DFT); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 441| 1|} fftw_codelet_hc2cfdft_10: 543| 1|void X(codelet_hc2cfdft_10) (planner *p) { 544| 1| X(khc2c_register) (p, hc2cfdft_10, &desc, HC2C_VIA_DFT); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 545| 1|} fftw_codelet_hc2cfdft_12: 643| 1|void X(codelet_hc2cfdft_12) (planner *p) { 644| 1| X(khc2c_register) (p, hc2cfdft_12, &desc, HC2C_VIA_DFT); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 645| 1|} fftw_codelet_hc2cfdft_16: 906| 1|void X(codelet_hc2cfdft_16) (planner *p) { 907| 1| X(khc2c_register) (p, hc2cfdft_16, &desc, HC2C_VIA_DFT); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 908| 1|} fftw_codelet_hc2cfdft_2: 130| 1|void X(codelet_hc2cfdft_2) (planner *p) { 131| 1| X(khc2c_register) (p, hc2cfdft_2, &desc, HC2C_VIA_DFT); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 132| 1|} fftw_codelet_hc2cfdft_20: 1152| 1|void X(codelet_hc2cfdft_20) (planner *p) { 1153| 1| X(khc2c_register) (p, hc2cfdft_20, &desc, HC2C_VIA_DFT); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 1154| 1|} fftw_codelet_hc2cfdft_32: 1980| 1|void X(codelet_hc2cfdft_32) (planner *p) { 1981| 1| X(khc2c_register) (p, hc2cfdft_32, &desc, HC2C_VIA_DFT); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 1982| 1|} fftw_codelet_hc2cfdft_4: 215| 1|void X(codelet_hc2cfdft_4) (planner *p) { 216| 1| X(khc2c_register) (p, hc2cfdft_4, &desc, HC2C_VIA_DFT); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 217| 1|} fftw_codelet_hc2cfdft_6: 336| 1|void X(codelet_hc2cfdft_6) (planner *p) { 337| 1| X(khc2c_register) (p, hc2cfdft_6, &desc, HC2C_VIA_DFT); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 338| 1|} fftw_codelet_hc2cfdft_8: 434| 1|void X(codelet_hc2cfdft_8) (planner *p) { 435| 1| X(khc2c_register) (p, hc2cfdft_8, &desc, HC2C_VIA_DFT); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 436| 1|} fftw_codelet_hf2_16: 833| 1|void X(codelet_hf2_16) (planner *p) { 834| 1| X(khc2hc_register) (p, hf2_16, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 835| 1|} fftw_codelet_hf2_20: 1094| 1|void X(codelet_hf2_20) (planner *p) { 1095| 1| X(khc2hc_register) (p, hf2_20, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 1096| 1|} fftw_codelet_hf2_25: 1617| 1|void X(codelet_hf2_25) (planner *p) { 1618| 1| X(khc2hc_register) (p, hf2_25, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 1619| 1|} fftw_codelet_hf2_32: 1890| 1|void X(codelet_hf2_32) (planner *p) { 1891| 1| X(khc2hc_register) (p, hf2_32, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 1892| 1|} fftw_codelet_hf2_4: 197| 1|void X(codelet_hf2_4) (planner *p) { 198| 1| X(khc2hc_register) (p, hf2_4, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 199| 1|} fftw_codelet_hf2_5: 261| 1|void X(codelet_hf2_5) (planner *p) { 262| 1| X(khc2hc_register) (p, hf2_5, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 263| 1|} fftw_codelet_hf2_8: 387| 1|void X(codelet_hf2_8) (planner *p) { 388| 1| X(khc2hc_register) (p, hf2_8, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 389| 1|} fftw_codelet_hf_10: 486| 1|void X(codelet_hf_10) (planner *p) { 487| 1| X(khc2hc_register) (p, hf_10, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 488| 1|} fftw_codelet_hf_12: 578| 1|void X(codelet_hf_12) (planner *p) { 579| 1| X(khc2hc_register) (p, hf_12, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 580| 1|} fftw_codelet_hf_15: 813| 1|void X(codelet_hf_15) (planner *p) { 814| 1| X(khc2hc_register) (p, hf_15, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 815| 1|} fftw_codelet_hf_16: 793| 1|void X(codelet_hf_16) (planner *p) { 794| 1| X(khc2hc_register) (p, hf_16, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 795| 1|} fftw_codelet_hf_2: 114| 1|void X(codelet_hf_2) (planner *p) { 115| 1| X(khc2hc_register) (p, hf_2, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 116| 1|} fftw_codelet_hf_20: 1047| 1|void X(codelet_hf_20) (planner *p) { 1048| 1| X(khc2hc_register) (p, hf_20, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 1049| 1|} fftw_codelet_hf_25: 1569| 1|void X(codelet_hf_25) (planner *p) { 1570| 1| X(khc2hc_register) (p, hf_25, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 1571| 1|} fftw_codelet_hf_3: 163| 1|void X(codelet_hf_3) (planner *p) { 164| 1| X(khc2hc_register) (p, hf_3, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 165| 1|} fftw_codelet_hf_32: 1806| 1|void X(codelet_hf_32) (planner *p) { 1807| 1| X(khc2hc_register) (p, hf_32, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 1808| 1|} fftw_codelet_hf_4: 193| 1|void X(codelet_hf_4) (planner *p) { 194| 1| X(khc2hc_register) (p, hf_4, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 195| 1|} fftw_codelet_hf_5: 250| 1|void X(codelet_hf_5) (planner *p) { 251| 1| X(khc2hc_register) (p, hf_5, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 252| 1|} fftw_codelet_hf_6: 292| 1|void X(codelet_hf_6) (planner *p) { 293| 1| X(khc2hc_register) (p, hf_6, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 294| 1|} fftw_codelet_hf_64: 4102| 1|void X(codelet_hf_64) (planner *p) { 4103| 1| X(khc2hc_register) (p, hf_64, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 4104| 1|} fftw_codelet_hf_7: 351| 1|void X(codelet_hf_7) (planner *p) { 352| 1| X(khc2hc_register) (p, hf_7, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 353| 1|} fftw_codelet_hf_8: 373| 1|void X(codelet_hf_8) (planner *p) { 374| 1| X(khc2hc_register) (p, hf_8, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 375| 1|} fftw_codelet_hf_9: 484| 1|void X(codelet_hf_9) (planner *p) { 485| 1| X(khc2hc_register) (p, hf_9, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 486| 1|} fftw_codelet_r2cfII_10: 191| 1|void X(codelet_r2cfII_10) (planner *p) { X(kr2c_register) (p, r2cfII_10, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 192| 1|} fftw_codelet_r2cfII_12: 222| 1|void X(codelet_r2cfII_12) (planner *p) { X(kr2c_register) (p, r2cfII_12, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 223| 1|} fftw_codelet_r2cfII_15: 294| 1|void X(codelet_r2cfII_15) (planner *p) { X(kr2c_register) (p, r2cfII_15, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 295| 1|} fftw_codelet_r2cfII_16: 309| 1|void X(codelet_r2cfII_16) (planner *p) { X(kr2c_register) (p, r2cfII_16, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 310| 1|} fftw_codelet_r2cfII_2: 83| 1|void X(codelet_r2cfII_2) (planner *p) { X(kr2c_register) (p, r2cfII_2, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 84| 1|} fftw_codelet_r2cfII_20: 391| 1|void X(codelet_r2cfII_20) (planner *p) { X(kr2c_register) (p, r2cfII_20, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 392| 1|} fftw_codelet_r2cfII_25: 773| 1|void X(codelet_r2cfII_25) (planner *p) { X(kr2c_register) (p, r2cfII_25, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 774| 1|} fftw_codelet_r2cfII_3: 93| 1|void X(codelet_r2cfII_3) (planner *p) { X(kr2c_register) (p, r2cfII_3, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 94| 1|} fftw_codelet_r2cfII_32: 683| 1|void X(codelet_r2cfII_32) (planner *p) { X(kr2c_register) (p, r2cfII_32, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 684| 1|} fftw_codelet_r2cfII_4: 97| 1|void X(codelet_r2cfII_4) (planner *p) { X(kr2c_register) (p, r2cfII_4, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 98| 1|} fftw_codelet_r2cfII_5: 123| 1|void X(codelet_r2cfII_5) (planner *p) { X(kr2c_register) (p, r2cfII_5, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 124| 1|} fftw_codelet_r2cfII_6: 114| 1|void X(codelet_r2cfII_6) (planner *p) { X(kr2c_register) (p, r2cfII_6, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 115| 1|} fftw_codelet_r2cfII_64: 1545| 1|void X(codelet_r2cfII_64) (planner *p) { X(kr2c_register) (p, r2cfII_64, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 1546| 1|} fftw_codelet_r2cfII_7: 145| 1|void X(codelet_r2cfII_7) (planner *p) { X(kr2c_register) (p, r2cfII_7, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 146| 1|} fftw_codelet_r2cfII_8: 159| 1|void X(codelet_r2cfII_8) (planner *p) { X(kr2c_register) (p, r2cfII_8, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 160| 1|} fftw_codelet_r2cfII_9: 220| 1|void X(codelet_r2cfII_9) (planner *p) { X(kr2c_register) (p, r2cfII_9, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 221| 1|} fftw_codelet_r2cf_10: 197| 1|void X(codelet_r2cf_10) (planner *p) { X(kr2c_register) (p, r2cf_10, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 198| 1|} fftw_codelet_r2cf_11: 225| 1|void X(codelet_r2cf_11) (planner *p) { X(kr2c_register) (p, r2cf_11, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 226| 1|} fftw_codelet_r2cf_12: 215| 1|void X(codelet_r2cf_12) (planner *p) { X(kr2c_register) (p, r2cf_12, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 216| 1|} fftw_codelet_r2cf_128: 3241| 1|void X(codelet_r2cf_128) (planner *p) { X(kr2c_register) (p, r2cf_128, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 3242| 1|} fftw_codelet_r2cf_13: 358| 1|void X(codelet_r2cf_13) (planner *p) { X(kr2c_register) (p, r2cf_13, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 359| 1|} fftw_codelet_r2cf_14: 260| 1|void X(codelet_r2cf_14) (planner *p) { X(kr2c_register) (p, r2cf_14, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 261| 1|} fftw_codelet_r2cf_15: 301| 1|void X(codelet_r2cf_15) (planner *p) { X(kr2c_register) (p, r2cf_15, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 302| 1|} fftw_codelet_r2cf_16: 285| 1|void X(codelet_r2cf_16) (planner *p) { X(kr2c_register) (p, r2cf_16, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 286| 1|} fftw_codelet_r2cf_2: 83| 1|void X(codelet_r2cf_2) (planner *p) { X(kr2c_register) (p, r2cf_2, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 84| 1|} fftw_codelet_r2cf_20: 359| 1|void X(codelet_r2cf_20) (planner *p) { X(kr2c_register) (p, r2cf_20, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 360| 1|} fftw_codelet_r2cf_25: 733| 1|void X(codelet_r2cf_25) (planner *p) { X(kr2c_register) (p, r2cf_25, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 734| 1|} fftw_codelet_r2cf_3: 93| 1|void X(codelet_r2cf_3) (planner *p) { X(kr2c_register) (p, r2cf_3, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 94| 1|} fftw_codelet_r2cf_32: 607| 1|void X(codelet_r2cf_32) (planner *p) { X(kr2c_register) (p, r2cf_32, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 608| 1|} fftw_codelet_r2cf_4: 95| 1|void X(codelet_r2cf_4) (planner *p) { X(kr2c_register) (p, r2cf_4, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 96| 1|} fftw_codelet_r2cf_5: 123| 1|void X(codelet_r2cf_5) (planner *p) { X(kr2c_register) (p, r2cf_5, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 124| 1|} fftw_codelet_r2cf_6: 125| 1|void X(codelet_r2cf_6) (planner *p) { X(kr2c_register) (p, r2cf_6, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 126| 1|} fftw_codelet_r2cf_64: 1405| 1|void X(codelet_r2cf_64) (planner *p) { X(kr2c_register) (p, r2cf_64, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 1406| 1|} fftw_codelet_r2cf_7: 145| 1|void X(codelet_r2cf_7) (planner *p) { X(kr2c_register) (p, r2cf_7, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 146| 1|} fftw_codelet_r2cf_8: 151| 1|void X(codelet_r2cf_8) (planner *p) { X(kr2c_register) (p, r2cf_8, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 152| 1|} fftw_codelet_r2cf_9: 214| 1|void X(codelet_r2cf_9) (planner *p) { X(kr2c_register) (p, r2cf_9, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 215| 1|} fftw_codelet_e01_8: 184| 1|void X(codelet_e01_8) (planner *p) { X(kr2r_register) (p, e01_8, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 185| 1|} fftw_codelet_e10_8: 185| 1|void X(codelet_e10_8) (planner *p) { X(kr2r_register) (p, e10_8, &desc); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 186| 1|} fftw_rdft2_vrank_geq1_register: 209| 1|{ 210| | /* FIXME: Should we try other vecloop_dim values? */ 211| 1| static const int buddies[] = { 1, -1 }; 212| 1| size_t i; 213| | 214| 3| for (i = 0; i < NELEM(buddies); ++i) ------------------ | | 90| 3|#define NELEM(array) ((sizeof(array) / sizeof((array)[0]))) ------------------ | Branch (214:18): [True: 2, False: 1] ------------------ 215| 2| REGISTER_SOLVER(p, mksolver(buddies[i], buddies, NELEM(buddies))); ------------------ | | 95| 2|#define REGISTER_SOLVER(p, s) X(solver_register)(p, s) | | ------------------ | | | | 78| 2|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 2|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 216| 1|} vrank-geq1-rdft2.c:mksolver: 199| 2|{ 200| 2| static const solver_adt sadt = { PROBLEM_RDFT2, mkplan, 0 }; 201| 2| S *slv = MKSOLVER(S, &sadt); ------------------ | | 604| 2|#define MKSOLVER(type, adt) (type *)X(mksolver)(sizeof(type), adt) | | ------------------ | | | | 78| 2|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 2|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 202| 2| slv->vecloop_dim = vecloop_dim; 203| 2| slv->buddies = buddies; 204| 2| slv->nbuddies = nbuddies; 205| 2| return &(slv->super); 206| 2|} fftw_rdft_vrank_geq1_register: 211| 1|{ 212| | /* FIXME: Should we try other vecloop_dim values? */ 213| 1| static const int buddies[] = { 1, -1 }; 214| 1| size_t i; 215| | 216| 3| for (i = 0; i < NELEM(buddies); ++i) ------------------ | | 90| 3|#define NELEM(array) ((sizeof(array) / sizeof((array)[0]))) ------------------ | Branch (216:18): [True: 2, False: 1] ------------------ 217| 2| REGISTER_SOLVER(p, mksolver(buddies[i], buddies, NELEM(buddies))); ------------------ | | 95| 2|#define REGISTER_SOLVER(p, s) X(solver_register)(p, s) | | ------------------ | | | | 78| 2|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 2|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 218| 1|} fftw_rdft_vrank3_transpose_register: 769| 1|{ 770| 1| unsigned i; 771| 1| static const transpose_adt *const adts[] = { 772| 1| &adt_gcd, &adt_cut, 773| 1| &adt_toms513 774| 1| }; 775| 4| for (i = 0; i < sizeof(adts) / sizeof(adts[0]); ++i) ------------------ | Branch (775:18): [True: 3, False: 1] ------------------ 776| 3| REGISTER_SOLVER(p, mksolver(adts[i])); ------------------ | | 95| 3|#define REGISTER_SOLVER(p, s) X(solver_register)(p, s) | | ------------------ | | | | 78| 3|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 3|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 777| 1|} vrank3-transpose.c:applicable_gcd: 229| 27|{ 230| 27| INT n = p->vecsz->dims[dim0].n; 231| 27| INT m = p->vecsz->dims[dim1].n; 232| 27| INT d, vl, vs; 233| 27| get_transpose_vec(p, dim2, &vl, &vs); 234| 27| d = gcd(n, m); 235| 27| *nbuf = n * (m / d) * vl; 236| 27| return (!NO_SLOWP(plnr) /* FIXME: not really SLOW for large 1d ffts */ ------------------ | | 688| 54|#define NO_SLOWP(plnr) (PLNR_L(plnr) & NO_SLOW) | | ------------------ | | | | 674| 27|#define PLNR_L(plnr) ((plnr)->flags.l) | | ------------------ ------------------ | Branch (236:14): [True: 0, False: 27] ------------------ 237| 0| && n != m ------------------ | Branch (237:10): [True: 0, False: 0] ------------------ 238| 0| && d > 1 ------------------ | Branch (238:10): [True: 0, False: 0] ------------------ 239| 0| && Ntuple_transposable(p->vecsz->dims + dim0, ------------------ | Branch (239:10): [True: 0, False: 0] ------------------ 240| 0| p->vecsz->dims + dim1, 241| 0| vl, vs)); 242| 27|} vrank3-transpose.c:get_transpose_vec: 149| 81|{ 150| 81| if (p->vecsz->rnk == 2) { ------------------ | Branch (150:10): [True: 0, False: 81] ------------------ 151| 0| *vl = 1; *vs = 1; 152| 0| } 153| 81| else { 154| 81| *vl = p->vecsz->dims[dim2].n; 155| 81| *vs = p->vecsz->dims[dim2].is; /* == os */ 156| 81| } 157| 81|} vrank3-transpose.c:gcd: 61| 27|{ 62| 27| INT r; 63| 27| do { 64| 27| r = a % b; 65| 27| a = b; 66| 27| b = r; 67| 27| } while (r != 0); ------------------ | Branch (67:15): [True: 0, False: 27] ------------------ 68| | 69| 27| return a; 70| 27|} vrank3-transpose.c:Ntuple_transposable: 75| 402|{ 76| 402| return (vs == 1 && b->is == vl && a->os == vl && ------------------ | Branch (76:14): [True: 402, False: 0] | Branch (76:25): [True: 402, False: 0] | Branch (76:40): [True: 402, False: 0] ------------------ 77| 402| ((a->n == b->n && a->is == b->os ------------------ | Branch (77:9): [True: 0, False: 402] | Branch (77:25): [True: 0, False: 0] ------------------ 78| 0| && a->is >= b->n && a->is % vl == 0) ------------------ | Branch (78:12): [True: 0, False: 0] | Branch (78:29): [True: 0, False: 0] ------------------ 79| 402| || (a->is == b->n * vl && b->os == a->n * vl))); ------------------ | Branch (79:12): [True: 402, False: 0] | Branch (79:34): [True: 402, False: 0] ------------------ 80| 402|} vrank3-transpose.c:applicable_cut: 371| 27|{ 372| 27| INT n = p->vecsz->dims[dim0].n; 373| 27| INT m = p->vecsz->dims[dim1].n; 374| 27| INT vl, vs; 375| 27| get_transpose_vec(p, dim2, &vl, &vs); 376| 27| *nbuf = 0; /* always small enough to be non-UGLY (?) */ 377| 27| A(MINBUFDIV <= CUT_NSRCH); /* assumed to avoid inf. loops below */ 378| 27| return (!NO_SLOWP(plnr) /* FIXME: not really SLOW for large 1d ffts? */ ------------------ | | 688| 54|#define NO_SLOWP(plnr) (PLNR_L(plnr) & NO_SLOW) | | ------------------ | | | | 674| 27|#define PLNR_L(plnr) ((plnr)->flags.l) | | ------------------ ------------------ | Branch (378:14): [True: 0, False: 27] ------------------ 379| 0| && n != m ------------------ | Branch (379:10): [True: 0, False: 0] ------------------ 380| | 381| | /* Don't call transpose-cut recursively (avoid inf. loops): 382| | the non-square sub-transpose produced when !cut1 383| | should always have gcd(n,m) >= min(CUT_NSRCH,n,m), 384| | for which transpose-gcd is applicable */ 385| 0| && (cut1(n, m, vl) ------------------ | Branch (385:11): [True: 0, False: 0] ------------------ 386| 0| || gcd(n, m) < X(imin)(MINBUFDIV, X(imin)(n,m))) ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ || gcd(n, m) < X(imin)(MINBUFDIV, X(imin)(n,m))) ------------------ | | 111| 0|#define MINBUFDIV 9 /* min factor by which buffer is smaller than data */ ------------------ || gcd(n, m) < X(imin)(MINBUFDIV, X(imin)(n,m))) ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ | Branch (386:7): [True: 0, False: 0] ------------------ 387| | 388| 0| && Ntuple_transposable(p->vecsz->dims + dim0, ------------------ | Branch (388:10): [True: 0, False: 0] ------------------ 389| 0| p->vecsz->dims + dim1, 390| 0| vl, vs)); 391| 27|} vrank3-transpose.c:applicable_toms513: 662| 27|{ 663| 27| INT n = p->vecsz->dims[dim0].n; 664| 27| INT m = p->vecsz->dims[dim1].n; 665| 27| INT vl, vs; 666| 27| get_transpose_vec(p, dim2, &vl, &vs); 667| 27| *nbuf = 2*vl 668| 27| + ((n + m) / 2 * sizeof(char) + sizeof(R) - 1) / sizeof(R); 669| 27| return (!NO_SLOWP(plnr) ------------------ | | 688| 54|#define NO_SLOWP(plnr) (PLNR_L(plnr) & NO_SLOW) | | ------------------ | | | | 674| 27|#define PLNR_L(plnr) ((plnr)->flags.l) | | ------------------ ------------------ | Branch (669:14): [True: 0, False: 27] ------------------ 670| 0| && (vl > 8 || !NO_UGLYP(plnr)) /* UGLY for small vl */ ------------------ | | 689| 0|#define NO_UGLYP(plnr) (PLNR_L(plnr) & NO_UGLY) | | ------------------ | | | | 674| 0|#define PLNR_L(plnr) ((plnr)->flags.l) | | ------------------ ------------------ | Branch (670:11): [True: 0, False: 0] | Branch (670:21): [True: 0, False: 0] ------------------ 671| 0| && n != m ------------------ | Branch (671:10): [True: 0, False: 0] ------------------ 672| 0| && Ntuple_transposable(p->vecsz->dims + dim0, ------------------ | Branch (672:10): [True: 0, False: 0] ------------------ 673| 0| p->vecsz->dims + dim1, 674| 0| vl, vs)); 675| 27|} vrank3-transpose.c:mksolver: 761| 3|{ 762| 3| static const solver_adt sadt = { PROBLEM_RDFT, mkplan, 0 }; 763| 3| S *slv = MKSOLVER(S, &sadt); ------------------ | | 604| 3|#define MKSOLVER(type, adt) (type *)X(mksolver)(sizeof(type), adt) | | ------------------ | | | | 78| 3|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 3|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 764| 3| slv->adt = adt; 765| 3| return &(slv->super); 766| 3|} vrank3-transpose.c:mkplan: 723| 1.00k|{ 724| 1.00k| const S *ego = (const S *) ego_; 725| 1.00k| const problem_rdft *p; 726| 1.00k| int dim0, dim1, dim2; 727| 1.00k| INT nbuf, vs; 728| 1.00k| P *pln; 729| | 730| 1.00k| static const plan_adt padt = { 731| 1.00k| X(rdft_solve), awake, print, destroy ------------------ | | 78| 1.00k|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1.00k|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 732| 1.00k| }; 733| | 734| 1.00k| if (!applicable(ego_, p_, plnr, &dim0, &dim1, &dim2, &nbuf)) ------------------ | Branch (734:10): [True: 1.00k, False: 0] ------------------ 735| 1.00k| return (plan *) 0; 736| | 737| 0| p = (const problem_rdft *) p_; 738| 0| pln = MKPLAN_RDFT(P, &padt, ego->adt->apply); ------------------ | | 73| 0| (type *)X(mkplan_rdft)(sizeof(type), adt, apply) | | ------------------ | | | | 78| 0|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 739| | 740| 0| pln->n = p->vecsz->dims[dim0].n; 741| 0| pln->m = p->vecsz->dims[dim1].n; 742| 0| get_transpose_vec(p, dim2, &pln->vl, &vs); 743| 0| pln->nbuf = nbuf; 744| 0| pln->d = gcd(pln->n, pln->m); 745| 0| pln->nd = pln->n / pln->d; 746| 0| pln->md = pln->m / pln->d; 747| 0| pln->slv = ego; 748| | 749| 0| X(ops_zero)(&pln->super.super.ops); /* mkcldrn is responsible for ops */ ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 750| | 751| 0| pln->cld1 = pln->cld2 = pln->cld3 = 0; 752| 0| if (!ego->adt->mkcldrn(p, plnr, pln)) { ------------------ | Branch (752:10): [True: 0, False: 0] ------------------ 753| 0| X(plan_destroy_internal)(&(pln->super.super)); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 754| 0| return 0; 755| 0| } 756| | 757| 0| return &(pln->super.super); 758| 0|} vrank3-transpose.c:applicable: 117| 1.00k|{ 118| 1.00k| const S *ego = (const S *) ego_; 119| 1.00k| const problem_rdft *p = (const problem_rdft *) p_; 120| | 121| 1.00k| return (1 ------------------ | Branch (121:14): [True: 1.00k, Folded] ------------------ 122| 1.00k| && p->I == p->O ------------------ | Branch (122:10): [True: 519, False: 486] ------------------ 123| 519| && p->sz->rnk == 0 ------------------ | Branch (123:10): [True: 519, False: 0] ------------------ 124| 519| && (p->vecsz->rnk == 2 || p->vecsz->rnk == 3) ------------------ | Branch (124:11): [True: 0, False: 519] | Branch (124:33): [True: 483, False: 36] ------------------ 125| | 126| 483| && pickdim(p->vecsz, dim0, dim1, dim2) ------------------ | Branch (126:10): [True: 483, False: 0] ------------------ 127| | 128| | /* UGLY if vecloop in wrong order for locality */ 129| 483| && (!NO_UGLYP(plnr) || ------------------ | | 689| 966|#define NO_UGLYP(plnr) (PLNR_L(plnr) & NO_UGLY) | | ------------------ | | | | 674| 483|#define PLNR_L(plnr) ((plnr)->flags.l) | | ------------------ ------------------ | Branch (129:11): [True: 0, False: 483] ------------------ 130| 483| p->vecsz->rnk == 2 || ------------------ | Branch (130:4): [True: 0, False: 483] ------------------ 131| 483| X(iabs)(p->vecsz->dims[*dim2].is) ------------------ | | 78| 483|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 483|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ | Branch (131:4): [True: 483, False: 0] ------------------ 132| 483| < X(imax)(X(iabs)(p->vecsz->dims[*dim0].is), ------------------ | | 78| 483|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 483|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ < X(imax)(X(iabs)(p->vecsz->dims[*dim0].is), ------------------ | | 78| 483|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 483|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 133| 483| X(iabs)(p->vecsz->dims[*dim0].os))) ------------------ | | 78| 483|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 483|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 134| | 135| | /* SLOW if non-square */ 136| 483| && (!NO_SLOWP(plnr) ------------------ | | 688| 966|#define NO_SLOWP(plnr) (PLNR_L(plnr) & NO_SLOW) | | ------------------ | | | | 674| 483|#define PLNR_L(plnr) ((plnr)->flags.l) | | ------------------ ------------------ | Branch (136:11): [True: 0, False: 483] ------------------ 137| 483| || p->vecsz->dims[*dim0].n == p->vecsz->dims[*dim1].n) ------------------ | Branch (137:7): [True: 81, False: 402] ------------------ 138| | 139| 81| && ego->adt->applicable(p, plnr, *dim0,*dim1,*dim2,nbuf) ------------------ | Branch (139:10): [True: 0, False: 81] ------------------ 140| | 141| | /* buffers too big are UGLY */ 142| 0| && ((!NO_UGLYP(plnr) && !CONSERVE_MEMORYP(plnr)) ------------------ | | 689| 0|#define NO_UGLYP(plnr) (PLNR_L(plnr) & NO_UGLY) | | ------------------ | | | | 674| 0|#define PLNR_L(plnr) ((plnr)->flags.l) | | ------------------ ------------------ && ((!NO_UGLYP(plnr) && !CONSERVE_MEMORYP(plnr)) ------------------ | | 697| 0|#define CONSERVE_MEMORYP(plnr) (PLNR_L(plnr) & CONSERVE_MEMORY) | | ------------------ | | | | 674| 0|#define PLNR_L(plnr) ((plnr)->flags.l) | | ------------------ ------------------ | Branch (142:12): [True: 0, False: 0] | Branch (142:31): [True: 0, False: 0] ------------------ 143| 0| || *nbuf <= MAXBUF ------------------ | | 112| 0|#define MAXBUF 65536 /* maximum non-ugly buffer */ ------------------ | Branch (143:7): [True: 0, False: 0] ------------------ 144| 0| || *nbuf * MINBUFDIV <= X(tensor_sz)(p->vecsz)) ------------------ | | 111| 0|#define MINBUFDIV 9 /* min factor by which buffer is smaller than data */ ------------------ || *nbuf * MINBUFDIV <= X(tensor_sz)(p->vecsz)) ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ | Branch (144:7): [True: 0, False: 0] ------------------ 145| 1.00k| ); 146| 1.00k|} vrank3-transpose.c:pickdim: 91| 483|{ 92| 483| int dim0, dim1; 93| | 94| 483| for (dim0 = 0; dim0 < s->rnk; ++dim0) ------------------ | Branch (94:21): [True: 483, False: 0] ------------------ 95| 966| for (dim1 = 0; dim1 < s->rnk; ++dim1) { ------------------ | Branch (95:26): [True: 966, False: 0] ------------------ 96| 966| int dim2 = 3 - dim0 - dim1; 97| 966| if (dim0 == dim1) continue; ------------------ | Branch (97:13): [True: 483, False: 483] ------------------ 98| 483| if ((s->rnk == 2 || s->dims[dim2].is == s->dims[dim2].os) ------------------ | Branch (98:21): [True: 0, False: 483] | Branch (98:36): [True: 483, False: 0] ------------------ 99| 483| && transposable(s->dims + dim0, s->dims + dim1, ------------------ | Branch (99:9): [True: 483, False: 0] ------------------ 100| 483| s->rnk == 2 ? (INT)1 : s->dims[dim2].n, ------------------ | Branch (100:8): [True: 0, False: 483] ------------------ 101| 483| s->rnk == 2 ? (INT)1 : s->dims[dim2].is)) { ------------------ | Branch (101:8): [True: 0, False: 483] ------------------ 102| 483| *pdim0 = dim0; 103| 483| *pdim1 = dim1; 104| 483| *pdim2 = dim2; 105| 483| return 1; 106| 483| } 107| 483| } 108| 0| return 0; 109| 483|} vrank3-transpose.c:transposable: 85| 483|{ 86| 483| return ((a->n == b->n && a->os == b->is && a->is == b->os) ------------------ | Branch (86:15): [True: 81, False: 402] | Branch (86:31): [True: 81, False: 0] | Branch (86:49): [True: 81, False: 0] ------------------ 87| 402| || Ntuple_transposable(a, b, vl, vs)); ------------------ | Branch (87:17): [True: 402, False: 0] ------------------ 88| 483|} fftw_reodft_conf_standard: 43| 1|{ 44| 1| X(solvtab_exec)(s, p); ------------------ | | 78| 1|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 45| 1|} fftw_redft00e_r2hc_pad_register: 195| 1|{ 196| 1| REGISTER_SOLVER(p, mksolver()); ------------------ | | 95| 1|#define REGISTER_SOLVER(p, s) X(solver_register)(p, s) | | ------------------ | | | | 78| 1|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 197| 1|} redft00e-r2hc-pad.c:mksolver: 188| 1|{ 189| 1| static const solver_adt sadt = { PROBLEM_RDFT, mkplan, 0 }; 190| 1| S *slv = MKSOLVER(S, &sadt); ------------------ | | 604| 1|#define MKSOLVER(type, adt) (type *)X(mksolver)(sizeof(type), adt) | | ------------------ | | | | 78| 1|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 191| 1| return &(slv->super); 192| 1|} redft00e-r2hc-pad.c:mkplan: 119| 335|{ 120| 335| P *pln; 121| 335| const problem_rdft *p; 122| 335| plan *cld = (plan *) 0, *cldcpy; 123| 335| R *buf = (R *) 0; 124| 335| INT n; 125| 335| INT vl, ivs, ovs; 126| 335| opcnt ops; 127| | 128| 335| static const plan_adt padt = { 129| 335| X(rdft_solve), awake, print, destroy ------------------ | | 78| 335|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 335|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 130| 335| }; 131| | 132| 335| if (!applicable(ego_, p_, plnr)) ------------------ | Branch (132:10): [True: 335, False: 0] ------------------ 133| 335| goto nada; 134| | 135| 0| p = (const problem_rdft *) p_; 136| | 137| 0| n = p->sz->dims[0].n - 1; 138| 0| A(n > 0); 139| 0| buf = (R *) MALLOC(sizeof(R) * (2*n), BUFFERS); ------------------ | | 306| 0|#define MALLOC(n, what) X(malloc_plain)(n) | | ------------------ | | | | 78| 0|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 140| | 141| 0| cld = X(mkplan_d)(plnr,X(mkproblem_rdft_1_d)(X(mktensor_1d)(2*n,1,1), ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ cld = X(mkplan_d)(plnr,X(mkproblem_rdft_1_d)(X(mktensor_1d)(2*n,1,1), ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ cld = X(mkplan_d)(plnr,X(mkproblem_rdft_1_d)(X(mktensor_1d)(2*n,1,1), ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 142| 0| X(mktensor_0d)(), ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 143| 0| buf, buf, R2HC)); ------------------ | | 48| 0|#define R2HC R2HC00 ------------------ 144| 0| if (!cld) ------------------ | Branch (144:10): [True: 0, False: 0] ------------------ 145| 0| goto nada; 146| | 147| 0| X(tensor_tornk1)(p->vecsz, &vl, &ivs, &ovs); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 148| 0| cldcpy = 149| 0| X(mkplan_d)(plnr, ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 150| 0| X(mkproblem_rdft_1_d)(X(mktensor_0d)(), ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ X(mkproblem_rdft_1_d)(X(mktensor_0d)(), ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 151| 0| X(mktensor_1d)(n+1,1, ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 152| 0| p->sz->dims[0].os), 153| 0| buf, TAINT(p->O, ovs), R2HC)); ------------------ | | 1036| 0|#define TAINT(p, s) (p) ------------------ buf, TAINT(p->O, ovs), R2HC)); ------------------ | | 48| 0|#define R2HC R2HC00 ------------------ 154| 0| if (!cldcpy) ------------------ | Branch (154:10): [True: 0, False: 0] ------------------ 155| 0| goto nada; 156| | 157| 0| X(ifree)(buf); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 158| | 159| 0| pln = MKPLAN_RDFT(P, &padt, apply); ------------------ | | 73| 0| (type *)X(mkplan_rdft)(sizeof(type), adt, apply) | | ------------------ | | | | 78| 0|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 160| | 161| 0| pln->n = n; 162| 0| pln->is = p->sz->dims[0].is; 163| 0| pln->cld = cld; 164| 0| pln->cldcpy = cldcpy; 165| 0| pln->vl = vl; 166| 0| pln->ivs = ivs; 167| 0| pln->ovs = ovs; 168| | 169| 0| X(ops_zero)(&ops); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 170| 0| ops.other = n + 2*n; /* loads + stores (input -> buf) */ 171| | 172| 0| X(ops_zero)(&pln->super.super.ops); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 173| 0| X(ops_madd2)(pln->vl, &ops, &pln->super.super.ops); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 174| 0| X(ops_madd2)(pln->vl, &cld->ops, &pln->super.super.ops); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 175| 0| X(ops_madd2)(pln->vl, &cldcpy->ops, &pln->super.super.ops); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 176| | 177| 0| return &(pln->super.super); 178| | 179| 335| nada: 180| 335| X(ifree0)(buf); ------------------ | | 78| 335|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 335|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 181| 335| if (cld) ------------------ | Branch (181:10): [True: 0, False: 335] ------------------ 182| 0| X(plan_destroy_internal)(cld); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 183| 335| return (plan *)0; 184| 0|} redft00e-r2hc-pad.c:applicable: 114| 335|{ 115| 335| return (!NO_SLOWP(plnr) && applicable0(ego, p)); ------------------ | | 688| 670|#define NO_SLOWP(plnr) (PLNR_L(plnr) & NO_SLOW) | | ------------------ | | | | 674| 335|#define PLNR_L(plnr) ((plnr)->flags.l) | | ------------------ ------------------ | Branch (115:14): [True: 0, False: 335] | Branch (115:33): [True: 0, False: 0] ------------------ 116| 335|} fftw_reodft00e_splitradix_register: 352| 1|{ 353| 1| REGISTER_SOLVER(p, mksolver()); ------------------ | | 95| 1|#define REGISTER_SOLVER(p, s) X(solver_register)(p, s) | | ------------------ | | | | 78| 1|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 354| 1|} reodft00e-splitradix.c:mksolver: 345| 1|{ 346| 1| static const solver_adt sadt = { PROBLEM_RDFT, mkplan, 0 }; 347| 1| S *slv = MKSOLVER(S, &sadt); ------------------ | | 604| 1|#define MKSOLVER(type, adt) (type *)X(mksolver)(sizeof(type), adt) | | ------------------ | | | | 78| 1|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 348| 1| return &(slv->super); 349| 1|} reodft00e-splitradix.c:mkplan: 266| 335|{ 267| 335| P *pln; 268| 335| const problem_rdft *p; 269| 335| plan *clde, *cldo; 270| 335| R *buf; 271| 335| INT n, n0; 272| 335| opcnt ops; 273| 335| int inplace_odd; 274| | 275| 335| static const plan_adt padt = { 276| 335| X(rdft_solve), awake, print, destroy ------------------ | | 78| 335|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 335|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 277| 335| }; 278| | 279| 335| if (!applicable(ego_, p_, plnr)) ------------------ | Branch (279:10): [True: 335, False: 0] ------------------ 280| 335| return (plan *)0; 281| | 282| 0| p = (const problem_rdft *) p_; 283| | 284| 0| n = (n0 = p->sz->dims[0].n) + (p->kind[0] == REDFT00 ? (INT)-1 : (INT)1); ------------------ | Branch (284:37): [True: 0, False: 0] ------------------ 285| 0| A(n > 0 && n % 2 == 0); 286| 0| buf = (R *) MALLOC(sizeof(R) * (n/2), BUFFERS); ------------------ | | 306| 0|#define MALLOC(n, what) X(malloc_plain)(n) | | ------------------ | | | | 78| 0|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 287| | 288| 0| inplace_odd = p->kind[0]==RODFT00 && p->I == p->O; ------------------ | Branch (288:20): [True: 0, False: 0] | Branch (288:43): [True: 0, False: 0] ------------------ 289| 0| clde = X(mkplan_d)(plnr, X(mkproblem_rdft_1_d)( ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ clde = X(mkplan_d)(plnr, X(mkproblem_rdft_1_d)( ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 290| 0| X(mktensor_1d)(n0-n/2, 2*p->sz->dims[0].is, ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 291| 0| inplace_odd ? p->sz->dims[0].is ------------------ | Branch (291:10): [True: 0, False: 0] ------------------ 292| 0| : p->sz->dims[0].os), 293| 0| X(mktensor_0d)(), ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 294| 0| TAINT(p->I ------------------ | | 1036| 0|#define TAINT(p, s) (p) ------------------ 295| 0| + p->sz->dims[0].is * (p->kind[0]==RODFT00), 296| 0| p->vecsz->rnk ? p->vecsz->dims[0].is : 0), 297| 0| TAINT(p->O ------------------ | | 1036| 0|#define TAINT(p, s) (p) ------------------ 298| 0| + p->sz->dims[0].is * inplace_odd, 299| 0| p->vecsz->rnk ? p->vecsz->dims[0].os : 0), 300| 0| p->kind[0])); 301| 0| if (!clde) { ------------------ | Branch (301:10): [True: 0, False: 0] ------------------ 302| 0| X(ifree)(buf); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 303| 0| return (plan *)0; 304| 0| } 305| | 306| 0| cldo = X(mkplan_d)(plnr, X(mkproblem_rdft_1_d)( ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ cldo = X(mkplan_d)(plnr, X(mkproblem_rdft_1_d)( ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 307| 0| X(mktensor_1d)(n/2, 1, 1), ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 308| 0| X(mktensor_0d)(), ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 309| 0| buf, buf, R2HC)); ------------------ | | 48| 0|#define R2HC R2HC00 ------------------ 310| 0| X(ifree)(buf); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 311| 0| if (!cldo) ------------------ | Branch (311:10): [True: 0, False: 0] ------------------ 312| 0| return (plan *)0; 313| | 314| 0| pln = MKPLAN_RDFT(P, &padt, p->kind[0] == REDFT00 ? apply_e : apply_o); ------------------ | | 73| 0| (type *)X(mkplan_rdft)(sizeof(type), adt, apply) | | ------------------ | | | | 78| 0|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ | | | Branch (73:45): [True: 0, False: 0] | | ------------------ ------------------ 315| | 316| 0| pln->n = n; 317| 0| pln->is = p->sz->dims[0].is; 318| 0| pln->os = p->sz->dims[0].os; 319| 0| pln->clde = clde; 320| 0| pln->cldo = cldo; 321| 0| pln->td = 0; 322| | 323| 0| X(tensor_tornk1)(p->vecsz, &pln->vl, &pln->ivs, &pln->ovs); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 324| | 325| 0| X(ops_zero)(&ops); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 326| 0| ops.other = n/2; 327| 0| ops.add = (p->kind[0]==REDFT00 ? (INT)2 : (INT)0) + ------------------ | Branch (327:17): [True: 0, False: 0] ------------------ 328| 0| (n/2-1)/2 * 6 + ((n/2)%2==0) * 2; 329| 0| ops.mul = 1 + (n/2-1)/2 * 6 + ((n/2)%2==0) * 2; 330| | 331| | /* tweak ops.other so that r2hc-pad is used for small sizes, which 332| | seems to be a lot faster on my machine: */ 333| 0| ops.other += 256; 334| | 335| 0| X(ops_zero)(&pln->super.super.ops); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 336| 0| X(ops_madd2)(pln->vl, &ops, &pln->super.super.ops); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 337| 0| X(ops_madd2)(pln->vl, &clde->ops, &pln->super.super.ops); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 338| 0| X(ops_madd2)(pln->vl, &cldo->ops, &pln->super.super.ops); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 339| | 340| 0| return &(pln->super.super); 341| 0|} reodft00e-splitradix.c:applicable: 261| 335|{ 262| 335| return (!NO_SLOWP(plnr) && applicable0(ego, p)); ------------------ | | 688| 670|#define NO_SLOWP(plnr) (PLNR_L(plnr) & NO_SLOW) | | ------------------ | | | | 674| 335|#define PLNR_L(plnr) ((plnr)->flags.l) | | ------------------ ------------------ | Branch (262:14): [True: 0, False: 335] | Branch (262:33): [True: 0, False: 0] ------------------ 263| 335|} fftw_reodft010e_r2hc_register: 408| 1|{ 409| 1| REGISTER_SOLVER(p, mksolver()); ------------------ | | 95| 1|#define REGISTER_SOLVER(p, s) X(solver_register)(p, s) | | ------------------ | | | | 78| 1|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 410| 1|} reodft010e-r2hc.c:mksolver: 401| 1|{ 402| 1| static const solver_adt sadt = { PROBLEM_RDFT, mkplan, 0 }; 403| 1| S *slv = MKSOLVER(S, &sadt); ------------------ | | 604| 1|#define MKSOLVER(type, adt) (type *)X(mksolver)(sizeof(type), adt) | | ------------------ | | | | 78| 1|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 404| 1| return &(slv->super); 405| 1|} reodft010e-r2hc.c:mkplan: 337| 335|{ 338| 335| P *pln; 339| 335| const problem_rdft *p; 340| 335| plan *cld; 341| 335| R *buf; 342| 335| INT n; 343| 335| opcnt ops; 344| | 345| 335| static const plan_adt padt = { 346| 335| X(rdft_solve), awake, print, destroy ------------------ | | 78| 335|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 335|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 347| 335| }; 348| | 349| 335| if (!applicable(ego_, p_, plnr)) ------------------ | Branch (349:10): [True: 335, False: 0] ------------------ 350| 335| return (plan *)0; 351| | 352| 0| p = (const problem_rdft *) p_; 353| | 354| 0| n = p->sz->dims[0].n; 355| 0| buf = (R *) MALLOC(sizeof(R) * n, BUFFERS); ------------------ | | 306| 0|#define MALLOC(n, what) X(malloc_plain)(n) | | ------------------ | | | | 78| 0|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 356| | 357| 0| cld = X(mkplan_d)(plnr, X(mkproblem_rdft_1_d)(X(mktensor_1d)(n, 1, 1), ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ cld = X(mkplan_d)(plnr, X(mkproblem_rdft_1_d)(X(mktensor_1d)(n, 1, 1), ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ cld = X(mkplan_d)(plnr, X(mkproblem_rdft_1_d)(X(mktensor_1d)(n, 1, 1), ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 358| 0| X(mktensor_0d)(), ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 359| 0| buf, buf, R2HC)); ------------------ | | 48| 0|#define R2HC R2HC00 ------------------ 360| 0| X(ifree)(buf); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 361| 0| if (!cld) ------------------ | Branch (361:10): [True: 0, False: 0] ------------------ 362| 0| return (plan *)0; 363| | 364| 0| switch (p->kind[0]) { 365| 0| case REDFT01: pln = MKPLAN_RDFT(P, &padt, apply_re01); break; ------------------ | | 73| 0| (type *)X(mkplan_rdft)(sizeof(type), adt, apply) | | ------------------ | | | | 78| 0|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ | Branch (365:3): [True: 0, False: 0] ------------------ 366| 0| case REDFT10: pln = MKPLAN_RDFT(P, &padt, apply_re10); break; ------------------ | | 73| 0| (type *)X(mkplan_rdft)(sizeof(type), adt, apply) | | ------------------ | | | | 78| 0|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ | Branch (366:3): [True: 0, False: 0] ------------------ 367| 0| case RODFT01: pln = MKPLAN_RDFT(P, &padt, apply_ro01); break; ------------------ | | 73| 0| (type *)X(mkplan_rdft)(sizeof(type), adt, apply) | | ------------------ | | | | 78| 0|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ | Branch (367:3): [True: 0, False: 0] ------------------ 368| 0| case RODFT10: pln = MKPLAN_RDFT(P, &padt, apply_ro10); break; ------------------ | | 73| 0| (type *)X(mkplan_rdft)(sizeof(type), adt, apply) | | ------------------ | | | | 78| 0|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ | Branch (368:3): [True: 0, False: 0] ------------------ 369| 0| default: A(0); return (plan*)0; ------------------ | Branch (369:3): [True: 0, False: 0] ------------------ 370| 0| } 371| | 372| 0| pln->n = n; 373| 0| pln->is = p->sz->dims[0].is; 374| 0| pln->os = p->sz->dims[0].os; 375| 0| pln->cld = cld; 376| 0| pln->td = 0; 377| 0| pln->kind = p->kind[0]; 378| | 379| 0| X(tensor_tornk1)(p->vecsz, &pln->vl, &pln->ivs, &pln->ovs); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 380| | 381| 0| X(ops_zero)(&ops); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 382| 0| ops.other = 4 + (n-1)/2 * 10 + (1 - n % 2) * 5; 383| 0| if (p->kind[0] == REDFT01 || p->kind[0] == RODFT01) { ------------------ | Branch (383:10): [True: 0, False: 0] | Branch (383:35): [True: 0, False: 0] ------------------ 384| 0| ops.add = (n-1)/2 * 6; 385| 0| ops.mul = (n-1)/2 * 4 + (1 - n % 2) * 2; 386| 0| } 387| 0| else { /* 10 transforms */ 388| 0| ops.add = (n-1)/2 * 2; 389| 0| ops.mul = 1 + (n-1)/2 * 6 + (1 - n % 2) * 2; 390| 0| } 391| | 392| 0| X(ops_zero)(&pln->super.super.ops); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 393| 0| X(ops_madd2)(pln->vl, &ops, &pln->super.super.ops); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 394| 0| X(ops_madd2)(pln->vl, &cld->ops, &pln->super.super.ops); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 395| | 396| 0| return &(pln->super.super); 397| 0|} reodft010e-r2hc.c:applicable: 332| 335|{ 333| 335| return (!NO_SLOWP(plnr) && applicable0(ego, p)); ------------------ | | 688| 670|#define NO_SLOWP(plnr) (PLNR_L(plnr) & NO_SLOW) | | ------------------ | | | | 674| 335|#define PLNR_L(plnr) ((plnr)->flags.l) | | ------------------ ------------------ | Branch (333:14): [True: 0, False: 335] | Branch (333:33): [True: 0, False: 0] ------------------ 334| 335|} fftw_reodft11e_r2hc_odd_register: 298| 1|{ 299| 1| REGISTER_SOLVER(p, mksolver()); ------------------ | | 95| 1|#define REGISTER_SOLVER(p, s) X(solver_register)(p, s) | | ------------------ | | | | 78| 1|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 300| 1|} reodft11e-r2hc-odd.c:mksolver: 291| 1|{ 292| 1| static const solver_adt sadt = { PROBLEM_RDFT, mkplan, 0 }; 293| 1| S *slv = MKSOLVER(S, &sadt); ------------------ | | 604| 1|#define MKSOLVER(type, adt) (type *)X(mksolver)(sizeof(type), adt) | | ------------------ | | | | 78| 1|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 294| 1| return &(slv->super); 295| 1|} reodft11e-r2hc-odd.c:mkplan: 241| 335|{ 242| 335| P *pln; 243| 335| const problem_rdft *p; 244| 335| plan *cld; 245| 335| R *buf; 246| 335| INT n; 247| 335| opcnt ops; 248| | 249| 335| static const plan_adt padt = { 250| 335| X(rdft_solve), awake, print, destroy ------------------ | | 78| 335|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 335|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 251| 335| }; 252| | 253| 335| if (!applicable(ego_, p_, plnr)) ------------------ | Branch (253:10): [True: 335, False: 0] ------------------ 254| 335| return (plan *)0; 255| | 256| 0| p = (const problem_rdft *) p_; 257| | 258| 0| n = p->sz->dims[0].n; 259| 0| buf = (R *) MALLOC(sizeof(R) * n, BUFFERS); ------------------ | | 306| 0|#define MALLOC(n, what) X(malloc_plain)(n) | | ------------------ | | | | 78| 0|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 260| | 261| 0| cld = X(mkplan_d)(plnr, X(mkproblem_rdft_1_d)(X(mktensor_1d)(n, 1, 1), ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ cld = X(mkplan_d)(plnr, X(mkproblem_rdft_1_d)(X(mktensor_1d)(n, 1, 1), ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ cld = X(mkplan_d)(plnr, X(mkproblem_rdft_1_d)(X(mktensor_1d)(n, 1, 1), ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 262| 0| X(mktensor_0d)(), ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 263| 0| buf, buf, R2HC)); ------------------ | | 48| 0|#define R2HC R2HC00 ------------------ 264| 0| X(ifree)(buf); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 265| 0| if (!cld) ------------------ | Branch (265:10): [True: 0, False: 0] ------------------ 266| 0| return (plan *)0; 267| | 268| 0| pln = MKPLAN_RDFT(P, &padt, p->kind[0]==REDFT11 ? apply_re11:apply_ro11); ------------------ | | 73| 0| (type *)X(mkplan_rdft)(sizeof(type), adt, apply) | | ------------------ | | | | 78| 0|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ | | | Branch (73:45): [True: 0, False: 0] | | ------------------ ------------------ 269| 0| pln->n = n; 270| 0| pln->is = p->sz->dims[0].is; 271| 0| pln->os = p->sz->dims[0].os; 272| 0| pln->cld = cld; 273| 0| pln->kind = p->kind[0]; 274| | 275| 0| X(tensor_tornk1)(p->vecsz, &pln->vl, &pln->ivs, &pln->ovs); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 276| | 277| 0| X(ops_zero)(&ops); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 278| 0| ops.add = n - 1; 279| 0| ops.mul = n; 280| 0| ops.other = 4*n; 281| | 282| 0| X(ops_zero)(&pln->super.super.ops); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 283| 0| X(ops_madd2)(pln->vl, &ops, &pln->super.super.ops); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 284| 0| X(ops_madd2)(pln->vl, &cld->ops, &pln->super.super.ops); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 285| | 286| 0| return &(pln->super.super); 287| 0|} reodft11e-r2hc-odd.c:applicable: 236| 335|{ 237| 335| return (!NO_SLOWP(plnr) && applicable0(ego, p)); ------------------ | | 688| 670|#define NO_SLOWP(plnr) (PLNR_L(plnr) & NO_SLOW) | | ------------------ | | | | 674| 335|#define PLNR_L(plnr) ((plnr)->flags.l) | | ------------------ ------------------ | Branch (237:14): [True: 0, False: 335] | Branch (237:33): [True: 0, False: 0] ------------------ 238| 335|} fftw_reodft11e_radix2_r2hc_register: 511| 1|{ 512| 1| REGISTER_SOLVER(p, mksolver()); ------------------ | | 95| 1|#define REGISTER_SOLVER(p, s) X(solver_register)(p, s) | | ------------------ | | | | 78| 1|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 513| 1|} reodft11e-radix2.c:mksolver: 504| 1|{ 505| 1| static const solver_adt sadt = { PROBLEM_RDFT, mkplan, 0 }; 506| 1| S *slv = MKSOLVER(S, &sadt); ------------------ | | 604| 1|#define MKSOLVER(type, adt) (type *)X(mksolver)(sizeof(type), adt) | | ------------------ | | | | 78| 1|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 507| 1| return &(slv->super); 508| 1|} reodft11e-radix2.c:mkplan: 448| 335|{ 449| 335| P *pln; 450| 335| const problem_rdft *p; 451| 335| plan *cld; 452| 335| R *buf; 453| 335| INT n; 454| 335| opcnt ops; 455| | 456| 335| static const plan_adt padt = { 457| 335| X(rdft_solve), awake, print, destroy ------------------ | | 78| 335|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 335|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 458| 335| }; 459| | 460| 335| if (!applicable(ego_, p_, plnr)) ------------------ | Branch (460:10): [True: 335, False: 0] ------------------ 461| 335| return (plan *)0; 462| | 463| 0| p = (const problem_rdft *) p_; 464| | 465| 0| n = p->sz->dims[0].n; 466| 0| buf = (R *) MALLOC(sizeof(R) * n, BUFFERS); ------------------ | | 306| 0|#define MALLOC(n, what) X(malloc_plain)(n) | | ------------------ | | | | 78| 0|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 467| | 468| 0| cld = X(mkplan_d)(plnr, X(mkproblem_rdft_1_d)(X(mktensor_1d)(n/2, 1, 1), ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ cld = X(mkplan_d)(plnr, X(mkproblem_rdft_1_d)(X(mktensor_1d)(n/2, 1, 1), ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ cld = X(mkplan_d)(plnr, X(mkproblem_rdft_1_d)(X(mktensor_1d)(n/2, 1, 1), ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 469| 0| X(mktensor_1d)(2, n/2, n/2), ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 470| 0| buf, buf, R2HC)); ------------------ | | 48| 0|#define R2HC R2HC00 ------------------ 471| 0| X(ifree)(buf); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 472| 0| if (!cld) ------------------ | Branch (472:10): [True: 0, False: 0] ------------------ 473| 0| return (plan *)0; 474| | 475| 0| pln = MKPLAN_RDFT(P, &padt, p->kind[0]==REDFT11 ? apply_re11:apply_ro11); ------------------ | | 73| 0| (type *)X(mkplan_rdft)(sizeof(type), adt, apply) | | ------------------ | | | | 78| 0|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ | | | Branch (73:45): [True: 0, False: 0] | | ------------------ ------------------ 476| 0| pln->n = n; 477| 0| pln->is = p->sz->dims[0].is; 478| 0| pln->os = p->sz->dims[0].os; 479| 0| pln->cld = cld; 480| 0| pln->td = pln->td2 = 0; 481| 0| pln->kind = p->kind[0]; 482| | 483| 0| X(tensor_tornk1)(p->vecsz, &pln->vl, &pln->ivs, &pln->ovs); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 484| | 485| 0| X(ops_zero)(&ops); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 486| 0| ops.add = 2 + (n/2 - 1)/2 * 20; 487| 0| ops.mul = 6 + (n/2 - 1)/2 * 16; 488| 0| ops.other = 4*n + 2 + (n/2 - 1)/2 * 6; 489| 0| if ((n/2) % 2 == 0) { ------------------ | Branch (489:10): [True: 0, False: 0] ------------------ 490| 0| ops.add += 4; 491| 0| ops.mul += 8; 492| 0| ops.other += 4; 493| 0| } 494| | 495| 0| X(ops_zero)(&pln->super.super.ops); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 496| 0| X(ops_madd2)(pln->vl, &ops, &pln->super.super.ops); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 497| 0| X(ops_madd2)(pln->vl, &cld->ops, &pln->super.super.ops); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 498| | 499| 0| return &(pln->super.super); 500| 0|} reodft11e-radix2.c:applicable: 443| 335|{ 444| 335| return (!NO_SLOWP(plnr) && applicable0(ego, p)); ------------------ | | 688| 670|#define NO_SLOWP(plnr) (PLNR_L(plnr) & NO_SLOW) | | ------------------ | | | | 674| 335|#define PLNR_L(plnr) ((plnr)->flags.l) | | ------------------ ------------------ | Branch (444:14): [True: 0, False: 335] | Branch (444:33): [True: 0, False: 0] ------------------ 445| 335|} fftw_rodft00e_r2hc_pad_register: 193| 1|{ 194| 1| REGISTER_SOLVER(p, mksolver()); ------------------ | | 95| 1|#define REGISTER_SOLVER(p, s) X(solver_register)(p, s) | | ------------------ | | | | 78| 1|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 195| 1|} rodft00e-r2hc-pad.c:mksolver: 186| 1|{ 187| 1| static const solver_adt sadt = { PROBLEM_RDFT, mkplan, 0 }; 188| 1| S *slv = MKSOLVER(S, &sadt); ------------------ | | 604| 1|#define MKSOLVER(type, adt) (type *)X(mksolver)(sizeof(type), adt) | | ------------------ | | | | 78| 1|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 1|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 189| 1| return &(slv->super); 190| 1|} rodft00e-r2hc-pad.c:mkplan: 117| 335|{ 118| 335| P *pln; 119| 335| const problem_rdft *p; 120| 335| plan *cld = (plan *) 0, *cldcpy; 121| 335| R *buf = (R *) 0; 122| 335| INT n; 123| 335| INT vl, ivs, ovs; 124| 335| opcnt ops; 125| | 126| 335| static const plan_adt padt = { 127| 335| X(rdft_solve), awake, print, destroy ------------------ | | 78| 335|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 335|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 128| 335| }; 129| | 130| 335| if (!applicable(ego_, p_, plnr)) ------------------ | Branch (130:10): [True: 335, False: 0] ------------------ 131| 335| goto nada; 132| | 133| 0| p = (const problem_rdft *) p_; 134| | 135| 0| n = p->sz->dims[0].n + 1; 136| 0| A(n > 0); 137| 0| buf = (R *) MALLOC(sizeof(R) * (2*n), BUFFERS); ------------------ | | 306| 0|#define MALLOC(n, what) X(malloc_plain)(n) | | ------------------ | | | | 78| 0|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 138| | 139| 0| cld = X(mkplan_d)(plnr,X(mkproblem_rdft_1_d)(X(mktensor_1d)(2*n,1,1), ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ cld = X(mkplan_d)(plnr,X(mkproblem_rdft_1_d)(X(mktensor_1d)(2*n,1,1), ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ cld = X(mkplan_d)(plnr,X(mkproblem_rdft_1_d)(X(mktensor_1d)(2*n,1,1), ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 140| 0| X(mktensor_0d)(), ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 141| 0| buf, buf, R2HC)); ------------------ | | 48| 0|#define R2HC R2HC00 ------------------ 142| 0| if (!cld) ------------------ | Branch (142:10): [True: 0, False: 0] ------------------ 143| 0| goto nada; 144| | 145| 0| X(tensor_tornk1)(p->vecsz, &vl, &ivs, &ovs); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 146| 0| cldcpy = 147| 0| X(mkplan_d)(plnr, ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 148| 0| X(mkproblem_rdft_1_d)(X(mktensor_0d)(), ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ X(mkproblem_rdft_1_d)(X(mktensor_0d)(), ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 149| 0| X(mktensor_1d)(n-1,-1, ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 150| 0| p->sz->dims[0].os), 151| 0| buf+2*n-1,TAINT(p->O, ovs), R2HC)); ------------------ | | 1036| 0|#define TAINT(p, s) (p) ------------------ buf+2*n-1,TAINT(p->O, ovs), R2HC)); ------------------ | | 48| 0|#define R2HC R2HC00 ------------------ 152| 0| if (!cldcpy) ------------------ | Branch (152:10): [True: 0, False: 0] ------------------ 153| 0| goto nada; 154| | 155| 0| X(ifree)(buf); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 156| | 157| 0| pln = MKPLAN_RDFT(P, &padt, apply); ------------------ | | 73| 0| (type *)X(mkplan_rdft)(sizeof(type), adt, apply) | | ------------------ | | | | 78| 0|# define X(name) CONCAT(fftw_, name) | | | | ------------------ | | | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | | | ------------------ | | ------------------ ------------------ 158| | 159| 0| pln->n = n; 160| 0| pln->is = p->sz->dims[0].is; 161| 0| pln->cld = cld; 162| 0| pln->cldcpy = cldcpy; 163| 0| pln->vl = vl; 164| 0| pln->ivs = ivs; 165| 0| pln->ovs = ovs; 166| | 167| 0| X(ops_zero)(&ops); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 168| 0| ops.other = n-1 + 2*n; /* loads + stores (input -> buf) */ 169| | 170| 0| X(ops_zero)(&pln->super.super.ops); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 171| 0| X(ops_madd2)(pln->vl, &ops, &pln->super.super.ops); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 172| 0| X(ops_madd2)(pln->vl, &cld->ops, &pln->super.super.ops); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 173| 0| X(ops_madd2)(pln->vl, &cldcpy->ops, &pln->super.super.ops); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 174| | 175| 0| return &(pln->super.super); 176| | 177| 335| nada: 178| 335| X(ifree0)(buf); ------------------ | | 78| 335|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 335|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 179| 335| if (cld) ------------------ | Branch (179:10): [True: 0, False: 335] ------------------ 180| 0| X(plan_destroy_internal)(cld); ------------------ | | 78| 0|# define X(name) CONCAT(fftw_, name) | | ------------------ | | | | 63| 0|#define CONCAT(prefix, name) prefix ## name | | ------------------ ------------------ 181| 335| return (plan *)0; 182| 0|} rodft00e-r2hc-pad.c:applicable: 112| 335|{ 113| 335| return (!NO_SLOWP(plnr) && applicable0(ego, p)); ------------------ | | 688| 670|#define NO_SLOWP(plnr) (PLNR_L(plnr) & NO_SLOW) | | ------------------ | | | | 674| 335|#define PLNR_L(plnr) ((plnr)->flags.l) | | ------------------ ------------------ | Branch (113:14): [True: 0, False: 335] | Branch (113:33): [True: 0, False: 0] ------------------ 114| 335|} LLVMFuzzerTestOneInput: 15| 319|extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { 16| | // For reference, fftw_compelx is defined: 17| | // typedef double fftw_complex[2] 18| 319| if (size < 1) { ------------------ | Branch (18:7): [True: 0, False: 319] ------------------ 19| 0| return 0; 20| 0| } 21| 319| uint8_t ARRAY_SIZE = (*data) % 250 + 1; 22| 319| data++; 23| 319| size -= 1; 24| 319| if ((ARRAY_SIZE * sizeof(double) * 2) > size) { ------------------ | Branch (24:7): [True: 34, False: 285] ------------------ 25| 34| return 0; 26| 34| } 27| 285| fftw_complex signal[ARRAY_SIZE]; 28| 285| fftw_complex result[ARRAY_SIZE]; 29| | 30| | // Seed the signals with fuzz data 31| 285| memcpy(signal, data, (ARRAY_SIZE * sizeof(double) * 2)); 32| | 33| 285| fftw_plan plan = 34| 285| fftw_plan_dft_1d(ARRAY_SIZE, signal, result, FFTW_FORWARD, FFTW_ESTIMATE); ------------------ | | 489| 285|#define FFTW_FORWARD (-1) ------------------ fftw_plan_dft_1d(ARRAY_SIZE, signal, result, FFTW_FORWARD, FFTW_ESTIMATE); ------------------ | | 502| 285|#define FFTW_ESTIMATE (1U << 6) ------------------ 35| | 36| 285| fftw_execute(plan); 37| 285| fftw_destroy_plan(plan); 38| | 39| 285| return 0; 40| 319|}