rotate_gcc.cc 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374
  1. /*
  2. * Copyright 2015 The LibYuv Project Authors. All rights reserved.
  3. *
  4. * Use of this source code is governed by a BSD-style license
  5. * that can be found in the LICENSE file in the root of the source
  6. * tree. An additional intellectual property rights grant can be found
  7. * in the file PATENTS. All contributing project authors may
  8. * be found in the AUTHORS file in the root of the source tree.
  9. */
  10. #include "libyuv/rotate_row.h"
  11. #include "libyuv/row.h"
  12. #ifdef __cplusplus
  13. namespace libyuv {
  14. extern "C" {
  15. #endif
  16. // This module is for GCC x86 and x64.
  17. #if !defined(LIBYUV_DISABLE_X86) && \
  18. (defined(__x86_64__) || (defined(__i386__) && !defined(_MSC_VER)))
  19. // Transpose 8x8. 32 or 64 bit, but not NaCL for 64 bit.
  20. #if defined(HAS_TRANSPOSEWX8_SSSE3)
  21. void TransposeWx8_SSSE3(const uint8* src,
  22. int src_stride,
  23. uint8* dst,
  24. int dst_stride,
  25. int width) {
  26. asm volatile(
  27. // Read in the data from the source pointer.
  28. // First round of bit swap.
  29. LABELALIGN
  30. "1: \n"
  31. "movq (%0),%%xmm0 \n"
  32. "movq (%0,%3),%%xmm1 \n"
  33. "lea (%0,%3,2),%0 \n"
  34. "punpcklbw %%xmm1,%%xmm0 \n"
  35. "movq (%0),%%xmm2 \n"
  36. "movdqa %%xmm0,%%xmm1 \n"
  37. "palignr $0x8,%%xmm1,%%xmm1 \n"
  38. "movq (%0,%3),%%xmm3 \n"
  39. "lea (%0,%3,2),%0 \n"
  40. "punpcklbw %%xmm3,%%xmm2 \n"
  41. "movdqa %%xmm2,%%xmm3 \n"
  42. "movq (%0),%%xmm4 \n"
  43. "palignr $0x8,%%xmm3,%%xmm3 \n"
  44. "movq (%0,%3),%%xmm5 \n"
  45. "lea (%0,%3,2),%0 \n"
  46. "punpcklbw %%xmm5,%%xmm4 \n"
  47. "movdqa %%xmm4,%%xmm5 \n"
  48. "movq (%0),%%xmm6 \n"
  49. "palignr $0x8,%%xmm5,%%xmm5 \n"
  50. "movq (%0,%3),%%xmm7 \n"
  51. "lea (%0,%3,2),%0 \n"
  52. "punpcklbw %%xmm7,%%xmm6 \n"
  53. "neg %3 \n"
  54. "movdqa %%xmm6,%%xmm7 \n"
  55. "lea 0x8(%0,%3,8),%0 \n"
  56. "palignr $0x8,%%xmm7,%%xmm7 \n"
  57. "neg %3 \n"
  58. // Second round of bit swap.
  59. "punpcklwd %%xmm2,%%xmm0 \n"
  60. "punpcklwd %%xmm3,%%xmm1 \n"
  61. "movdqa %%xmm0,%%xmm2 \n"
  62. "movdqa %%xmm1,%%xmm3 \n"
  63. "palignr $0x8,%%xmm2,%%xmm2 \n"
  64. "palignr $0x8,%%xmm3,%%xmm3 \n"
  65. "punpcklwd %%xmm6,%%xmm4 \n"
  66. "punpcklwd %%xmm7,%%xmm5 \n"
  67. "movdqa %%xmm4,%%xmm6 \n"
  68. "movdqa %%xmm5,%%xmm7 \n"
  69. "palignr $0x8,%%xmm6,%%xmm6 \n"
  70. "palignr $0x8,%%xmm7,%%xmm7 \n"
  71. // Third round of bit swap.
  72. // Write to the destination pointer.
  73. "punpckldq %%xmm4,%%xmm0 \n"
  74. "movq %%xmm0,(%1) \n"
  75. "movdqa %%xmm0,%%xmm4 \n"
  76. "palignr $0x8,%%xmm4,%%xmm4 \n"
  77. "movq %%xmm4,(%1,%4) \n"
  78. "lea (%1,%4,2),%1 \n"
  79. "punpckldq %%xmm6,%%xmm2 \n"
  80. "movdqa %%xmm2,%%xmm6 \n"
  81. "movq %%xmm2,(%1) \n"
  82. "palignr $0x8,%%xmm6,%%xmm6 \n"
  83. "punpckldq %%xmm5,%%xmm1 \n"
  84. "movq %%xmm6,(%1,%4) \n"
  85. "lea (%1,%4,2),%1 \n"
  86. "movdqa %%xmm1,%%xmm5 \n"
  87. "movq %%xmm1,(%1) \n"
  88. "palignr $0x8,%%xmm5,%%xmm5 \n"
  89. "movq %%xmm5,(%1,%4) \n"
  90. "lea (%1,%4,2),%1 \n"
  91. "punpckldq %%xmm7,%%xmm3 \n"
  92. "movq %%xmm3,(%1) \n"
  93. "movdqa %%xmm3,%%xmm7 \n"
  94. "palignr $0x8,%%xmm7,%%xmm7 \n"
  95. "sub $0x8,%2 \n"
  96. "movq %%xmm7,(%1,%4) \n"
  97. "lea (%1,%4,2),%1 \n"
  98. "jg 1b \n"
  99. : "+r"(src), // %0
  100. "+r"(dst), // %1
  101. "+r"(width) // %2
  102. : "r"((intptr_t)(src_stride)), // %3
  103. "r"((intptr_t)(dst_stride)) // %4
  104. : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
  105. "xmm7");
  106. }
  107. #endif // defined(HAS_TRANSPOSEWX8_SSSE3)
  108. // Transpose 16x8. 64 bit
  109. #if defined(HAS_TRANSPOSEWX8_FAST_SSSE3)
  110. void TransposeWx8_Fast_SSSE3(const uint8* src,
  111. int src_stride,
  112. uint8* dst,
  113. int dst_stride,
  114. int width) {
  115. asm volatile(
  116. // Read in the data from the source pointer.
  117. // First round of bit swap.
  118. LABELALIGN
  119. "1: \n"
  120. "movdqu (%0),%%xmm0 \n"
  121. "movdqu (%0,%3),%%xmm1 \n"
  122. "lea (%0,%3,2),%0 \n"
  123. "movdqa %%xmm0,%%xmm8 \n"
  124. "punpcklbw %%xmm1,%%xmm0 \n"
  125. "punpckhbw %%xmm1,%%xmm8 \n"
  126. "movdqu (%0),%%xmm2 \n"
  127. "movdqa %%xmm0,%%xmm1 \n"
  128. "movdqa %%xmm8,%%xmm9 \n"
  129. "palignr $0x8,%%xmm1,%%xmm1 \n"
  130. "palignr $0x8,%%xmm9,%%xmm9 \n"
  131. "movdqu (%0,%3),%%xmm3 \n"
  132. "lea (%0,%3,2),%0 \n"
  133. "movdqa %%xmm2,%%xmm10 \n"
  134. "punpcklbw %%xmm3,%%xmm2 \n"
  135. "punpckhbw %%xmm3,%%xmm10 \n"
  136. "movdqa %%xmm2,%%xmm3 \n"
  137. "movdqa %%xmm10,%%xmm11 \n"
  138. "movdqu (%0),%%xmm4 \n"
  139. "palignr $0x8,%%xmm3,%%xmm3 \n"
  140. "palignr $0x8,%%xmm11,%%xmm11 \n"
  141. "movdqu (%0,%3),%%xmm5 \n"
  142. "lea (%0,%3,2),%0 \n"
  143. "movdqa %%xmm4,%%xmm12 \n"
  144. "punpcklbw %%xmm5,%%xmm4 \n"
  145. "punpckhbw %%xmm5,%%xmm12 \n"
  146. "movdqa %%xmm4,%%xmm5 \n"
  147. "movdqa %%xmm12,%%xmm13 \n"
  148. "movdqu (%0),%%xmm6 \n"
  149. "palignr $0x8,%%xmm5,%%xmm5 \n"
  150. "palignr $0x8,%%xmm13,%%xmm13 \n"
  151. "movdqu (%0,%3),%%xmm7 \n"
  152. "lea (%0,%3,2),%0 \n"
  153. "movdqa %%xmm6,%%xmm14 \n"
  154. "punpcklbw %%xmm7,%%xmm6 \n"
  155. "punpckhbw %%xmm7,%%xmm14 \n"
  156. "neg %3 \n"
  157. "movdqa %%xmm6,%%xmm7 \n"
  158. "movdqa %%xmm14,%%xmm15 \n"
  159. "lea 0x10(%0,%3,8),%0 \n"
  160. "palignr $0x8,%%xmm7,%%xmm7 \n"
  161. "palignr $0x8,%%xmm15,%%xmm15 \n"
  162. "neg %3 \n"
  163. // Second round of bit swap.
  164. "punpcklwd %%xmm2,%%xmm0 \n"
  165. "punpcklwd %%xmm3,%%xmm1 \n"
  166. "movdqa %%xmm0,%%xmm2 \n"
  167. "movdqa %%xmm1,%%xmm3 \n"
  168. "palignr $0x8,%%xmm2,%%xmm2 \n"
  169. "palignr $0x8,%%xmm3,%%xmm3 \n"
  170. "punpcklwd %%xmm6,%%xmm4 \n"
  171. "punpcklwd %%xmm7,%%xmm5 \n"
  172. "movdqa %%xmm4,%%xmm6 \n"
  173. "movdqa %%xmm5,%%xmm7 \n"
  174. "palignr $0x8,%%xmm6,%%xmm6 \n"
  175. "palignr $0x8,%%xmm7,%%xmm7 \n"
  176. "punpcklwd %%xmm10,%%xmm8 \n"
  177. "punpcklwd %%xmm11,%%xmm9 \n"
  178. "movdqa %%xmm8,%%xmm10 \n"
  179. "movdqa %%xmm9,%%xmm11 \n"
  180. "palignr $0x8,%%xmm10,%%xmm10 \n"
  181. "palignr $0x8,%%xmm11,%%xmm11 \n"
  182. "punpcklwd %%xmm14,%%xmm12 \n"
  183. "punpcklwd %%xmm15,%%xmm13 \n"
  184. "movdqa %%xmm12,%%xmm14 \n"
  185. "movdqa %%xmm13,%%xmm15 \n"
  186. "palignr $0x8,%%xmm14,%%xmm14 \n"
  187. "palignr $0x8,%%xmm15,%%xmm15 \n"
  188. // Third round of bit swap.
  189. // Write to the destination pointer.
  190. "punpckldq %%xmm4,%%xmm0 \n"
  191. "movq %%xmm0,(%1) \n"
  192. "movdqa %%xmm0,%%xmm4 \n"
  193. "palignr $0x8,%%xmm4,%%xmm4 \n"
  194. "movq %%xmm4,(%1,%4) \n"
  195. "lea (%1,%4,2),%1 \n"
  196. "punpckldq %%xmm6,%%xmm2 \n"
  197. "movdqa %%xmm2,%%xmm6 \n"
  198. "movq %%xmm2,(%1) \n"
  199. "palignr $0x8,%%xmm6,%%xmm6 \n"
  200. "punpckldq %%xmm5,%%xmm1 \n"
  201. "movq %%xmm6,(%1,%4) \n"
  202. "lea (%1,%4,2),%1 \n"
  203. "movdqa %%xmm1,%%xmm5 \n"
  204. "movq %%xmm1,(%1) \n"
  205. "palignr $0x8,%%xmm5,%%xmm5 \n"
  206. "movq %%xmm5,(%1,%4) \n"
  207. "lea (%1,%4,2),%1 \n"
  208. "punpckldq %%xmm7,%%xmm3 \n"
  209. "movq %%xmm3,(%1) \n"
  210. "movdqa %%xmm3,%%xmm7 \n"
  211. "palignr $0x8,%%xmm7,%%xmm7 \n"
  212. "movq %%xmm7,(%1,%4) \n"
  213. "lea (%1,%4,2),%1 \n"
  214. "punpckldq %%xmm12,%%xmm8 \n"
  215. "movq %%xmm8,(%1) \n"
  216. "movdqa %%xmm8,%%xmm12 \n"
  217. "palignr $0x8,%%xmm12,%%xmm12 \n"
  218. "movq %%xmm12,(%1,%4) \n"
  219. "lea (%1,%4,2),%1 \n"
  220. "punpckldq %%xmm14,%%xmm10 \n"
  221. "movdqa %%xmm10,%%xmm14 \n"
  222. "movq %%xmm10,(%1) \n"
  223. "palignr $0x8,%%xmm14,%%xmm14 \n"
  224. "punpckldq %%xmm13,%%xmm9 \n"
  225. "movq %%xmm14,(%1,%4) \n"
  226. "lea (%1,%4,2),%1 \n"
  227. "movdqa %%xmm9,%%xmm13 \n"
  228. "movq %%xmm9,(%1) \n"
  229. "palignr $0x8,%%xmm13,%%xmm13 \n"
  230. "movq %%xmm13,(%1,%4) \n"
  231. "lea (%1,%4,2),%1 \n"
  232. "punpckldq %%xmm15,%%xmm11 \n"
  233. "movq %%xmm11,(%1) \n"
  234. "movdqa %%xmm11,%%xmm15 \n"
  235. "palignr $0x8,%%xmm15,%%xmm15 \n"
  236. "sub $0x10,%2 \n"
  237. "movq %%xmm15,(%1,%4) \n"
  238. "lea (%1,%4,2),%1 \n"
  239. "jg 1b \n"
  240. : "+r"(src), // %0
  241. "+r"(dst), // %1
  242. "+r"(width) // %2
  243. : "r"((intptr_t)(src_stride)), // %3
  244. "r"((intptr_t)(dst_stride)) // %4
  245. : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
  246. "xmm7", "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", "xmm13", "xmm14",
  247. "xmm15");
  248. }
  249. #endif // defined(HAS_TRANSPOSEWX8_FAST_SSSE3)
  250. // Transpose UV 8x8. 64 bit.
  251. #if defined(HAS_TRANSPOSEUVWX8_SSE2)
  252. void TransposeUVWx8_SSE2(const uint8* src,
  253. int src_stride,
  254. uint8* dst_a,
  255. int dst_stride_a,
  256. uint8* dst_b,
  257. int dst_stride_b,
  258. int width) {
  259. asm volatile(
  260. // Read in the data from the source pointer.
  261. // First round of bit swap.
  262. LABELALIGN
  263. "1: \n"
  264. "movdqu (%0),%%xmm0 \n"
  265. "movdqu (%0,%4),%%xmm1 \n"
  266. "lea (%0,%4,2),%0 \n"
  267. "movdqa %%xmm0,%%xmm8 \n"
  268. "punpcklbw %%xmm1,%%xmm0 \n"
  269. "punpckhbw %%xmm1,%%xmm8 \n"
  270. "movdqa %%xmm8,%%xmm1 \n"
  271. "movdqu (%0),%%xmm2 \n"
  272. "movdqu (%0,%4),%%xmm3 \n"
  273. "lea (%0,%4,2),%0 \n"
  274. "movdqa %%xmm2,%%xmm8 \n"
  275. "punpcklbw %%xmm3,%%xmm2 \n"
  276. "punpckhbw %%xmm3,%%xmm8 \n"
  277. "movdqa %%xmm8,%%xmm3 \n"
  278. "movdqu (%0),%%xmm4 \n"
  279. "movdqu (%0,%4),%%xmm5 \n"
  280. "lea (%0,%4,2),%0 \n"
  281. "movdqa %%xmm4,%%xmm8 \n"
  282. "punpcklbw %%xmm5,%%xmm4 \n"
  283. "punpckhbw %%xmm5,%%xmm8 \n"
  284. "movdqa %%xmm8,%%xmm5 \n"
  285. "movdqu (%0),%%xmm6 \n"
  286. "movdqu (%0,%4),%%xmm7 \n"
  287. "lea (%0,%4,2),%0 \n"
  288. "movdqa %%xmm6,%%xmm8 \n"
  289. "punpcklbw %%xmm7,%%xmm6 \n"
  290. "neg %4 \n"
  291. "lea 0x10(%0,%4,8),%0 \n"
  292. "punpckhbw %%xmm7,%%xmm8 \n"
  293. "movdqa %%xmm8,%%xmm7 \n"
  294. "neg %4 \n"
  295. // Second round of bit swap.
  296. "movdqa %%xmm0,%%xmm8 \n"
  297. "movdqa %%xmm1,%%xmm9 \n"
  298. "punpckhwd %%xmm2,%%xmm8 \n"
  299. "punpckhwd %%xmm3,%%xmm9 \n"
  300. "punpcklwd %%xmm2,%%xmm0 \n"
  301. "punpcklwd %%xmm3,%%xmm1 \n"
  302. "movdqa %%xmm8,%%xmm2 \n"
  303. "movdqa %%xmm9,%%xmm3 \n"
  304. "movdqa %%xmm4,%%xmm8 \n"
  305. "movdqa %%xmm5,%%xmm9 \n"
  306. "punpckhwd %%xmm6,%%xmm8 \n"
  307. "punpckhwd %%xmm7,%%xmm9 \n"
  308. "punpcklwd %%xmm6,%%xmm4 \n"
  309. "punpcklwd %%xmm7,%%xmm5 \n"
  310. "movdqa %%xmm8,%%xmm6 \n"
  311. "movdqa %%xmm9,%%xmm7 \n"
  312. // Third round of bit swap.
  313. // Write to the destination pointer.
  314. "movdqa %%xmm0,%%xmm8 \n"
  315. "punpckldq %%xmm4,%%xmm0 \n"
  316. "movlpd %%xmm0,(%1) \n" // Write back U channel
  317. "movhpd %%xmm0,(%2) \n" // Write back V channel
  318. "punpckhdq %%xmm4,%%xmm8 \n"
  319. "movlpd %%xmm8,(%1,%5) \n"
  320. "lea (%1,%5,2),%1 \n"
  321. "movhpd %%xmm8,(%2,%6) \n"
  322. "lea (%2,%6,2),%2 \n"
  323. "movdqa %%xmm2,%%xmm8 \n"
  324. "punpckldq %%xmm6,%%xmm2 \n"
  325. "movlpd %%xmm2,(%1) \n"
  326. "movhpd %%xmm2,(%2) \n"
  327. "punpckhdq %%xmm6,%%xmm8 \n"
  328. "movlpd %%xmm8,(%1,%5) \n"
  329. "lea (%1,%5,2),%1 \n"
  330. "movhpd %%xmm8,(%2,%6) \n"
  331. "lea (%2,%6,2),%2 \n"
  332. "movdqa %%xmm1,%%xmm8 \n"
  333. "punpckldq %%xmm5,%%xmm1 \n"
  334. "movlpd %%xmm1,(%1) \n"
  335. "movhpd %%xmm1,(%2) \n"
  336. "punpckhdq %%xmm5,%%xmm8 \n"
  337. "movlpd %%xmm8,(%1,%5) \n"
  338. "lea (%1,%5,2),%1 \n"
  339. "movhpd %%xmm8,(%2,%6) \n"
  340. "lea (%2,%6,2),%2 \n"
  341. "movdqa %%xmm3,%%xmm8 \n"
  342. "punpckldq %%xmm7,%%xmm3 \n"
  343. "movlpd %%xmm3,(%1) \n"
  344. "movhpd %%xmm3,(%2) \n"
  345. "punpckhdq %%xmm7,%%xmm8 \n"
  346. "sub $0x8,%3 \n"
  347. "movlpd %%xmm8,(%1,%5) \n"
  348. "lea (%1,%5,2),%1 \n"
  349. "movhpd %%xmm8,(%2,%6) \n"
  350. "lea (%2,%6,2),%2 \n"
  351. "jg 1b \n"
  352. : "+r"(src), // %0
  353. "+r"(dst_a), // %1
  354. "+r"(dst_b), // %2
  355. "+r"(width) // %3
  356. : "r"((intptr_t)(src_stride)), // %4
  357. "r"((intptr_t)(dst_stride_a)), // %5
  358. "r"((intptr_t)(dst_stride_b)) // %6
  359. : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
  360. "xmm7", "xmm8", "xmm9");
  361. }
  362. #endif // defined(HAS_TRANSPOSEUVWX8_SSE2)
  363. #endif // defined(__x86_64__) || defined(__i386__)
  364. #ifdef __cplusplus
  365. } // extern "C"
  366. } // namespace libyuv
  367. #endif