build_base.sh 24 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588
  1. #!/bin/bash
  2. set -ex
  3. # Get an updated config.sub and config.guess
  4. cp $BUILD_PREFIX/share/libtool/build-aux/config.* .
  5. # The LTO/PGO information was sourced from @pitrou and the Debian rules file in:
  6. # http://http.debian.net/debian/pool/main/p/python3.6/python3.6_3.6.2-2.debian.tar.xz
  7. # https://packages.debian.org/source/sid/python3.6
  8. # or:
  9. # http://bazaar.launchpad.net/~doko/python/pkg3.5-debian/view/head:/rules#L255
  10. # .. but upstream regrtest.py now has --pgo (since >= 3.6) and skips tests that are:
  11. # "not helpful for PGO".
  12. VER=${PKG_VERSION%.*}
  13. TCLTK_VER=${tk}
  14. # Disables some PGO/LTO but not optimizations.
  15. QUICK_BUILD=no
  16. _buildd_static=build-static
  17. _buildd_shared=build-shared
  18. _ENABLE_SHARED=--enable-shared
  19. # We *still* build a shared lib here for non-static embedded use cases
  20. _DISABLE_SHARED=--disable-shared
  21. # Hack to allow easily comparing static vs shared interpreter performance
  22. # .. hack because we just build it shared in both the build-static and
  23. # build-shared directories.
  24. # Yes this hack is a bit confusing, sorry about that.
  25. if [[ ${PY_INTERP_LINKAGE_NATURE} == shared ]]; then
  26. _DISABLE_SHARED=--enable-shared
  27. _ENABLE_SHARED=--enable-shared
  28. fi
  29. # For debugging builds, set this to no to disable profile-guided optimization
  30. if [[ ${DEBUG_C} == yes ]]; then
  31. _OPTIMIZED=no
  32. else
  33. _OPTIMIZED=yes
  34. fi
  35. # ppc64le cdt need to be rebuilt with files in powerpc64le-conda-linux-gnu instead of powerpc64le-conda_cos7-linux-gnu. In the mean time:
  36. if [ "$(uname -m)" = "ppc64le" ]; then
  37. cp --force --archive --update --link $BUILD_PREFIX/powerpc64le-conda_cos7-linux-gnu/. $BUILD_PREFIX/powerpc64le-conda-linux-gnu
  38. fi
  39. export PKG_CONFIG_PATH=${PKG_CONFIG_PATH:-}:${PREFIX}/lib/pkgconfig:$BUILD_PREFIX/$BUILD/sysroot/usr/lib64/pkgconfig:$BUILD_PREFIX/$BUILD/sysroot/usr/share/pkgconfig
  40. # Since these take very long to build in our emulated ci, disable for now
  41. if [[ ${CONDA_FORGE} == yes ]]; then
  42. if [[ ${target_platform} == linux-aarch64 ]]; then
  43. _OPTIMIZED=no
  44. fi
  45. if [[ ${target_platform} == linux-ppc64le ]]; then
  46. _OPTIMIZED=no
  47. fi
  48. fi
  49. declare -a _dbg_opts
  50. if [[ ${DEBUG_PY} == yes ]]; then
  51. # This Python will not be usable with non-debug Python modules.
  52. _dbg_opts+=(--with-pydebug)
  53. DBG=d
  54. else
  55. DBG=
  56. fi
  57. ABIFLAGS=${DBG}
  58. VERABI=${VER}${DBG}
  59. # Make sure the "python" value in conda_build_config.yaml is up to date.
  60. test "${PY_VER}" = "${VER}"
  61. # This is the mechanism by which we fall back to default gcc, but having it defined here
  62. # would probably break the build by using incorrect settings and/or importing files that
  63. # do not yet exist.
  64. unset _PYTHON_SYSCONFIGDATA_NAME
  65. unset _CONDA_PYTHON_SYSCONFIGDATA_NAME
  66. # Remove bzip2's shared library if present,
  67. # as we only want to link to it statically.
  68. # This is important in cases where conda
  69. # tries to update bzip2.
  70. find "${PREFIX}/lib" -name "libbz2*${SHLIB_EXT}*" | xargs rm -fv {}
  71. # Prevent lib/python${VER}/_sysconfigdata_*.py from ending up with full paths to these things
  72. # in _build_env because _build_env will not get found during prefix replacement, only _h_env_placeh ...
  73. AR=$(basename "${AR}")
  74. # CC must contain the string 'gcc' or else distutils thinks it is on macOS and uses '-R' to set rpaths.
  75. if [[ ${target_platform} == osx-* ]]; then
  76. CC=$(basename "${CC}")
  77. else
  78. CC=$(basename "${GCC}")
  79. _CCACHE=$(type -P ccache) || true
  80. if [[ ${_CCACHE} =~ ${BUILD_PREFIX}.* ]]; then
  81. CC="${_CCACHE} ${CC}"
  82. fi
  83. fi
  84. CXX=$(basename "${CXX}")
  85. RANLIB=$(basename "${RANLIB}")
  86. READELF=$(basename "${READELF}")
  87. if [[ ${HOST} =~ .*darwin.* ]] && [[ -n ${CONDA_BUILD_SYSROOT} ]]; then
  88. # Python's setup.py will figure out that this is a macOS sysroot.
  89. CFLAGS="-isysroot ${CONDA_BUILD_SYSROOT} "${CFLAGS}
  90. LDFLAGS="-isysroot ${CONDA_BUILD_SYSROOT} "${LDFLAGS}
  91. CPPFLAGS="-isysroot ${CONDA_BUILD_SYSROOT} "${CPPFLAGS}
  92. fi
  93. # Debian uses -O3 then resets it at the end to -O2 in _sysconfigdata.py
  94. if [[ ${_OPTIMIZED} = yes && ${target_platform} != osx-* ]]; then
  95. CPPFLAGS=$(echo "${CPPFLAGS}" | sed "s/-O2/-O3/g")
  96. CFLAGS=$(echo "${CFLAGS}" | sed "s/-O2/-O3/g")
  97. CXXFLAGS=$(echo "${CXXFLAGS}" | sed "s/-O2/-O3/g")
  98. fi
  99. if [[ ${CONDA_FORGE} == yes ]]; then
  100. ${SYS_PYTHON} ${RECIPE_DIR}/brand_python.py
  101. fi
  102. declare -a LTO_CFLAGS=()
  103. # Following is needed for building extensions like zlib
  104. CPPFLAGS=${CPPFLAGS}" -I${PREFIX}/include"
  105. re='^(.*)(-I[^ ]*)(.*)$'
  106. if [[ ${CFLAGS} =~ $re ]]; then
  107. CFLAGS="${BASH_REMATCH[1]}${BASH_REMATCH[3]}"
  108. fi
  109. # https://src.fedoraproject.org/rpms/python39/pull-request/9
  110. if [[ ${target_platform} =~ linux.* ]] && [[ ${_OPTIMIZED} == yes ]]; then
  111. CFLAGS_NODIST="${CFLAGS_NODIST} -fno-semantic-interposition"
  112. fi
  113. # Force rebuild to avoid:
  114. # ../work/Modules/unicodename_db.h:24118:30: note: (near initialization for 'code_hash')
  115. # ../work/Modules/unicodename_db.h:24118:33: warning: excess elements in scalar initializer
  116. # 0, 0, 12018, 0, 0, 0, 0, 0, 4422, 4708, 3799, 119358, 119357, 0, 120510,
  117. # ^~~~
  118. # This should have been fixed by https://github.com/python/cpython/commit/7c69c1c0fba8c1c8ff3969bce4c1135736a4cc58
  119. # .. but that appears incomplete. In particular, the generated files contain:
  120. # /* this file was generated by Tools/unicode/makeunicodedata.py 3.2 */
  121. # .. yet the PR updated to version of makeunicodedata.py to 3.3
  122. # rm -f Modules/unicodedata_db.h Modules/unicodename_db.h
  123. # ${SYS_PYTHON} ${SRC_DIR}/Tools/unicode/makeunicodedata.py
  124. # .. instead we revert this commit for now.
  125. export CPPFLAGS CFLAGS CXXFLAGS LDFLAGS
  126. if [[ ${target_platform} == osx-* ]]; then
  127. sed -i -e "s/@OSX_ARCH@/$ARCH/g" Lib/distutils/unixccompiler.py
  128. fi
  129. if [[ "${CONDA_BUILD_CROSS_COMPILATION}" == "1" ]]; then
  130. # Build the exact same Python for the build machine. It would be nice (and might be
  131. # possible already?) to be able to make this just an 'exact' pinned build dependency
  132. # of a split-package?
  133. BUILD_PYTHON_PREFIX=${PWD}/build-python-install
  134. mkdir build-python-build
  135. pushd build-python-build
  136. (unset CPPFLAGS LDFLAGS;
  137. export CC=${CC_FOR_BUILD} \
  138. CXX=${CXX_FOR_BUILD} \
  139. CPP="${CC_FOR_BUILD} -E" \
  140. CFLAGS="-O2" \
  141. AR="$(${CC_FOR_BUILD} --print-prog-name=ar)" \
  142. RANLIB="$(${CC_FOR_BUILD} --print-prog-name=ranlib)" \
  143. LD="$(${CC_FOR_BUILD} --print-prog-name=ld)" && \
  144. ${SRC_DIR}/configure --build=${BUILD} \
  145. --host=${BUILD} \
  146. --prefix=${BUILD_PYTHON_PREFIX} \
  147. --with-ensurepip=no \
  148. --with-tzpath=${PREFIX}/share/zoneinfo \
  149. --with-platlibdir=lib && \
  150. make -j${CPU_COUNT} && \
  151. make install)
  152. export PATH=${BUILD_PYTHON_PREFIX}/bin:${PATH}
  153. ln -s ${BUILD_PYTHON_PREFIX}/bin/python${VER} ${BUILD_PYTHON_PREFIX}/bin/python
  154. popd
  155. echo "ac_cv_file__dev_ptmx=yes" > config.site
  156. echo "ac_cv_file__dev_ptc=yes" >> config.site
  157. echo "ac_cv_pthread=yes" >> config.site
  158. echo "ac_cv_little_endian_double=yes" >> config.site
  159. if [[ ${target_platform} == osx-arm64 ]]; then
  160. echo "ac_cv_aligned_required=no" >> config.site
  161. echo "ac_cv_file__dev_ptc=no" >> config.site
  162. echo "ac_cv_pthread_is_default=yes" >> config.site
  163. echo "ac_cv_working_tzset=yes" >> config.site
  164. echo "ac_cv_pthread_system_supported=yes" >> config.site
  165. fi
  166. export CONFIG_SITE=${PWD}/config.site
  167. # This is needed for libffi:
  168. export PKG_CONFIG_PATH=${PREFIX}/lib/pkgconfig
  169. fi
  170. # This causes setup.py to query the sysroot directories from the compiler, something which
  171. # IMHO should be done by default anyway with a flag to disable it to workaround broken ones.
  172. # Technically, setting _PYTHON_HOST_PLATFORM causes setup.py to consider it cross_compiling
  173. if [[ -n ${HOST} ]]; then
  174. if [[ ${HOST} =~ .*darwin.* ]]; then
  175. # Even if BUILD is .*darwin.* you get better isolation by cross_compiling (no /usr/local)
  176. IFS='-' read -r host_arch host_os host_kernel <<<"${HOST}"
  177. export _PYTHON_HOST_PLATFORM=darwin-${host_arch}
  178. else
  179. IFS='-' read -r host_arch host_vendor host_os host_libc <<<"${HOST}"
  180. export _PYTHON_HOST_PLATFORM=${host_os}-${host_arch}
  181. fi
  182. fi
  183. if [[ ${target_platform} == osx-* ]]; then
  184. # TODO: check with LLVM 12 if the following hack is needed.
  185. # https://reviews.llvm.org/D76461 may have fixed the need for the following hack.
  186. echo '#!/bin/bash' > $BUILD_PREFIX/bin/$HOST-llvm-ar
  187. echo "$BUILD_PREFIX/bin/llvm-ar --format=darwin" '"$@"' >> $BUILD_PREFIX/bin/$HOST-llvm-ar
  188. chmod +x $BUILD_PREFIX/bin/$HOST-llvm-ar
  189. echo "WARNING :: For some reason, configure finds libintl (gettext) in the BUILD_PREFIX on macOS."
  190. echo "WARNING :: to prevent this, removing BUILD_PREFIX/include/libintl.h"
  191. echo "WARNING :: and also setting ac_cv_lib_intl_textdomain=no"
  192. rm -f ${BUILD_PREFIX}/include/libintl.h
  193. export ac_cv_lib_intl_textdomain=no
  194. fi
  195. if [[ ${target_platform} == osx-64 ]]; then
  196. export MACHDEP=darwin
  197. export ac_sys_system=Darwin
  198. export ac_sys_release=13.4.0
  199. export MACOSX_DEFAULT_ARCH=x86_64
  200. export ARCHFLAGS="-arch x86_64"
  201. export CFLAGS="$CFLAGS $ARCHFLAGS"
  202. elif [[ ${target_platform} == osx-arm64 ]]; then
  203. export MACHDEP=darwin
  204. export ac_sys_system=Darwin
  205. export ac_sys_release=20.0.0
  206. export MACOSX_DEFAULT_ARCH=arm64
  207. export ARCHFLAGS="-arch arm64"
  208. export CFLAGS="$CFLAGS $ARCHFLAGS"
  209. elif [[ ${target_platform} == linux-* ]]; then
  210. export MACHDEP=linux
  211. export ac_sys_system=Linux
  212. export ac_sys_release=
  213. fi
  214. # Not used at present but we should run 'make test' and finish up TESTOPTS (see debians rules).
  215. declare -a TEST_EXCLUDES
  216. TEST_EXCLUDES+=(test_ensurepip test_venv)
  217. TEST_EXCLUDES+=(test_tcl test_codecmaps_cn test_codecmaps_hk
  218. test_codecmaps_jp test_codecmaps_kr test_codecmaps_tw
  219. test_normalization test_ossaudiodev test_socket)
  220. if [[ ! -f /dev/dsp ]]; then
  221. TEST_EXCLUDES+=(test_linuxaudiodev test_ossaudiodev)
  222. fi
  223. # hangs on Aarch64, see LP: #1264354
  224. if [[ ${CC} =~ .*-aarch64.* ]]; then
  225. TEST_EXCLUDES+=(test_faulthandler)
  226. fi
  227. if [[ ${CC} =~ .*-arm.* ]]; then
  228. TEST_EXCLUDES+=(test_ctypes)
  229. TEST_EXCLUDES+=(test_compiler)
  230. fi
  231. declare -a _common_configure_args
  232. _common_configure_args+=(--prefix=${PREFIX})
  233. _common_configure_args+=(--build=${BUILD})
  234. _common_configure_args+=(--host=${HOST})
  235. _common_configure_args+=(--enable-ipv6)
  236. _common_configure_args+=(--with-ensurepip=no)
  237. _common_configure_args+=(--with-tzpath=${PREFIX}/share/zoneinfo)
  238. _common_configure_args+=(--with-computed-gotos)
  239. _common_configure_args+=(--with-system-ffi)
  240. _common_configure_args+=(--enable-loadable-sqlite-extensions)
  241. _common_configure_args+=(--with-tcltk-includes="-I${PREFIX}/include")
  242. _common_configure_args+=("--with-tcltk-libs=-L${PREFIX}/lib -ltcl8.6 -ltk8.6")
  243. _common_configure_args+=(--with-platlibdir=lib)
  244. _common_configure_args+=(--with-openssl="${PREFIX}")
  245. if [[ ${target_platform} == osx-arm64 ]]; then
  246. _common_configure_args+=(--with-dtrace)
  247. fi
  248. _common_configure_args+=(PKG_CONFIG_LIBDIR="${PREFIX}/lib")
  249. _common_configure_args+=(PKG_CONFIG_PATH="${PREFIX}/lib")
  250. _common_configure_args+=(CPPFLAGS="${CPPFLAGS} -I${PREFIX}/include")
  251. _common_configure_args+=(CXXFLAGS="${CXXFLAGS} -I${PREFIX}/include")
  252. _common_configure_args+=(CFLAGS="${CFLAGS} -I${PREFIX}/include")
  253. _common_configure_args+=(LDFLAGS="${LDFLAGS} -L${PREFIX}/lib")
  254. _common_configure_args+=(CC="${CC}")
  255. _common_configure_args+=(CXX="${CXX}")
  256. _comoon_configure_args+=(CC_FOR_BUIL="${CC}")
  257. # Add more optimization flags for the static Python interpreter:
  258. declare -a PROFILE_TASK=()
  259. if [[ ${_OPTIMIZED} == yes ]]; then
  260. _common_configure_args+=(--with-lto)
  261. if [[ "$CONDA_BUILD_CROSS_COMPILATION" != "1" ]]; then
  262. _common_configure_args+=(--enable-optimizations)
  263. _MAKE_TARGET=profile-opt
  264. # To speed up build times during testing (1):
  265. if [[ ${QUICK_BUILD} == yes ]]; then
  266. echo "WARNING :: Setting empty PROFILE_TASK as QUICK_BUILD set"
  267. _PROFILE_TASK+=(PROFILE_TASK="")
  268. else
  269. # From talking to Steve Dower, who implemented pgo/pgo-extended, it is really not worth
  270. # it to run pgo-extended (which runs the whole test-suite). The --pgo set of tests are
  271. # curated specifically to be useful/appropriate for pgo instrumentation.
  272. # _PROFILE_TASK+=(PROFILE_TASK="-m test --pgo-extended")
  273. _PROFILE_TASK+=(PROFILE_TASK="-m test --pgo")
  274. fi
  275. fi
  276. if [[ ${CC} =~ .*gcc.* ]]; then
  277. LTO_CFLAGS+=(-fuse-linker-plugin)
  278. LTO_CFLAGS+=(-ffat-lto-objects)
  279. # -flto must come after -flto-partition due to the replacement code
  280. # TODO :: Replace the replacement code using conda-build's in-build regex replacement.
  281. LTO_CFLAGS+=(-flto-partition=none)
  282. LTO_CFLAGS+=(-flto)
  283. else
  284. # TODO :: Check if -flto=thin gives better results. It is about faster
  285. # compilation rather than faster execution so probably not:
  286. # http://clang.llvm.org/docs/ThinLTO.html
  287. # http://blog.llvm.org/2016/06/thinlto-scalable-and-incremental-lto.html
  288. LTO_CFLAGS+=(-flto)
  289. # -flto breaks the check to determine whether float word ordering is bigendian
  290. # see:
  291. # https://bugs.python.org/issue28015
  292. # https://bugs.python.org/issue38527
  293. # manually specify this setting
  294. export ax_cv_c_float_words_bigendian=no
  295. fi
  296. export CFLAGS="${CFLAGS} ${LTO_CFLAGS[@]}"
  297. else
  298. _MAKE_TARGET=
  299. fi
  300. mkdir -p ${_buildd_shared}
  301. pushd ${_buildd_shared}
  302. set +e
  303. ${SRC_DIR}/configure "${_common_configure_args[@]}" \
  304. "${_dbg_opts[@]}" \
  305. --oldincludedir=${BUILD_PREFIX}/${HOST}/sysroot/usr/include \
  306. --enable-shared
  307. if [[ $? != 0 ]]; then
  308. echo "ERROR :: configure of shared python failed. config.log contains:"
  309. cat config.log
  310. exit 1
  311. fi
  312. set +e
  313. popd
  314. mkdir -p ${_buildd_static}
  315. pushd ${_buildd_static}
  316. ${SRC_DIR}/configure "${_common_configure_args[@]}" \
  317. "${_dbg_opts[@]}" \
  318. -oldincludedir=${BUILD_PREFIX}/${HOST}/sysroot/usr/include \
  319. ${_DISABLE_SHARED} "${_PROFILE_TASK[@]}"
  320. popd
  321. if [[ ${target_platform} == linux-ppc64le ]]; then
  322. # Travis has issues with long logs
  323. make -j${CPU_COUNT} -C ${_buildd_static} \
  324. EXTRA_CFLAGS="${EXTRA_CFLAGS}" \
  325. ${_MAKE_TARGET} "${_PROFILE_TASK[@]}" 2>&1 >make-static.log
  326. else
  327. make -j${CPU_COUNT} -C ${_buildd_static} \
  328. EXTRA_CFLAGS="${EXTRA_CFLAGS}" \
  329. ${_MAKE_TARGET} "${_PROFILE_TASK[@]}" 2>&1 | tee make-static.log
  330. fi
  331. if rg "Failed to build these modules" make-static.log; then
  332. echo "(static) :: Failed to build some modules, check the log"
  333. exit 1
  334. fi
  335. if [[ ${target_platform} == linux-ppc64le ]]; then
  336. # Travis has issues with long logs
  337. make -j${CPU_COUNT} -C ${_buildd_shared} \
  338. EXTRA_CFLAGS="${EXTRA_CFLAGS}" 2>&1 >make-shared.log
  339. elif [[ ${target_platform} == osx-* ]]; then
  340. # Additional env vars required for osx builds.
  341. # The "-undefined" flag allows for undefined symbols.
  342. env LDFLAGS="${LDFLAGS} -Xlinker -undefined -Xlinker dynamic_lookup" \
  343. make -j${CPU_COUNT} -C ${_buildd_shared} \
  344. CROSS_COMPILE=no \
  345. # This is the key fix for ensuring libpython*.dylib is built:
  346. BLDSHARED="${CC} -shared" \
  347. V=1 \
  348. _PYTHON_HOST_PLATFORM="${_PYTHON_HOST_PLATFORM}" \
  349. EXTRA_CFLAGS="${EXTRA_CFLAGS}" 2>&1 | tee make-shared.log
  350. else
  351. make -j${CPU_COUNT} -C ${_buildd_shared} \
  352. EXTRA_CFLAGS="${EXTRA_CFLAGS}" 2>&1 | tee make-shared.log
  353. fi
  354. if rg "Failed to build these modules" make-shared.log; then
  355. echo "(shared) :: Failed to build some modules, check the log"
  356. exit 1
  357. fi
  358. # build a static library with PIC objects and without LTO/PGO
  359. make -j${CPU_COUNT} -C ${_buildd_shared} \
  360. EXTRA_CFLAGS="${EXTRA_CFLAGS}" \
  361. LIBRARY=libpython${VERABI}-pic.a libpython${VERABI}-pic.a
  362. make -C ${_buildd_static} install
  363. declare -a _FLAGS_REPLACE=()
  364. if [[ ${target_platform} != osx-* ]]; then
  365. if [[ -n ${_CCACHE} ]]; then
  366. _FLAGS_REPLACE+=("${_CCACHE}"); _FLAGS_REPLACE+=("")
  367. fi
  368. fi
  369. _FLAGS_REPLACE+=("-L."); _FLAGS_REPLACE+=("")
  370. # 3 entries as this can be split over two lines.
  371. _FLAGS_REPLACE+=("-isysroot ${CONDA_BUILD_SYSROOT}"); _FLAGS_REPLACE+=("")
  372. _FLAGS_REPLACE+=("-isysroot"); _FLAGS_REPLACE+=("")
  373. _FLAGS_REPLACE+=("${CONDA_BUILD_SYSROOT}"); _FLAGS_REPLACE+=("")
  374. # fdebug-prefix-map for python work dir is useless for extensions
  375. _FLAGS_REPLACE+=("-fdebug-prefix-map=$SRC_DIR=/usr/local/src/conda/python-$PKG_VERSION"); _FLAGS_REPLACE+=("")
  376. _FLAGS_REPLACE+=("-fdebug-prefix-map=$PREFIX=/usr/local/src/conda-prefix"); _FLAGS_REPLACE+=("")
  377. if [[ ${_OPTIMIZED} == yes ]]; then
  378. _FLAGS_REPLACE+=("-O3"); _FLAGS_REPLACE+=("-O2")
  379. _FLAGS_REPLACE+=("-fprofile-use"); _FLAGS_REPLACE+=("")
  380. _FLAGS_REPLACE+=("-fprofile-correction"); _FLAGS_REPLACE+=("")
  381. for _LTO_CFLAG in "${LTO_CFLAGS[@]}"; do
  382. _FLAGS_REPLACE+=("${_LTO_CFLAG}"); _FLAGS_REPLACE+=("")
  383. done
  384. fi
  385. # Install the shared library (for people who embed Python only, e.g. GDB).
  386. # Linking module extensions to this on Linux is redundant (but harmless).
  387. # Linking module extensions to this on Darwin is harmful (multiply defined symbols).
  388. cp -pf ${_buildd_shared}/libpython*${SHLIB_EXT}* ${PREFIX}/lib/
  389. if [[ ${target_platform} =~ .*linux.* ]]; then
  390. ln -sf ${PREFIX}/lib/libpython${VERABI}${SHLIB_EXT}.1.0 ${PREFIX}/lib/libpython${VERABI}${SHLIB_EXT}
  391. fi
  392. SYSCONFIG=$(find ${_buildd_static}/$(cat ${_buildd_static}/pybuilddir.txt) -name "_sysconfigdata*.py" -print0)
  393. cat ${SYSCONFIG} | ${SYS_PYTHON} "${RECIPE_DIR}"/replace-word-pairs.py \
  394. "${_FLAGS_REPLACE[@]}" \
  395. > ${PREFIX}/lib/python${VER}/$(basename ${SYSCONFIG})
  396. MAKEFILE=$(find ${PREFIX}/lib/python${VER}/ -path "*config-*/Makefile" -print0)
  397. cp ${MAKEFILE} /tmp/Makefile-$$
  398. cat /tmp/Makefile-$$ | ${SYS_PYTHON} "${RECIPE_DIR}"/replace-word-pairs.py \
  399. "${_FLAGS_REPLACE[@]}" \
  400. > ${MAKEFILE}
  401. # Check to see that our differences took.
  402. # echo diff -urN ${SYSCONFIG} ${PREFIX}/lib/python${VER}/$(basename ${SYSCONFIG})
  403. # diff -urN ${SYSCONFIG} ${PREFIX}/lib/python${VER}/$(basename ${SYSCONFIG})
  404. # Python installs python${VER}m and python${VER}, one as a hardlink to the other. conda-build breaks these
  405. # by copying. Since the executable may be static it may be very large so change one to be a symlink
  406. # of the other. In this case, python${VER}m will be the symlink.
  407. if [[ -f ${PREFIX}/bin/python${VER}m ]]; then
  408. rm -f ${PREFIX}/bin/python${VER}m
  409. ln -s ${PREFIX}/bin/python${VER} ${PREFIX}/bin/python${VER}m
  410. fi
  411. ln -s ${PREFIX}/bin/python${VER} ${PREFIX}/bin/python
  412. ln -s ${PREFIX}/bin/pydoc${VER} ${PREFIX}/bin/pydoc
  413. # Exclude test data from the base package to save space
  414. # though keep `support` as some things use that.
  415. # TODO :: Make a subpackage for this once we implement multi-level testing.
  416. pushd ${PREFIX}/lib/python${VER}
  417. mkdir test_keep
  418. mv test/__init__.py test/support test/test_support* test/test_script_helper* test_keep/
  419. # This will be put into the regr-testsuite package.
  420. mkdir ${SRC_DIR}/test.backup || true
  421. mv test ${SRC_DIR}/test.backup/
  422. if [[ $(uname) == Darwin ]]; then
  423. rsync */test ${SRC_DIR}/test.backup/
  424. else
  425. cp -rnf --parents */test ${SRC_DIR}/test.backup/
  426. fi
  427. cp -rf */test ${SRC_DIR}/test.backup/
  428. rm -rf test */test
  429. mv test_keep test
  430. popd
  431. # Size reductions:
  432. pushd ${PREFIX}
  433. if [[ -f lib/libpython${VERABI}.a ]]; then
  434. chmod +w lib/libpython${VERABI}.a
  435. ${STRIP} -S lib/libpython${VERABI}.a
  436. fi
  437. CONFIG_LIBPYTHON=$(find lib/python${VER}/config-${VERABI}* -name "libpython${VERABI}.a")
  438. if [[ -f lib/libpython${VERABI}.a ]] && [[ -f ${CONFIG_LIBPYTHON} ]]; then
  439. chmod +w ${CONFIG_LIBPYTHON}
  440. rm ${CONFIG_LIBPYTHON}
  441. fi
  442. popd
  443. # OLD_HOST is with CentOS version in them. When building this recipe
  444. # with the compilers from conda-forge OLD_HOST != HOST, but when building
  445. # with the compilers from defaults OLD_HOST == HOST. Both cases are handled in the
  446. # code below
  447. case "$target_platform" in
  448. linux-64)
  449. OLD_HOST=$(echo ${HOST} | sed -e 's/-conda-/-conda_cos6-/g')
  450. ;;
  451. linux-*)
  452. OLD_HOST=$(echo ${HOST} | sed -e 's/-conda-/-conda_cos7-/g')
  453. ;;
  454. *)
  455. OLD_HOST=$HOST
  456. ;;
  457. esac
  458. # Copy sysconfig that gets recorded to a non-default name
  459. # using the new compilers with python will require setting _PYTHON_SYSCONFIGDATA_NAME
  460. # to the name of this file (minus the .py extension)
  461. pushd "${PREFIX}"/lib/python${VER}
  462. # On Python 3.5 _sysconfigdata.py was getting copied in here and compiled for some reason.
  463. # This breaks our attempt to find the right one as recorded_name.
  464. find lib-dynload -name "_sysconfigdata*.py*" -exec rm {} \;
  465. recorded_name=$(find . -name "_sysconfigdata*.py")
  466. our_compilers_name=_sysconfigdata_$(echo ${HOST} | sed -e 's/[.-]/_/g').py
  467. # So we can see if anything has significantly diverged by looking in a built package.
  468. cp ${recorded_name} ${recorded_name}.orig
  469. cp ${recorded_name} sysconfigfile
  470. # fdebug-prefix-map for python work dir is useless for extensions
  471. sed -i.bak "s@-fdebug-prefix-map=$SRC_DIR=/usr/local/src/conda/python-$PKG_VERSION@@g" sysconfigfile
  472. sed -i.bak "s@-fdebug-prefix-map=$PREFIX=/usr/local/src/conda-prefix@@g" sysconfigfile
  473. # Append the conda-forge zoneinfo to the end
  474. sed -i.bak "s@zoneinfo'@zoneinfo:$PREFIX/share/tzinfo'@g" sysconfigfile
  475. # Remove osx sysroot as it depends on the build machine
  476. # sed -i.bak "s@-isysroot ${CONDA_BUILD_SYSROOT}@@g" sysconfigfile
  477. # Remove unfilled config option
  478. sed -i.bak "s/@SGI_ABI@//g" sysconfigfile
  479. cp sysconfigfile ${our_compilers_name}
  480. sed -i.bak "s@${HOST}@${OLD_HOST}@g" sysconfigfile
  481. old_compiler_name=_sysconfigdata_$(echo ${OLD_HOST} | sed -e 's/[.-]/_/g').py
  482. cp sysconfigfile ${old_compiler_name}
  483. if [[ "$target_platform" == linux-64 ]]; then
  484. HOST_COS=no
  485. if [[ "$HOST" == *_cos6* ]]; then
  486. HOST_COS=$(echo $HOST | sed -e 's/_cos6/_cos7/g')
  487. elif [[ "$OLD_HOST" == *_cos6* ]]; then
  488. HOST_COS=$(echo $OLD_HOST | sed -e 's/_cos6/_cos7/g')
  489. elif [[ "$HOST" == *_cos7* ]]; then
  490. HOST_COS=$(echo $HOST | sed -e 's/_cos7/_cos6/g')
  491. elif [[ "$OLD_HOST" == *_cos7* ]]; then
  492. HOST_COS=$(echo $OLD_HOST | sed -e 's/_cos7/_cos6/g')
  493. fi
  494. if [[ ${HOST_COS} == *cos* ]]; then
  495. cp sysconfigfile sysconfigfile_alt
  496. cos_compiler_name=_sysconfigdata_$(echo ${HOST_COS} | sed -e 's/[.-]/_/g').py
  497. sed -i.bak "s@${OLD_HOST}@${HOST_COS}@g" sysconfigfile_alt
  498. cp sysconfigfile_alt ${cos_compiler_name}
  499. fi
  500. fi
  501. # For system gcc remove the triple
  502. sed -i.bak "s@$OLD_HOST-c++@g++@g" sysconfigfile
  503. sed -i.bak "s@$OLD_HOST-@@g" sysconfigfile
  504. if [[ "$target_platform" == linux* ]]; then
  505. # For linux, make sure the system gcc uses our linker
  506. sed -i.bak "s@-pthread@-pthread -B $PREFIX/compiler_compat@g" sysconfigfile
  507. fi
  508. # Don't set -march and -mtune for system gcc
  509. sed -i.bak "s@-march=[^( |\\\"|\\\')]*@@g" sysconfigfile
  510. sed -i.bak "s@-mtune=[^( |\\\"|\\\')]*@@g" sysconfigfile
  511. # Remove these flags that older compilers and linkers may not know
  512. for flag in "-fstack-protector-strong" "-ffunction-sections" "-pipe" "-fno-plt" \
  513. "-ftree-vectorize" "-Wl,--sort-common" "-Wl,--as-needed" "-Wl,-z,relro" \
  514. "-Wl,-z,now" "-Wl,--disable-new-dtags" "-Wl,--gc-sections" "-Wl,-O2" \
  515. "-fPIE" "-ftree-vectorize" "-mssse3" "-Wl,-pie" "-Wl,-dead_strip_dylibs" \
  516. "-Wl,-headerpad_max_install_names"; do
  517. sed -i.bak "s@$flag@@g" sysconfigfile
  518. done
  519. # Cleanup some extra spaces from above
  520. sed -r -i.bak "s/' +'/''/g" sysconfigfile
  521. cp sysconfigfile $recorded_name
  522. rm sysconfigfile
  523. rm sysconfigfile.bak
  524. popd
  525. if [[ ${HOST} =~ .*linux.* ]]; then
  526. mkdir -p ${PREFIX}/compiler_compat
  527. ln -s ${PREFIX}/bin/${HOST}-ld ${PREFIX}/compiler_compat/ld
  528. echo "Files in this folder are to enhance backwards compatibility of anaconda software with older compilers." > ${PREFIX}/compiler_compat/README
  529. echo "See: https://github.com/conda/conda/issues/6030 for more information." >> ${PREFIX}/compiler_compat/README
  530. fi
  531. # There are some strange distutils files around. Delete them
  532. rm -rf ${PREFIX}/lib/python${VER}/distutils/command/*.exe
  533. python -c "import compileall,os;compileall.compile_dir(os.environ['PREFIX'])"
  534. rm ${PREFIX}/lib/libpython${VER}.a