From c32f5c8adba60ac3b7530c83814e13ed3167fc1e Mon Sep 17 00:00:00 2001 From: "Eric N. Vander Weele" Date: Fri, 26 Jul 2019 13:28:28 -0400 Subject: [PATCH 1/2] bpo-37690: Simplify linking of shared libraries on the AIX OS Have the approach of building shared libraries on the AIX operating system be similar to that of a System V system. The primary benefit of this change is the elimination of custom AIX paths and reducing the changes at `./configure` to affect just the `LDSHARED` environment variable. For background context, AIX sees shared libraries as fully linked and resolved, where symbol references are resolved at link-time and cannot be rebound at load-time. System V resolves all global symbols by the run-time linker. Thus, conventional shared libraries in AIX cannot have undefined symbols, while System V can. However, AIX does allow for run-time linking in allowing symbols to be undefined until load-time. Therefore, this change affects how linking of shared libraries are performed on AIX to behave similarly to that of System V. Given that symbols are now going to be allowed to be undefined for AIX, all the code paths for generating exported symbols and the related wrapper scripts go away. The real magic is in the `-G` flag for `LDSHARED`. Effectively, `-G` is equivalent to specifying the following: * -berok: Suppress errors even if there are unresolved symbols * -brtl: Enable run-time linking * -bnortllib: Do not include a reference to the run-time linker * -bnosymbolic: Assigns 'nosymbolic' attribute to most symbols (i.e., can be rebound) * -bnoautoexp: Prevent auto exportation of any symbols * -bM:SRE: Set the module type to reusable (i.e., require a private copy of the data area for each process). --- .gitignore | 1 - Lib/sysconfig.py | 5 -- Makefile.pre.in | 18 +--- Misc/README.AIX | 38 ++------- Modules/ld_so_aix.in | 195 ------------------------------------------- Modules/makexp_aix | 81 ------------------ configure | 21 +---- configure.ac | 14 +--- 8 files changed, 10 insertions(+), 363 deletions(-) delete mode 100644 Modules/ld_so_aix.in delete mode 100755 Modules/makexp_aix diff --git a/.gitignore b/.gitignore index 9445ef1e2c5252..3b02617c79de1d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,5 @@ # added for local development .buildaix/ -Modules/python.exp buildaix/ installp/ .gitignore diff --git a/Lib/sysconfig.py b/Lib/sysconfig.py index e76e6927cb1ff3..882b1cb9bddf8d 100644 --- a/Lib/sysconfig.py +++ b/Lib/sysconfig.py @@ -375,11 +375,6 @@ def _generate_posix_vars(): if hasattr(e, "strerror"): msg = msg + " (%s)" % e.strerror raise OSError(msg) - # On AIX, there are wrong paths to the linker scripts in the Makefile - # -- these paths are relative to the Python source, but when installed - # the scripts are in another directory. - if _PYTHON_BUILD: - vars['BLDSHARED'] = vars['LDSHARED'] # There's a chicken-and-egg situation on OS X with regards to the # _sysconfigdata module after the changes introduced by #15298: diff --git a/Makefile.pre.in b/Makefile.pre.in index 363a4eb9d2cb29..c7447832e9ce84 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1578,21 +1578,6 @@ libainstall: @DEF_MAKE_RULE@ python-config $(INSTALL_SCRIPT) $(srcdir)/install-sh $(DESTDIR)$(LIBPL)/install-sh $(INSTALL_SCRIPT) python-config.py $(DESTDIR)$(LIBPL)/python-config.py $(INSTALL_SCRIPT) python-config $(DESTDIR)$(BINDIR)/python$(LDVERSION)-config - @if [ -s Modules/python.exp -a \ - "`echo $(MACHDEP) | sed 's/^\(...\).*/\1/'`" = "aix" ]; then \ - echo; echo "Installing support files for building shared extension modules on AIX:"; \ - $(INSTALL_DATA) Modules/python.exp \ - $(DESTDIR)$(LIBPL)/python.exp; \ - echo; echo "$(LIBPL)/python.exp"; \ - $(INSTALL_SCRIPT) $(srcdir)/Modules/makexp_aix \ - $(DESTDIR)$(LIBPL)/makexp_aix; \ - echo "$(LIBPL)/makexp_aix"; \ - $(INSTALL_SCRIPT) Modules/ld_so_aix \ - $(DESTDIR)$(LIBPL)/ld_so_aix; \ - echo "$(LIBPL)/ld_so_aix"; \ - echo; echo "See Misc/AIX-NOTES for details."; \ - else true; \ - fi # Install the dynamically loadable modules # This goes into $(exec_prefix) @@ -1787,8 +1772,7 @@ distclean: clobber if test "$$file" != "$(srcdir)/Lib/test/data/README"; then rm "$$file"; fi; \ done -rm -f core Makefile Makefile.pre config.status Modules/Setup.local \ - Modules/ld_so_aix Modules/python.exp Misc/python.pc \ - Misc/python-embed.pc Misc/python-config.sh + Misc/python.pc Misc/python-embed.pc Misc/python-config.sh -rm -f python*-gdb.py # Issue #28258: set LC_ALL to avoid issues with Estonian locale. # Expansion is performed here by shell (spawned by make) itself before diff --git a/Misc/README.AIX b/Misc/README.AIX index 92ad0293e65850..13150f22f30147 100644 --- a/Misc/README.AIX +++ b/Misc/README.AIX @@ -28,13 +28,17 @@ There are various aliases for the native compiler. The recommended alias for compiling Python is 'xlc_r', which provides a better level of compatibility and handles thread initialization properly. -It is a good idea to add the '-qmaxmem=70000' option, otherwise the -compiler considers various files too complex to optimize. +It is a good idea to add the '-qmaxmem=-1' option, otherwise the +compiler is limited by the amount of memory allocated for performing +specific, memory-intensive optimizations. The value '-1' permits as +much memory as needed (i.e., not limited). If you find that +compilation fails due to exceeding available system resources, +previous recommendations of '70000' have been advised. Compiling with xlc: cd Python-3.2 -CC=xlc_r OPT="-O2 -qmaxmem=70000" ./configure --without-computed-gotos --enable-shared +CC=xlc_r OPT="-O2 -qmaxmem=-1" ./configure --without-computed-gotos --enable-shared make Note: @@ -100,31 +104,3 @@ Those issues are currently affecting Python on AIX: * issue 9920: minor arithmetic issues in cmath * issue 11215: test_fileio fails - - - -====================================================================== - Implementation details for developers ----------------------------------------------------------------------- - -Python and python modules can now be built as shared libraries on AIX -as usual. - -AIX shared libraries require that an "export" and "import" file be -provided at compile time to list all extern symbols which may be -shared between modules. The "export" file (named python.exp) for the -modules and the libraries that belong to the Python core is created by -the "makexp_aix" script before performing the link of the python -binary. It lists all global symbols (exported during the link) of the -modules and the libraries that make up the python executable. - -When shared library modules (.so files) are made, a second shell -script is invoked. This script is named "ld_so_aix" and is also -provided with the distribution in the Modules subdirectory. This -script acts as an "ld" wrapper which hides the explicit management of -"export" and "import" files; it adds the appropriate arguments (in the -appropriate order) to the link command that creates the shared module. -Among other things, it specifies that the "python.exp" file is an -"import" file for the shared module. - -This mechanism should be transparent. diff --git a/Modules/ld_so_aix.in b/Modules/ld_so_aix.in deleted file mode 100644 index f4eab40b6b8219..00000000000000 --- a/Modules/ld_so_aix.in +++ /dev/null @@ -1,195 +0,0 @@ -#!/bin/sh -# -# ======================================================================== -# FILE: ld_so_aix -# TYPE: executable, uses makexp_aix -# SYSTEM: AIX -# -# DESCRIPTION: Creates a shareable .o from a set of pre-compiled -# (unshared) .o files -# -# USAGE: ld_so_aix [CC] [arguments] -# -# ARGUMENTS: Same as for "ld". The following arguments are processed -# or supplied by this script (those marked with an asterisk -# can be overridden from command line): -# -# Argument Default value -# (*) -o [OutputFileName] -o shr.o -# (*) -e [EntryPointLabel] -e init[OutputBaseName] -# (*) -bE:[ExportFile] -bE:[OutputBaseName].exp -# (*) -bI:[ImportFile] -bI:./python.exp -# -bM:[ModuleType] -bM:SRE -# -bhalt:[Number] -bhalt:4 -# -T[Number] -T512 -# -H[Number] -H512 -# -lm -# -# The compiler specific ("-lc" or "-lc_r", "-lpthreads",...) -# arguments will be automatically passed to "ld" according -# to the CC command provided as a first argument to this -# script. Usually, the same CC command was used to produce -# the pre-compiled .o file(s). -# -# NOTES: 1. Since "ld_so_aix" was originally written for building -# shared modules for the Python interpreter, the -e and -# -bI default values match Python's conventions. In -# Python, the entry point for a shared module is based -# on the module's name (e.g., the "mathmodule" will -# expect an entry point of "initmath"). -# 2. The script accepts multiple .o or .a input files and -# creates a single (shared) output file. The export list -# that is created is based on the output file's basename -# with the suffix ".exp". -# 3. The resulting shared object file is left in the -# current directory. -# 4. Uncommenting the "echo" lines gives detailed output -# about the commands executed in the script. -# -# -# HISTORY: Oct-1996 -- Support added for multiple .o files -- -# -- and optional arguments processing. -- -# Chris Myers (myers@tc.cornell.edu), Keith Kwok -# (kkwok@tc.cornell.edu) and Vladimir Marangozov -# -# Aug-6-1996 -- Take care of the compiler specific -- -# -- args by leaving CC to invoke "ld". -- -# Vladimir Marangozov -# -# Jul-1-1996 -- Make sure to use /usr/ccs/bin/ld -- -# -- Use makexp_aix for the export list. -- -# Vladimir Marangozov (Vladimir.Marangozov@imag.fr) -# -# Manus Hand (mhand@csn.net) -- Initial code -- 6/24/96 -# ======================================================================== -# - -usage="Usage: ld_so_aix [CC command] [ld arguments]" -if test ! -n "$*"; then - echo $usage; exit 2 -fi - -makexp=`dirname $0`/makexp_aix -test -x "${makexp}" || makexp="@abs_srcdir@/makexp_aix" - -# Check for existence of compiler. -CC=$1; shift -whichcc=`which $CC` - -if test ! -x "$whichcc"; then - echo "ld_so_aix: Compiler '$CC' not found; exiting." - exit 2 -fi - -if test ! -n "$*"; then - echo $usage; exit 2 -fi - -# Default import file for Python -# Can be overridden by providing a -bI: argument. -impfile="./python.exp" - -# Parse arguments -while test -n "$1" -do - case "$1" in - -e | -Wl,-e) - if test -z "$2"; then - echo "ld_so_aix: The -e flag needs a parameter; exiting."; exit 2 - else - shift; entry=$1 - fi - ;; - -e* | -Wl,-e*) - entry=`echo $1 | sed -e "s/-Wl,//" -e "s/-e//"` - ;; - -o) - if test -z "$2"; then - echo "ld_so_aix: The -o flag needs a parameter; exiting."; exit 2 - else - shift; objfile=$1 - fi - ;; - -o*) - objfile=`echo $1 | sed "s/-o//"` - ;; - -bI:* | -Wl,-bI:*) - impfile=`echo $1 | sed -e "s/-Wl,//" -e "s/-bI://"` - ;; - -bE:* | -Wl,-bE:*) - expfile=`echo $1 | sed -e "s/-Wl,//" -e "s/-bE://"` - ;; - *.o | *.a) - objs="$objs $1" - args="$args $1" - ;; - -bM:* | -Wl,-bM:* | -H* | -Wl,-H* | -T* | -Wl,-T* | -lm) - ;; - *) - args="$args $1" - ;; - esac - shift -done - -if test "$objfile" = "libpython@VERSION@@ABIFLAGS@.so"; then - ldsocoremode="true" -fi - -if test -z "$objs"; then - echo "ld_so_aix: No input files; exiting." - exit 2 -elif test ! -r "$impfile" -a -z "$ldsocoremode"; then - echo "ld_so_aix: Import file '$impfile' not found or not readable; exiting." - exit 2 -fi - -# If -o wasn't specified, assume "-o shr.o" -if test -z "$objfile"; then - objfile=shr.o -fi - -filename=`basename $objfile | sed "s/\.[^.]*$//"` - -# If -bE: wasn't specified, assume "-bE:$filename.exp" -if test -z "$expfile"; then - expfile="$filename.exp" -fi - -# Default entry symbol for Python modules = init[modulename] -# Can be overridden by providing a -e argument. -if test -z "$entry"; then - entry=PyInit_`echo $filename | sed "s/module.*//"` -fi - -#echo "ld_so_aix: Debug info section" -#echo " -> output file : $objfile" -#echo " -> import file : $impfile" -#echo " -> export file : $expfile" -#echo " -> entry point : $entry" -#echo " -> object files: $objs" -#echo " -> CC arguments: $args" - -if test -z "$ldsocoremode"; then - CCOPT="-Wl,-e$entry -Wl,-bE:$expfile -Wl,-bI:$impfile -Wl,-bhalt:4" -else - CCOPT="-Wl,-bnoentry -Wl,-bE:$expfile -Wl,-bhalt:4" -fi -CCOPT="$CCOPT -Wl,-bM:SRE -Wl,-T512 -Wl,-H512 -Wl,-brtl -Wl,-bnortllib -lm -o $objfile" - -CCARGS="$args" - -# Export list generation. -#echo $makexp $expfile "$objfile" $objs -$makexp $expfile "$objfile" $objs - -# Perform the link. -#echo $CC $CCOPT $CCARGS -$CC $CCOPT $CCARGS -retval=$? - -# Delete the module's export list file. -# Comment this line if you need it. -rm -f $expfile - -exit $retval diff --git a/Modules/makexp_aix b/Modules/makexp_aix deleted file mode 100755 index cb349c28757396..00000000000000 --- a/Modules/makexp_aix +++ /dev/null @@ -1,81 +0,0 @@ -#!/bin/sh -# -# =========================================================================== -# FILE: makexp_aix -# TYPE: standalone executable -# SYSTEM: AIX 3.2.5 and AIX 4 -# -# DESCRIPTION: This script creates an export list of ALL global symbols -# from a list of object or archive files. -# -# USAGE: makexp_aix "" ... -# -# where: -# is the target export list filename. -# is the path/file string to be appended -# after the "#!" symbols in the first line of the -# export file. Passing "" means deferred resolution. -# is an object (.o) or an archive file (.a). -# -# HISTORY: -# 3-Apr-1998 -- remove C++ entries of the form Class::method -# Vladimir Marangozov -# -# 1-Jul-1996 -- added header information -# Vladimir Marangozov -# -# 28-Jun-1996 -- initial code -# Vladimir Marangozov (Vladimir.Marangozov@imag.fr) -# ========================================================================== - -# Variables -expFileName=$1 -toAppendStr=$2 -shift; shift; -inputFiles=$* -automsg="Generated automatically by makexp_aix" -notemsg="NOTE: lists _all_ global symbols defined in the above file(s)." -curwdir=`pwd` - -# Create the export file and setup the header info -echo "#!"$toAppendStr > $expFileName -echo "*" >> $expFileName -echo "* $automsg (`date -u`)" >> $expFileName -echo "*" >> $expFileName -echo "* Base Directory: $curwdir" >> $expFileName -echo "* Input File(s) : $inputFiles" >> $expFileName -echo "*" >> $expFileName -echo "* $notemsg" >> $expFileName -echo "*" >> $expFileName - -# Extract the symbol list using 'nm' which produces quite -# different output under AIX 4 than under AIX 3.2.5. -# The following handles both versions by using a common flagset. -# Here are some hidden tricks: -# 1. Use /usr/ccs/bin/nm. Relevant to AIX 3.2.5 which has -# another version under /usr/ucb/bin/nm. -# 2. Use the -B flag to have a standard BSD representation -# of the symbol list on both AIX 3.2.5 and AIX 4. The "-B" -# flag is missing in the AIX 3.2.5 online usage help of 'nm'. -# 3. Use the -x flag to have a hex representation of the symbol -# values. This fills the leading whitespaces on AIX 4, -# thus simplifying the sed statement. -# 4. Eliminate all entries except those with either "B", "D" -# or "T" key letters. We are interested only in the global -# (extern) BSS, DATA and TEXT symbols. With the same statement -# we eliminate object member lines relevant to AIX 4. -# 5. Eliminate entries containing a dot. We can have a dot only -# as a symbol prefix, but such symbols are undefined externs. -# 6. Eliminate everything including the key letter, so that we're -# left with just the symbol name. -# 7. Eliminate all entries containing two colons, like Class::method -# - -# Use -X32_64 if it appears to be implemented in this version of 'nm'. -NM=/usr/ccs/bin/nm -xopt=-X32_64 -$NM -e $xopt $1 >/dev/null 2>&1 || xopt="" - -$NM -Bex $xopt $inputFiles \ -| sed -e '/ [^BDT] /d' -e '/\./d' -e 's/.* [BDT] //' -e '/::/d' \ -| sort | uniq >> $expFileName diff --git a/configure b/configure index 4cea98e3652896..972c6b5e314e4b 100755 --- a/configure +++ b/configure @@ -5754,8 +5754,6 @@ LDVERSION="$VERSION" # If CXX is set, and if it is needed to link a main function that was # compiled with CXX, LINKCC is CXX instead. Always using CXX is undesirable: # python might then depend on the C++ runtime -# This is altered for AIX in order to build the export list before -# linking. { $as_echo "$as_me:${as_lineno-$LINENO}: checking LINKCC" >&5 $as_echo_n "checking LINKCC... " >&6; } @@ -5763,13 +5761,6 @@ if test -z "$LINKCC" then LINKCC='$(PURIFY) $(MAINCC)' case $ac_sys_system in - AIX*) - exp_extra="\"\"" - if test $ac_sys_release -ge 5 -o \ - $ac_sys_release -eq 4 -a `uname -r` -ge 2 ; then - exp_extra="." - fi - LINKCC="\$(srcdir)/Modules/makexp_aix Modules/python.exp $exp_extra \$(LIBRARY); $LINKCC";; QNX*) # qcc must be used because the other compilers do not # support -N. @@ -9357,8 +9348,7 @@ if test -z "$LDSHARED" then case $ac_sys_system/$ac_sys_release in AIX*) - BLDSHARED="Modules/ld_so_aix \$(CC) -bI:Modules/python.exp" - LDSHARED="\$(LIBPL)/ld_so_aix \$(CC) -bI:\$(LIBPL)/python.exp" + LDSHARED='$(CC) -G' ;; SunOS/5*) if test "$GCC" = "yes" ; then @@ -9529,7 +9519,6 @@ $as_echo_n "checking LINKFORSHARED... " >&6; } if test -z "$LINKFORSHARED" then case $ac_sys_system/$ac_sys_release in - AIX*) LINKFORSHARED='-Wl,-bE:Modules/python.exp -lld';; hp*|HP*) LINKFORSHARED="-Wl,-E -Wl,+s";; # LINKFORSHARED="-Wl,-E -Wl,+s -Wl,+b\$(BINLIBDEST)/lib-dynload";; @@ -17373,8 +17362,6 @@ fi # generate output files ac_config_files="$ac_config_files Makefile.pre Misc/python.pc Misc/python-embed.pc Misc/python-config.sh" -ac_config_files="$ac_config_files Modules/ld_so_aix" - cat >confcache <<\_ACEOF # This file is a shell script that caches the results of configure # tests run on this system so they can be shared between configure @@ -18076,7 +18063,6 @@ do "Misc/python.pc") CONFIG_FILES="$CONFIG_FILES Misc/python.pc" ;; "Misc/python-embed.pc") CONFIG_FILES="$CONFIG_FILES Misc/python-embed.pc" ;; "Misc/python-config.sh") CONFIG_FILES="$CONFIG_FILES Misc/python-config.sh" ;; - "Modules/ld_so_aix") CONFIG_FILES="$CONFIG_FILES Modules/ld_so_aix" ;; *) as_fn_error $? "invalid argument: \`$ac_config_target'" "$LINENO" 5;; esac @@ -18633,11 +18619,6 @@ $as_echo "$as_me: $ac_file is unchanged" >&6;} esac - - case $ac_file$ac_mode in - "Modules/ld_so_aix":F) chmod +x Modules/ld_so_aix ;; - - esac done # for ac_tag diff --git a/configure.ac b/configure.ac index b9759e12f89f74..7b690323dbd68e 100644 --- a/configure.ac +++ b/configure.ac @@ -1006,21 +1006,12 @@ LDVERSION="$VERSION" # If CXX is set, and if it is needed to link a main function that was # compiled with CXX, LINKCC is CXX instead. Always using CXX is undesirable: # python might then depend on the C++ runtime -# This is altered for AIX in order to build the export list before -# linking. AC_SUBST(LINKCC) AC_MSG_CHECKING(LINKCC) if test -z "$LINKCC" then LINKCC='$(PURIFY) $(MAINCC)' case $ac_sys_system in - AIX*) - exp_extra="\"\"" - if test $ac_sys_release -ge 5 -o \ - $ac_sys_release -eq 4 -a `uname -r` -ge 2 ; then - exp_extra="." - fi - LINKCC="\$(srcdir)/Modules/makexp_aix Modules/python.exp $exp_extra \$(LIBRARY); $LINKCC";; QNX*) # qcc must be used because the other compilers do not # support -N. @@ -2513,8 +2504,7 @@ if test -z "$LDSHARED" then case $ac_sys_system/$ac_sys_release in AIX*) - BLDSHARED="Modules/ld_so_aix \$(CC) -bI:Modules/python.exp" - LDSHARED="\$(LIBPL)/ld_so_aix \$(CC) -bI:\$(LIBPL)/python.exp" + LDSHARED='$(CC) -G' ;; SunOS/5*) if test "$GCC" = "yes" ; then @@ -2681,7 +2671,6 @@ AC_MSG_CHECKING(LINKFORSHARED) if test -z "$LINKFORSHARED" then case $ac_sys_system/$ac_sys_release in - AIX*) LINKFORSHARED='-Wl,-bE:Modules/python.exp -lld';; hp*|HP*) LINKFORSHARED="-Wl,-E -Wl,+s";; # LINKFORSHARED="-Wl,-E -Wl,+s -Wl,+b\$(BINLIBDEST)/lib-dynload";; @@ -5620,7 +5609,6 @@ AC_DEFINE(PY_SSL_DEFAULT_CIPHERS, 1) # generate output files AC_CONFIG_FILES(Makefile.pre Misc/python.pc Misc/python-embed.pc Misc/python-config.sh) -AC_CONFIG_FILES([Modules/ld_so_aix], [chmod +x Modules/ld_so_aix]) AC_OUTPUT echo "creating Modules/Setup.local" >&AS_MESSAGE_FD From d1d5d62f03a9055f49d5fb313402371af711a255 Mon Sep 17 00:00:00 2001 From: "Eric N. Vander Weele" Date: Fri, 26 Jul 2019 13:41:39 -0400 Subject: [PATCH 2/2] Add `Misc/News.d/next/` entry Ensure that the summary of the change will be rendered in release notes. --- Misc/NEWS.d/next/Build/2019-07-26-13-41-35.bpo-37690.sVdcJk.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Build/2019-07-26-13-41-35.bpo-37690.sVdcJk.rst diff --git a/Misc/NEWS.d/next/Build/2019-07-26-13-41-35.bpo-37690.sVdcJk.rst b/Misc/NEWS.d/next/Build/2019-07-26-13-41-35.bpo-37690.sVdcJk.rst new file mode 100644 index 00000000000000..cf3152972b4605 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2019-07-26-13-41-35.bpo-37690.sVdcJk.rst @@ -0,0 +1,2 @@ +Simplify linking of shared libraries on the AIX to be similar to that of a +System V system. Patch by Eric N. Vander Weele.