Issue #437 - Add static Python 3.12

.github/packaging/specs-python.wxs.in

@@ -13,17 +13,27 @@
     <Property Id="ARPDISPLAYVERSION" Value="@DISPLAY_VERSION@" />
 
     <StandardDirectory Id="ProgramFiles6432Folder">
-      <Directory Id="INSTALLFOLDER" Name="specs">
-        <Component Id="MainExecutable" Guid="B2C3D4E5-F6A7-8901-BCDE-F12345678902">
-          <File Id="SpecsExe" Source="specs\bin\Release\specs.exe" KeyPath="yes" />
-          <Environment Id="PATH" Name="PATH" Value="[INSTALLFOLDER]"
-                       Permanent="no" Part="last" Action="set" System="yes" />
-        </Component>
-      </Directory>
+      <Directory Id="INSTALLFOLDER" Name="specs" />
     </StandardDirectory>
 
+    <!-- The main executable; carries the PATH environment entry. -->
+    <ComponentGroup Id="MainExecutable" Directory="INSTALLFOLDER">
+      <Component Guid="B2C3D4E5-F6A7-8901-BCDE-F12345678902">
+        <File Id="SpecsExe" Source="msi-stage\specs.exe" KeyPath="yes" />
+        <Environment Id="PATH" Name="PATH" Value="[INSTALLFOLDER]"
+                     Permanent="no" Part="last" Action="set" System="yes" />
+      </Component>
+    </ComponentGroup>
+
+    <!-- The bundled Python runtime: pythonXY.dll, the C extension modules in
+         DLLs\, and the standard library in Lib\. specs.exe loads the DLL from
+         its own directory and points Python's home there at runtime, so no
+         system Python installation is required. This component group is harvested
+         by heat from the msi-stage directory. -->
+
     <Feature Id="MainFeature" Title="specs (Python 3.12)" Level="1">
-      <ComponentRef Id="MainExecutable" />
+      <ComponentGroupRef Id="MainExecutable" />
+      <ComponentGroupRef Id="PythonRuntime" />
     </Feature>
   </Package>
 </Wix>

.github/packaging/specs.spec.in

@@ -6,7 +6,7 @@ License:        MIT
 URL:            https://github.com/yoavnir/specs2016
 Source0:        specs-%{version}.tar.gz
 
-# Disable automatic dependency detection since Python is statically linked
+# Disable automatic dependency detection since Python is bundled
 AutoReqProv: no
 
 %description
@@ -20,12 +20,15 @@ multiple lines into single lines or vice versa.
 %install
 mkdir -p %{buildroot}/usr/local/bin
 mkdir -p %{buildroot}/usr/share/specs
+mkdir -p %{buildroot}/usr/share/doc/specs
 mkdir -p %{buildroot}/usr/lib/specs
 mkdir -p %{buildroot}/etc/bash_completion.d
 install -m 755 specs %{buildroot}/usr/local/bin/specs
 install -m 755 specs-autocomplete %{buildroot}/usr/local/bin/specs-autocomplete
 install -m 644 specs.1.gz %{buildroot}/usr/share/specs/specs.1.gz
 install -m 644 specs-completion.bash %{buildroot}/etc/bash_completion.d/specs
+install -m 644 docs/LICENSE %{buildroot}/usr/share/doc/specs/LICENSE
+install -m 644 docs/PYTHON_LICENSE %{buildroot}/usr/share/doc/specs/PYTHON_LICENSE
 cp -r python %{buildroot}/usr/lib/specs/
 
 %post
@@ -94,4 +97,6 @@ fi
 /usr/local/bin/specs-autocomplete
 /usr/share/specs/specs.1.gz
 /etc/bash_completion.d/specs
+/usr/share/doc/specs/LICENSE
+/usr/share/doc/specs/PYTHON_LICENSE
 /usr/lib/specs/python

.github/workflows/release.yml

@@ -83,18 +83,34 @@ jobs:
         cp specs.1.gz rpmbuild/SOURCES/specs-${SPECS_VERSION#v}/
         cp .github/packaging/specs-completion.bash rpmbuild/SOURCES/specs-${SPECS_VERSION#v}/
 
-    - name: Bundle Python stdlib for RPM
+    - name: Bundle Python stdlib and shared library for RPM
       run: |
-        PYVER=$(python3 -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')")
+        PYVER=$(python3.12 -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')")
+        # Use the platlibdir reported by the bundled Python (e.g. "lib" on
+        # Debian/Ubuntu, "lib64" on Fedora/RHEL-based) so the bundled stdlib
+        # lands in the directory that libpython will actually search.
+        PLATLIBDIR=$(python3.12 -c "import sys; print(sys.platlibdir)")
         SRCDIR=rpmbuild/SOURCES/specs-${SPECS_VERSION#v}
-        mkdir -p ${SRCDIR}/python/lib/python${PYVER}
-        cp -r /usr/lib/python${PYVER}/* ${SRCDIR}/python/lib/python${PYVER}/
+        mkdir -p ${SRCDIR}/python/${PLATLIBDIR}/python${PYVER}
+        cp -r /usr/${PLATLIBDIR}/python${PYVER}/* ${SRCDIR}/python/${PLATLIBDIR}/python${PYVER}/ 2>/dev/null || \
+        cp -r /usr/lib/python${PYVER}/* ${SRCDIR}/python/${PLATLIBDIR}/python${PYVER}/
+        # Copy the shared libpython library into the same platlibdir subtree
+        cp -L /usr/${PLATLIBDIR}/libpython${PYVER}.so.1.0 ${SRCDIR}/python/${PLATLIBDIR}/ 2>/dev/null || \
+        cp -L /usr/lib/x86_64-linux-gnu/libpython${PYVER}.so.1.0 ${SRCDIR}/python/${PLATLIBDIR}/ 2>/dev/null || \
+        cp -L /usr/lib/aarch64-linux-gnu/libpython${PYVER}.so.1.0 ${SRCDIR}/python/${PLATLIBDIR}/ 2>/dev/null || true
         # Remove unnecessary files to reduce package size
-        find ${SRCDIR}/python/lib/python${PYVER} -type d -name __pycache__ -exec rm -rf {} + 2>/dev/null || true
-        find ${SRCDIR}/python/lib/python${PYVER} -type f -name "*.pyc" -delete
-        find ${SRCDIR}/python/lib/python${PYVER} -type f -name "*.pyo" -delete
-        rm -rf ${SRCDIR}/python/lib/python${PYVER}/site-packages
-        rm -rf ${SRCDIR}/python/lib/python${PYVER}/dist-packages
+        find ${SRCDIR}/python/${PLATLIBDIR}/python${PYVER} -type d -name __pycache__ -exec rm -rf {} + 2>/dev/null || true
+        find ${SRCDIR}/python/${PLATLIBDIR}/python${PYVER} -type f -name "*.pyc" -delete
+        find ${SRCDIR}/python/${PLATLIBDIR}/python${PYVER} -type f -name "*.pyo" -delete
+        rm -rf ${SRCDIR}/python/${PLATLIBDIR}/python${PYVER}/site-packages
+        rm -rf ${SRCDIR}/python/${PLATLIBDIR}/python${PYVER}/dist-packages
+
+    - name: Copy license files for RPM
+      run: |
+        SRCDIR=rpmbuild/SOURCES/specs-${SPECS_VERSION#v}
+        mkdir -p ${SRCDIR}/docs
+        cp LICENSE ${SRCDIR}/docs/
+        cp PYTHON_LICENSE ${SRCDIR}/docs/
 
     - name: Create RPM tarball
       run: |
@@ -151,12 +167,51 @@ jobs:
         mkdir -p pkg-root/usr/local/bin
         mkdir -p pkg-root/usr/local/share/man/man1
         mkdir -p pkg-root/usr/local/share/zsh/site-functions
+        mkdir -p pkg-root/usr/local/share/doc/specs
         cp specs/exe/specs pkg-root/usr/local/bin/
         cp specs/exe/specs-autocomplete pkg-root/usr/local/bin/
         chmod 755 pkg-root/usr/local/bin/specs
         chmod 755 pkg-root/usr/local/bin/specs-autocomplete
         cp specs.1.gz pkg-root/usr/local/share/man/man1/
         cp .github/packaging/specs-completion.zsh pkg-root/usr/local/share/zsh/site-functions/_specs
+        cp LICENSE pkg-root/usr/local/share/doc/specs/
+        cp PYTHON_LICENSE pkg-root/usr/local/share/doc/specs/
+
+    - name: Bundle Python dylib and stdlib for pkg
+      run: |
+        PYVER=$(python3.12 -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')")
+        # Resolve the bundled install prefix used by setup.py (-> PYTHON_STDLIB_PATH)
+        BUNDLE_PREFIX=/usr/local/lib/specs/python
+        LIBDIR=$(python3.12 -c "import sysconfig; print(sysconfig.get_config_var('LIBDIR'))")
+        STDLIB=$(python3.12 -c "import sysconfig; print(sysconfig.get_path('stdlib'))")
+        mkdir -p pkg-root${BUNDLE_PREFIX}/lib/python${PYVER}
+        # Locate and copy the shared libpython dylib (cp -L resolves any symlink
+        # into the framework's real Mach-O so the bundled copy is standalone)
+        SRC_DYLIB="${LIBDIR}/libpython${PYVER}.dylib"
+        if [ ! -e "${SRC_DYLIB}" ]; then
+          SRC_DYLIB=$(find "$(python3.12 -c "import sys; print(sys.base_prefix)")" -name "libpython${PYVER}.dylib" | head -n1)
+        fi
+        cp -L "${SRC_DYLIB}" pkg-root${BUNDLE_PREFIX}/lib/libpython${PYVER}.dylib
+        # Copy the standard library (includes lib-dynload extension modules)
+        cp -R "${STDLIB}/" pkg-root${BUNDLE_PREFIX}/lib/python${PYVER}/
+        # Trim files that are not needed at runtime to reduce package size
+        find pkg-root${BUNDLE_PREFIX}/lib/python${PYVER} -type d -name __pycache__ -exec rm -rf {} + 2>/dev/null || true
+        find pkg-root${BUNDLE_PREFIX}/lib/python${PYVER} -type f -name "*.pyc" -delete
+        rm -rf pkg-root${BUNDLE_PREFIX}/lib/python${PYVER}/site-packages
+        rm -rf pkg-root${BUNDLE_PREFIX}/lib/python${PYVER}/test
+        # Repoint the binary at the bundled dylib so it no longer needs system Python
+        OLDREF=$(otool -L pkg-root/usr/local/bin/specs | awk 'NR>1 && (/libpython'${PYVER}'/ || /Python\.framework/){print $1; exit}')
+        NEWREF=${BUNDLE_PREFIX}/lib/libpython${PYVER}.dylib
+        echo "Repointing libpython reference: ${OLDREF} -> ${NEWREF}"
+        install_name_tool -change "${OLDREF}" "${NEWREF}" pkg-root/usr/local/bin/specs
+        install_name_tool -id "${NEWREF}" pkg-root${BUNDLE_PREFIX}/lib/libpython${PYVER}.dylib
+        chmod 755 pkg-root${BUNDLE_PREFIX}/lib/libpython${PYVER}.dylib
+        # install_name_tool invalidates the (ad-hoc) code signature; re-sign so the
+        # dynamic loader will accept the binaries on Apple Silicon.
+        codesign --force --sign - pkg-root${BUNDLE_PREFIX}/lib/libpython${PYVER}.dylib
+        codesign --force --sign - pkg-root/usr/local/bin/specs
+        # Sanity check that the reference now points at the bundled copy
+        otool -L pkg-root/usr/local/bin/specs | grep -i "libpython${PYVER}"
 
     - name: Prepare pkg scripts
       run: |
@@ -320,9 +375,76 @@ jobs:
       shell: bash
       run: cp specs/bin/Release/specs.exe specs-${{ steps.version.outputs.display }}-python312-windows-x64.exe
 
+    - name: Stage MSI payload (bundle Python runtime)
+      shell: bash
+      run: |
+        # Use forward slashes so the paths are safe to use inside bash
+        PREFIX=$(python -c "import sys; print(sys.base_prefix.replace(chr(92), '/'))")
+        PYVER=$(python -c "import sys; print(f'{sys.version_info.major}{sys.version_info.minor}')")
+        mkdir -p msi-stage
+        cp specs/bin/Release/specs.exe msi-stage/
+        # The Python DLLs must sit next to specs.exe so Windows loads them from
+        # the application directory without any system Python installation.
+        cp "${PREFIX}/python${PYVER}.dll" msi-stage/
+        cp "${PREFIX}/python3.dll" msi-stage/ 2>/dev/null || true
+        cp "${PREFIX}/vcruntime140.dll" msi-stage/ 2>/dev/null || true
+        cp "${PREFIX}/vcruntime140_1.dll" msi-stage/ 2>/dev/null || true
+        # The C extension modules (.pyd) and the pure-Python standard library.
+        cp -r "${PREFIX}/DLLs" msi-stage/DLLs
+        cp -r "${PREFIX}/Lib" msi-stage/Lib
+        cp LICENSE msi-stage/LICENSE.txt
+        cp PYTHON_LICENSE msi-stage/PYTHON_LICENSE.txt
+        # Trim files that are not needed at runtime to reduce package size
+        find msi-stage/Lib -type d -name __pycache__ -exec rm -rf {} + 2>/dev/null || true
+        rm -rf msi-stage/Lib/site-packages
+        rm -rf msi-stage/Lib/test
+        rm -rf msi-stage/Lib/idlelib
+        rm -rf msi-stage/Lib/tkinter
+        rm -rf msi-stage/Lib/turtledemo
+
     - name: Install WiX Toolset
       run: dotnet tool install --global wix --version 4.0.6
 
+    - name: Harvest Python runtime files (generate WiX fragment)
+      shell: pwsh
+      run: |
+        # Enumerate every file under msi-stage\ except specs.exe and emit a
+        # WiX v4 fragment with one Component/File per file, each with an
+        # explicit unique Id derived from the full relative path so that files
+        # with the same name in different directories (e.g. __init__.py) do not
+        # collide.  heat.exe is a separate NuGet package in WiX v4 and not
+        # available via the dotnet global tool, so we generate the fragment here.
+        $root     = Resolve-Path "msi-stage"
+        $rootLen  = $root.Path.Length
+        $files    = Get-ChildItem -Path $root -Recurse -File |
+                    Where-Object { $_.Name -ne "specs.exe" }
+        $lines = [System.Collections.Generic.List[string]]::new()
+        $lines.Add('<?xml version="1.0" encoding="UTF-8"?>')
+        $lines.Add('<Wix xmlns="http://wixtoolset.org/schemas/v4/wxs">')
+        $lines.Add('  <Fragment>')
+        $lines.Add('    <ComponentGroup Id="PythonRuntime" Directory="INSTALLFOLDER">')
+        foreach ($f in $files) {
+          # Path relative to the workspace root, e.g. msi-stage\Lib\os.py
+          $rel = $f.FullName.Substring((Get-Location).Path.Length + 1)
+          # Path relative to msi-stage\, e.g. Lib\os.py  (used as Subdirectory)
+          $sub = $f.FullName.Substring($rootLen + 1)
+          # Unique XML identifier: replace every non-alphanumeric char with '_'
+          $id  = "f_" + ($sub -replace '[^A-Za-z0-9]', '_')
+          # Subdirectory of the file relative to msi-stage (empty for top-level)
+          $dir = Split-Path $sub -Parent
+          if ($dir) {
+            $lines.Add("      <Component Id=`"cmp_$id`" Subdirectory=`"$dir`">")
+          } else {
+            $lines.Add("      <Component Id=`"cmp_$id`">")
+          }
+          $lines.Add("        <File Id=`"$id`" Source=`"$rel`" />")
+          $lines.Add("      </Component>")
+        }
+        $lines.Add('    </ComponentGroup>')
+        $lines.Add('  </Fragment>')
+        $lines.Add('</Wix>')
+        $lines | Set-Content -Encoding UTF8 msi-stage.wxs
+
     - name: Create WiX source
       shell: bash
       run: |
@@ -333,7 +455,7 @@ jobs:
 
     - name: Build MSI
       shell: bash
-      run: wix build -o specs-${{ steps.version.outputs.display }}-python312.msi specs-python.wxs
+      run: wix build -o specs-${{ steps.version.outputs.display }}-python312.msi specs-python.wxs msi-stage.wxs
 
     - name: Upload MSI artifact
       uses: actions/upload-artifact@v7
@@ -430,17 +552,29 @@ jobs:
         cp .github/packaging/postinst deb-root/DEBIAN/
         cp .github/packaging/postrm deb-root/DEBIAN/
 
-    - name: Bundle Python stdlib for DEB
+    - name: Bundle Python stdlib and shared library for DEB
       run: |
-        PYVER=$(python3 -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')")
-        mkdir -p deb-root/usr/lib/specs/python/lib/python${PYVER}
-        cp -r /usr/lib/python${PYVER}/* deb-root/usr/lib/specs/python/lib/python${PYVER}/
+        PYVER=$(python3.12 -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')")
+        # Use the platlibdir reported by the bundled Python so the stdlib lands
+        # in the directory that libpython will actually search at runtime.
+        PLATLIBDIR=$(python3.12 -c "import sys; print(sys.platlibdir)")
+        mkdir -p deb-root/usr/lib/specs/python/${PLATLIBDIR}/python${PYVER}
+        mkdir -p deb-root/usr/share/doc/specs
+        cp -r /usr/${PLATLIBDIR}/python${PYVER}/* deb-root/usr/lib/specs/python/${PLATLIBDIR}/python${PYVER}/ 2>/dev/null || \
+        cp -r /usr/lib/python${PYVER}/* deb-root/usr/lib/specs/python/${PLATLIBDIR}/python${PYVER}/
+        # Copy the shared libpython library into the same platlibdir subtree
+        cp -L /usr/lib/x86_64-linux-gnu/libpython${PYVER}.so.1.0 deb-root/usr/lib/specs/python/${PLATLIBDIR}/ 2>/dev/null || \
+        cp -L /usr/lib/aarch64-linux-gnu/libpython${PYVER}.so.1.0 deb-root/usr/lib/specs/python/${PLATLIBDIR}/ 2>/dev/null || \
+        cp -L /usr/lib/libpython${PYVER}.so.1.0 deb-root/usr/lib/specs/python/${PLATLIBDIR}/ 2>/dev/null || true
+        # Copy license files
+        install -m 644 LICENSE deb-root/usr/share/doc/specs/LICENSE
+        install -m 644 PYTHON_LICENSE deb-root/usr/share/doc/specs/PYTHON_LICENSE
         # Remove unnecessary files to reduce package size
-        find deb-root/usr/lib/specs/python/lib/python${PYVER} -type d -name __pycache__ -exec rm -rf {} + 2>/dev/null || true
-        find deb-root/usr/lib/specs/python/lib/python${PYVER} -type f -name "*.pyc" -delete
-        find deb-root/usr/lib/specs/python/lib/python${PYVER} -type f -name "*.pyo" -delete
-        rm -rf deb-root/usr/lib/specs/python/lib/python${PYVER}/site-packages
-        rm -rf deb-root/usr/lib/specs/python/lib/python${PYVER}/dist-packages
+        find deb-root/usr/lib/specs/python/${PLATLIBDIR}/python${PYVER} -type d -name __pycache__ -exec rm -rf {} + 2>/dev/null || true
+        find deb-root/usr/lib/specs/python/${PLATLIBDIR}/python${PYVER} -type f -name "*.pyc" -delete
+        find deb-root/usr/lib/specs/python/${PLATLIBDIR}/python${PYVER} -type f -name "*.pyo" -delete
+        rm -rf deb-root/usr/lib/specs/python/${PLATLIBDIR}/python${PYVER}/site-packages
+        rm -rf deb-root/usr/lib/specs/python/${PLATLIBDIR}/python${PYVER}/dist-packages
 
     - name: Build DEB
       run: |

PYTHON_LICENSE

@@ -0,0 +1,42 @@
+PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
+
+1. This LICENSE AGREEMENT is between the Python Software Foundation ("PSF"), and
+   the Individual or Organization ("Licensee") accessing and otherwise using this
+   software ("Python") in source or binary form and its associated documentation.
+
+2. Subject to the terms and conditions of this License Agreement, PSF hereby
+   grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce,
+   analyze, test, perform and/or display publicly, prepare derivative works,
+   distribute, and otherwise use Python alone or in any derivative
+   version, provided, however, that PSF's License Agreement and PSF's notice of
+   copyright, i.e., "Copyright © 2001-2023 Python Software Foundation; All Rights
+   Reserved" are retained in Python alone or in any derivative version
+   prepared by Licensee.
+
+3. In the event Licensee prepares a derivative work that is based on or
+   incorporates Python or any part thereof, and wants to make the
+   derivative work available to others as provided herein, then Licensee hereby
+   agrees to include in any such work a brief summary of the changes made to Python.
+
+4. PSF is making Python available to Licensee on an "AS IS" basis.
+   PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED.  BY WAY OF
+   EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND DISCLAIMS ANY REPRESENTATION OR
+   WARRANTY OF MERCHANTABILITY OR FITNESS FOR ANY PARTICULAR PURPOSE OR THAT THE
+   USE OF PYTHON WILL NOT INFRINGE ANY THIRD PARTY RIGHTS.
+
+5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON
+   FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS A RESULT OF
+   MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, OR ANY DERIVATIVE
+   THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF.
+
+6. This License Agreement will automatically terminate upon a material breach of
+   its terms and conditions.
+
+7. Nothing in this License Agreement shall be deemed to create any relationship
+   of agency, partnership, or joint venture between PSF and Licensee.  This License
+   Agreement does not grant permission to use PSF trademarks or trade name in a
+   trademark sense to endorse or promote products or services of Licensee, or any
+   third party.
+
+8. By copying, installing or otherwise using Python, Licensee agrees
+   to be bound by the terms and conditions of this License Agreement.

README.md

@@ -60,7 +60,7 @@ For detailed build instructions covering Linux, Mac OS, and Windows (both `make`
 Known Issues
 ============
 * Regular expression grammars other than the default `ECMAScript` don't work except on Mac OS.
-* On Windows with Python support, `python312.dll` must be in the path (or Python 3.12 must be installed).
+* The Python-enabled Windows MSI bundles its own Python 3.12 runtime, so no system Python is required. The standalone Python-enabled `.exe` (downloaded on its own, outside the MSI) still needs `python312.dll` on the path (or Python 3.12 installed).
 
 Contributing
 ============