python 3.8 support (#201)

christopherhesse · web-flow · commit bf21ddbdc5fb · 2020-04-25T21:11:07.000-07:00
diff --git a/.travis.yml b/.travis.yml
@@ -6,9 +6,9 @@ cache:
     - $HOME/roms/
 language: cpp # Needed for ccache
 env:
-  - PYVER=3.5
   - PYVER=3.6
   - PYVER=3.7
+  - PYVER=3.8
 os:
   - linux
   - osx
@@ -21,15 +21,15 @@ matrix:
     - os: linux
       env:
         - CROSS=win64
-        - PYVER=3.5
+        - PYVER=3.6
     - os: linux
       env:
         - CROSS=win64
-        - PYVER=3.6
+        - PYVER=3.7
     - os: linux
       env:
         - CROSS=win64
-        - PYVER=3.7
+        - PYVER=3.8
 
 before_script:
   - mkdir -p $HOME/roms
@@ -40,6 +40,7 @@ script:
     else
     TAG=$TRAVIS_OS_NAME
     fi
+    TAG=${TAG}-v2
   - >
     if [ "$TRAVIS_OS_NAME" == "linux" ]; then
     docker run
@@ -60,7 +61,7 @@ script:
     if [ "$TRAVIS_OS_NAME" == "osx" ]; then
     brew update > /dev/null
     brew install qt5 capnp lua@5.1 ccache
-    curl -o miniconda.sh https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
+    curl -o miniconda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
     bash miniconda.sh -b
     export PATH=/usr/local/opt/ccache/libexec:$HOME/miniconda3/bin:$PATH
     hash -r
diff --git a/CHANGES.md b/CHANGES.md
@@ -1,5 +1,10 @@
 # Changelog
 
+## 0.8.0
+
+* add python 3.8 support
+* drop python 3.5 due to build issues on windows
+
 ## 0.7.1
 
 * fix discrete and multi-discrete action space filtering
diff --git a/README.md b/README.md
@@ -12,9 +12,9 @@ Supported platforms:
 
 Supported Pythons:
 
-- 3.5
 - 3.6
 - 3.7
+- 3.8
 
 Each game integration has files listing memory locations for in-game variables, reward functions based on those variables, episode end conditions, savestates at the beginning of levels and a file containing hashes of ROMs that work with these files.
 
diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-0.7.1
+0.8.0
diff --git a/docker/build_images.sh b/docker/build_images.sh
@@ -0,0 +1,8 @@
+#!/bin/sh
+
+cd "$(dirname $0)/../"
+./docker/linux/build_scripts/prefetch.sh OPENSSL CURL
+docker build . --tag openai/retro-build:linux-v2 --file docker/linux/Dockerfile
+docker push openai/retro-build:linux-v2
+docker build . --tag openai/retro-build:win64-v2 --file docker/windows/Dockerfile
+docker push openai/retro-build:win64-v2
diff --git a/docker/linux/Dockerfile b/docker/linux/Dockerfile
@@ -26,6 +26,10 @@ RUN /opt/python/cp35-cp35m/bin/python3.5 -m venv ~/venv3.5 && \
     . ~/venv3.7/bin/activate && \
     pip install wheel && \
     pip install google-auth google-cloud-storage pytest requests && \
+    /opt/python/cp38-cp38/bin/python3.8 -m venv ~/venv3.8 && \
+    . ~/venv3.8/bin/activate && \
+    pip install wheel && \
+    pip install google-auth google-cloud-storage pytest requests && \
     rm -rf ~/.cache && \
     ln -s ~/venv$PYVER ~/venv && \
     echo "source /root/venv\$PYVER/bin/activate" > ~/.bash_profile
diff --git a/docker/linux/build_scripts/build.sh b/docker/linux/build_scripts/build.sh
@@ -43,8 +43,7 @@ yum -y update
 
 # EPEL support
 yum -y install wget
-# https://dl.fedoraproject.org/pub/epel/5/x86_64/epel-release-5-4.noarch.rpm
-cp $MY_DIR/epel-release-5-4.noarch.rpm .
+wget -q http://archives.fedoraproject.org/pub/archive/epel/5/x86_64/epel-release-5-4.noarch.rpm
 check_sha256sum epel-release-5-4.noarch.rpm $EPEL_RPM_HASH
 
 # Dev toolset (for LLVM and other projects requiring C++11 support)
diff --git a/docker/linux/build_scripts/build_env.sh b/docker/linux/build_scripts/build_env.sh
@@ -1,13 +1,13 @@
 # source me
 
 PYTHON_DOWNLOAD_URL=https://www.python.org/ftp/python
-CPYTHON_VERSIONS="3.5.6 3.6.7 3.7.1"
+CPYTHON_VERSIONS="3.5.6 3.6.7 3.7.1 3.8.2"
 
 # openssl version to build, with expected sha256 hash of .tar.gz
 # archive.
 OPENSSL_ROOT=openssl-1.0.2p
 OPENSSL_HASH=50a98e07b1a89eb8f6a99477f262df71c6fa7bef77df4dc83025a2845c827d00
-OPENSSL_DOWNLOAD_URL=https://www.openssl.org/source
+OPENSSL_DOWNLOAD_URL=https://www.openssl.org/source/old/1.0.2
 
 # Update to slightly newer, verified Git commit:
 # https://github.com/NixOS/patchelf/commit/2a9cefd7d637d160d12dc7946393778fa8abbc58
diff --git a/docker/linux/build_scripts/build_utils.sh b/docker/linux/build_scripts/build_utils.sh
@@ -92,8 +92,8 @@ function build_cpythons {
     check_var $GET_PIP_URL
     curl -fsSLO $GET_PIP_URL
     # Import public keys used to verify downloaded Python source tarballs.
-    # https://www.python.org/static/files/pubkeys.txt
-    gpg --import ${MY_DIR}/cpython-pubkeys.txt
+    curl -fsSLO https://www.python.org/static/files/pubkeys.txt
+    gpg --import pubkeys.txt
     for py_ver in $@; do
         build_cpython $py_ver
     done
diff --git a/docker/linux/build_scripts/prefetch.sh b/docker/linux/build_scripts/prefetch.sh
@@ -2,7 +2,7 @@
 # Prefetch tarballs so they don't need to be fetched in the container (which has
 # very old tools).
 #
-# usage: prefetch.sh <output_dir> [name ...]
+# usage: prefetch.sh [name ...]
 set -ex
 
 MY_DIR=$(dirname "${BASH_SOURCE[0]}")
diff --git a/docker/linux/build_scripts/py36-requirements.txt b/docker/linux/build_scripts/py36-requirements.txt
@@ -8,7 +8,7 @@ auditwheel==1.9.0 \
     --hash=sha256:fe8cbf74d16e7d1a89fd96b689f8e16f196edfb66cc98eb8a345bc03e28fed63
 # this package required for auditwheel
 pyelftools==0.25 \
-    --hash=sha256:89c6da6f56280c37a5ff33468591ba9a124e17d71fe42de971818cbff46c1b24
+    --hash=sha256:1ec62792b31d9d4e5399a686a515a31091be84e307c4680fd3627121500a1be7
 # this package required for auditwheel
 typing==3.6.6 \
     --hash=sha256:a4c8473ce11a65999c8f59cb093e70686b6c84c98df58c1dae9b3b196089858a \
diff --git a/docker/scripts/build_qt5.sh b/docker/scripts/build_qt5.sh
@@ -88,4 +88,4 @@ fi
 	-no-sql-odbc \
 	-no-harfbuzz \
 	-no-openssl \
-	-no-dbus && make -j2 install
+	-no-dbus && make -j$(nproc) install
diff --git a/docker/scripts/install_python.sh b/docker/scripts/install_python.sh
@@ -16,15 +16,30 @@ function install_python {
 
 	curl -LO https://www.python.org/ftp/python/$MAJOR.$MINOR.$PATCH/$PLATFORM/dev.msi
 	7z x dev.msi
-	mkdir -p $ROOT/lib/python$MAJOR.$MINOR include
+	mkdir -p $ROOT/lib/python$MAJOR.$MINOR include include/cpython include/internal
 	mv libs_python.lib $ROOT/lib/python$MAJOR.$MINOR/python$MAJOR$MINOR.lib
 	mv libs_python_stable.lib $ROOT/lib/python$MAJOR.$MINOR/python$MAJOR.lib
 	mv libs_libpython.a $ROOT/lib/python$MAJOR.$MINOR/libpython.a
+	find . -maxdepth 1 -name include_cpython_\*.h -print0 | xargs -0 -n1 -I% bash -c 'mv % $(echo % | sed -e "s,_,/," | sed -e "s,_,/,")'
+	find . -maxdepth 1 -name include_internal_\*.h -print0 | xargs -0 -n1 -I% bash -c 'mv % $(echo % | sed -e "s,_,/," | sed -e "s,_,/,")'
 	find . -maxdepth 1 -name include_\*.h -print0 | xargs -0 -n1 -I% bash -c 'mv % $(echo % | sed -e "s,_,/,")'
 	mv include $ROOT/include/python$MAJOR.$MINOR
 	rm dev.msi
+
+	# python 3.8 is missing this file, create it from the dll
+	# https://github.com/openai/retro/pull/201#issuecomment-619326417
+	if [[ ! -f $ROOT/lib/python$MAJOR.$MINOR/libpython.a ]]
+	then
+		curl -LO https://www.python.org/ftp/python/$MAJOR.$MINOR.$PATCH/$PLATFORM/core.msi
+		7z x core.msi
+		apt-get install --yes mingw-w64-tools
+		gendef python.dll
+		/usr/bin/x86_64-w64-mingw32-dlltool --as-flags=--64 -m i386:x86-64 -k --output-lib $ROOT/lib/python$MAJOR.$MINOR/libpython.a --input-def python.def
+		rm *.dll *.def core.msi
+	fi
 }
 
 install_python 3 5 4
 install_python 3 6 6
 install_python 3 7 0
+install_python 3 8 2
diff --git a/docker/windows/Dockerfile b/docker/windows/Dockerfile
@@ -3,7 +3,7 @@ FROM ubuntu:xenial
 SHELL ["/bin/bash", "-c"]
 ARG ARCH=x86_64
 ARG BITS=64
-ENV PYVER=3.5
+ENV PYVER=3.6
 RUN apt-get update
 RUN DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
         build-essential ccache cmake curl g++-mingw-w64-`echo $ARCH | tr _ -` \
@@ -16,7 +16,8 @@ RUN apt-add-repository -y ppa:deadsnakes/ppa && \
     DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
         libpython3.5-dev python3.5-venv \
         libpython3.6-dev python3.6-venv \
-        libpython3.7-dev python3.7-venv && \
+        libpython3.7-dev python3.7-venv \
+        libpython3.8-dev python3.8-venv && \
     apt-get clean
 
 RUN python3.5 -m venv ~/venv3.5 && \
@@ -26,11 +27,15 @@ RUN python3.5 -m venv ~/venv3.5 && \
     python3.6 -m venv ~/venv3.6 && \
     . ~/venv3.6/bin/activate && \
     pip install wheel && \
-    pip install google-cloud google-resumable-media pytest requests && \
+    pip install google-auth google-cloud-storage pytest requests && \
     python3.7 -m venv ~/venv3.7 && \
     . ~/venv3.7/bin/activate && \
     pip install wheel && \
-    pip install google-cloud google-resumable-media pytest requests && \
+    pip install google-auth google-cloud-storage pytest requests && \
+    python3.8 -m venv ~/venv3.8 && \
+    . ~/venv3.8/bin/activate && \
+    pip install wheel && \
+    pip install google-auth google-cloud-storage pytest requests && \
     rm -rf ~/.cache && \
     ln -s ~/venv$PYVER ~/venv && \
     echo "source /root/venv\$PYVER/bin/activate" > ~/.bash_profile
diff --git a/setup.py b/setup.py
@@ -8,6 +8,9 @@
 
 VERSION_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'VERSION')
 
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+README = open(os.path.join(SCRIPT_DIR, "README.md")).read()
+
 if not os.path.exists(os.path.join(os.path.dirname(__file__), '.git')):
     use_scm_version = False
     shutil.copy('VERSION', 'retro/VERSION.txt')
@@ -65,11 +68,13 @@ def run(self):
 
 kwargs = {}
 if tuple(int(v) for v in setuptools_version.split('.')[:3]) >= (24, 2, 0):
-    kwargs['python_requires'] = '>=3.5.0'
+    kwargs['python_requires'] = '>=3.6.0'
 
 
 setup(
     name='gym-retro',
+    long_description=README,
+    long_description_content_type="text/markdown",
     author='OpenAI',
     author_email='csh@openai.com',
     url='https://github.com/openai/retro',
diff --git a/travis.py b/travis.py
@@ -6,6 +6,9 @@
 import sys
 import re
 import shutil
+from google.oauth2 import service_account
+from google.cloud import storage
+
 
 
 class Fold:
@@ -26,9 +29,6 @@ def call(cmd):
 
 
 def get_bucket():
-    from google.oauth2 import service_account
-    from google.cloud import storage
-
     credentials_info = json.loads(os.environ['GOOGLE_APPLICATION_CREDENTIALS_DATA'])
     credentials = service_account.Credentials.from_service_account_info(credentials_info)
     client = storage.Client(credentials=credentials, project=credentials_info['project_id'])
@@ -67,8 +67,9 @@ def main():
         if os_name == 'osx':
             cmake_options = ['-DCMAKE_PREFIX_PATH=/usr/local/opt/qt', '-DBUILD_UI=ON']
         elif os_name == 'linux':
+            include_suffix = "m" if float(os.environ['PYVER']) < 3.8 else ""
             cmake_options = ['-DBUILD_MANYLINUX=ON',
-                             '-DPYTHON_INCLUDE_DIR=%s/include/python%sm' % (sys.base_prefix, os.environ['PYVER'])]
+                             '-DPYTHON_INCLUDE_DIR=%s/include/python%s%s' % (sys.base_prefix, os.environ['PYVER'], include_suffix)]
             if cross in ('win32', 'win64'):
                 cmake_options = ['-DCMAKE_TOOLCHAIN_FILE=docker/cmake/%s.cmake' % cross, '-DBUILD_UI=ON']
             if cross == 'win32':

Original file line number	Diff line number	Diff line change
`@@ -2,7 +2,7 @@`
`2`	`2`	`# Prefetch tarballs so they don't need to be fetched in the container (which has`
`3`	`3`	`# very old tools).`
`4`	`4`	`#`
`5`		`-# usage: prefetch.sh <output_dir> [name ...]`
	`5`	`+# usage: prefetch.sh [name ...]`
`6`	`6`	`set -ex`
`7`	`7`
`8`	`8`	`MY_DIR=$(dirname "${BASH_SOURCE[0]}")`