Merge pull request #550 from analogdevicesinc/python-setup-fixes

Python setup fixes

Author: rgetz

.travis.yml

@@ -95,14 +95,18 @@ matrix:
         - TRIGGER_NEXT_BUILD=true
 
 addons:
-  artifacts: true
   ssh_known_hosts:
     secure: "q0dQ9MrhnOCo031McxSdQIqw4N6tEyAZLq7mdbWkAJcXOh/CX58NjFvcdSfamleDUYRmg7KpCZPPgnbx2JtqVvWJv8aNnr67CE1GIjRP1Fxh2WaKS+VK+I6rro7GwCO2C9d+uffCt63LfZKqddF1T7vMI2IgHcu9irc5LCuw6Wo="
 
 before_install:
-  - if [[ "$ARCH" == "arm" ]] ; then ./CI/travis/setup_qemu_for_arm.sh ${OS_VERSION} ; fi
+  - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
+      if [[ "$ARCH" == "arm" ]] ; then
+        ./CI/travis/setup_qemu_for_arm.sh ${OS_VERSION} ;
+      else
+        ./CI/travis/before_install_linux "$OS_TYPE" ;
+      fi
+    fi
   - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then ./CI/travis/before_install_darwin ; fi
-  - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then ./CI/travis/before_install_linux "$OS_TYPE" ; fi
   - if [[ -n "$COVERITY_SCAN_PROJECT_NAME" ]] ; then echo -n | openssl s_client -connect scan.coverity.com:443 | sed -ne '/-BEGIN CERTIFICATE-/,/-END CERTIFICATE-/p' | sudo tee -a /etc/ssl/certs/ca- ; fi
   - if [ -n "$COVERITY_SCAN_PROJECT_NAME" -a "$TRAVIS_EVENT_TYPE" == "cron" ] ; then curl -s 'https://scan.coverity.com/scripts/travisci_build_coverity_scan.sh' | bash || true ; fi
 

CI/travis/before_install_linux

@@ -3,16 +3,36 @@
 . CI/travis/lib.sh
 
 install_sphinx() {
-	if ! command_exists pip3 ; then
-		pip install sphinx
-		pip install sphinx-rtd-theme
-	else
-		pip3 install sphinx
-		pip3 install sphinx-rtd-theme
+	if command_exists python ; then
+		python --version
+		command -v python
+		python -m pip install -U pip
+		python -m pip install -U setuptools
+		python -m pip install sphinx
+		python -m pip install sphinx-rtd-theme
 	fi
+}
 
+install_pyenv() {
+	if ! command_exists pyenv ; then
+		echo installing pyenv
+		git clone git://github.com/yyuu/pyenv.git $HOME/.pyenv
+		export PATH="$HOME/.pyenv/bin:$PATH"
+		export PYENV_ROOT="$HOME/.pyenv"
+	fi
 }
 
+install_python() {
+	echo "### installing python"
+	command -v pyenv
+	add_python_path
+	pyenv install --list | grep "^[[:space:]]*[0-9]"
+	pyenv install 3.6.3
+	pyenv global 3.6.3
+	add_python_path
+}
+
+
 handle_centos() {
 	# needed for man2html and a few other popular tools
 	yum search epel-release
@@ -24,25 +44,40 @@ handle_centos() {
 	yum -y install cmake libxml2-devel libusb1-devel libaio-devel \
 		bzip2 gzip rpm rpm-build redhat-lsb-core
 
-	# CENTOS 6 doesn't include python 3, it's too old.
-	if is_centos_at_least_ver "7" ; then
-		yum -y install python3-pip
+	# needed for building python with pyenv
+	yum install -y  gcc gcc-c++ make git patch openssl-devel zlib-devel readline-devel sqlite-devel bzip2-devel
+
+	# CentOS 6 & 7 don't work with doc, or the latest python.
+	# install_pyenv
+	# install_python
+
+	if [ "$(get_version | head -c 1)" = "7" ] ; then
+		# install Cmake3, and make it the default
+		yum -y install cmake3
+		alternatives --install /usr/local/bin/cmake cmake /usr/bin/cmake 10 \
+			--slave /usr/local/bin/ctest ctest /usr/bin/ctest \
+			--slave /usr/local/bin/cpack cpack /usr/bin/cpack \
+			--slave /usr/local/bin/ccmake ccmake /usr/bin/ccmake \
+			--family cmake
+		alternatives --install /usr/local/bin/cmake cmake /usr/bin/cmake3 20 \
+			--slave /usr/local/bin/ctest ctest /usr/bin/ctest3 \
+			--slave /usr/local/bin/cpack cpack /usr/bin/cpack3 \
+			--slave /usr/local/bin/ccmake ccmake /usr/bin/ccmake3 \
+			--family cmake
 	fi
 
 	if is_centos_at_least_ver "8" ; then
 		# On CentOS 8, avahi-devel & doxygen are in this repo; enable it
+		yum -y install yum-utils
 		yum config-manager --set-enabled PowerTools
+		# On CentOS 6 & 7, have issues building or packaging doc
+		yum -y install python3 doxygen man2html
+		install_sphinx
 	else
 		# On CentOS 8, cdk-devel (Curses Development Kit) does not exist yet
 		yum -y install ncurses-devel cdk-devel
 	fi
 
-	# CENTOS 6 will has issues with sphinx
-	if is_centos_at_least_ver "7" ; then
-		yum -y install doxygen man2html
-		install_sphinx
-	fi
-
 	yum -y install avahi-devel
 }
 
@@ -56,15 +91,25 @@ handle_ubuntu_docker() {
 
 handle_default() {
 	sudo apt-get -qq update
+	sudo apt-get install -y apt-utils
 	sudo DEBIAN_FRONTEND=noninteractive apt-get install -y cmake graphviz \
 		libaio-dev libavahi-client-dev \
 		libavahi-common-dev libusb-1.0-0-dev libxml2-dev rpm tar \
-		bzip2 gzip flex bison git lsb-release python3-pip libncurses5-dev libcdk5-dev
+		bzip2 gzip flex bison git lsb-release libncurses5-dev libcdk5-dev
+
+	# Most of these should be here, but are needed for building python by pyenv
+	sudo DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential \
+		libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev \
+		wget curl llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev \
+		libffi-dev liblzma-dev python-openssl git
 
-	if [ -n "${GH_DOC_TOKEN}" ] ; then
+	if ! is_arm ; then
+		install_pyenv
+		install_python
 		sudo apt-get install -y doxygen man2html
 		install_sphinx
 	fi
+	
 	if [ `sudo apt-cache search libserialport-dev | wc -l` -gt 0 ] ; then
 		sudo apt-get install -y libserialport-dev
 	fi

CI/travis/deploy

@@ -7,4 +7,4 @@ upload_file_to_swdownloads libiio ${RELEASE_PKG_FILE_DEB} ${TARGET_DEB} .deb
 upload_file_to_swdownloads libiio ${RELEASE_PKG_FILE_RPM} ${TARGET_RPM} .rpm
 upload_file_to_swdownloads libiio ${RELEASE_PKG_FILE_TGZ} ${TARGET_TGZ} .tar.gz
 upload_file_to_swdownloads libiio ${RELEASE_PKG_FILE_PKG} ${TARGET_PKG} .pkg
-
+remove_old_pkgs            libiio

CI/travis/lib.sh

@@ -27,6 +27,48 @@ echo_red()   { printf "\033[1;31m$*\033[m\n"; }
 echo_green() { printf "\033[1;32m$*\033[m\n"; }
 echo_blue()  { printf "\033[1;34m$*\033[m\n"; }
 
+backtrace() {
+	# shell backtraces only work on bash
+	if [ ! -z "${BASH}" ] ; then
+		local i=
+		i=${#FUNCNAME[@]}
+		((--i))
+
+		while (( i >= 0 ))
+		do
+			echo "${BASH_SOURCE[$i]}:${BASH_LINENO[$i]}.${FUNCNAME[$i]}()"
+			i=$((i - 1))
+		done
+	fi
+}
+
+add_python_path() {
+	echo "adding Python to the path"
+	if [ -d "$HOME/.pyenv/bin" -a "$(echo "$PATH" | grep .pyenv/bin | wc -c)" -eq "0" ] ; then
+		echo "adding $HOME/.pyenv/bin to path"
+		export PATH="$HOME/.pyenv/bin:$PATH"
+	fi
+	if [ -z "${PYENV_SHELL}" ] ; then
+		echo init pyenv
+		eval "$(pyenv init -)"
+	fi
+	if [ -d /opt/pyenv/versions/3.6.3/bin -a "$(echo "$PATH" | grep opt/pyenv/versions | wc -c)" -eq "0" ] ; then
+		echo adding python on opt to PATH
+		export PATH="/opt/pyenv/versions/3.6.3/bin:$PATH"
+	fi
+	if [ -d /root/.pyenv/versions/3.6.3/bin -a "$(echo "$PATH" | grep root/.pyenv/versions | wc -c)" -eq "0" ] ; then
+		echo adding python on root/.pyenv to PATH
+		export PATH="/root/.pyenv/versions/3.6.3/bin:$PATH"
+	fi
+	if ! command_exists python ; then
+		echo No python on path
+		echo "$PATH"
+	else
+		python --version
+		command -v python
+	fi
+}
+
 get_script_path() {
 	local script="$1"
 
@@ -180,14 +222,30 @@ brew_install_if_not_exists() {
 }
 
 sftp_cmd_pipe() {
-	sftp "${EXTRA_SSH}" "${SSHUSER}@${SSHHOST}"
+	sftp -o "StrictHostKeyChecking no" "${EXTRA_SSH}" "${SSHUSER}@${SSHHOST}"
+}
+
+sftp_run_cmds() {
+	local x=5
+	while [ "${x}" -gt "0" ] ; do
+		sftp -o "StrictHostKeyChecking no" "${EXTRA_SSH}" "${SSHUSER}@${SSHHOST}" 0< "$1" && break;
+		echo_red "failed to ssh, trying again"
+		x=$((x  - 1))
+		sleep 10
+	done
+	if [ "${x}" -eq "0" ] ; then
+		echo_red "failed to upload files"
+		return 1;
+	fi
+	return 0
 }
 
 sftp_rm_artifact() {
 	local artifact="$1"
 	sftp_cmd_pipe <<-EOF
 		cd ${DEPLOY_TO}
 		rm ${artifact}
+		bye
 	EOF
 }
 
@@ -196,16 +254,26 @@ sftp_upload() {
 	local TO="$2"
 	local LATE="$3"
 
-	sftp_cmd_pipe <<-EOF
-		cd ${DEPLOY_TO}
+	if [ -n "${LATE}" ] ; then
+		sftp_cmd_pipe <<-EOF
+			cd ${DEPLOY_TO}
 
-		put ${FROM} ${TO}
-		ls -l ${TO}
+			put ${FROM} ${TO}
+			ls -l ${TO}
 
-		symlink ${TO} ${LATE}
-		ls -l ${LATE}
-		bye
-	EOF
+			symlink ${TO} ${LATE}
+			ls -l ${LATE}
+			bye
+		EOF
+	else
+		sftp_cmd_pipe <<-EOF
+			cd ${DEPLOY_TO}
+
+			put ${FROM} ${TO}
+			ls -l ${TO}
+			bye
+		EOF
+	fi
 }
 
 upload_file_to_swdownloads() {
@@ -245,26 +313,106 @@ upload_file_to_swdownloads() {
 	echo and "${branch}_${LIBNAME}${LDIST}${EXT}"
 	ssh -V
 
-	for rmf in ${TO} ${LATE} ; do
-		sftp_rm_artifact "${rmf}" || \
-			echo_blue "Could not delete ${rmf}"
-	done
+	local tmpfl=$(mktemp)
+	echo "cd ${DEPLOY_TO}" > "${tmpfl}"
+	echo "rm ${TO}" >> "${tmpfl}"
+	echo "rm ${LATE}" >> "${tmpfl}"
+	echo "put ${FROM} ${TO}" >> "${tmpfl}"
+	echo "symlink ${TO} ${LATE}" >> "${tmpfl}"
+	echo "ls -l ${TO}" >> "${tmpfl}"
+	echo "ls -l ${LATE}" >> "${tmpfl}"
+	echo "bye"  >> "${tmpfl}"
 
-	sftp_upload "${FROM}" "${TO}" "${LATE}" || {
-		echo_red "Failed to upload artifact from '${FROM}', to '${TO}', symlink '${LATE}'"
-		return 1
-	}
+	sftp_run_cmds "${tmpfl}"
+	rm "${tmpfl}"
+
+	return 0
+}
 
+remove_old_pkgs() {
 	# limit things to a few files, so things don't grow forever
-	if [ "${EXT}" = ".deb" ] ; then
-		for files in $(ssh "${EXTRA_SSH}" "${SSHUSER}@${SSHHOST}" \
+	# we only do this on one build so simultaneous builds don't clobber each other
+	if [ -z "${GH_DOC_TOKEN}" ] ; then
+		return 0
+	fi
+
+	if [ -z "${TRAVIS_BUILD_DIR}" ] ; then
+		echo "TRAVIS_BUILD_DIR not set"
+		return 0
+	fi
+
+	if [ ! -d "${TRAVIS_BUILD_DIR}/.git" ] ; then
+		echo "No ${TRAVIS_BUILD_DIR}/.git to operate git on"
+		return 0
+	fi
+
+	local LIBNAME=$1
+	local old=
+
+	echo "Remove old packages from ${LIBNAME}"
+
+	if [ -n "$TRAVIS_PULL_REQUEST_BRANCH" ] ; then
+		local branch="$TRAVIS_PULL_REQUEST_BRANCH"
+	else
+		local branch="$TRAVIS_BRANCH"
+	fi
+
+	local GLOB=${DEPLOY_TO}/${branch}_${LIBNAME}-*
+
+	# putting everything into a file, and connecting once decreases the chances
+	# for ssh issues, connections happen once, not every single file
+	local tmpfl=$(mktemp)
+	echo "cd ${DEPLOY_TO}" > "${tmpfl}"
+	for files in $(ssh -o "StrictHostKeyChecking no" "${EXTRA_SSH}" "${SSHUSER}@${SSHHOST}" \
 			"ls -lt ${GLOB}" | tail -n +100 | awk '{print $NF}')
-		do
-			ssh "${EXTRA_SSH}" "${SSHUSER}@${SSHHOST}" \
-				"rm ${DEPLOY_TO}/${files}" || \
-				return 1
-		done
+	do
+		echo "rm ${files}" >> "${tmpfl}"
+	done
+	echo "bye" >> "${tmpfl}"
+	# if it is only cd & bye, skip it
+	if [ "$(wc -l "${tmpfl}" | awk '{print $1}')" -gt "2" ] ; then
+		sftp_run_cmds "${tmpfl}"
 	fi
+	rm "${tmpfl}"
+	# provide an index so people can find files.
+	ssh -o "StrictHostKeyChecking no" "${EXTRA_SSH}" "${SSHUSER}@${SSHHOST}" \
+			"ls -lt ${DEPLOY_TO}" | grep "${LIBNAME}" > "${LIBNAME}_index.html"
+	echo "ls captured"
+
+	echo "cd ${DEPLOY_TO}" > "${tmpfl}"
+	# prune old / removed branches, leave things are are tags/branches
+	for old in $(sed 's/-> .*$//' libiio_index.html | \
+			awk  '{print $NF}' | grep -v master | sort | \
+			sed "s/_libiio-0.[0-9][0-9].g[a-z0-9]*-/ %% /" | \
+			grep "%%" | awk '{print $1}' | sort -u)
+	do
+		if [ "$(git --git-dir "${TRAVIS_BUILD_DIR}/.git" ls-remote --heads origin "${old}" | wc -l)" -ne "0" ] ; then
+			echo "${old} is a branch"
+		else
+			if [ "$(git --git-dir "${TRAVIS_BUILD_DIR}/.git" ls-remote --tags origin "${old}" | wc -l)" -ne "0" ] ; then
+				echo "${old} is a tag"
+			else
+				echo "${old} can be removed"
+				echo "rm ${old}_${LIBNAME}-*" >> "${tmpfl}"
+				echo "rm ${old}_latest_${LIBNAME}-*" >> "${tmpfl}"
+				echo "rm ${old}_lastest_${LIBNAME}-*" >> "${tmpfl}"
+			fi
+		fi
+	done
+	# cap things at 15, so we don't exceed the time
+	sed -i 16q "${tmpfl}"
+	echo "bye" >> "${tmpfl}"
+	# if it is only cd & bye, skip it
+	if [ "$(wc -l "${tmpfl}" | awk '{print $1}')" -gt "2" ] ; then
+		sftp_run_cmds "${tmpfl}"
+	fi
+	rm "${tmpfl}"
+
+	#Now that we removed things, do it again
+	rm "${LIBNAME}_index.html"
+	ssh -o "StrictHostKeyChecking no" "${EXTRA_SSH}" "${SSHUSER}@${SSHHOST}" \
+			"ls -lt ${DEPLOY_TO}" | grep "${LIBNAME}" > "${LIBNAME}_index.html"
+	sftp_upload "${LIBNAME}_index.html" "${LIBNAME}_index.html"
 
 	return 0
 }
@@ -347,6 +495,11 @@ is_centos_at_least_ver() {
 	version_ge "$(get_version)" "$1"
 }
 
+is_arm() {
+	[ "$(dpkg --print-architecture)" = "armhf" ] || return 1
+	test "$(dpkg --print-architecture)" = "armhf"
+}
+
 print_github_api_rate_limits() {
 	# See https://developer.github.com/v3/rate_limit/
 	# Note: Accessing this endpoint does not count against your REST API rate limit.