Update to Python 3.11 (#15544)

Closes #12064
Closes #12551
Closes #15577
Closes #15167

Summary of the issue:
Python needs to be updated to 3.11, as Python 3.7 is EOL.
Python pip dependencies need to be updated to match the python upgrade.
typing_extensions is no longer needed.

Description of user facing changes
Performance and security enhancements from dependency upgrades.

Description of development approach
Updates python in build scripts
Updates pip dependencies
Set Windows 8.1 (Blue) as minimum windows version
Update references to python and windows version in docs
nose is replaced by the unittest module with xmlrunner to generate XML test output
drop python optimization to level 0 from level 1. level 1 removes asserts.

Author: seanbudd

appveyor.yml

@@ -20,7 +20,7 @@ init:
       "INIT, $(Get-Date -Format 'o')"| Out-File ../timing.csv -Append
 
 environment:
- PY_PYTHON: 3.7-32
+ PY_PYTHON: 3.11-32
  secure_authenticode_pass:
   secure: Way+hJyhbiLG/cmCo4+dHHzS5DiSvk/45o6frnIQ27GBX6nVDsh7jwQ7fSnqxBRP
  secure_ssh_pass:

appveyor/scripts/tests/unitTests.ps1

@@ -1,6 +1,6 @@
 $outDir = (Resolve-Path .\testOutput\unit\)
 $unitTestsXml = "$outDir\unitTests.xml"
-.\rununittests.bat --with-xunit --xunit-file="$unitTestsXml"
+.\rununittests.bat --output-file "$unitTestsXml"
 if($LastExitCode -ne 0) {
 	Set-AppveyorBuildVariable "testFailExitCode" $LastExitCode
 	Add-AppveyorMessage "FAIL: Unit tests. See test results for more information."

appx/sconscript

@@ -61,8 +61,7 @@ else: # not for submission, just side-loadable
 signExec=env['signExec'] if env['certFile'] else None
 
 # Files from NVDA's distribution that cannot be included in the appx due to policy or security restrictions
-excludedDistFiles=[
-	'nvda_eoaProxy.exe',
+excludedDistFiles = [
 	'nvda_service.exe',
 	'nvda_slave.exe',
 	'nvda_noUIAccess.exe',
@@ -107,4 +106,3 @@ if signExec and not isStoreSubmission:
 	env.AddPostAction(appx,signExec)
 
 Return(['appx'])
-

nvdaHelper/archBuild_sconscript

@@ -1,16 +1,7 @@
-###
-#This file is a part of the NVDA project.
-#URL: http://www.nvda-project.org/
-#Copyright 2006-2010 NVDA contributers.
-#This program is free software: you can redistribute it and/or modify
-#it under the terms of the GNU General Public License version 2.0, as published by
-#the Free Software Foundation.
-#This program is distributed in the hope that it will be useful,
-#but WITHOUT ANY WARRANTY; without even the implied warranty of
-#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
-#This license can be found at:
-#http://www.gnu.org/licenses/old-licenses/gpl-2.0.html
-###
+# A part of NonVisual Desktop Access (NVDA)
+# Copyright (C) 2006-2023 NV Access Limited
+# This file may be used under the terms of the GNU General Public License, version 2 or later.
+# For more details see: https://www.gnu.org/licenses/gpl-2.0.html
 
 import os
 import shutil
@@ -87,7 +78,7 @@ env.Append(
 		'UNICODE',
 		'_CRT_SECURE_NO_DEPRECATE',
 		('LOGLEVEL', '${nvdaHelperLogLevel}'),
-		('_WIN32_WINNT', '_WIN32_WINNT_WIN7'),
+		('_WIN32_WINNT', '_WIN32_WINNT_WINBLUE'),
 		# NOMINMAX: prevent minwindef.h min/max macro definition, which unexpectedly override developer
 		# expectations
 		'NOMINMAX',
@@ -102,10 +93,10 @@ env.Append(CPPPATH=[
 	'#/miscDeps/include',
 	Dir('.').abspath
 ])
-if TARGET_ARCH == "arm64":
-	subsystem = "/subsystem:windows,6.02"
-else:
-	subsystem = "/subsystem:windows,6.01"
+
+# Windows 8.1 (blue)
+subsystem = "/subsystem:windows,6.03"
+
 env.Append(
 	LINKFLAGS=[
 		'/incremental:no',

nvdaHelper/local/nvdaHelperLocal.cpp

@@ -1,15 +1,8 @@
 /*
-This file is a part of the NVDA project.
-URL: http://www.nvda-project.org/
-Copyright 2008-2014 NV Access Limited.
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License version 2.0, as published by
-    the Free Software Foundation.
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
-This license can be found at:
-http://www.gnu.org/licenses/old-licenses/gpl-2.0.html
+A part of NonVisual Desktop Access (NVDA)
+Copyright (C) 2008-2023 NV Access Limited
+This file may be used under the terms of the GNU General Public License, version 2 or later.
+For more details see: https://www.gnu.org/licenses/gpl-2.0.html
 */
 
 #include <cstdio>
@@ -28,20 +21,6 @@ decltype(&SendMessageW) real_SendMessageW = nullptr;
 decltype(&SendMessageTimeoutW) real_SendMessageTimeoutW = nullptr;
 decltype(&OpenClipboard) real_OpenClipboard = nullptr;
 
-typedef struct _RPC_SECURITY_QOS_V5_W {
-  unsigned long Version;
-  unsigned long Capabilities;
-  unsigned long IdentityTracking;
-  unsigned long ImpersonationType;
-  unsigned long AdditionalSecurityInfoType;
-  union
-      {
-      RPC_HTTP_TRANSPORT_CREDENTIALS_W *HttpCredentials;
-      } u;
-  void *Sid;
-  unsigned int EffectiveOnly;
-  void *ServerSecurityDescriptor;
-} RPC_SECURITY_QOS_V5_W, *PRPC_SECURITY_QOS_V5_W;
 
 handle_t createRemoteBindingHandle(wchar_t* uuidString) {
 	RPC_STATUS rpcStatus;

nvdaHelper/readme.md

@@ -70,7 +70,7 @@ You should still build on the command line to verify errors.
   - Press next
 -  Specify Debug Configuration Settings
   - Build command line: `scons source`
-  - Preprocessor definitions (/D): `WIN32;_WINDOWS;_USRDLL;NVDAHELPER_EXPORTS;UNICODE;_CRT_SECURE_NO_DEPRECATE;LOGLEVEL=15;_WIN32_WINNT=_WIN32_WINNT_WIN7;NOMINMAX`
+  - Preprocessor definitions (/D): `WIN32;_WINDOWS;_USRDLL;NVDAHELPER_EXPORTS;UNICODE;_CRT_SECURE_NO_DEPRECATE;LOGLEVEL=15;_WIN32_WINNT=_WIN32_WINNT_WINBLUE;NOMINMAX`
   - Include search paths (/I): `../include;../miscDeps/include;./;../build\x86_64;../include/minhook/include`
   - Forced Included files (/FI): `winuser.h`
   - Press next
@@ -90,7 +90,7 @@ You should still build on the command line to verify errors.
   ```
   cl /Fobuild\x86\vbufBackends\gecko_ia2\gecko_ia2.obj /c build\x86\vbufBackends\gecko_ia2\gecko_ia2.cpp
   /TP /EHsc /nologo /std:c++20 /permissive- /Od /MT /W3 /WX
-  /DUNICODE /D_CRT_SECURE_NO_DEPRECATE /DLOGLEVEL=15 /D_WIN32_WINNT=_WIN32_WINNT_WIN7 /DNOMINMAX /DNDEBUG
+  /DUNICODE /D_CRT_SECURE_NO_DEPRECATE /DLOGLEVEL=15 /D_WIN32_WINNT=_WIN32_WINNT_WINBLUE /DNOMINMAX /DNDEBUG
   /Iinclude /Imiscdeps\include /Ibuild\x86
   /Z7
   ```

projectDocs/dev/createDevEnvironment.md

@@ -34,7 +34,7 @@ The NVDA source depends on several other packages to run correctly.
 ### Installed Dependencies
 The following dependencies need to be installed on your system:
 
-* [Python](https://www.python.org/), version 3.7, 32 bit
+* [Python](https://www.python.org/), version 3.11, 32 bit
 	* Use latest minor version if possible.
 * Microsoft Visual Studio 2019 or 2022:
 	* To replicate the production build environment, use the [version of Visual Studio 2019 that AppVeyor is using](https://www.appveyor.com/docs/windows-images-software/#visual-studio-2019). 

requirements.txt

@@ -4,37 +4,30 @@ SCons==4.5.2
 # NVDA's runtime dependencies
 comtypes==1.2.0
 pyserial==3.5
-wxPython==4.1.1
-git+https://github.com/DiffSK/configobj@3e2f4cc#egg=configobj
+wxPython==4.2.1
+git+https://github.com/DiffSK/configobj@e2ba4457c4651fa54f8d59d8dcdd3da950e956b8#egg=configobj
 requests==2.31.0
-# Required to use a pinned old version for requests.
-# py2exe fails to compile properly without this.
-# This can be removed when upgrading py2exe to 0.13+ and python to 3.8+.
-# https://github.com/Ousret/charset_normalizer/issues/253
-charset-normalizer==2.1.1
+# Pillow is an implicit dependency and requires zlib and jpeg by default, but we don't need it
+Pillow==10.0.1 -C "zlib=disable" -C "jpeg=disable"
 
 #NVDA_DMP requires diff-match-patch
 fast_diff_match_patch==2.0.1
 
 # Packaging NVDA
-py2exe==0.11.1.0
+git+https://github.com/py2exe/py2exe@4e7b2b2c60face592e67cb1bc935172a20fa371d#egg=py2exe
 
-# For building developer documentation
-sphinx==5.1.1
-sphinx_rtd_theme==1.0.0
+# Creating XML unit test reports
+unittest-xml-reporting==3.2.0
 
-# Requirements for type checking.
-# typing_extensions is incorporated in py3.8+, also available via mypy
-typing_extensions==4.3.0
+# For building developer documentation
+sphinx==7.2.6
+sphinx_rtd_theme==1.3.0
 
 # Requirements for automated linting
 flake8 ~= 3.7.7
 flake8-tabs == 2.1.0
 
-# Requirements for unit tests
-nose==1.3.7
-
 # Requirements for system tests
-robotframework==3.2.2
-robotremoteserver==1.1
-robotframework-screencaplibrary==1.5.0
+robotframework==6.1.1
+robotremoteserver==1.1.1
+robotframework-screencaplibrary==1.6.0

rununittests.bat

@@ -4,5 +4,7 @@ set here=%hereOrig%
 if #%hereOrig:~-1%# == #\# set here=%hereOrig:~0,-1%
 set scriptsDir=%here%\venvUtils
 set unitTestsPath=%here%\tests\unit
+set testOutput=%here%\testOutput\unit
+md %testOutput%
 
-call "%scriptsDir%\venvCmd.bat" py -m nose -sv --traverse-namespace -w "%unitTestsPath%" %*
+call "%scriptsDir%\venvCmd.bat" py -m xmlrunner discover -v -s "%unitTestsPath%" -t "%here%" --output-file "%testOutput%\unitTests.xml" %*

sconstruct

@@ -23,8 +23,8 @@ if nvdaVenv != virtualEnv:
 	print("Warning: SCons launched within a custom Python virtual environment.")
 
 # Variables for storing required version of Python, and the version which is used to run this script.
-requiredPythonMajor ="3"
-requiredPythonMinor = "7"
+requiredPythonMajor = "3"
+requiredPythonMinor = "11"
 requiredPythonArchitecture = "32bit"
 installedPythonMajor = str(sys.version_info.major)
 installedPythonMinor = str(sys.version_info.minor)
@@ -308,16 +308,12 @@ def NVDADistGenerator(target, source, env, for_signature):
 	# which scons treats as an error code.	
 	and None]
 
-	buildCmd = ["cd", source[0].path, "&&",
-		sys.executable]
+	buildCmd = ["cd", source[0].path, "&&", sys.executable]
 	if release:
 		buildCmd.append("-O")
 	# Issue errors about str(bytes_instance), str(bytearray_instance)
 	buildCmd.append("-bb")
-	buildCmd.extend(("setup.py", "-vv", "build", "--build-base", buildDir.abspath,
-		"py2exe", "--dist-dir", target[0].abspath))
-	if release:
-		buildCmd.append("-O1")
+	buildCmd.append("setup.py")
 	if env.get("uiAccess"):
 		buildCmd.append("--enable-uiAccess")
 
@@ -346,7 +342,7 @@ def NVDADistGenerator(target, source, env, for_signature):
 				action.append(Copy(target[0], file.path))
 
 	if certFile:
-		for prog in "nvda_noUIAccess.exe", "nvda_uiAccess.exe", "nvda_slave.exe", "nvda_eoaProxy.exe":
+		for prog in "nvda_noUIAccess.exe", "nvda_uiAccess.exe", "nvda_slave.exe":
 			action.append(lambda target,source,env, progByVal=prog: signExec([target[0].File(progByVal)],source,env))
 
 	action.extend((