feat(pip-compile): Support compiling dependencies from pyproject.toml (renovatebot#37457)

Author: mbudnek

lib/modules/manager/pip-compile/extract.spec.ts

@@ -1,3 +1,4 @@
+import { codeBlock } from 'common-tags';
 import upath from 'upath';
 import { GlobalConfig } from '../../../config/global';
 import type { RepoGlobalConfig } from '../../../config/types';
@@ -36,8 +37,8 @@ describe('modules/manager/pip-compile/extract', () => {
   });
 
   describe('extractPackageFile()', () => {
-    it('returns object for requirements.in', () => {
-      const packageFile = extractPackageFile(
+    it('returns object for requirements.in', async () => {
+      const packageFile = await extractPackageFile(
         Fixtures.get('requirementsWithHashes.txt'),
         'requirements.in',
         {},
@@ -46,8 +47,8 @@ describe('modules/manager/pip-compile/extract', () => {
       expect(packageFile?.deps[0]).toHaveProperty('depName', 'attrs');
     });
 
-    it('returns object for setup.py', () => {
-      const packageFile = extractPackageFile(
+    it('returns object for setup.py', async () => {
+      const packageFile = await extractPackageFile(
         Fixtures.get('setup.py', '../pip_setup'),
         'lib/setup.py',
         {},
@@ -56,15 +57,47 @@ describe('modules/manager/pip-compile/extract', () => {
       expect(packageFile?.deps[0]).toHaveProperty('depName', 'celery');
     });
 
+    it('returns object for pyproject.toml', async () => {
+      const pyproject = codeBlock`
+        [build-system]
+        requires = ["setuptools", "wheel"]
+        build-backend = "setuptools.build_meta"
+
+        [project]
+        name = "test-project"
+        requires-python = ">=3.11"
+        version = "1.2.3"
+        description = "Test project for pip-compile with setuptools"
+        readme = "README.md"
+        dependencies = [
+          "aiohttp",
+          "pydantic>=2.0.0",
+        ]
+
+        [project.optional-dependencies]
+        dev = [
+          "black",
+          "flake8",
+        ]
+      `;
+      const packageFile = await extractPackageFile(
+        pyproject,
+        'pyproject.toml',
+        {},
+      );
+      expect(packageFile).toHaveProperty('deps');
+      expect(packageFile?.deps[0]).toHaveProperty('depType', 'requires-python');
+      expect(packageFile?.deps[1]).toHaveProperty('depName', 'aiohttp');
+    });
+
     it.each([
       'random.py',
       'app.cfg',
       'already_locked.txt',
       // TODO(not7cd)
-      'pyproject.toml',
       'setup.cfg',
-    ])('returns null on not supported package files', (file: string) => {
-      expect(extractPackageFile('some content', file, {})).toBeNull();
+    ])('returns null on not supported package files', async (file: string) => {
+      expect(await extractPackageFile('some content', file, {})).toBeNull();
     });
   });
 

lib/modules/manager/pip-compile/extract.ts

@@ -3,6 +3,7 @@ import { logger } from '../../../logger';
 import { coerceArray } from '../../../util/array';
 import { readLocalFile } from '../../../util/fs';
 import { ensureLocalPath } from '../../../util/fs/util';
+import { extractPackageFile as extractPep621File } from '../pep621/extract';
 import { extractPackageFile as extractRequirementsFile } from '../pip_requirements/extract';
 import { extractPackageFile as extractSetupPyFile } from '../pip_setup';
 import type {
@@ -19,18 +20,20 @@ import {
   sortPackageFiles,
 } from './utils';
 
-export function extractPackageFile(
+export async function extractPackageFile(
   content: string,
   packageFile: string,
   _config: ExtractConfig,
-): PackageFileContent | null {
+): Promise<PackageFileContent | null> {
   logger.trace('pip-compile.extractPackageFile()');
   const manager = matchManager(packageFile);
   switch (manager) {
     case 'pip_setup':
       return extractSetupPyFile(content, packageFile, _config);
     case 'pip_requirements':
       return extractRequirementsFile(content);
+    case 'pep621':
+      return await extractPep621File(content, packageFile, _config);
     case 'unknown':
       logger.warn(
         { packageFile },
@@ -134,7 +137,7 @@ export async function extractAllPackageFiles(
         continue;
       }
 
-      const packageFileContent = extractPackageFile(
+      const packageFileContent = await extractPackageFile(
         content,
         packageFile,
         config,

lib/modules/manager/pip-compile/readme.md

@@ -32,10 +32,11 @@ In turn `pip-compile` manager will find all source files and parse them as packa
 
 The following files are currently supported:
 
-| Source filename | Manager            |
-| --------------: | ------------------ |
-|      `setup.py` | `pip_setup`        |
-|          `*.in` | `pip_requirements` |
+|  Source filename | Manager            |
+| ---------------: | ------------------ |
+| `pyproject.toml` | `pep621`           |
+|       `setup.py` | `pip_setup`        |
+|           `*.in` | `pip_requirements` |
 
 Example header: