G-Research · marcin-krystianc · Mar 18, 2025 · Mar 17, 2025
diff --git a/.github/scripts/generate_index.py b/.github/scripts/generate_index.py
@@ -0,0 +1,133 @@
+import os
+import sys
+import re
+import itertools
+import requests
+import hashlib
+
+from urllib.parse import quote
+from pathlib import Path
+from github import Github
+from typing import List, Dict
+
+HTML_TEMPLATE = """<!DOCTYPE html>
+ <html>
+ <head>
+     <title>{package_name}</title>
+ </head>
+ <body>
+     <h1>{package_name}</h1>
+     {package_links}
+ </body>
+ </html>
+"""
+
+def normalize(name):
+    """Normalize package name according to PEP 503."""
+    return re.sub(r"[-_.]+", "-", name).lower()
+
+def calculate_sha256(file_path):
+    with open(file_path, "rb") as f:
+        digest = hashlib.file_digest(f, "sha256")
+
+    return digest.hexdigest()
+
+class PackageIndexBuilder:
+    def __init__(self, token: str, repo_name: str, output_dir: str):
+        self.github = Github(token)
+        self.repo = self.github.get_repo(repo_name)
+        self.output_dir = Path(output_dir)
+        self.packages: Dict[str, List[Dict]] = {}
+
+        # Set up authenticated session
+        self.session = requests.Session()
+        self.session.headers.update({
+            "Authorization": f"token {token}",
+            "Accept": "application/octet-stream",
+        })
+
+    def collect_packages(self):
+
+        print ("Query release assets")
+
+        for release in self.repo.get_releases():
+            for asset in release.get_assets():
+                if asset.name.endswith(('.whl', '.tar.gz')):
+                    package_name = normalize(asset.name.split('-')[0])
+                    if package_name not in self.packages:
+                        self.packages[package_name] = []
+
+                    self.packages[package_name].append({
+                        'filename': asset.name,
+                        'url': asset.url,
+                        'size': asset.size,
+                        'upload_time': asset.created_at.strftime('%Y-%m-%d %H:%M:%S'),
+                    })
+
+    def generate_index_html(self):
+        # Generate main index
+        package_list = self.packages.keys()
+        main_index = HTML_TEMPLATE.format(
+            package_name="Simple Package Index",
+            package_links="\n".join([f'<a href="{x}/">{x}</a><br/>' for x in package_list])
+        )
+
+        with open(self.output_dir / "index.html", "w") as f:
+            f.write(main_index)
+
+        for package, assets in self.packages.items():
+
+            package_dir = self.output_dir / package
+            package_dir.mkdir(exist_ok=True)
+
+            # Generate package-specific index.html
+            file_links = []
+            assets = sorted(assets, key=lambda x: x["filename"])
+            for filename, items in itertools.groupby(assets, key=lambda x: x["filename"]):
+                url = next(items)['url']
+
+                # Download the file
+                with open(package_dir / filename, 'wb') as f:
+                    print (f"Downloading '{filename}' from '{url}'")
+                    response = self.session.get(url, stream=True)
+                    response.raise_for_status()
+                    for chunk in response.iter_content(chunk_size=8192):
+                        if chunk:
+                            f.write(chunk)
+
+                sha256_hash = calculate_sha256(package_dir / filename)
+                file_links.append(f'<a href="{quote(filename)}#sha256={sha256_hash}">{filename}</a><br/>')
+
+            package_index = HTML_TEMPLATE.format(
+                package_name=f"Links for {package}",
+                package_links="\n".join(file_links)
+            )
+
+            with open(package_dir / "index.html", "w") as f:
+                f.write(package_index)
+
+    def build(self):
+        # Create output directory
+        self.output_dir.mkdir(parents=True, exist_ok=True)
+
+        # Collect and generate
+        self.collect_packages()
+        self.generate_index_html()
+
+
+def main():
+    # Get environment variables
+    token = os.environ.get("GITHUB_TOKEN")
+    repo = os.environ.get("GITHUB_REPOSITORY")
+    print (repo)
+    output_dir = os.environ.get("OUTPUT_DIR", "dist")
+
+    if not all([token, repo]):
+        print ("Missing required environment variables")
+        sys.exit(1)
+
+    builder = PackageIndexBuilder(token, repo, output_dir)
+    builder.build()
+
+if __name__ == "__main__":
+    main()
diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml
@@ -0,0 +1,97 @@
+# .github/workflows/build-wheels.yml
+name: Build and Package Wheels
+
+on:
+  pull_request:
+  push:
+
+env:
+  LIBRDKAFKA_VERSION: v2.8.0-gr
+
+jobs:
+
+  build-linux-x64:
+    name: Build wheels for Linux x64
+    runs-on: ubuntu-latest
+    env:
+      OS_NAME: linux
+      ARCH: x64
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Build wheels
+        run: |
+          ./tools/wheels/build-wheels.sh "${LIBRDKAFKA_VERSION#v}" wheelhouse
+      - uses: actions/upload-artifact@v4
+        with:
+          name: wheels-${{ env.OS_NAME }}-${{ env.ARCH }}
+          path: wheelhouse/confluent_kafka*.whl
+
+  build-windows:
+    name: Build wheels for Windows
+    runs-on: windows-latest
+    env:
+      OS_NAME: windows
+      ARCH: x64
+      CHERE_INVOKING: yes
+      MSYSTEM: UCRT64
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup MSYS2
+        uses: msys2/setup-msys2@v2
+      - name: Build wheels
+        shell: bash
+        run: |
+          ./tools/mingw-w64/msys2-dependencies.sh
+          bash tools/mingw-w64/semaphore_commands.sh
+          bash tools/wheels/install-librdkafka.sh ${LIBRDKAFKA_VERSION#v} dest
+          tools/wheels/build-wheels.bat x64 win_amd64 dest wheelhouse
+      - uses: actions/upload-artifact@v4
+        with:
+          name: wheels-${{ env.OS_NAME }}-${{ env.ARCH }}
+          path: wheelhouse/confluent_kafka*.whl
+
+  publish_pypi_index:
+    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
+    name: Build a PyPI-compatible index
+    needs: [build-linux-x64, build-windows]
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      actions: read
+      packages: read
+      pages: write
+      id-token: write
+    steps:    
+      - uses: actions/checkout@v2
+      - uses: actions/download-artifact@v4
+        with:
+          path: artifacts
+          pattern: wheels-*
+          merge-multiple: true
+
+      - name: Create release
+        uses: softprops/action-gh-release@v2
+        with:
+          files: |
+            artifacts/confluent_kafka*
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Generate Package Index
+        run: |        
+          python -m pip install --upgrade pip
+          pip install PyGithub
+          python .github/scripts/generate_index.py
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          OUTPUT_DIR: dist
+
+      - name: Upload Site Artifact
+        uses: actions/upload-pages-artifact@v3
+        with:
+          path: 'dist'
+
+      - name: Deploy to GitHub Pages
+        uses: actions/deploy-pages@v4
diff --git a/pyproject.toml b/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "confluent-kafka"
-version = "2.8.0"
+version = "2.8.0+gr"
 description = "Confluent's Python client for Apache Kafka"
 classifiers = [
     "Development Status :: 5 - Production/Stable",

diff --git a/tools/wheels/install-librdkafka.sh b/tools/wheels/install-librdkafka.sh
@@ -19,7 +19,19 @@ echo "$0: Installing librdkafka $VER to $DEST"
 [[ -d "$DEST" ]] || mkdir -p "$DEST"
 pushd "$DEST"
 
-curl -L -o lrk$VER.zip https://www.nuget.org/api/v2/package/librdkafka.redist/$VER
+# Check if variable exists
+if [ -z "${GITHUB_TOKEN}" ]; then
+    echo "Error: GITHUB_TOKEN is not set"
+    exit 1
+fi
+
+curl -H "Authorization: Bearer ${GITHUB_TOKEN}" \
+ -H "Accept: application/vnd.github.v3+json" \
+ -L \
+ -o lrk$VER.zip \
+https://nuget.pkg.github.com/G-Research/download/librdkafka.redist/$VER/librdkafka.redist.$VER.nupkg
+
+#curl -L -o lrk$VER.zip https://www.nuget.org/api/v2/package/librdkafka.redist/$VER
 
 unzip lrk$VER.zip