Skip to content

Commit 03e1333

Browse files
marcin-krystiancmarcin-krystianc
authored
Workflow to build and publish gr-patched version of librdkafka/confluent-kafka (#1)
1 parent 0e5706b commit 03e1333

File tree

4 files changed

+229
-2
lines changed

4 files changed

+229
-2
lines changed

‎.github/scripts/generate_index.py‎

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
import os
2+
import sys
3+
from pathlib import Path
4+
from github import Github
5+
from typing import List, Dict
6+
import itertools
7+
import requests
8+
9+
HTML_TEMPLATE = """<!DOCTYPE html>
10+
<html>
11+
<head>
12+
<title>{package_name}</title>
13+
</head>
14+
<body>
15+
<h1>{package_name}</h1>
16+
{package_links}
17+
</body>
18+
</html>
19+
"""
20+
21+
class PackageIndexBuilder:
22+
def __init__(self, token: str, repo_name: str, output_dir: str):
23+
self.github = Github(token)
24+
self.repo_name = repo_name
25+
self.output_dir = Path(output_dir)
26+
self.packages: Dict[str, List[Dict]] = {}
27+
28+
# Set up authenticated session
29+
self.session = requests.Session()
30+
self.session.headers.update({
31+
"Authorization": f"token {token}",
32+
"Accept": "application/octet-stream",
33+
})
34+
35+
def collect_packages(self):
36+
37+
print ("Query release assets")
38+
repo = self.github.get_repo(self.repo_name)
39+
40+
for release in repo.get_releases():
41+
for asset in release.get_assets():
42+
if asset.name.endswith(('.whl', '.tar.gz')):
43+
package_name = asset.name.split('-')[0].replace('_', '-')
44+
if package_name not in self.packages:
45+
self.packages[package_name] = []
46+
47+
self.packages[package_name].append({
48+
'filename': asset.name,
49+
'url': asset.url,
50+
'size': asset.size,
51+
'upload_time': asset.created_at.strftime('%Y-%m-%d %H:%M:%S'),
52+
})
53+
54+
def generate_index_html(self):
55+
# Generate main index
56+
package_list = self.packages.keys()
57+
main_index = HTML_TEMPLATE.format(
58+
package_name="Simple Package Index",
59+
package_links="\n".join([f'<a href="{x}/">{x}</a><br/>' for x in package_list])
60+
)
61+
62+
with open(self.output_dir / "index.html", "w") as f:
63+
f.write(main_index)
64+
65+
for package, assets in self.packages.items():
66+
67+
package_dir = self.output_dir / package
68+
package_dir.mkdir(exist_ok=True)
69+
70+
# Generate package-specific index.html
71+
file_links = []
72+
assets = sorted(assets, key=lambda x: x["filename"])
73+
for filename, items in itertools.groupby(assets, key=lambda x: x["filename"]):
74+
file_links.append(f'<a href="./{filename}">{filename}</a><br/>')
75+
url = next(items)['url']
76+
77+
# Download the file
78+
with open(package_dir / filename, 'wb') as f:
79+
print (f"Downloading '{filename}' from '{url}'")
80+
response = self.session.get(url, stream=True)
81+
response.raise_for_status()
82+
for chunk in response.iter_content(chunk_size=8192):
83+
if chunk:
84+
f.write(chunk)
85+
86+
package_index = HTML_TEMPLATE.format(
87+
package_name=package,
88+
package_links="\n".join(file_links)
89+
)
90+
91+
with open(package_dir / "index.html", "w") as f:
92+
f.write(package_index)
93+
94+
def build(self):
95+
# Create output directory
96+
self.output_dir.mkdir(parents=True, exist_ok=True)
97+
98+
# Collect and generate
99+
self.collect_packages()
100+
self.generate_index_html()
101+
102+
103+
def main():
104+
# Get environment variables
105+
token = os.environ.get("GITHUB_TOKEN")
106+
repo = os.environ.get("GITHUB_REPOSITORY")
107+
print (repo)
108+
output_dir = os.environ.get("OUTPUT_DIR", "dist")
109+
110+
if not all([token, repo]):
111+
print ("Missing required environment variables")
112+
sys.exit(1)
113+
114+
builder = PackageIndexBuilder(token, repo, output_dir)
115+
builder.build()
116+
117+
if __name__ == "__main__":
118+
main()

‎.github/workflows/package.yml‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
# .github/workflows/build-wheels.yml
2+
name: Build and Package Wheels
3+
4+
on:
5+
pull_request:
6+
push:
7+
8+
env:
9+
LIBRDKAFKA_VERSION: v2.6.1-gr
10+
11+
jobs:
12+
13+
build-linux-x64:
14+
name: Build wheels for Linux x64
15+
runs-on: ubuntu-latest
16+
env:
17+
OS_NAME: linux
18+
ARCH: x64
19+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
20+
steps:
21+
- uses: actions/checkout@v4
22+
- name: Build wheels
23+
run: |
24+
./tools/wheels/build-wheels.sh "${LIBRDKAFKA_VERSION#v}" wheelhouse
25+
- uses: actions/upload-artifact@v4
26+
with:
27+
name: wheels-${{ env.OS_NAME }}-${{ env.ARCH }}
28+
path: wheelhouse/confluent_kafka*.whl
29+
30+
build-windows:
31+
name: Build wheels for Windows
32+
runs-on: windows-latest
33+
env:
34+
OS_NAME: windows
35+
ARCH: x64
36+
CHERE_INVOKING: yes
37+
MSYSTEM: UCRT64
38+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
39+
steps:
40+
- uses: actions/checkout@v4
41+
- name: Setup MSYS2
42+
uses: msys2/setup-msys2@v2
43+
- name: Build wheels
44+
shell: bash
45+
run: |
46+
./tools/mingw-w64/msys2-dependencies.sh
47+
bash tools/mingw-w64/semaphore_commands.sh
48+
bash tools/wheels/install-librdkafka.sh ${LIBRDKAFKA_VERSION#v} dest
49+
tools/wheels/build-wheels.bat x64 win_amd64 dest wheelhouse
50+
- uses: actions/upload-artifact@v4
51+
with:
52+
name: wheels-${{ env.OS_NAME }}-${{ env.ARCH }}
53+
path: wheelhouse/confluent_kafka*.whl
54+
55+
publish_pypi_index:
56+
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
57+
name: Build a PyPI-compatible index
58+
needs: [build-linux-x64, build-windows]
59+
runs-on: ubuntu-latest
60+
permissions:
61+
contents: write
62+
actions: read
63+
packages: read
64+
pages: write
65+
id-token: write
66+
steps:
67+
- uses: actions/checkout@v2
68+
- uses: actions/download-artifact@v4
69+
with:
70+
path: artifacts
71+
pattern: wheels-*
72+
merge-multiple: true
73+
74+
- name: Create release
75+
uses: softprops/action-gh-release@v2
76+
with:
77+
files: |
78+
artifacts/confluent_kafka*
79+
env:
80+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
81+
82+
- name: Generate Package Index
83+
run: |
84+
python -m pip install --upgrade pip
85+
pip install PyGithub
86+
python .github/scripts/generate_index.py
87+
env:
88+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
89+
OUTPUT_DIR: dist
90+
91+
- name: Upload Site Artifact
92+
uses: actions/upload-pages-artifact@v3
93+
with:
94+
path: 'dist'
95+
96+
- name: Deploy to GitHub Pages
97+
uses: actions/deploy-pages@v4

‎pyproject.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
44

55
[project]
66
name = "confluent-kafka"
7-
version = "2.6.1"
7+
version = "2.6.1+gr"
88
description = "Confluent's Python client for Apache Kafka"
99
classifiers = [
1010
"Development Status :: 5 - Production/Stable",

‎tools/wheels/install-librdkafka.sh‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,19 @@ echo "$0: Installing librdkafka $VER to $DEST"
1919
[[ -d "$DEST" ]] || mkdir -p "$DEST"
2020
pushd "$DEST"
2121

22-
curl -L -o lrk$VER.zip https://www.nuget.org/api/v2/package/librdkafka.redist/$VER
22+
# Check if variable exists
23+
if [ -z "${GITHUB_TOKEN}" ]; then
24+
echo "Error: GITHUB_TOKEN is not set"
25+
exit 1
26+
fi
27+
28+
curl -H "Authorization: Bearer ${GITHUB_TOKEN}" \
29+
-H "Accept: application/vnd.github.v3+json" \
30+
-L \
31+
-o lrk$VER.zip \
32+
https://nuget.pkg.github.com/G-Research/download/librdkafka.redist/$VER/librdkafka.redist.$VER.nupkg
33+
34+
#curl -L -o lrk$VER.zip https://www.nuget.org/api/v2/package/librdkafka.redist/$VER
2335

2436
unzip lrk$VER.zip
2537

0 commit comments

Comments
 (0)