update (#30)

Author: Jeremy Dai

.github/workflows/auto-update.yml

@@ -3,7 +3,7 @@ name: Scrape and Update Repo
 permissions:
   contents: write
   pull-requests: write
-  issues: read
+  issues: write
 
 on:
   issues:
@@ -38,20 +38,48 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install requests
+          pip install requests boto3 jsonschema mcp loguru pydantic ruamel-yaml
 
       - name: Create local directory
         run: mkdir -p local
 
       - name: Run Script with Issue Body
+        id: run_script
         env:
           ISSUE_BODY: ${{ github.event.issue.body }}
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          AWS_REGION: us-east-1
         shell: /usr/bin/bash -e {0}
+        continue-on-error: true
         run: |
           echo "Running get_manifest.py with issue body"
-          python scripts/get_manifest.py "$ISSUE_BODY"
+          mkdir -p local
+          if python scripts/get_manifest.py "$ISSUE_BODY"; then
+            echo "script_success=true" >> $GITHUB_OUTPUT
+          else
+            echo "script_success=false" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Comment on issue
+        uses: actions/github-script@v6
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            const issue_number = context.issue.number;
+            const run_id = context.runId;
+            const repo = context.repo;
+            const run_url = `https://github.com/${repo.owner}/${repo.repo}/actions/runs/${run_id}`;
+            
+            await github.rest.issues.createComment({
+              owner: repo.owner,
+              repo: repo.repo,
+              issue_number: issue_number,
+              body: `🤖 Processing your MCP server submission...\n\nTrack progress here: [GitHub Action Run #${run_id}](${run_url})`
+            });
 
       - name: Create and push new branch
+        if: steps.run_script.outputs.script_success == 'true'
         env:
           ISSUE_NUMBER: ${{ github.event.issue.number }}
         shell: /usr/bin/bash -e {0}
@@ -61,6 +89,65 @@ jobs:
           git config user.name "GitHub Action"
           git config user.email "[email protected]"
           git checkout -b "$BRANCH_NAME"
-          git add -f local/readme.md  # Force add despite .gitignore TODO: change to real json file
-          git commit -m "Update repo with README from issue #$ISSUE_NUMBER" || echo "No changes to commit"
-          git push origin "$BRANCH_NAME" --force  # Push to the new branch
+          git add -f local/ mcp-registry/  # Add all files generated by the script
+          git commit -m "Update repo with server manifest from issue #$ISSUE_NUMBER" || echo "No changes to commit"
+          git push origin "$BRANCH_NAME" --force  # Push to the new branch
+          echo "branch_name=$BRANCH_NAME" >> $GITHUB_ENV
+
+      - name: Create Pull Request
+        id: create_pr
+        if: steps.run_script.outputs.script_success == 'true'
+        uses: peter-evans/create-pull-request@v5
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          commit-message: "Add server from issue #${{ github.event.issue.number }}"
+          title: "Add server from issue #${{ github.event.issue.number }}"
+          body: |
+            Automated PR from issue #${{ github.event.issue.number }}
+            
+            Server URL: ${{ github.event.issue.body }}
+          branch: ${{ env.branch_name }}
+          base: main
+
+      - name: Comment on issue with success
+        if: steps.run_script.outputs.script_success == 'true'
+        uses: actions/github-script@v6
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            const issue_number = context.issue.number;
+            const repo = context.repo;
+            const pr_number = '${{ steps.create_pr.outputs.pull-request-number }}';
+            const pr_url = pr_number ? `https://github.com/${repo.owner}/${repo.repo}/pull/${pr_number}` : '';
+            
+            let body = '';
+            if (pr_number) {
+              body = `✅ Processing complete!\n\nA pull request has been created with the server manifest: [PR #${pr_number}](${pr_url})`;
+            } else {
+              body = `⚠️ Processing completed, but no pull request was created. Please check the action logs for details.`;
+            }
+            
+            await github.rest.issues.createComment({
+              owner: repo.owner,
+              repo: repo.repo,
+              issue_number: issue_number,
+              body: body
+            });
+            
+      - name: Comment on issue with failure
+        if: steps.run_script.outputs.script_success == 'false'
+        uses: actions/github-script@v6
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            const issue_number = context.issue.number;
+            const repo = context.repo;
+            const run_id = context.runId;
+            const run_url = `https://github.com/${repo.owner}/${repo.repo}/actions/runs/${run_id}`;
+            
+            await github.rest.issues.createComment({
+              owner: repo.owner,
+              repo: repo.repo,
+              issue_number: issue_number,
+              body: `❌ Failed to process the server manifest. Please check the [action logs](${run_url}) for details.`
+            });

scripts/get_manifest.py

@@ -66,7 +66,8 @@ def _extract_server_info_from_url(self, repo_url: str) -> Dict[str, str]:
             else:
                 # Fallback to repo name if structure is unexpected
                 server_name = parts[4]  # 'servers'
-                logger.warning(f"Could not find server name in URL: {repo_url}")
+                logger.warning(
+                    f"Could not find server name in URL: {repo_url}")
         else:
             # Third-party repo: use repo name as server name
             server_name = parts[4]
@@ -344,7 +345,8 @@ def _extract_with_llms(self, prompt: tuple[str, str]) -> Dict:
         try:
             response = self.client.converse(
                 modelId=LLMModel.CLAUDE_3_7_SONNET,
-                system=[{"text": "You are a helpful assistant for README analysis."}],
+                system=[
+                    {"text": "You are a helpful assistant for README analysis."}],
                 messages=[
                     {
                         "role": "user",
@@ -364,7 +366,8 @@ def _extract_with_llms(self, prompt: tuple[str, str]) -> Dict:
                 content = response["output"]["message"]["content"]
 
                 # Find the first text item
-                text_items = [item.get("text") for item in content if "text" in item]
+                text_items = [item.get("text")
+                              for item in content if "text" in item]
                 if text_items:
                     try:
                         # Look for JSON content within the text
@@ -377,18 +380,22 @@ def _extract_with_llms(self, prompt: tuple[str, str]) -> Dict:
                             # If not, try to find JSON in the text (it might be surrounded by other text)
                             import re
 
-                            json_match = re.search(r"(\{.*\})", text_content, re.DOTALL)
+                            json_match = re.search(
+                                r"(\{.*\})", text_content, re.DOTALL)
                             if json_match:
                                 return json.loads(json_match.group(1))
                             else:
-                                logger.error(f"No JSON content found in response: {text_content[:100]}...")
+                                logger.error(
+                                    f"No JSON content found in response: {text_content[:100]}...")
                                 return self._get_minimal_manifest()
                     except (json.JSONDecodeError, ValueError) as e:
-                        logger.error(f"Failed to parse JSON from response: {e}")
+                        logger.error(
+                            f"Failed to parse JSON from response: {e}")
                 else:
                     logger.error("No text items found in response content")
             else:
-                logger.error(f"Unexpected response structure: {response.keys()}")
+                logger.error(
+                    f"Unexpected response structure: {response.keys()}")
 
             # If we get here, something went wrong with the parsing
             return self._get_minimal_manifest()
@@ -437,7 +444,8 @@ def generate_manifest(self, repo_url: str, server_name: Optional[str] = None) ->
 
         # If server info doesn't have a description, extract from README
         if not server_info["desc"]:
-            server_info["desc"] = self._extract_description_from_readme(readme_content)
+            server_info["desc"] = self._extract_description_from_readme(
+                readme_content)
 
         # Get prompt as tuple and extract manifest
         prompt = self._create_prompt(repo_url, readme_content)
@@ -457,19 +465,24 @@ def generate_manifest(self, repo_url: str, server_name: Optional[str] = None) ->
             manifest["description"] = server_info["desc"]
 
         # Categorize the server
-        sample_server = {"name": manifest.get("name", ""), "description": manifest.get("description", "")}
+        sample_server = {"name": manifest.get(
+            "name", ""), "description": manifest.get("description", "")}
 
-        categorized_servers = asyncio.run(self.categorize_servers([sample_server]))
+        categorized_servers = asyncio.run(
+            self.categorize_servers([sample_server]))
         if categorized_servers:
-            manifest["categories"] = [categorized_servers[0].get("category", "Unknown")]
+            manifest["categories"] = [
+                categorized_servers[0].get("category", "Unknown")]
             manifest["tags"] = manifest.get("tags", [])
             logger.info(f"Server categorized as: {manifest['categories'][0]}")
 
         # Sort installations by priority before running
-        manifest["installations"] = self._filter_and_sort_installations(manifest.get("installations", {}))
+        manifest["installations"] = self._filter_and_sort_installations(
+            manifest.get("installations", {}))
         logger.info(f"Server installations: {manifest['installations']}")
         try:
-            capabilities = asyncio.run(self._run_server_and_extract_capabilities(manifest))
+            capabilities = asyncio.run(
+                self._run_server_and_extract_capabilities(manifest))
             if capabilities:
                 manifest.update(capabilities)
         except Exception as e:
@@ -483,16 +496,21 @@ async def _run_server_and_extract_capabilities(self, manifest: dict[str, Any]) -
         env_vars = {}
         if envs:
             for k, v in envs.items():
-                env_vars[k] = manifest.get("arguments", {}).get(k, {}).get("example", v)
+                env_vars[k] = manifest.get("arguments", {}).get(
+                    k, {}).get("example", v)
         await mcp_client.connect_to_server(installation["command"], installation["args"], env_vars)
         result = {}
         try:
             tools = await mcp_client.list_tools()
-            result["tools"] = [json.loads(tool.model_dump_json()) for tool in tools.tools]  # to avoid $schema field
+            # to avoid $schema field
+            result["tools"] = [json.loads(tool.model_dump_json())
+                               for tool in tools.tools]
             prompts = await mcp_client.list_prompts()
-            result["prompts"] = [json.loads(prompt.model_dump_json()) for prompt in prompts.prompts]
+            result["prompts"] = [json.loads(
+                prompt.model_dump_json()) for prompt in prompts.prompts]
             resources = await mcp_client.list_resources()
-            result["resources"] = [json.loads(resource.model_dump_json()) for resource in resources.resources]
+            result["resources"] = [json.loads(
+                resource.model_dump_json()) for resource in resources.resources]
         except Exception as e:
             traceback.print_exc()
             logger.error(f"Failed to list tools: {e}")
@@ -502,9 +520,12 @@ async def _run_server_and_extract_capabilities(self, manifest: dict[str, Any]) -
         return result
 
     def _filter_and_sort_installations(self, installations: dict[str, dict[str, Any]]):
-        priority = {"uvx": 0, "npm": 1, "python": 2, "docker": 3, "cli": 4, "custom": 5}
-        filtered_installations = {k: v for k, v in installations.items() if k in priority}
-        sorted_installations = sorted(filtered_installations.items(), key=lambda x: priority.get(x[0], 6))
+        priority = {"uvx": 0, "npm": 1, "python": 2,
+                    "docker": 3, "cli": 4, "custom": 5}
+        filtered_installations = {k: v for k,
+                                  v in installations.items() if k in priority}
+        sorted_installations = sorted(
+            filtered_installations.items(), key=lambda x: priority.get(x[0], 6))
         return dict(sorted_installations)
 
 
@@ -540,7 +561,8 @@ def main(repo_url: str, is_official: bool = False):
                     existing_manifest["name"] = new_name
                     with open(new_filename, "w", encoding="utf-8") as file:
                         json.dump(existing_manifest, file, indent=2)
-                    logger.info(f"Previous community manifest saved to {new_filename}")
+                    logger.info(
+                        f"Previous community manifest saved to {new_filename}")
 
         with open(filename, "w", encoding="utf-8") as file:
             json.dump(manifest, file, indent=2)
@@ -558,9 +580,21 @@ def main(repo_url: str, is_official: bool = False):
         sys.exit(1)
 
     repo_url = sys.argv[1].strip()
-    is_official = bool(sys.argv[2]) if len(sys.argv) > 2 else False
+
+    # Check if the URL is a simple URL without protocol
     if not repo_url.startswith(("http://", "https://")):
-        logger.error("Error: URL must start with http:// or https://")
+        # Add https:// if it's a github.com URL without protocol
+        if repo_url.startswith("github.com"):
+            repo_url = "https://" + repo_url
+        # Check if it's a full URL without protocol
+        elif "github.com" in repo_url:
+            repo_url = "https://" + repo_url
+
+    is_official = bool(sys.argv[2]) if len(sys.argv) > 2 else False
+
+    # Validate URL format
+    if not "github.com" in repo_url:
+        logger.error("Error: URL must be a GitHub URL")
         sys.exit(1)
 
     logger.add(sys.stderr, format="{time} {level} {message}", level="INFO")