refactor

README.md

@@ -53,17 +53,14 @@ refactoring guidance.
 ## Running through Coze OpenAPI
 1. Set .env file for configuration on ABCoder's working directory. Taking Coze as an example:
 ```
-# Base directory for other XXX_DIRs, default to current directory
-WORK_DIR=./
+# cache for repo，AST and so on
+WORK_DIR=tmp_abcoder
 
-# DIRs below support both relative and absolute paths, and relative paths are relative to WORK_DIR
-REPO_DIR=tmp_repos
-CACHE_DIR=tmp_caches
-TOOLS_DIR=tools
-EXCLUDE_DIRS=target,gen-codes
+# exclude dirs for repo parsing, separated by comma
+EXCLUDE_DIRS=target,gen-codes 
 
-# coze|ollama 
-API_TYPE=coze 
+# LLM's api type
+API_TYPE=coze # coze|ollama 
 
 # LLM's output language
 LANGUAGE=zh 
@@ -89,6 +86,11 @@ cargo run --bin cmd compress https://xxx.git
    3. Call the LLM to compress the repository codes, and refresh the AST for each call.
 You can stop the process at anytime after step 2. You can restart the compressing by running the same command.
 
+5. Export the compressed results
+```
+cargo run --bin cmd export https://xxx.git --out-dir {OUTPUT_DIR}
+```
+
 # Status Update
 
 The system is designed to automatically fetch the latest data from Github upon triggering relevant tasks, ensuring the

script/make_parser.sh

@@ -15,11 +15,6 @@
 
 root=$(dirname $(realpath $(dirname $0)))
 
-# make go_ast
-cd $root/src/compress/golang/plugin
-go build -o ../../../../tools/parser/go_ast .
-
-
 # make lang
 cd $root/src/lang
 go build -o ../../tools/parser/lang .

src/bin/cmd.rs

@@ -15,9 +15,9 @@
 use std::{env, panic, process, thread, time::Duration};
 
 use ABCoder::{
-    compress::{compress::compress_all, types::types::CodeCache},
-    config::CONFIG,
-    repo,
+    compress::compress::compress_all,
+    export::{self, ExportOptions},
+    parse::{self, CompressOptions},
 };
 
 #[derive(Clone, Debug)]
@@ -28,14 +28,8 @@ struct Options {
 
 #[derive(Clone, Debug)]
 enum Action {
-    Compress(CompressAction),
-}
-
-#[derive(Clone, Debug, Default)]
-struct CompressAction {
-    export_compress: bool,
-    force_update_ast: bool,
-    not_load_external_symbol: bool,
+    Compress(CompressOptions),
+    Export(ExportOptions),
 }
 
 fn main() {
@@ -46,22 +40,30 @@ fn main() {
     match options.action {
         Action::Compress(cmp) => {
             if cmp.force_update_ast {
-                merge_compress(&options.repo_path, &cmp);
+                merge_repo(&options.repo_path, &cmp);
             }
             compress(&options.repo_path, &cmp);
-            if cmp.export_compress {
-                export_compress(&options.repo_path, &cmp);
-            }
+        }
+        Action::Export(exp) => {
+            export(&options.repo_path, &exp);
         }
     }
 }
 
 const USAGE: &str = "Usage: ABCoder <Action> <RepoPath> [Flags]
-Action: compress
-    compress: compress the repo. Including flags:
-      --export-compress: export the compress result
-      --force-update-ast: force parsing repo and merge the previous result 
-      --not-load-external-symbol: not load external external symbols to speed up parsing";
+RepoPath: the path of the repo to compress. Can be a local path or a git url.
+Actions: compress|export
+compress: compress the repo. Including flags:
+    --parse-only: only parse the repo, not compress it
+    --export-compress: export the compress result
+    --force-update-ast: force parsing repo and merge the previous result 
+    --not-load-external-symbol: not load external external symbols to speed up parsing
+    --no-need-comment: not need comment in symbol content (only works for Go now)
+export: export the result to csv or markdown (default). Including flags:
+    --csv: export the compress result to csv
+    --out-dir <path>: output directory path, default is $WORK_DIR
+    --public-only: only export the public symbols
+";
 
 fn parse_options() -> Options {
     let args: Vec<String> = env::args().collect();
@@ -72,27 +74,51 @@ fn parse_options() -> Options {
 
     let action = match args[1].as_str() {
         "compress" => {
-            let mut compress_action = CompressAction::default();
+            let mut compress_action = CompressOptions::default();
             if args.len() > 3 {
                 for i in 3..args.len() {
                     match args[i].as_str() {
-                        "--export-compress" => {
-                            compress_action.export_compress = true;
-                        }
                         "--force-update-ast" => {
                             compress_action.force_update_ast = true;
                         }
                         "--not-load-external-symbol" => {
                             compress_action.not_load_external_symbol = true;
                         }
+                        "--no-need-comment" => {
+                            compress_action.no_need_comment = true;
+                        }
                         _ => {}
                     }
                 }
             }
-
             Action::Compress(compress_action)
         }
 
+        "export" => {
+            let mut opts = ExportOptions::default();
+            if args.len() > 3 {
+                for i in 3..args.len() {
+                    match args[i].as_str() {
+                        "--out-dir" => {
+                            if args.len() <= i + 1 {
+                                println!("--out-dir must specify a value");
+                                process::exit(1);
+                            }
+                            opts.output = Some(args[i + 1].clone());
+                        }
+                        "--csv" => {
+                            opts.csv = true;
+                        }
+                        "--public-only" => {
+                            opts.public_only = true;
+                        }
+                        _ => {}
+                    }
+                }
+            }
+            Action::Export(opts)
+        }
+
         _ => {
             println!("{}", USAGE);
             process::exit(1);
@@ -105,17 +131,22 @@ fn parse_options() -> Options {
     }
 }
 
-fn compress(repo_path: &String, cmp: &CompressAction) {
+fn compress(repo_path: &String, cmp: &CompressOptions) {
     // recoverable logic
     let run = || {
         // get the repo
-        let repo = repo::get_repo(repo_path, !cmp.not_load_external_symbol);
+        let repo = parse::get_repo(repo_path, &cmp);
         if let Err(err) = repo {
             println!("get repo error: {:?}", err);
             process::exit(1);
         }
 
         let mut repo = repo.unwrap();
+        repo.save_to_cache();
+        println!("successfully parsed repo: {}", repo.id);
+        if cmp.parse_only {
+            return;
+        }
 
         // compress the repo
         println!("compressing repo: {}", repo.id);
@@ -142,9 +173,9 @@ fn compress(repo_path: &String, cmp: &CompressAction) {
     }
 }
 
-fn export_compress(repo_path: &String, cmp: &CompressAction) {
+fn export(repo_path: &String, cmp: &ExportOptions) {
     // get the repo
-    let repo = repo::get_repo(repo_path, !cmp.not_load_external_symbol);
+    let repo = parse::get_repo(repo_path, &CompressOptions::default());
     if let Err(err) = repo {
         println!("get repo error: {:?}", err);
 
@@ -155,24 +186,22 @@ fn export_compress(repo_path: &String, cmp: &CompressAction) {
 
     // export the compress
     println!("export repo: {}", repo.id);
-    repo::export_repo(&mut repo);
-    // save the compressed repo
-    repo.save_to_cache();
+    export::export_repo(&mut repo, cmp);
 
-    println!("successfully compressed repo: {}", repo.id);
+    println!("successfully exported repo: {}", repo.id);
 }
 
-fn merge_compress(repo_path: &String, opts: &CompressAction) {
+fn merge_repo(repo_path: &String, cmp: &CompressOptions) {
     // get old repo
-    let repo = repo::get_repo(repo_path, !opts.not_load_external_symbol);
+    let repo = parse::get_repo(repo_path, &cmp);
     if let Err(err) = repo {
         println!("get repo error: {:?}", err);
         process::exit(1);
     }
     let mut repo = repo.unwrap();
 
     // parse new repo
-    let nrepo = repo::force_parse_repo(repo_path, !opts.not_load_external_symbol);
+    let nrepo = parse::force_parse_repo(repo_path, &cmp);
     if let Err(err) = nrepo {
         println!("parse repo error: {:?}", err);
         process::exit(1);

src/compress/compress.rs

@@ -1,11 +1,11 @@
 // Copyright 2025 CloudWeGo Authors
-// 
+//
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
-// 
+//
 //     https://www.apache.org/licenses/LICENSE-2.0
-// 
+//
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -40,7 +40,11 @@ pub fn from_json(id: &str, json: &str) -> Result<Repository, Box<dyn Error>> {
         f.id = id.to_string();
     }
     if f.graph.is_none() {
-        f.build_graph();
+        //return err
+        return Err(Box::new(std::io::Error::new(
+            std::io::ErrorKind::Other,
+            "graph is None",
+        )));
     }
     f.save_to_cache();
     Ok(f)
@@ -171,18 +175,20 @@ pub async fn cascade_compress_variable(
         return;
     }
     let var_node = var_node.unwrap();
-    for (i, v) in var_node.references.iter().enumerate() {
-        if i >= MAX_REFERS {
-            eprintln!("too many references for {:?}", id);
-            break;
-        }
-        let c = repo.get_id_content(v);
-        if c.is_none() {
-            eprintln!("{:?} node is not found", v);
-            continue;
+    if let Some(nfs) = &var_node.references {
+        for (i, v) in nfs.iter().enumerate() {
+            if i >= MAX_REFERS {
+                eprintln!("too many references for {:?}", id);
+                break;
+            }
+            let c = repo.get_id_content(&v.id());
+            if c.is_none() {
+                eprintln!("{:?} node is not found", v);
+                continue;
+            }
+            let elem = c.unwrap();
+            refs.push(elem);
         }
-        let elem = c.unwrap();
-        refs.push(elem);
     }
 
     // compress type if any