Merge main into package (#19)

* fix doc req

* merge general_ref into main (#18)

* modify simple code example

* refactor code

---------

Co-authored-by: Kay (Zekuan) Liu <[email protected]>
Co-authored-by: Junhao Li <[email protected]>

Author: 3 people

docs/examples/configs/config_GC_FedAvg.yaml

@@ -11,6 +11,7 @@
 # setup
   device: 'cpu'
   seed: 10
+  seed_split_data: 42
 
 # model_parameters
   num_trainers: 10

docs/examples/configs/config_GC_FedProx.yaml

@@ -11,6 +11,7 @@
 # setup:
   device: 'cpu'
   seed: 10
+  seed_split_data: 42
 
 # model_parameters:
   num_trainers: 10

docs/examples/configs/config_GC_GCFL.yaml

@@ -11,6 +11,7 @@
 # setup:
   device: 'cpu'
   seed: 10
+  seed_split_data: 42
 
 # model_parameters:
   num_trainers: 10

docs/examples/configs/config_GC_SelfTrain.yaml

@@ -11,6 +11,7 @@
 # setup
   device: 'cpu'
   seed: 10
+  seed_split_data: 42
 
 # model_parameters
   num_trainers: 10

docs/examples/intro_FedGCN.py

@@ -19,7 +19,7 @@
 
 ray.init()
 
-from fedgraph.data_process_nc import load_data
+from fedgraph.data_process import FedGCN_load_data
 from fedgraph.server_class import Server
 from fedgraph.trainer_class import Trainer_General
 from fedgraph.utils_nc import (
@@ -62,7 +62,7 @@
 # tutorial <https://pytorch-geometric.readthedocs.io/en/latest/notes
 # /create_dataset.html>`__ in PyG.
 
-features, adj, labels, idx_train, idx_val, idx_test = load_data(args.dataset)
+features, adj, labels, idx_train, idx_val, idx_test = FedGCN_load_data(args.dataset)
 class_num = labels.max().item() + 1
 
 if args.dataset in ["simulate", "cora", "citeseer", "pubmed", "reddit"]:

docs/examples/intro_GC.py

@@ -23,7 +23,7 @@
 
 sys.path.append("../fedgraph")
 sys.path.append("../../")
-from fedgraph.data_process_gc import *
+from fedgraph.data_process import data_loader_GC
 from fedgraph.federated_methods import (
     run_GC_fedavg,
     run_GC_fedprox,
@@ -113,7 +113,6 @@
 # Here we prepare the data for the experiment.
 # The data is split into training and test sets, and then the training set
 # is further split into training and validation sets.
-# The statistics of the data on trainers are also computed and saved.
 # The user can also use their own dataset and dataloader.
 # The expected format of the dataset is a dictionary with the keys as the trainer names.
 # For each trainer, the value `data[trainer]` is a tuple with 4 elements: (dataloader, num_node_features, num_graph_labels, train_size)
@@ -126,32 +125,9 @@
 """ using original features """
 print("Preparing data (original features) ...")
 
-if args.is_multiple_dataset:
-    splited_data, df_stats = load_multiple_datasets(
-        datapath=args.datapath,
-        dataset_group=args.dataset,
-        batch_size=args.batch_size,
-        convert_x=args.convert_x,
-        seed=seed_split_data,
-    )
-else:
-    splited_data, df_stats = load_single_dataset(
-        args.datapath,
-        args.dataset,
-        num_trainer=args.num_trainers,
-        batch_size=args.batch_size,
-        convert_x=args.convert_x,
-        seed=seed_split_data,
-        overlap=args.overlap,
-    )
+splited_data = data_loader_GC(args)
 print("Data prepared.")
 
-if args.save_files:
-    outdir_stats = os.path.join(outdir, f"stats_train_data.csv")
-    df_stats.to_csv(outdir_stats)
-    print(f"The statistics of the data are written to {outdir_stats}")
-
-
 #######################################################################
 # Setup server and trainers
 # ------------

docs/examples/simple_code_example.py

@@ -17,10 +17,8 @@
 import yaml
 
 sys.path.append("../fedgraph")
-sys.path.append("../../")
-from fedgraph.data_process_gc import load_single_dataset
-from fedgraph.federated_methods import run_FedGCN, run_GC, run_LP
-from fedgraph.utils_nc import federated_data_loader
+from fedgraph.data_process import data_loader
+from fedgraph.federated_methods import run_fedgraph
 
 #######################################################################
 # Specify the task
@@ -44,30 +42,14 @@
 
 with open(config_file_path, "r") as f:
     config = attridict(yaml.safe_load(f))
+config.fedgraph_task = fedgraph_task
 
 print(config)
 
-if fedgraph_task == "FedGCN":
-    data = federated_data_loader(config)
-elif fedgraph_task == "GC":
-    seed_split_data = 42  # seed for splitting data must be fixed
-    data, _ = load_single_dataset(
-        config.datapath,
-        dataset=config.dataset,
-        num_trainer=config.num_trainers,
-        batch_size=config.batch_size,
-        convert_x=config.convert_x,
-        seed=seed_split_data,
-        overlap=config.overlap,
-    )
+data = data_loader(config)  # Load federated data
 
 #######################################################################
 # Run FedGCN method
 # ------------
 
-if fedgraph_task == "FedGCN":
-    run_FedGCN(config, data)
-elif fedgraph_task == "GC":
-    run_GC(config, data)
-elif fedgraph_task == "LP":
-    run_LP(config)
+run_fedgraph(config, data)

docs/fedgraph.data_process_gc.rst

@@ -1,4 +1,4 @@
-Data Process for Node Classification
+Data Process
 ============
 
 .. automodule:: fedgraph.data_process

docs/fedgraph.data_process_nc.rst

@@ -18,3 +18,4 @@ attridict
 fedgraph
 dtaidistance
 gdown
+pandas