Merge remote-tracking branch 'origin/develop' into feat/cloak

Author: noel2004

Cargo.lock

@@ -35,6 +35,7 @@ coordinator_tool:
 	go build -ldflags "-X scroll-tech/common/version.ZkVersion=${ZK_VERSION}" -o $(PWD)/build/bin/coordinator_tool ./cmd/tool
 
 localsetup: coordinator_api ## Local setup: build coordinator_api, copy config, and setup releases
+	mkdir -p build/bin/conf
 	@echo "Copying configuration files..."
 	cp -r $(PWD)/conf/config.json $(PWD)/build/bin/conf/config.template.json
 	@echo "Setting up releases..."

coordinator/Makefile

@@ -7,7 +7,7 @@ if [ -z "${SCROLL_ZKVM_VERSION}" ]; then
 fi
 
 # set ASSET_DIR by reading from config.json
-CONFIG_FILE="bin/conf/config.json"
+CONFIG_FILE="bin/conf/config.template.json"
 if [ ! -f "$CONFIG_FILE" ]; then
   echo "Config file $CONFIG_FILE not found"
   exit 1

coordinator/build/setup_releases.sh

@@ -147,3 +147,20 @@ func SetDynamicFeature(feats string) {
 	defer freeCString(cFeats)
 	C.set_dynamic_feature(cFeats)
 }
+
+// UnivTaskCompatibilityFix calls the universal task compatibility fix function
+func UniversalTaskCompatibilityFix(taskJSON string) (string, error) {
+	cTaskJSON := goToCString(taskJSON)
+	defer freeCString(cTaskJSON)
+
+	resultPtr := C.univ_task_compatibility_fix(cTaskJSON)
+	if resultPtr == nil {
+		return "", fmt.Errorf("univ_task_compatibility_fix failed")
+	}
+
+	// Convert result to Go string and free C memory
+	result := C.GoString(resultPtr)
+	C.release_string(resultPtr)
+
+	return result, nil
+}

coordinator/internal/logic/libzkp/lib.go

@@ -58,5 +58,7 @@ void release_string(char* string_ptr);
 
 void set_dynamic_feature(const char* feats);
 
+// Universal task compatibility fix function
+char* univ_task_compatibility_fix(char* task_json);
 
 #endif /* LIBZKP_H */

coordinator/internal/logic/libzkp/libzkp.h

@@ -213,6 +213,14 @@ func (bp *BatchProverTask) Assign(ctx *gin.Context, getTaskParameter *coordinato
 			return nil, ErrCoordinatorInternalFailure
 		}
 		proverTask.Metadata = metadata
+
+		if isCompatibilityFixingVersion(taskCtx.ProverVersion) {
+			log.Info("Apply compatibility fixing for prover", "version", taskCtx.ProverVersion)
+			if err := fixCompatibility(taskMsg); err != nil {
+				log.Error("apply compatibility failure", "err", err)
+				return nil, ErrCoordinatorInternalFailure
+			}
+		}
 	}
 
 	// Store session info.

coordinator/internal/logic/provertask/batch_prover_task.go

@@ -211,6 +211,14 @@ func (bp *BundleProverTask) Assign(ctx *gin.Context, getTaskParameter *coordinat
 		// bundle proof require snark
 		taskMsg.UseSnark = true
 		proverTask.Metadata = metadata
+
+		if isCompatibilityFixingVersion(taskCtx.ProverVersion) {
+			log.Info("Apply compatibility fixing for prover", "version", taskCtx.ProverVersion)
+			if err := fixCompatibility(taskMsg); err != nil {
+				log.Error("apply compatibility failure", "err", err)
+				return nil, ErrCoordinatorInternalFailure
+			}
+		}
 	}
 
 	// Store session info.

coordinator/internal/logic/provertask/bundle_prover_task.go

@@ -15,6 +15,7 @@ import (
 	"gorm.io/gorm"
 
 	"scroll-tech/common/types/message"
+	"scroll-tech/common/version"
 
 	"scroll-tech/coordinator/internal/config"
 	"scroll-tech/coordinator/internal/logic/libzkp"
@@ -219,6 +220,23 @@ func (b *BaseProverTask) applyUniversal(schema *coordinatorType.GetTaskSchema) (
 	return schema, []byte(metadata), nil
 }
 
+const CompatibilityVersion = "4.5.43"
+
+func isCompatibilityFixingVersion(ver string) bool {
+	return !version.CheckScrollRepoVersion(ver, CompatibilityVersion)
+}
+
+func fixCompatibility(schema *coordinatorType.GetTaskSchema) error {
+
+	fixedTask, err := libzkp.UniversalTaskCompatibilityFix(schema.TaskData)
+	if err != nil {
+		return err
+	}
+	schema.TaskData = fixedTask
+
+	return nil
+}
+
 func newGetTaskCounterVec(factory promauto.Factory, taskType string) *prometheus.CounterVec {
 	getTaskCounterInitOnce.Do(func() {
 		getTaskCounterVec = factory.NewCounterVec(prometheus.CounterOpts{

coordinator/internal/logic/provertask/prover_task.go

@@ -132,7 +132,7 @@ func setupCoordinator(t *testing.T, proversPerSession uint8, coordinatorURL stri
 func setEnv(t *testing.T) {
 	var err error
 
-	version.Version = "v4.4.89"
+	version.Version = "v4.5.45"
 
 	glogger := log.NewGlogHandler(log.StreamHandler(os.Stderr, log.LogfmtFormat()))
 	glogger.Verbosity(log.LvlInfo)

coordinator/test/api_test.go

@@ -48,6 +48,47 @@ pub fn checkout_chunk_task(
     )?)
 }
 
+/// Convert the universal task json into compatible form for old prover
+pub fn univ_task_compatibility_fix(task_json: &str) -> eyre::Result<String> {
+    use scroll_zkvm_types::proof::VmInternalStarkProof;
+
+    let task: tasks::ProvingTask = serde_json::from_str(task_json)?;
+    let aggregated_proofs: Vec<VmInternalStarkProof> = task
+        .aggregated_proofs
+        .into_iter()
+        .map(|proof| VmInternalStarkProof {
+            proofs: proof.proofs,
+            public_values: proof.public_values,
+        })
+        .collect();
+
+    #[derive(Serialize)]
+    struct CompatibleProvingTask {
+        /// seralized witness which should be written into stdin first
+        pub serialized_witness: Vec<Vec<u8>>,
+        /// aggregated proof carried by babybear fields, should be written into stdin
+        /// followed `serialized_witness`
+        pub aggregated_proofs: Vec<VmInternalStarkProof>,
+        /// Fork name specify
+        pub fork_name: String,
+        /// The vk of app which is expcted to prove this task
+        pub vk: Vec<u8>,
+        /// An identifier assigned by coordinator, it should be kept identify for the
+        /// same task (for example, using chunk, batch and bundle hashes)
+        pub identifier: String,
+    }
+
+    let compatible_u_task = CompatibleProvingTask {
+        serialized_witness: task.serialized_witness,
+        aggregated_proofs,
+        fork_name: task.fork_name,
+        vk: task.vk,
+        identifier: task.identifier,
+    };
+
+    Ok(serde_json::to_string(&compatible_u_task)?)
+}
+
 /// Generate required staff for proving tasks
 /// return (pi_hash, metadata, task)
 pub fn gen_universal_task(