[PIR]add patch header template (#68481)

* add patch header template

* fix ci

Author: changeyoung98

paddle/fluid/pir/serialize_deserialize/CMakeLists.txt

@@ -11,11 +11,28 @@ if(LINUX)
   link_libraries(stdc++fs)
 endif()
 
-add_definitions(-DPADDLE_ROOT="${PADDLE_SOURCE_DIR}")
-add_definitions(
-  -DPATCH_PATH="../../../../../python/paddle/pir/serialize_deserialize/patch")
+file(GLOB_RECURSE YAML_PATCH_FILES "*.yaml")
+# change pir version when new patches are added
+add_definitions(-DDEVELOP_VERSION=1)
+add_definitions(-DRELEASE_VERSION=1)
+set(TEMPLATE_FILE ${CMAKE_CURRENT_SOURCE_DIR}/patch/template.h.in)
+set(PATCH_HEADER ${CMAKE_CURRENT_BINARY_DIR}/patch/patch.h)
+
+configure_file(${TEMPLATE_FILE} ${PATCH_HEADER} @ONLY)
+file(WRITE "${PATCH_HEADER}"
+     "#include <map>\n#include <string>\n\n"
+     "const std::map<std::string, std::string> yaml_files = {\n")
+
+foreach(PATCH_FILE ${YAML_PATCH_FILES})
+  get_filename_component(FILENAME "${PATCH_FILE}" NAME_WE)
+  file(READ ${PATCH_FILE} FILE_CONTENT)
+  set(CONTENT "R\"(${FILE_CONTENT})\"")
+  file(APPEND "${PATCH_HEADER}" "{ \"${FILENAME}\", ${CONTENT} },\n")
+endforeach()
+
+file(APPEND "${PATCH_HEADER}" "};\n")
 
 cc_library(
   pir_save_load
-  SRCS ${SERIALIZE_DESERIALIZE_CPP_SOURCES}
+  SRCS ${SERIALIZE_DESERIALIZE_CPP_SOURCES} ${PATCH_HEADER}
   DEPS op_dialect phi json yaml)

paddle/fluid/pir/serialize_deserialize/include/patch_util.h

@@ -35,6 +35,7 @@ Json ParseAttrPatches(const YAML::Node &root);
 
 Json ParseTypePatches(const YAML::Node &root);
 
-Json YamlParser(const std::string &yaml_file);
+/* Yaml file is set to be empty by default. It's only used for testing. */
+Json YamlParser(const std::string &version, const std::string &yaml_file = "");
 
 }  // namespace pir

paddle/fluid/pir/serialize_deserialize/include/schema.h

@@ -82,9 +82,17 @@ namespace pir {
 #define NULL_TYPE "NULL"
 
 // special op compress
-
 #define PARAMETEROP "p"
 
+// actions for patch
+#define DELETE "DEL"
+#define ADD "ADD"
+#define UPDATE "UPD"
+#define NEW_NAME "NN"
+#define ADD_ATTRS "ADD_A"
+#define ADD_OPRESULTS_ATTRS "ADD_OA"
+#define PATCH "patch"
+
 std::pair<std::string, std::string> GetContentSplitByDot(
     const std::string& str);
 
@@ -109,6 +117,4 @@ class DialectIdMap {
   std::unordered_map<std::string, std::string> DecompressDialect;
 };
 
-uint64_t GetPirVersion();
-uint64_t GetMaxReleasePirVersion();
 }  // namespace pir

paddle/fluid/pir/serialize_deserialize/include/version_compat.h

@@ -34,9 +34,11 @@ class PatchBuilder {
   PatchBuilder& operator=(const PatchBuilder&) = delete;
   PatchBuilder& operator=(PatchBuilder&&);
 
-  void IR_API BuildPatch(const std::string& path,
-                         uint64_t pir_version,
-                         uint64_t max_version);
+  /* Patch patch is set to empty by default. It is only used for testing.
+   */
+  void IR_API BuildPatch(uint64_t pir_version,
+                         uint64_t max_version,
+                         const std::string& path = "");
   /* If file_version != pir_vefrsion, set file_version for finding patch yamls.
    */
   void SetFileVersion(const uint64_t version) { file_version_ = version; }

python/paddle/pir/serialize_deserialize/patch/1.yaml renamed to paddle/fluid/pir/serialize_deserialize/patch/1.yaml

@@ -140,11 +140,24 @@ type_patches:
 ```
 
 ## pir_version 配置说明
-### C++端版本号管理
-- 版本号管理在C++端，通过宏PIR_VERSION进行管理。
-- pir_version 定义PIR的版本迭代，每次PIR进行更新并新增patch文件后，版本号会顺序递增。与Paddle的主版本号解耦，可以独立迭代。
-- 定义GetPirVersion函数获取当前的版本号：在"paddle/fluid/pir/serialize_deserialize/patch"路径下进行yaml文件查询，如果存在"0.yaml"则为develop版本，pir_verison为0；否则查找到的yaml文件名最大值即为当前的pir_version。
-- ReadModule和WriteModule参数中的pir_version设为默认值，可以不用传递。pir_version 函数默认值为0，为develop版本下的值，进入函数后会获取当前的版本号。
+### C++端版本号管理与CMake配置
+- 版本号管理在C++端，在CMakeList.txt中配置。
+- PIR版本号定义PIR的版本迭代，版本号与yaml文件名强相关。每次PIR进行更新并新增patch文件后，patch文件名顺序递增，版本号同时顺序递增。与Paddle的主版本号解耦，可以独立迭代。
+  ```cmake
+  # change pir version when new patches are added
+  add_definitions(-DDEVELOP_VERSION=1)
+  add_definitions(-DRELEASE_VERSION=1)
+  ```
+
+  ```tree
+  ├─patch
+  │  ├─0.yaml
+  │  └─1.yaml
+  ```
+  - RELEASE_VERSION 为已发布的版本中PIR版本号，即为patch yaml文件名的最大值。
+  - DEVELOP_VERSION 为当前develop分支下的PIR版本号，若存在未发布的新增patch，配置在`0.yaml`中，且当前的develop pir 版本号为0。
+
+- ReadModule和WriteModule参数中的pir_version设为默认值，可以不用传递。pir_version 函数默认值为-1，进入函数后会获取CMake中配置的当前的PIR版本号。
 
 ### Python端
 - Paddle的主版本号定义在Python端，与PIR version不产生关联。Python端不再需要获取和传入pir_version，直接使用默认值即可。

python/paddle/pir/serialize_deserialize/patch/Readme.md renamed to paddle/fluid/pir/serialize_deserialize/patch/Readme.md

@@ -0,0 +1,7 @@
+#pragma once
+#include <map>
+#include <string>
+
+const std::map<std::string, std::string> yaml_files = {
+    @FILE_CONTENTS@
+};

paddle/fluid/pir/serialize_deserialize/patch/template.h.in

@@ -72,7 +72,7 @@ bool ReadModule(const std::string& file_path,
   std::ifstream f(file_path);
   Json data = Json::parse(f);
   if (pir_version < 0) {
-    pir_version = GetPirVersion();
+    pir_version = DEVELOP_VERSION;
     VLOG(6) << "pir_version is null, get pir_version: " << pir_version;
   }
 
@@ -85,17 +85,15 @@ bool ReadModule(const std::string& file_path,
     if (file_version != (uint64_t)pir_version) {
       builder.SetFileVersion(file_version);
       // Set max_version to the max version number of release pir plus 1.
-      auto max_version = GetMaxReleasePirVersion() + 1;
+      auto max_version = RELEASE_VERSION + 1;
       // If pir_version_ is not 0, we will build patch from file_version_ to
       // pir_version_; If pir_version_ is 0, we will first build patch from
       // file_version_ to max_version, and then add 0.yaml to the end.
       auto version = pir_version == 0 ? max_version : pir_version;
       VLOG(6) << "file_version: " << file_version
               << ", pir_version: " << pir_version
               << ", final_version: " << version;
-      std::filesystem::path patch_path = std::filesystem::path(PATCH_PATH);
-      VLOG(8) << "Patch path: " << patch_path;
-      builder.BuildPatch(patch_path.string(), version, max_version);
+      builder.BuildPatch(version, max_version);
     }
   } else {
     PADDLE_THROW(common::errors::InvalidArgument("Invalid model file."));

paddle/fluid/pir/serialize_deserialize/src/interface.cc

@@ -240,9 +240,8 @@ pir::Operation* ProgramReader::ReadOp(Json* op_json) {
     VLOG(8) << op_name << " has been patched: " << *op_json;
     // Apply patch to op name
     // This happens when changing an op into another dialect
-    if (op_patch.contains("NEW_NAME")) {
-      std::string new_name =
-          op_patch.at("NEW_NAME").template get<std::string>();
+    if (op_patch.contains(NEW_NAME)) {
+      std::string new_name = op_patch.at(NEW_NAME).template get<std::string>();
       VLOG(8) << "change op name from " << op_name << " to " << new_name;
       op_name = new_name;
       op_json->at(ID) = op_name;
@@ -334,8 +333,8 @@ pir::AttributeMap ProgramReader::ReadAttributesMap(
     const std::unordered_map<std::string, Json>& attr_patch) {
   pir::AttributeMap attributes;
   // Add new attribute from patch
-  if (attr_patch.count("A_ADD")) {
-    for (auto& attr_json : attr_patch.at("A_ADD")) {
+  if (attr_patch.count(ADD_ATTRS)) {
+    for (auto& attr_json : attr_patch.at(ADD_ATTRS)) {
       attrs_json->insert(attrs_json->end(), attr_json);
     }
     VLOG(8) << "attr has been added: " << *attrs_json;
@@ -358,8 +357,8 @@ pir::AttributeMap ProgramReader::ReadAttributesMap(
   }
   VLOG(6) << "Finish Read pir::AttributeMap.";
   // Add new opresult attribute from patch
-  if (attr_patch.count("OA_ADD")) {
-    for (auto& attr_json : attr_patch.at("OA_ADD")) {
+  if (attr_patch.count(ADD_OPRESULTS_ATTRS)) {
+    for (auto& attr_json : attr_patch.at(ADD_OPRESULTS_ATTRS)) {
       opresult_attrs_json->insert(opresult_attrs_json->end(), attr_json);
     }
     VLOG(8) << "opresult attr has been added: " << *opresult_attrs_json;

paddle/fluid/pir/serialize_deserialize/src/ir_deserialize.cc

@@ -20,6 +20,7 @@
 #include <vector>
 #include "paddle/fluid/pir/dialect/operator/ir/op_attribute.h"
 #include "paddle/fluid/pir/serialize_deserialize/include/schema.h"
+#include "paddle/fluid/pir/serialize_deserialize/patch/patch.h"
 #include "paddle/phi/common/data_type.h"
 #include "paddle/pir/include/core/builtin_attribute.h"
 #include "paddle/pir/include/core/builtin_type.h"
@@ -262,8 +263,8 @@ Json ParseOpPairPatches(const YAML::Node &root) {
       VLOG(8) << "Op_pair_name: " << name;
       j_patch["op_pair"].push_back(op_name);
     }
-    j_patch["patch"] = Json::object();
-    j_patch["patch"]["op_pair"] = j_patch["op_pair"];
+    j_patch[PATCH] = Json::object();
+    j_patch[PATCH]["op_pair"] = j_patch["op_pair"];
     // parse actions
     auto actions = node["actions"];
     for (size_t j = 0; j < actions.size(); j++) {
@@ -278,20 +279,20 @@ Json ParseOpPairPatches(const YAML::Node &root) {
         Json j_add_out;
         j_add_out[ID] = out_id;
         j_add_out[TYPE_TYPE] = BuildTypeJsonPatch(action);
-        j_patch["patch"][OPRESULTS]["ADD"].push_back(j_add_out);
+        j_patch[PATCH][OPRESULTS][ADD].push_back(j_add_out);
         Json j_add_in;
         j_add_in[ID] = in_id;
-        j_patch["patch"][OPOPERANDS]["ADD"].push_back(j_add_in);
+        j_patch[PATCH][OPOPERANDS][ADD].push_back(j_add_in);
       } else if (action_name == "delete_value") {
         VLOG(8) << "Patch for deleting values.";
         int out_id = action["object"][0].as<int>();
         int in_id = action["object"][1].as<int>();
         Json j_del_out;
         j_del_out[ID] = out_id;
-        j_patch["patch"][OPRESULTS]["DELETE"].push_back(j_del_out);
+        j_patch[PATCH][OPRESULTS][DELETE].push_back(j_del_out);
         Json j_del_in;
         j_del_in[ID] = in_id;
-        j_patch["patch"][OPOPERANDS]["DELETE"].push_back(j_del_in);
+        j_patch[PATCH][OPOPERANDS][DELETE].push_back(j_del_in);
       }
     }
     json_patch.push_back(j_patch);
@@ -314,7 +315,7 @@ Json ParseOpPatches(const YAML::Node &root) {
     VLOG(8) << "Parse patches for " << op_name;
     Json j_patch;
     j_patch["op_name"] = op_name;
-    j_patch["patch"] = Json::object();
+    j_patch[PATCH] = Json::object();
     // parse actions
     auto actions = node["actions"];
 
@@ -335,10 +336,10 @@ Json ParseOpPatches(const YAML::Node &root) {
         j_attr[ATTR_TYPE] = BuildAttrJsonPatch(action);
         if (action_name == "add_attr") {
           Json j_add = Json::object();
-          j_add["ADD"] = j_attr;
-          j_patch["patch"][ATTRS].push_back(j_add);
+          j_add[ADD] = j_attr;
+          j_patch[PATCH][ATTRS].push_back(j_add);
         } else {
-          j_patch["patch"][ATTRS].push_back(j_attr);
+          j_patch[PATCH][ATTRS].push_back(j_attr);
         }
       } else if (action_name == "add_output_attr" ||
                  action_name == "modify_output_attr" ||
@@ -350,10 +351,10 @@ Json ParseOpPatches(const YAML::Node &root) {
         j_attr[ATTR_TYPE] = BuildAttrJsonPatch(action);
         if (action_name == "add_output_attr") {
           Json j_add = Json::object();
-          j_add["ADD"] = j_attr;
-          j_patch["patch"][OPRESULTS_ATTRS].push_back(j_add);
+          j_add[ADD] = j_attr;
+          j_patch[PATCH][OPRESULTS_ATTRS].push_back(j_add);
         } else {
-          j_patch["patch"][OPRESULTS_ATTRS].push_back(j_attr);
+          j_patch[PATCH][OPRESULTS_ATTRS].push_back(j_attr);
         }
       } else if (action_name == "modify_attr_name" ||
                  action_name == "modify_output_attr_name") {
@@ -362,35 +363,35 @@ Json ParseOpPatches(const YAML::Node &root) {
         std::string new_name = action["default"].as<std::string>();
         Json j_attr;
         j_attr[NAME] = old_name;
-        j_attr["NEW_NAME"] = new_name;
+        j_attr[NEW_NAME] = new_name;
         std::string col =
             action_name == "modify_attr_name" ? ATTRS : OPRESULTS_ATTRS;
-        j_patch["patch"][col].push_back(j_attr);
+        j_patch[PATCH][col].push_back(j_attr);
       } else if (action_name == "delete_input") {
         VLOG(8) << "Patch for delete_input";
         Json j_input;
         int op_id = action["object"].as<int>();
         j_input[ID] = op_id;
-        j_patch["patch"][OPOPERANDS]["DELETE"].push_back(j_input);
+        j_patch[PATCH][OPOPERANDS][DELETE].push_back(j_input);
       } else if (action_name == "add_output") {
         VLOG(8) << "Patch for add_output";
         Json j_output;
         int op_id = action["object"].as<int>();
         j_output[ID] = op_id;
         j_output[TYPE_TYPE] = BuildTypeJsonPatch(action);
-        j_patch["patch"][OPRESULTS]["ADD"].push_back(j_output);
+        j_patch[PATCH][OPRESULTS][ADD].push_back(j_output);
       } else if (action_name == "modify_output_type") {
         VLOG(8) << "Patch for modify_output_type";
         int op_id = action["object"].as<int>();
         Json j_type;
         j_type[ID] = op_id;
         j_type[TYPE_TYPE] = BuildTypeJsonPatch(action);
-        j_patch["patch"][OPRESULTS]["UPDATE"].push_back(j_type);
+        j_patch[PATCH][OPRESULTS][UPDATE].push_back(j_type);
       } else if (action_name == "modify_name") {
         VLOG(8) << "Patch for modify_name";
         std::string op_name = action["default"].as<std::string>();
         GetCompressOpName(&op_name);
-        j_patch["patch"]["NEW_NAME"] = op_name;
+        j_patch[PATCH][NEW_NAME] = op_name;
       }
     }
     json_patch.push_back(j_patch);
@@ -410,15 +411,15 @@ Json ParseTypePatches(const YAML::Node &root) {
     VLOG(8) << "Type name after compressing: " << type_name;
     Json j_patch;
     j_patch["type_name"] = type_name;
-    j_patch["patch"] = Json::object();
+    j_patch[PATCH] = Json::object();
     auto actions = node["actions"];
     for (size_t j = 0; j < actions.size(); j++) {
       YAML::Node action = actions[j];
       std::string action_name = action["action"].as<std::string>();
       if (action_name == "modify_name") {
-        j_patch["patch"]["NEW_NAME"] = GetTypeName(action);
+        j_patch[PATCH][NEW_NAME] = GetTypeName(action);
       } else if (action_name == "delete_type") {
-        j_patch["patch"]["NEW_NAME"] = "";
+        j_patch[PATCH][NEW_NAME] = "";
       }
     }
     json_patch.push_back(j_patch);
@@ -439,31 +440,36 @@ Json ParseAttrPatches(const YAML::Node &root) {
     VLOG(8) << attr_name;
     Json j_patch;
     j_patch["attr_name"] = attr_name;
-    j_patch["patch"] = Json::object();
+    j_patch[PATCH] = Json::object();
     auto actions = node["actions"];
     for (size_t j = 0; j < actions.size(); j++) {
       YAML::Node action = actions[j];
       std::string action_name = action["action"].as<std::string>();
       if (action_name == "modify_name") {
-        j_patch["patch"]["NEW_NAME"] = GetAttrName(action);
+        j_patch[PATCH][NEW_NAME] = GetAttrName(action);
       } else if (action_name == "delete_attr") {
-        j_patch["patch"]["NEW_NAME"] = "";
+        j_patch[PATCH][NEW_NAME] = "";
       }
     }
     json_patch.push_back(j_patch);
   }
   return json_patch;
 }
 
-Json YamlParser(const std::string &yaml_file) {
+Json YamlParser(const std::string &version, const std::string &yaml_file) {
+  YAML::Node root;
   std::ifstream fin;
-  VLOG(8) << yaml_file;
-  fin.open(yaml_file);
-  if (!fin) {
-    VLOG(8) << yaml_file << " is not fin and return empty. ";
-    return Json::object();
+  if (yaml_file.empty()) {
+    root = YAML::Load(yaml_files.at(version));
+  } else {
+    VLOG(8) << yaml_file;
+    fin.open(yaml_file);
+    if (!fin) {
+      VLOG(8) << yaml_file << " is not fin and return empty. ";
+      return Json::object();
+    }
+    root = YAML::Load(fin);
   }
-  YAML::Node root = YAML::Load(fin);
   Json json_patch;
   if (!root.IsDefined()) {
     VLOG(8) << "Not defined";
@@ -484,7 +490,9 @@ Json YamlParser(const std::string &yaml_file) {
   Yaml attr_patch = root["attr_patches"];
   json_patch["attr_patches"] = ParseAttrPatches(attr_patch);
   VLOG(8) << "Finish attr json_patch: " << json_patch;
-  fin.close();
+  if (fin) {
+    fin.close();
+  }
   return json_patch;
 }
 }  // namespace pir