Include Cache: add HeaderCacheService

.bazelrc

@@ -3,8 +3,8 @@ build --tool_java_language_version=21 --tool_java_runtime_version=21
 
 # Delete test data packages, needed for bazel integration tests. Update by running the following command:
 # bazel run @rules_bazel_integration_test//tools:update_deleted_packages
-build --deleted_packages=.bazelbsp,.bazelbsp/aspects,aspect/testing/tests/src/com/google/idea/blaze/aspect/integration/testdata,clwb/tests/projects/clang_cl,clwb/tests/projects/clang_cl/main,clwb/tests/projects/execution/main,clwb/tests/projects/external_includes/main,clwb/tests/projects/llvm_toolchain/main,clwb/tests/projects/llvm_toolchain/wasm,clwb/tests/projects/query_sync/main,clwb/tests/projects/simple/main,clwb/tests/projects/target_compatible/main,clwb/tests/projects/virtual_includes/.clwb/aspects,clwb/tests/projects/virtual_includes/lib/impl_deps,clwb/tests/projects/virtual_includes/lib/raw_files,clwb/tests/projects/virtual_includes/lib/strip_absolut,clwb/tests/projects/virtual_includes/lib/strip_relative,clwb/tests/projects/virtual_includes/main,examples/cpp/simple_project/.clwb/aspects,examples/cpp/simple_project/src,examples/cpp/simple_project/src/lib,examples/go/with_go_source,examples/go/with_go_source/otherlib,examples/go/with_go_source/testa,examples/go/with_go_source/testb,examples/go/with_proto,examples/go/with_proto/go,examples/go/with_proto/go/external,examples/go/with_proto/go/lib,examples/go/with_proto/proto,examples/java/greetings_project,examples/java/greetings_project/.ijwb/aspects,examples/java/greetings_project/greeting_lib,examples/java/greetings_project/hello,examples/java/greetings_project/hi,examples/kotlin/simple_project,examples/python/simple_code_generator/example,examples/python/simple_code_generator/generated,examples/python/simple_code_generator/lib,examples/python/simple_code_generator/rules,examples/python/with_numpy,examples/python/with_numpy/app,examples/python/with_numpy/lib,examples/scala/with_bzlmod/hello,ijwb/tests/projects/simple,testing/test_deps/projects,testing/test_deps/projects/java_and_deps,testing/test_deps/projects/java_and_deps/deps/no_ide,testing/test_deps/projects/java_and_deps/deps/top_level_lib_1,testing/test_deps/projects/java_and_deps/deps/top_level_lib_2,testing/test_deps/projects/java_and_deps/deps/transitive_dep_lib,testing/test_deps/projects/java_and_deps/project,testing/test_deps/projects/java_and_deps/project/java/com/example/sample/nested,testing/test_deps/projects/simple_java,testing/test_deps/projects/simple_java/java/com/example/sample/nested,testing/test_deps/projects/simple_proto/external,testing/test_deps/projects/simple_proto/project
-query --deleted_packages=.bazelbsp,.bazelbsp/aspects,aspect/testing/tests/src/com/google/idea/blaze/aspect/integration/testdata,clwb/tests/projects/clang_cl,clwb/tests/projects/clang_cl/main,clwb/tests/projects/execution/main,clwb/tests/projects/external_includes/main,clwb/tests/projects/llvm_toolchain/main,clwb/tests/projects/llvm_toolchain/wasm,clwb/tests/projects/query_sync/main,clwb/tests/projects/simple/main,clwb/tests/projects/target_compatible/main,clwb/tests/projects/virtual_includes/.clwb/aspects,clwb/tests/projects/virtual_includes/lib/impl_deps,clwb/tests/projects/virtual_includes/lib/raw_files,clwb/tests/projects/virtual_includes/lib/strip_absolut,clwb/tests/projects/virtual_includes/lib/strip_relative,clwb/tests/projects/virtual_includes/main,examples/cpp/simple_project/.clwb/aspects,examples/cpp/simple_project/src,examples/cpp/simple_project/src/lib,examples/go/with_go_source,examples/go/with_go_source/otherlib,examples/go/with_go_source/testa,examples/go/with_go_source/testb,examples/go/with_proto,examples/go/with_proto/go,examples/go/with_proto/go/external,examples/go/with_proto/go/lib,examples/go/with_proto/proto,examples/java/greetings_project,examples/java/greetings_project/.ijwb/aspects,examples/java/greetings_project/greeting_lib,examples/java/greetings_project/hello,examples/java/greetings_project/hi,examples/kotlin/simple_project,examples/python/simple_code_generator/example,examples/python/simple_code_generator/generated,examples/python/simple_code_generator/lib,examples/python/simple_code_generator/rules,examples/python/with_numpy,examples/python/with_numpy/app,examples/python/with_numpy/lib,examples/scala/with_bzlmod/hello,ijwb/tests/projects/simple,testing/test_deps/projects,testing/test_deps/projects/java_and_deps,testing/test_deps/projects/java_and_deps/deps/no_ide,testing/test_deps/projects/java_and_deps/deps/top_level_lib_1,testing/test_deps/projects/java_and_deps/deps/top_level_lib_2,testing/test_deps/projects/java_and_deps/deps/transitive_dep_lib,testing/test_deps/projects/java_and_deps/project,testing/test_deps/projects/java_and_deps/project/java/com/example/sample/nested,testing/test_deps/projects/simple_java,testing/test_deps/projects/simple_java/java/com/example/sample/nested,testing/test_deps/projects/simple_proto/external,testing/test_deps/projects/simple_proto/project
+build --deleted_packages=clwb/tests/projects/clang_cl,clwb/tests/projects/clang_cl/main,clwb/tests/projects/execution/main,clwb/tests/projects/external_includes/main,clwb/tests/projects/llvm_toolchain/main,clwb/tests/projects/llvm_toolchain/wasm,clwb/tests/projects/protobuf,clwb/tests/projects/protobuf/main,clwb/tests/projects/protobuf/proto,clwb/tests/projects/simple/main,clwb/tests/projects/target_compatible/main,clwb/tests/projects/virtual_includes/lib/impl_deps,clwb/tests/projects/virtual_includes/lib/raw_files,clwb/tests/projects/virtual_includes/lib/strip_absolut,clwb/tests/projects/virtual_includes/lib/strip_relative,clwb/tests/projects/virtual_includes/main,examples/cpp/simple_project/src,examples/cpp/simple_project/src/lib,examples/python/simple_code_generator/example,examples/python/simple_code_generator/generated,examples/python/simple_code_generator/lib,examples/python/simple_code_generator/rules,examples/python/with_numpy,examples/python/with_numpy/app,examples/python/with_numpy/lib
+query --deleted_packages=clwb/tests/projects/clang_cl,clwb/tests/projects/clang_cl/main,clwb/tests/projects/execution/main,clwb/tests/projects/external_includes/main,clwb/tests/projects/llvm_toolchain/main,clwb/tests/projects/llvm_toolchain/wasm,clwb/tests/projects/protobuf,clwb/tests/projects/protobuf/main,clwb/tests/projects/protobuf/proto,clwb/tests/projects/simple/main,clwb/tests/projects/target_compatible/main,clwb/tests/projects/virtual_includes/lib/impl_deps,clwb/tests/projects/virtual_includes/lib/raw_files,clwb/tests/projects/virtual_includes/lib/strip_absolut,clwb/tests/projects/virtual_includes/lib/strip_relative,clwb/tests/projects/virtual_includes/main,examples/cpp/simple_project/src,examples/cpp/simple_project/src/lib,examples/python/simple_code_generator/example,examples/python/simple_code_generator/generated,examples/python/simple_code_generator/lib,examples/python/simple_code_generator/rules,examples/python/with_numpy,examples/python/with_numpy/app,examples/python/with_numpy/lib
 
 common --enable_bzlmod
 common --noincompatible_disallow_empty_glob

aspect/intellij_info_impl.bzl

@@ -344,7 +344,7 @@ def collect_cc_compilation_context(ctx, target):
     external_includes = getattr(compilation_context, "external_includes", depset()).to_list()
 
     return struct(
-        direct_headers = [artifact_location(it) for it in compilation_context.direct_headers],
+        headers = [artifact_location(it) for it in compilation_context.headers.to_list()],
         defines = compilation_context.defines.to_list(),
         includes = compilation_context.includes.to_list(),
         quote_includes = compilation_context.quote_includes.to_list(),

aspect/testing/tests/src/com/google/idea/blaze/aspect/cpp/cclibrary/CcLibraryTest.java

@@ -58,8 +58,8 @@ public void testCcLibrary() throws Exception {
     assertThat(compilationCtx.getDefinesList()).containsExactly("VERSION2");
     assertThat(compilationCtx.getSystemIncludesList()).contains(testRelative("foo/bar"));
     assertThat(compilationCtx.getQuoteIncludesList()).contains(".");
-    assertThat(relativePathsForArtifacts(compilationCtx.getDirectHeadersList()))
-        .containsExactly(testRelative("simple/simple.h"));
+    assertThat(relativePathsForArtifacts(compilationCtx.getHeadersList()))
+        .containsExactly(testRelative("simple/simple.h"), testRelative("simple/simple_textual.h"));
 
     // Can't test for this because the cc code stuffs source artifacts into
     // the output group

aspect/testing/tests/src/com/google/idea/blaze/aspect/general/artifacts/ArtifactTest.java

@@ -17,6 +17,7 @@
 
 import static com.google.common.truth.Truth.assertThat;
 
+import com.google.devtools.intellij.aspect.Common.ArtifactLocation;
 import com.google.devtools.intellij.ideinfo.IntellijIdeInfo.CIdeInfo;
 import com.google.devtools.intellij.ideinfo.IntellijIdeInfo.TargetIdeInfo;
 import com.google.idea.blaze.BazelIntellijAspectTest;
@@ -25,7 +26,9 @@
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
 
-/** Tests the artifact representation */
+/**
+ * Tests the artifact representation
+ */
 @RunWith(JUnit4.class)
 public class ArtifactTest extends BazelIntellijAspectTest {
 
@@ -78,4 +81,19 @@ public void testCorrectPathToTargetBuildFile() throws Exception {
     assertThat(buildFile.getIsExternal()).isTrue();
     assertThat(buildFile.getIsSource()).isTrue();
   }
+
+  @Test
+  public void testVirtualIncludesSymlinks() throws Exception {
+    final var ideInfo = getCIdeInfo("//main:main");
+
+    final var headers = ideInfo.getCompilationContext().getHeadersList();
+    assertThat(headers).hasSize(10);
+
+    final var virtualHeaders = headers.stream()
+        .filter(it -> it.getRelativePath().contains("virtual_includes"))
+        .toList();
+
+    assertThat(virtualHeaders).hasSize(3);
+    assertThat(virtualHeaders.stream().map(ArtifactLocation::getIsSource).distinct()).containsExactly(false);
+  }
 }

aspect/testing/tests/src/com/google/idea/blaze/aspect/general/artifacts/BUILD

@@ -19,6 +19,7 @@ java_test(
         "//aspect/testing/rules:IntellijAspectTest",
         "//aspect/testing/rules:intellij_aspect_test_fixture_java_proto",
         "//intellij_platform_sdk:test_libs",
+        "//proto:common_java_proto",
         "//proto:intellij_ide_info_java_proto",
         "//third_party/java/junit",
     ],

base/src/com/google/idea/blaze/base/ideinfo/CIdeInfo.java

@@ -107,7 +107,7 @@ public static abstract class CompilationContext implements ProtoWrapper<Intellij
 
     private static final CompilationContext EMPTY = builder().build();
 
-    public abstract ImmutableList<ArtifactLocation> directHeaders();
+    public abstract ImmutableList<ArtifactLocation> headers();
 
     public abstract ImmutableList<String> defines();
 
@@ -119,7 +119,7 @@ public static abstract class CompilationContext implements ProtoWrapper<Intellij
 
     public static CIdeInfo.CompilationContext fromProto(IntellijIdeInfo.CIdeInfo.CompilationContext proto) {
       return builder()
-          .setDirectHeaders(ProtoWrapper.map(proto.getDirectHeadersList(), ArtifactLocation::fromProto))
+          .setHeaders(ProtoWrapper.map(proto.getHeadersList(), ArtifactLocation::fromProto))
           .setDefines(ProtoWrapper.internStrings(proto.getDefinesList()))
           .setIncludes(ProtoWrapper.map(proto.getIncludesList(), ExecutionRootPath::fromProto))
           .setQuoteIncludes(ProtoWrapper.map(proto.getQuoteIncludesList(), ExecutionRootPath::fromProto))
@@ -130,7 +130,7 @@ public static CIdeInfo.CompilationContext fromProto(IntellijIdeInfo.CIdeInfo.Com
     @Override
     public IntellijIdeInfo.CIdeInfo.CompilationContext toProto() {
       return IntellijIdeInfo.CIdeInfo.CompilationContext.newBuilder()
-          .addAllDirectHeaders(ProtoWrapper.mapToProtos(directHeaders()))
+          .addAllHeaders(ProtoWrapper.mapToProtos(headers()))
           .addAllDefines(defines())
           .addAllIncludes(ProtoWrapper.mapToProtos(includes()))
           .addAllQuoteIncludes(ProtoWrapper.mapToProtos(quoteIncludes()))
@@ -140,7 +140,7 @@ public IntellijIdeInfo.CIdeInfo.CompilationContext toProto() {
 
     public static Builder builder() {
       return new AutoValue_CIdeInfo_CompilationContext.Builder()
-          .setDirectHeaders(ImmutableList.of())
+          .setHeaders(ImmutableList.of())
           .setDefines(ImmutableList.of())
           .setIncludes(ImmutableList.of())
           .setQuoteIncludes(ImmutableList.of())
@@ -150,7 +150,7 @@ public static Builder builder() {
     @AutoValue.Builder
     public abstract static class Builder {
 
-      public abstract Builder setDirectHeaders(ImmutableList<ArtifactLocation> value);
+      public abstract Builder setHeaders(ImmutableList<ArtifactLocation> value);
 
       public abstract Builder setDefines(ImmutableList<String> value);
 

clwb/BUILD

@@ -161,7 +161,10 @@ clwb_headless_test(
 
 clwb_headless_test(
     name = "virtual_includes_headless_test",
-    srcs = ["tests/headlesstests/com/google/idea/blaze/clwb/VirtualIncludesTest.java"],
+    srcs = [
+        "tests/headlesstests/com/google/idea/blaze/clwb/VirtualIncludesCacheTest.java",
+        "tests/headlesstests/com/google/idea/blaze/clwb/VirtualIncludesTest.java",
+    ],
     project = "virtual_includes",
 )
 
@@ -207,29 +210,39 @@ clwb_headless_test(
     project = "clang_cl",
 )
 
+clwb_headless_test(
+    name = "protobuf_headless_test",
+    srcs = [
+        "tests/headlesstests/com/google/idea/blaze/clwb/ProtobufCacheTest.java",
+        "tests/headlesstests/com/google/idea/blaze/clwb/ProtobufTest.java",
+    ],
+    project = "protobuf",
+)
+
 test_suite(
     name = "headless_tests",
     tests = [
         ":clang_cl_headless_test",
         ":example_headless_test",
         ":execution_headless_test",
         ":external_includes_headless_test",
+        ":lib_cpp_headless_test",
         ":llvm_toolchain_headless_test",
+        ":protobuf_headless_test",
         ":simple_headless_test",
         ":target_compatible_headless_test",
         ":virtual_includes_headless_test",
-        ":lib_cpp_headless_test",
     ],
 )
 
 clwb_integration_test(
-  name = "copts_processor",
-  srcs = ["tests/integrationtests/com/google/idea/blaze/clwb/CoptsProcessorTest.kt"],
+    name = "copts_processor",
+    srcs = ["tests/integrationtests/com/google/idea/blaze/clwb/CoptsProcessorTest.kt"],
 )
 
 test_suite(
     name = "integration_tests",
     tests = [
-      ":copts_processor",
+        ":copts_processor",
     ],
 )

clwb/tests/headlesstests/com/google/idea/blaze/clwb/ProtobufCacheTest.java

@@ -0,0 +1,59 @@
+package com.google.idea.blaze.clwb;
+
+import static com.google.common.truth.Truth.assertThat;
+import static com.google.idea.blaze.clwb.base.Assertions.assertContainsHeader;
+import static com.google.idea.blaze.clwb.base.Assertions.assertCachedHeader;
+import static com.google.idea.blaze.clwb.base.Utils.setIncludesCacheEnabled;
+
+import com.google.idea.blaze.base.bazel.BazelVersion;
+import com.google.idea.blaze.clwb.base.ClwbHeadlessTestCase;
+import com.google.idea.testing.headless.BazelVersionRule;
+import com.google.idea.testing.headless.ProjectViewBuilder;
+import com.intellij.util.system.OS;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+@RunWith(JUnit4.class)
+public class ProtobufCacheTest extends ClwbHeadlessTestCase {
+
+  // protobuf requires bazel 7+
+  @Rule
+  public final BazelVersionRule bazelRule = new BazelVersionRule(7, 0);
+  // on windows clang-cl is required to compile protobuf and therefore also bazel 8+
+  @Rule
+  public final BazelVersionRule bazelWindowsRule = new BazelVersionRule(OS.Windows, 8, 0);
+
+  @Test
+  public void testClwb() {
+    setIncludesCacheEnabled(true);
+
+    final var errors = runSync(defaultSyncParams().build());
+    errors.assertNoErrors();
+
+    checkProto();
+  }
+
+  @Override
+  protected ProjectViewBuilder projectViewText(BazelVersion version) {
+    final var builder = super.projectViewText(version);
+
+    if (OS.CURRENT.equals(OS.Windows)) {
+      builder.addBuildFlag("--extra_toolchains=@local_config_cc//:cc-toolchain-x64_windows-clang-cl");
+      builder.addBuildFlag("--extra_execution_platforms=//:x64_windows-clang-cl");
+    }
+
+    return builder;
+  }
+
+  private void checkProto() {
+    final var compilerSettings = findFileCompilerSettings("main/main.cc");
+
+    final var headersSearchRoots = compilerSettings.getHeadersSearchRoots().getAllRoots();
+    assertThat(headersSearchRoots).isNotEmpty();
+
+    assertContainsHeader("proto/addressbook.pb.h", compilerSettings);
+    assertCachedHeader("proto/addressbook.pb.h", compilerSettings, myProject);
+  }
+}

clwb/tests/headlesstests/com/google/idea/blaze/clwb/ProtobufTest.java

@@ -0,0 +1,66 @@
+package com.google.idea.blaze.clwb;
+
+import static com.google.common.truth.Truth.assertThat;
+import static com.google.idea.blaze.clwb.base.Assertions.assertContainsHeader;
+import static com.google.idea.blaze.clwb.base.Utils.setIncludesCacheEnabled;
+
+import com.google.idea.blaze.base.bazel.BazelVersion;
+import com.google.idea.blaze.clwb.base.AllowedVfsRoot;
+import com.google.idea.blaze.clwb.base.ClwbHeadlessTestCase;
+import com.google.idea.testing.headless.BazelVersionRule;
+import com.google.idea.testing.headless.ProjectViewBuilder;
+import com.intellij.util.system.OS;
+import java.util.ArrayList;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+@RunWith(JUnit4.class)
+public class ProtobufTest extends ClwbHeadlessTestCase {
+
+  // protobuf requires bazel 7+
+  @Rule
+  public final BazelVersionRule bazelRule = new BazelVersionRule(7, 0);
+
+  // on windows clang-cl is required to compile protobuf and therefore also bazel 8+
+  @Rule
+  public final BazelVersionRule bazelWindowsRule = new BazelVersionRule(OS.Windows, 8, 0);
+
+  @Test
+  public void testClwb() {
+    setIncludesCacheEnabled(false);
+
+    final var errors = runSync(defaultSyncParams().build());
+    errors.assertNoErrors();
+
+    checkProto();
+  }
+
+  @Override
+  protected ProjectViewBuilder projectViewText(BazelVersion version) {
+    final var builder = super.projectViewText(version);
+
+    if (OS.CURRENT.equals(OS.Windows)) {
+      builder.addBuildFlag("--extra_toolchains=@local_config_cc//:cc-toolchain-x64_windows-clang-cl");
+      builder.addBuildFlag("--extra_execution_platforms=//:x64_windows-clang-cl");
+    }
+
+    return builder;
+  }
+
+  @Override
+  protected void addAllowedVfsRoots(ArrayList<AllowedVfsRoot> roots) {
+    super.addAllowedVfsRoots(roots);
+    roots.add(AllowedVfsRoot.bazelBinRecursive(myBazelInfo, "proto"));
+  }
+
+  private void checkProto() {
+    final var compilerSettings = findFileCompilerSettings("main/main.cc");
+
+    final var headersSearchRoots = compilerSettings.getHeadersSearchRoots().getAllRoots();
+    assertThat(headersSearchRoots).isNotEmpty();
+
+    assertContainsHeader("proto/addressbook.pb.h", compilerSettings);
+  }
+}