Skip to content
This repository was archived by the owner on Feb 4, 2020. It is now read-only.

Add pch support #317

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Add pch support #317

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
821d2c6
Add support of precompiled headers
izmmisha Jul 24, 2018
8409e35
Remove CalledWithPchError stuff
izmmisha Jul 24, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
66 changes: 47 additions & 19 deletions clcache/__main__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@
# `objectHash`: hash of the object in cache
ManifestEntry = namedtuple('ManifestEntry', ['includeFiles', 'includesContentHash', 'objectHash'])

CompilerArtifacts = namedtuple('CompilerArtifacts', ['objectFilePath', 'stdout', 'stderr'])
CompilerArtifacts = namedtuple('CompilerArtifacts', ['objectFilePath', 'pchFilePath', 'stdout', 'stderr'])

def printBinary(stream, rawData):
with OUTPUT_LOCK:
Expand Down Expand Up @@ -231,6 +231,14 @@ def allSectionsLocked(repository):
section.lock.release()


def readPCHHash(pchFile):
with open(pchFile+".clcache", 'r') as f:
return f.read()

def writePCHHash(pchFile, hash):
with open(pchFile+".clcache", 'w') as f:
f.write('{}'.format(hash))

class ManifestRepository(object):
# Bump this counter whenever the current manifest file format changes.
# E.g. changing the file format from {'oldkey': ...} to {'newkey': ...} requires
Expand Down Expand Up @@ -293,6 +301,10 @@ def getManifestHash(compilerBinary, commandLine, sourceFile):

additionalData = "{}|{}|{}".format(
compilerHash, commandLine, ManifestRepository.MANIFEST_FILE_FORMAT_VERSION)

if 'Yu' in arguments:
pchFile = CommandLineAnalyzer.getPchFileName(arguments)
additionalData += readPCHHash(pchFile)
return getFileHash(sourceFile, additionalData)

@staticmethod
Expand Down Expand Up @@ -366,6 +378,7 @@ def forPath(path):

class CompilerArtifactsSection(object):
OBJECT_FILE = 'object'
PCH_FILE = 'pch'
STDOUT_FILE = 'output.txt'
STDERR_FILE = 'stderr.txt'

Expand Down Expand Up @@ -395,6 +408,10 @@ def setEntry(self, key, artifacts):
if artifacts.objectFilePath is not None:
copyOrLink(artifacts.objectFilePath,
os.path.join(tempEntryDir, CompilerArtifactsSection.OBJECT_FILE))
if artifacts.pchFilePath is not None:
copyOrLink(artifacts.pchFilePath,
os.path.join(tempEntryDir, CompilerArtifactsSection.PCH_FILE))
writePCHHash(artifacts.pchFilePath, key)
setCachedCompilerConsoleOutput(os.path.join(tempEntryDir, CompilerArtifactsSection.STDOUT_FILE),
artifacts.stdout)
if artifacts.stderr != '':
Expand All @@ -408,6 +425,7 @@ def getEntry(self, key):
cacheEntryDir = self.cacheEntryDir(key)
return CompilerArtifacts(
os.path.join(cacheEntryDir, CompilerArtifactsSection.OBJECT_FILE),
os.path.join(cacheEntryDir, CompilerArtifactsSection.PCH_FILE),
getCachedCompilerConsoleOutput(os.path.join(cacheEntryDir, CompilerArtifactsSection.STDOUT_FILE)),
getCachedCompilerConsoleOutput(os.path.join(cacheEntryDir, CompilerArtifactsSection.STDERR_FILE))
)
Expand Down Expand Up @@ -776,9 +794,6 @@ def registerCallWithMultipleSourceFiles(self):
def numCallsWithPch(self):
return self._stats[Statistics.CALLS_WITH_PCH]

def registerCallWithPch(self):
self._stats[Statistics.CALLS_WITH_PCH] += 1

def numCallsForLinking(self):
return self._stats[Statistics.CALLS_FOR_LINKING]

Expand Down Expand Up @@ -871,10 +886,6 @@ class CalledForLinkError(AnalysisError):
pass


class CalledWithPchError(AnalysisError):
pass


class ExternalDebugInfoError(AnalysisError):
pass

Expand Down Expand Up @@ -1266,6 +1277,18 @@ def parseArgumentsAndInputFiles(cmdline):

return dict(arguments), inputFiles

@staticmethod
def getPchFileName(options):
if 'Fp' in options:
return options['Fp'][0]
if 'Yc' in options:
option = options['Yc'][0]
elif 'Yu' in options:
option = options['Yu'][0]
else:
return None
return basenameWithoutExtension(option) + '.pch'

@staticmethod
def analyze(cmdline: List[str]) -> Tuple[List[Tuple[str, str]], List[str]]:
options, inputFiles = CommandLineAnalyzer.parseArgumentsAndInputFiles(cmdline)
Expand Down Expand Up @@ -1294,9 +1317,6 @@ def analyze(cmdline: List[str]) -> Tuple[List[Tuple[str, str]], List[str]]:
if 'Zi' in options:
raise ExternalDebugInfoError()

if 'Yc' in options or 'Yu' in options:
raise CalledWithPchError()

if 'link' in options or 'c' not in options:
raise CalledForLinkError()

Expand All @@ -1316,6 +1336,11 @@ def analyze(cmdline: List[str]) -> Tuple[List[Tuple[str, str]], List[str]]:
# Generate from .c/.cpp filenames
objectFiles = [os.path.join(prefix, basenameWithoutExtension(f)) + '.obj' for f, _ in inputFiles]

if 'Yc' in options:
assert len(objectFiles) == 1
pchFile = CommandLineAnalyzer.getPchFileName(options)
objectFiles = [(objectFiles[0], pchFile)]

printTraceStatement("Compiler source files: {}".format(inputFiles))
printTraceStatement("Compiler object file: {}".format(objectFiles))
return inputFiles, objectFiles
Expand Down Expand Up @@ -1397,8 +1422,7 @@ def printStatistics(cache):
called for linking : {}
called for external debug : {}
called w/o source : {}
called w/ multiple sources : {}
called w/ PCH : {}""".strip()
called w/ multiple sources : {}""".strip()

with cache.statistics.lock, cache.statistics as stats, cache.configuration as cfg:
print(template.format(
Expand All @@ -1417,7 +1441,6 @@ def printStatistics(cache):
stats.numCallsForExternalDebugInfo(),
stats.numCallsWithoutSourceFile(),
stats.numCallsWithMultipleSourceFiles(),
stats.numCallsWithPch(),
))


Expand Down Expand Up @@ -1481,14 +1504,16 @@ def addObjectToCache(stats, cache, cachekey, artifacts):
printTraceStatement("Adding file {} to cache using key {}".format(artifacts.objectFilePath, cachekey))

cache.setEntry(cachekey, artifacts)
stats.registerCacheEntry(os.path.getsize(artifacts.objectFilePath))
stats.registerCacheEntry(os.path.getsize(artifacts.objectFilePath) +
os.path.getsize(artifacts.pchFilePath) if artifacts.pchFilePath else 0)

with cache.configuration as cfg:
return stats.currentCacheSize() >= cfg.maximumCacheSize()


def processCacheHit(cache, objectFile, cachekey):
printTraceStatement("Reusing cached object for key {} for object file {}".format(cachekey, objectFile))
objectFile, pchFile = objectFile if isinstance(objectFile, tuple) else (objectFile, None)

with cache.lockFor(cachekey):
with cache.statistics.lock, cache.statistics as stats:
Expand All @@ -1499,6 +1524,10 @@ def processCacheHit(cache, objectFile, cachekey):

cachedArtifacts = cache.getEntry(cachekey)
copyOrLink(cachedArtifacts.objectFilePath, objectFile)
if pchFile is not None:
copyOrLink(cachedArtifacts.pchFilePath, pchFile)
writePCHHash(pchFile, cachekey)

printTraceStatement("Finished. Exit code 0")
return 0, cachedArtifacts.stdout, cachedArtifacts.stderr, False

Expand Down Expand Up @@ -1611,9 +1640,6 @@ def processCompileRequest(cache, compiler, args):
except MultipleSourceFilesComplexError:
printTraceStatement("Cannot cache invocation as {}: multiple source files found".format(cmdLine))
updateCacheStatistics(cache, Statistics.registerCallWithMultipleSourceFiles)
except CalledWithPchError:
printTraceStatement("Cannot cache invocation as {}: precompiled headers in use".format(cmdLine))
updateCacheStatistics(cache, Statistics.registerCallWithPch)
except CalledForLinkError:
printTraceStatement("Cannot cache invocation as {}: called for linking".format(cmdLine))
updateCacheStatistics(cache, Statistics.registerCallForLinking)
Expand Down Expand Up @@ -1756,13 +1782,15 @@ def processNoDirect(cache, objectFile, compiler, cmdLine, environment):
def ensureArtifactsExist(cache, cachekey, reason, objectFile, compilerResult, extraCallable=None):
cleanupRequired = False
returnCode, compilerOutput, compilerStderr = compilerResult
objectFile, pchFile = objectFile if isinstance(objectFile, tuple) else (objectFile, None)

correctCompiliation = (returnCode == 0 and os.path.exists(objectFile))
with cache.lockFor(cachekey):
if not cache.hasEntry(cachekey):
with cache.statistics.lock, cache.statistics as stats:
reason(stats)
if correctCompiliation:
artifacts = CompilerArtifacts(objectFile, compilerOutput, compilerStderr)
artifacts = CompilerArtifacts(objectFile, pchFile, compilerOutput, compilerStderr)
cleanupRequired = addObjectToCache(stats, cache, cachekey, artifacts)
if extraCallable and correctCompiliation:
extraCallable()
Expand Down