public FakeProjectFilesystem(Clock clock, Path root, Set<Path> files) {
super(root);
// We use LinkedHashMap to preserve insertion order, so the
// behavior of this test is consistent across versions. (It also lets
// us write tests which explicitly test iterating over entries in
// different orders.)
fileContents = new LinkedHashMap<>();
fileLastModifiedTimes = new LinkedHashMap<>();
FileTime modifiedTime = FileTime.fromMillis(clock.currentTimeMillis());
for (Path file : files) {
fileContents.put(file, new byte[0]);
fileLastModifiedTimes.put(file, modifiedTime);
}
fileAttributes = new LinkedHashMap<>();
symLinks = new LinkedHashMap<>();
directories = new LinkedHashSet<>();
directories.add(Paths.get(""));
for (Path file : files) {
Path dir = file.getParent();
while (dir != null) {
directories.add(dir);
dir = dir.getParent();
}
}
this.clock = Preconditions.checkNotNull(clock);
// Generally, tests don't care whether files exist.
ignoreValidityOfPaths = true;
}
@Override
public void touch(Path fileToTouch) throws IOException {
if (exists(fileToTouch)) {
setLastModifiedTime(fileToTouch, FileTime.fromMillis(clock.currentTimeMillis()));
} else {
createNewFile(fileToTouch);
}
}
@Override
public void mkdirs(Path path) throws IOException {
for (int i = 0; i < path.getNameCount(); i++) {
Path subpath = path.subpath(0, i + 1);
directories.add(subpath);
fileLastModifiedTimes.put(subpath, FileTime.fromMillis(clock.currentTimeMillis()));
}
}
@Override
public void writeBytesToPath(byte[] bytes, Path path, FileAttribute<?>... attrs)
throws IOException {
Path normalizedPath = MorePaths.normalize(path);
fileContents.put(normalizedPath, Preconditions.checkNotNull(bytes));
fileAttributes.put(normalizedPath, ImmutableSet.copyOf(attrs));
Path directory = normalizedPath.getParent();
while (directory != null) {
directories.add(directory);
directory = directory.getParent();
}
fileLastModifiedTimes.put(normalizedPath, FileTime.fromMillis(clock.currentTimeMillis()));
}
private ImmutableMap<String, String> getBuildMetadata() throws IOException {
return ImmutableMap.<String, String>builder()
.put(
BuildInfo.METADATA_KEY_FOR_ADDITIONAL_INFO,
formatAdditionalArtifactInfo(
ImmutableMap.<String, String>builder()
.put("build_id", buildId.toString())
.put(
"timestamp",
String.valueOf(TimeUnit.MILLISECONDS.toSeconds(clock.currentTimeMillis())))
.putAll(artifactExtraData)
.build()))
.putAll(buildMetadata)
.build();
}
/**
* Timestamp event. A timestamped event cannot subsequently being posted and is useful only to
* pass its timestamp on to another posted event.
*/
public void timestamp(BuckEvent event) {
event.configure(clock.currentTimeMillis(), clock.nanoTime(), threadIdSupplier.get(), buildId);
}
/**
* Query Watchman for file change events. If too many events are pending or an error occurs an
* overflow event is posted to the EventBus signalling that events may have been lost (and so
* typically caches must be cleared to avoid inconsistency). Interruptions and IOExceptions are
* propagated to callers, but typically if overflow events are handled conservatively by
* subscribers then no other remedial action is required.
*/
@Override
public void postEvents(BuckEventBus buckEventBus) throws IOException, InterruptedException {
ProcessExecutor.LaunchedProcess watchmanProcess =
processExecutor.launchProcess(
ProcessExecutorParams.builder()
.addCommand("watchman", "--server-encoding=json", "--no-pretty", "-j")
.build());
try {
LOG.debug("Writing query to Watchman: %s", query);
watchmanProcess.getOutputStream().write(query.getBytes(Charsets.US_ASCII));
watchmanProcess.getOutputStream().close();
LOG.debug("Parsing JSON output from Watchman");
final long parseStartTimeMillis = clock.currentTimeMillis();
InputStream jsonInput = watchmanProcess.getInputStream();
if (LOG.isVerboseEnabled()) {
byte[] fullResponse = ByteStreams.toByteArray(jsonInput);
jsonInput.close();
jsonInput = new ByteArrayInputStream(fullResponse);
LOG.verbose("Full JSON: " + new String(fullResponse, Charsets.UTF_8).trim());
}
JsonParser jsonParser = objectMapper.getJsonFactory().createJsonParser(jsonInput);
PathEventBuilder builder = new PathEventBuilder();
JsonToken token = jsonParser.nextToken();
/*
* Watchman returns changes as an array of JSON objects with potentially unstable key
* ordering:
* {
* "files": [
* {
* "new": false,
* "exists": true,
* "name": "bin/buckd",
* },
* ]
* }
* A simple way to parse these changes is to collect the relevant values from each object
* in a builder and then build an event when the end of a JSON object is reached. When the end
* of the enclosing JSON object is processed the builder will not contain a complete event, so
* the object end token will be ignored.
*/
int eventCount = 0;
while (token != null) {
boolean shouldOverflow = false;
if (eventCount > overflow) {
LOG.warn(
"Received too many events from Watchmen (%d > overflow max %d), posting overflow "
+ "event and giving up.",
eventCount, overflow);
shouldOverflow = true;
} else {
long elapsedMillis = clock.currentTimeMillis() - parseStartTimeMillis;
if (elapsedMillis >= timeoutMillis) {
LOG.warn(
"Parsing took too long (timeout %d ms), posting overflow event and giving up.",
timeoutMillis);
shouldOverflow = true;
}
}
if (shouldOverflow) {
postWatchEvent(createOverflowEvent());
processExecutor.destroyLaunchedProcess(watchmanProcess);
processExecutor.waitForLaunchedProcess(watchmanProcess);
return;
}
switch (token) {
case FIELD_NAME:
String fieldName = jsonParser.getCurrentName();
switch (fieldName) {
case "is_fresh_instance":
// Force caches to be invalidated --- we have no idea what's happening.
Boolean newInstance = jsonParser.nextBooleanValue();
if (newInstance) {
LOG.info(
"Fresh watchman instance detected. "
+ "Posting overflow event to flush caches.");
postWatchEvent(createOverflowEvent());
}
break;
case "name":
builder.setPath(Paths.get(jsonParser.nextTextValue()));
break;
case "new":
if (jsonParser.nextBooleanValue()) {
builder.setCreationEvent();
}
break;
case "exists":
if (!jsonParser.nextBooleanValue()) {
builder.setDeletionEvent();
}
break;
case "error":
WatchmanWatcherException e =
new WatchmanWatcherException(jsonParser.nextTextValue());
LOG.error(
e, "Error in Watchman output. Posting an overflow event to flush the caches");
postWatchEvent(createOverflowEvent());
throw e;
case "warning":
String message = jsonParser.nextTextValue();
buckEventBus.post(
ConsoleEvent.warning("Watchman has produced a warning: %s", message));
LOG.warn("Watchman has produced a warning: %s", message);
break;
}
break;
case END_OBJECT:
if (builder.canBuild()) {
postWatchEvent(builder.build());
++eventCount;
}
builder = new PathEventBuilder();
break;
// $CASES-OMITTED$
default:
break;
}
token = jsonParser.nextToken();
}
int watchmanExitCode;
LOG.debug("Posted %d Watchman events. Waiting for subprocess to exit...", eventCount);
watchmanExitCode = processExecutor.waitForLaunchedProcess(watchmanProcess);
if (watchmanExitCode != 0) {
LOG.error("Watchman exited with error code %d", watchmanExitCode);
postWatchEvent(createOverflowEvent()); // Events may have been lost, signal overflow.
ByteArrayOutputStream buffer = new ByteArrayOutputStream();
ByteStreams.copy(watchmanProcess.getErrorStream(), buffer);
throw new WatchmanWatcherException(
"Watchman failed with exit code " + watchmanExitCode + ": " + buffer.toString());
} else {
LOG.debug("Watchman exited cleanly.");
}
} catch (InterruptedException e) {
LOG.warn(e, "Killing Watchman process on interrupted exception");
postWatchEvent(createOverflowEvent()); // Events may have been lost, signal overflow.
processExecutor.destroyLaunchedProcess(watchmanProcess);
processExecutor.waitForLaunchedProcess(watchmanProcess);
Thread.currentThread().interrupt();
throw e;
} catch (IOException e) {
LOG.error(e, "Killing Watchman process on I/O exception");
postWatchEvent(createOverflowEvent()); // Events may have been lost, signal overflow.
processExecutor.destroyLaunchedProcess(watchmanProcess);
processExecutor.waitForLaunchedProcess(watchmanProcess);
throw e;
}
}