@Override protected void loadRecentModel(long mostRecentModelGeneration) throws IOException { if (mostRecentModelGeneration <= modelGeneration) { return; } if (modelGeneration == NO_GENERATION) { log.info("Most recent generation {} is the first available one", mostRecentModelGeneration); } else { log.info( "Most recent generation {} is newer than current {}", mostRecentModelGeneration, modelGeneration); } File modelPMMLFile = File.createTempFile("model-", ".pmml.gz"); modelPMMLFile.deleteOnExit(); IOUtils.delete(modelPMMLFile); Config config = ConfigUtils.getDefaultConfig(); String instanceDir = config.getString("model.instance-dir"); String generationPrefix = Namespaces.getInstanceGenerationPrefix(instanceDir, mostRecentModelGeneration); String modelPMMLKey = generationPrefix + "model.pmml.gz"; Store.get().download(modelPMMLKey, modelPMMLFile); log.info("Loading model description from {}", modelPMMLKey); Pair<DecisionForest, Map<Integer, BiMap<String, Integer>>> forestAndCatalog = DecisionForestPMML.read(modelPMMLFile); IOUtils.delete(modelPMMLFile); log.info("Loaded model description"); modelGeneration = mostRecentModelGeneration; currentModel = new Generation(forestAndCatalog.getFirst(), forestAndCatalog.getSecond()); }
@Override protected void runSteps() throws IOException { String instanceDir = getInstanceDir(); int generationID = getGenerationID(); String generationPrefix = Namespaces.getInstanceGenerationPrefix(instanceDir, generationID); int lastGenerationID = getLastGenerationID(); File currentInputDir = Files.createTempDir(); currentInputDir.deleteOnExit(); File tempOutDir = Files.createTempDir(); tempOutDir.deleteOnExit(); try { Store store = Store.get(); store.downloadDirectory(generationPrefix + "inbound/", currentInputDir); if (lastGenerationID >= 0) { store.downloadDirectory( Namespaces.getInstanceGenerationPrefix(instanceDir, lastGenerationID) + "input/", currentInputDir); } List<Example> examples = new ArrayList<>(); Map<Integer, BiMap<String, Integer>> columnToCategoryNameToIDMapping = new HashMap<>(); new ReadInputs(currentInputDir, examples, columnToCategoryNameToIDMapping).call(); DecisionForest forest = DecisionForest.fromExamplesWithDefault(examples); DecisionForestPMML.write( new File(tempOutDir, "model.pmml.gz"), forest, columnToCategoryNameToIDMapping); store.uploadDirectory(generationPrefix + "input/", currentInputDir, false); store.uploadDirectory(generationPrefix, tempOutDir, false); } finally { IOUtils.deleteRecursively(currentInputDir); IOUtils.deleteRecursively(tempOutDir); } }