public void testSomethingWorks() throws Exception {
   Tuple<Path, Environment> env = createEnv(fs, temp);
   Path pluginDir = createPluginDir(temp);
   String pluginZip = createPlugin("fake", pluginDir);
   installPlugin(pluginZip, env.v1());
   assertPlugin("fake", pluginDir, env.v2());
 }
 public void testPlatformBinPermissions() throws Exception {
   assumeTrue("posix filesystem", isPosix);
   Tuple<Path, Environment> env = createEnv(fs, temp);
   Path pluginDir = createPluginDir(temp);
   Path platformDir = pluginDir.resolve("platform");
   Path platformNameDir = platformDir.resolve("linux-x86_64");
   Path platformBinDir = platformNameDir.resolve("bin");
   Files.createDirectories(platformBinDir);
   Path programFile = Files.createFile(platformBinDir.resolve("someprogram"));
   // a file created with Files.createFile() should not have execute permissions
   Set<PosixFilePermission> sourcePerms = Files.getPosixFilePermissions(programFile);
   assertFalse(sourcePerms.contains(PosixFilePermission.OWNER_EXECUTE));
   assertFalse(sourcePerms.contains(PosixFilePermission.GROUP_EXECUTE));
   assertFalse(sourcePerms.contains(PosixFilePermission.OTHERS_EXECUTE));
   String pluginZip = createPlugin("fake", pluginDir);
   installPlugin(pluginZip, env.v1());
   assertPlugin("fake", pluginDir, env.v2());
   // check that the installed program has execute permissions, even though the one added to the
   // plugin didn't
   Path installedPlatformBinDir =
       env.v2()
           .pluginsFile()
           .resolve("fake")
           .resolve("platform")
           .resolve("linux-x86_64")
           .resolve("bin");
   assertTrue(Files.isDirectory(installedPlatformBinDir));
   Path installedProgramFile = installedPlatformBinDir.resolve("someprogram");
   assertTrue(Files.isRegularFile(installedProgramFile));
   Set<PosixFilePermission> installedPerms = Files.getPosixFilePermissions(installedProgramFile);
   assertTrue(installedPerms.contains(PosixFilePermission.OWNER_EXECUTE));
   assertTrue(installedPerms.contains(PosixFilePermission.GROUP_EXECUTE));
   assertTrue(installedPerms.contains(PosixFilePermission.OTHERS_EXECUTE));
 }
Ejemplo n.º 3
0
      @Override
      public Query apply(Function input, Context context) throws IOException {
        Tuple<Reference, Literal> prepare = prepare(input);
        if (prepare == null) {
          return null;
        }
        String fieldName = prepare.v1().info().ident().columnIdent().fqn();
        Object value = prepare.v2().value();

        // FIXME: nobody knows how Strings can arrive here
        if (value instanceof String) {
          if (isPcrePattern(value)) {
            return new RegexQuery(new Term(fieldName, (String) value));
          } else {
            return toLuceneRegexpQuery(fieldName, BytesRefs.toBytesRef(value), context);
          }
        }

        if (value instanceof BytesRef) {
          if (isPcrePattern(value)) {
            return new RegexQuery(new Term(fieldName, (BytesRef) value));
          } else {
            return toLuceneRegexpQuery(fieldName, (BytesRef) value, context);
          }
        }

        throw new IllegalArgumentException("Can only use ~ with patterns of type string");
      }
Ejemplo n.º 4
0
 public Query toQuery(Reference reference, DataType type, Object value) {
   String columnName = reference.info().ident().columnIdent().fqn();
   QueryBuilderHelper builder = QueryBuilderHelper.forType(type);
   Tuple<?, ?> bounds = boundsFunction.apply(value);
   assert bounds != null;
   return builder.rangeQuery(columnName, bounds.v1(), bounds.v2(), includeLower, includeUpper);
 }
Ejemplo n.º 5
0
 @Override
 public void collect(int doc) throws IOException {
   BytesWrap parentId = typeCache.parentIdByDoc(doc);
   if (parentId == null) {
     return;
   }
   for (Tuple<IndexReader, IdReaderTypeCache> tuple : readers) {
     IndexReader indexReader = tuple.v1();
     IdReaderTypeCache idReaderTypeCache = tuple.v2();
     if (idReaderTypeCache
         == null) { // might be if we don't have that doc with that type in this reader
       continue;
     }
     int parentDocId = idReaderTypeCache.docById(parentId);
     if (parentDocId != -1 && !indexReader.isDeleted(parentDocId)) {
       OpenBitSet docIdSet = parentDocs().get(indexReader.getCoreCacheKey());
       if (docIdSet == null) {
         docIdSet = new OpenBitSet(indexReader.maxDoc());
         parentDocs.put(indexReader.getCoreCacheKey(), docIdSet);
       }
       docIdSet.fastSet(parentDocId);
       return;
     }
   }
 }
Ejemplo n.º 6
0
 @Override
 public Query apply(Function input, Context context) {
   Tuple<Reference, Literal> tuple = prepare(input);
   if (tuple == null) {
     return null;
   }
   return toQuery(tuple.v1(), tuple.v2().value(), context);
 }
 public void testBuiltinModule() throws Exception {
   Tuple<Path, Environment> env = createEnv(fs, temp);
   Path pluginDir = createPluginDir(temp);
   String pluginZip = createPlugin("lang-painless", pluginDir);
   UserException e = expectThrows(UserException.class, () -> installPlugin(pluginZip, env.v1()));
   assertTrue(e.getMessage(), e.getMessage().contains("is a system module"));
   assertInstallCleaned(env.v2());
 }
 private void clearSeenMappings(String index) {
   // clear seen mappings as well
   for (Tuple<String, String> tuple : seenMappings.keySet()) {
     if (tuple.v1().equals(index)) {
       seenMappings.remove(tuple);
     }
   }
 }
 public void testPluginsDirMissing() throws Exception {
   Tuple<Path, Environment> env = createEnv(fs, temp);
   Files.delete(env.v2().pluginsFile());
   Path pluginDir = createPluginDir(temp);
   String pluginZip = createPlugin("fake", pluginDir);
   installPlugin(pluginZip, env.v1());
   assertPlugin("fake", pluginDir, env.v2());
 }
Ejemplo n.º 10
0
 @Override
 public Query apply(Function input, Context context) throws IOException {
   Tuple<Reference, Literal> tuple = super.prepare(input);
   if (tuple == null) {
     return null;
   }
   return toQuery(tuple.v1(), tuple.v1().valueType(), tuple.v2().value());
 }
Ejemplo n.º 11
0
    @Override
    public List<Query> buildGroupedQueries(
        MultiMatchQueryBuilder.Type type,
        Map<String, Float> fieldNames,
        Object value,
        String minimumShouldMatch)
        throws IOException {
      Map<Analyzer, List<FieldAndFieldType>> groups = new HashMap<>();
      List<Tuple<String, Float>> missing = new ArrayList<>();
      for (Map.Entry<String, Float> entry : fieldNames.entrySet()) {
        String name = entry.getKey();
        MappedFieldType fieldType = context.fieldMapper(name);
        if (fieldType != null) {
          Analyzer actualAnalyzer = getAnalyzer(fieldType);
          name = fieldType.name();
          if (!groups.containsKey(actualAnalyzer)) {
            groups.put(actualAnalyzer, new ArrayList<>());
          }
          Float boost = entry.getValue();
          boost = boost == null ? Float.valueOf(1.0f) : boost;
          groups.get(actualAnalyzer).add(new FieldAndFieldType(name, fieldType, boost));
        } else {
          missing.add(new Tuple<>(name, entry.getValue()));
        }
      }
      List<Query> queries = new ArrayList<>();
      for (Tuple<String, Float> tuple : missing) {
        Query q =
            parseGroup(type.matchQueryType(), tuple.v1(), tuple.v2(), value, minimumShouldMatch);
        if (q != null) {
          queries.add(q);
        }
      }
      for (List<FieldAndFieldType> group : groups.values()) {
        if (group.size() > 1) {
          blendedFields = new FieldAndFieldType[group.size()];
          int i = 0;
          for (FieldAndFieldType fieldAndFieldType : group) {
            blendedFields[i++] = fieldAndFieldType;
          }
        } else {
          blendedFields = null;
        }
        /*
         * We have to pick some field to pass through the superclass so
         * we just pick the first field. It shouldn't matter because
         * fields are already grouped by their analyzers/types.
         */
        String representativeField = group.get(0).field;
        Query q =
            parseGroup(type.matchQueryType(), representativeField, 1f, value, minimumShouldMatch);
        if (q != null) {
          queries.add(q);
        }
      }

      return queries.isEmpty() ? null : queries;
    }
 public void testExistingPlugin() throws Exception {
   Tuple<Path, Environment> env = createEnv(fs, temp);
   Path pluginDir = createPluginDir(temp);
   String pluginZip = createPlugin("fake", pluginDir);
   installPlugin(pluginZip, env.v1());
   UserException e = expectThrows(UserException.class, () -> installPlugin(pluginZip, env.v1()));
   assertTrue(e.getMessage(), e.getMessage().contains("already exists"));
   assertInstallCleaned(env.v2());
 }
 public void testMissingDescriptor() throws Exception {
   Tuple<Path, Environment> env = createEnv(fs, temp);
   Path pluginDir = createPluginDir(temp);
   Files.createFile(pluginDir.resolve("fake.yml"));
   String pluginZip = writeZip(pluginDir, "elasticsearch");
   NoSuchFileException e =
       expectThrows(NoSuchFileException.class, () -> installPlugin(pluginZip, env.v1()));
   assertTrue(e.getMessage(), e.getMessage().contains("plugin-descriptor.properties"));
   assertInstallCleaned(env.v2());
 }
 public void testConfigNotDir() throws Exception {
   Tuple<Path, Environment> env = createEnv(fs, temp);
   Path pluginDir = createPluginDir(temp);
   Path configDir = pluginDir.resolve("config");
   Files.createFile(configDir);
   String pluginZip = createPlugin("fake", pluginDir);
   UserException e = expectThrows(UserException.class, () -> installPlugin(pluginZip, env.v1()));
   assertTrue(e.getMessage(), e.getMessage().contains("not a directory"));
   assertInstallCleaned(env.v2());
 }
 public void testConfig() throws Exception {
   Tuple<Path, Environment> env = createEnv(fs, temp);
   Path pluginDir = createPluginDir(temp);
   Path configDir = pluginDir.resolve("config");
   Files.createDirectory(configDir);
   Files.createFile(configDir.resolve("custom.yaml"));
   String pluginZip = createPlugin("fake", pluginDir);
   installPlugin(pluginZip, env.v1());
   assertPlugin("fake", pluginDir, env.v2());
 }
 public void testZipRelativeOutsideEntryName() throws Exception {
   Tuple<Path, Environment> env = createEnv(fs, temp);
   Path zip = createTempDir().resolve("broken.zip");
   try (ZipOutputStream stream = new ZipOutputStream(Files.newOutputStream(zip))) {
     stream.putNextEntry(new ZipEntry("elasticsearch/../blah"));
   }
   String pluginZip = zip.toUri().toURL().toString();
   IOException e = expectThrows(IOException.class, () -> installPlugin(pluginZip, env.v1()));
   assertTrue(e.getMessage(), e.getMessage().contains("resolving outside of plugin directory"));
 }
 public void remove(ClusterStateListener listener) {
   clusterStateListeners.remove(listener);
   for (Iterator<Tuple<Timeout, NotifyTimeout>> it = onGoingTimeouts.iterator(); it.hasNext(); ) {
     Tuple<Timeout, NotifyTimeout> tuple = it.next();
     if (tuple.v2().listener.equals(listener)) {
       tuple.v1().cancel();
       it.remove();
     }
   }
 }
 public void testBin() throws Exception {
   Tuple<Path, Environment> env = createEnv(fs, temp);
   Path pluginDir = createPluginDir(temp);
   Path binDir = pluginDir.resolve("bin");
   Files.createDirectory(binDir);
   Files.createFile(binDir.resolve("somescript"));
   String pluginZip = createPlugin("fake", pluginDir);
   installPlugin(pluginZip, env.v1());
   assertPlugin("fake", pluginDir, env.v2());
 }
 public void testSpaceInUrl() throws Exception {
   Tuple<Path, Environment> env = createEnv(fs, temp);
   Path pluginDir = createPluginDir(temp);
   String pluginZip = createPlugin("fake", pluginDir);
   Path pluginZipWithSpaces = createTempFile("foo bar", ".zip");
   try (InputStream in = new URL(pluginZip).openStream()) {
     Files.copy(in, pluginZipWithSpaces, StandardCopyOption.REPLACE_EXISTING);
   }
   installPlugin(pluginZipWithSpaces.toUri().toURL().toString(), env.v1());
   assertPlugin("fake", pluginDir, env.v2());
 }
 public void testMissingDirectory() throws Exception {
   Tuple<Path, Environment> env = createEnv(fs, temp);
   Path pluginDir = createPluginDir(temp);
   Files.createFile(pluginDir.resolve(PluginInfo.ES_PLUGIN_PROPERTIES));
   String pluginZip = writeZip(pluginDir, null);
   UserException e = expectThrows(UserException.class, () -> installPlugin(pluginZip, env.v1()));
   assertTrue(
       e.getMessage(),
       e.getMessage().contains("`elasticsearch` directory is missing in the plugin zip"));
   assertInstallCleaned(env.v2());
 }
Ejemplo n.º 21
0
 /**
  * We build a plugin manager instance which wait only for 30 seconds before raising an
  * ElasticsearchTimeoutException
  */
 private static PluginManager pluginManager(
     String pluginUrl, Tuple<Settings, Environment> initialSettings) throws IOException {
   if (!Files.exists(initialSettings.v2().pluginsFile())) {
     Files.createDirectories(initialSettings.v2().pluginsFile());
   }
   return new PluginManager(
       initialSettings.v2(),
       pluginUrl,
       PluginManager.OutputMode.SILENT,
       TimeValue.timeValueSeconds(30));
 }
Ejemplo n.º 22
0
 private void registerStaticColumns() {
   for (Tuple<String, DataType> column : staticColumns) {
     ReferenceInfo info =
         new ReferenceInfo(
             new ReferenceIdent(ident(), column.v1(), null), RowGranularity.DOC, column.v2());
     if (info.ident().isColumn()) {
       columns.add(info);
     }
     INFOS.put(info.ident().columnIdent(), info);
   }
 }
 @Override
 public int compare(Tuple<Text, Integer> o1, Tuple<Text, Integer> o2) {
   int cmp = o2.v2() - o1.v2();
   if (cmp != 0) {
     return cmp;
   }
   cmp = o2.v1().compareTo(o1.v1());
   if (cmp != 0) {
     return cmp;
   }
   return System.identityHashCode(o2) - System.identityHashCode(o1);
 }
 public void testIsolatedPlugins() throws Exception {
   Tuple<Path, Environment> env = createEnv(fs, temp);
   // these both share the same FakePlugin class
   Path pluginDir1 = createPluginDir(temp);
   String pluginZip1 = createPlugin("fake1", pluginDir1);
   installPlugin(pluginZip1, env.v1());
   Path pluginDir2 = createPluginDir(temp);
   String pluginZip2 = createPlugin("fake2", pluginDir2);
   installPlugin(pluginZip2, env.v1());
   assertPlugin("fake1", pluginDir1, env.v2());
   assertPlugin("fake2", pluginDir2, env.v2());
 }
 public void testBinContainsDir() throws Exception {
   Tuple<Path, Environment> env = createEnv(fs, temp);
   Path pluginDir = createPluginDir(temp);
   Path dirInBinDir = pluginDir.resolve("bin").resolve("foo");
   Files.createDirectories(dirInBinDir);
   Files.createFile(dirInBinDir.resolve("somescript"));
   String pluginZip = createPlugin("fake", pluginDir);
   UserException e = expectThrows(UserException.class, () -> installPlugin(pluginZip, env.v1()));
   assertTrue(
       e.getMessage(), e.getMessage().contains("Directories not allowed in bin dir for plugin"));
   assertInstallCleaned(env.v2());
 }
 public void testPluginsDirReadOnly() throws Exception {
   assumeTrue("posix and filesystem", isPosix && isReal);
   Tuple<Path, Environment> env = createEnv(fs, temp);
   Path pluginDir = createPluginDir(temp);
   try (PosixPermissionsResetter pluginsAttrs =
       new PosixPermissionsResetter(env.v2().pluginsFile())) {
     pluginsAttrs.setPermissions(new HashSet<>());
     String pluginZip = createPlugin("fake", pluginDir);
     IOException e = expectThrows(IOException.class, () -> installPlugin(pluginZip, env.v1()));
     assertTrue(e.getMessage(), e.getMessage().contains(env.v2().pluginsFile().toString()));
   }
   assertInstallCleaned(env.v2());
 }
 public void testJarHell() throws Exception {
   // jar hell test needs a real filesystem
   assumeTrue("real filesystem", isReal);
   Tuple<Path, Environment> environment = createEnv(fs, temp);
   Path pluginDirectory = createPluginDir(temp);
   writeJar(pluginDirectory.resolve("other.jar"), "FakePlugin");
   String pluginZip = createPlugin("fake", pluginDirectory); // adds plugin.jar with FakePlugin
   IllegalStateException e =
       expectThrows(
           IllegalStateException.class, () -> installPlugin(pluginZip, environment.v1(), true));
   assertTrue(e.getMessage(), e.getMessage().contains("jar hell"));
   assertInstallCleaned(environment.v2());
 }
  private ParsedDocument parseDocument(String index, String type, BytesReference doc) {
    MapperService mapperService = indexShard.mapperService();
    IndexService indexService = indexShard.indexService();

    // TODO: make parsing not dynamically create fields not in the original mapping
    Tuple<DocumentMapper, Boolean> docMapper = mapperService.documentMapperWithAutoCreate(type);
    ParsedDocument parsedDocument =
        docMapper.v1().parse(source(doc).type(type).flyweight(true)).setMappingsModified(docMapper);
    if (parsedDocument.mappingsModified()) {
      mappingUpdatedAction.updateMappingOnMaster(index, docMapper.v1(), indexService.indexUUID());
    }
    return parsedDocument;
  }
  private void installPlugin(MockTerminal terminal, boolean isBatch) throws Exception {
    Tuple<Path, Environment> env = createEnv(fs, temp);
    Path pluginDir = createPluginDir(temp);
    // if batch is enabled, we also want to add a security policy
    String pluginZip = createPlugin("fake", pluginDir, isBatch);

    Map<String, String> settings = new HashMap<>();
    settings.put("path.home", env.v1().toString());
    new InstallPluginCommand() {
      @Override
      void jarHellCheck(Path candidate, Path pluginsDir) throws Exception {}
    }.execute(terminal, pluginZip, isBatch, settings);
  }
  public void testIndexWithShadowReplicasCleansUp() throws Exception {
    Path dataPath = createTempDir();
    Settings nodeSettings = nodeSettings(dataPath);

    final int nodeCount = randomIntBetween(2, 5);
    logger.info("--> starting {} nodes", nodeCount);
    final List<String> nodes = internalCluster().startNodesAsync(nodeCount, nodeSettings).get();
    final String IDX = "test";
    final Tuple<Integer, Integer> numPrimariesAndReplicas = randomPrimariesAndReplicas(nodeCount);
    final int numPrimaries = numPrimariesAndReplicas.v1();
    final int numReplicas = numPrimariesAndReplicas.v2();
    logger.info(
        "--> creating index {} with {} primary shards and {} replicas",
        IDX,
        numPrimaries,
        numReplicas);

    Settings idxSettings =
        Settings.builder()
            .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, numPrimaries)
            .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, numReplicas)
            .put(IndexMetaData.SETTING_DATA_PATH, dataPath.toAbsolutePath().toString())
            .put(IndexMetaData.SETTING_SHADOW_REPLICAS, true)
            .put(IndexMetaData.SETTING_SHARED_FILESYSTEM, true)
            .build();

    prepareCreate(IDX).setSettings(idxSettings).addMapping("doc", "foo", "type=text").get();
    ensureGreen(IDX);

    client().prepareIndex(IDX, "doc", "1").setSource("foo", "bar").get();
    client().prepareIndex(IDX, "doc", "2").setSource("foo", "bar").get();
    flushAndRefresh(IDX);

    GetResponse gResp1 = client().prepareGet(IDX, "doc", "1").get();
    GetResponse gResp2 = client().prepareGet(IDX, "doc", "2").get();
    assertThat(gResp1.getSource().get("foo"), equalTo("bar"));
    assertThat(gResp2.getSource().get("foo"), equalTo("bar"));

    logger.info("--> performing query");
    SearchResponse resp = client().prepareSearch(IDX).setQuery(matchAllQuery()).get();
    assertHitCount(resp, 2);

    logger.info("--> deleting index " + IDX);
    assertAcked(client().admin().indices().prepareDelete(IDX));
    assertAllIndicesRemovedAndDeletionCompleted(
        internalCluster().getInstances(IndicesService.class));
    assertPathHasBeenCleared(dataPath);
    // TODO: uncomment the test below when https://github.com/elastic/elasticsearch/issues/17695 is
    // resolved.
    // assertIndicesDirsDeleted(nodes);
  }