/** {@inheritDoc} */
@Override
public final void removeX() throws GridException {
if (closed.get()) throw new NoSuchElementException("Iterator has been closed.");
try {
onRemove();
} catch (GridException e) {
if (!closed.get()) throw e;
}
}
/** {@inheritDoc} */
@Override
public final boolean hasNextX() throws GridException {
if (closed.get()) return false;
try {
return onHasNext();
} catch (GridException e) {
if (closed.get()) return false;
else throw e;
}
}
/** {@inheritDoc} */
@Override
public final T nextX() throws GridException {
if (closed.get()) return null;
try {
if (!onHasNext()) throw new NoSuchElementException();
return onNext();
} catch (GridException e) {
if (closed.get()) return null;
else throw e;
}
}
public static void main(String[] args) throws Exception {
final Config config = Config.loadFromDisk("nexus-importer");
final HTTPCache http = HttpClient.createHttpCache(config);
final XmlParser xmlParser = new XmlParser();
final BoneCPDataSource boneCp = config.createBoneCp();
XmlParser.debugXml = false;
ObjectManager<NexusServerDto, ActorRef<NexusServer>> serverManager =
new ObjectManager<>(
"Nexus server",
Collections.<NexusServerDto>emptySet(),
new ObjectFactory<NexusServerDto, ActorRef<NexusServer>>() {
public ActorRef<NexusServer> create(NexusServerDto server) {
final NexusClient client = new NexusClient(http, server.url);
String name = server.name;
return ObjectUtil.threadedActor(
name,
config.nexusUpdateInterval,
boneCp,
"Nexus Server: " + name,
new NexusServer(client, server, xmlParser));
}
});
final AtomicBoolean shouldRun = new AtomicBoolean(true);
config.addShutdownHook(currentThread(), shouldRun);
while (shouldRun.get()) {
try {
List<NexusServerDto> newKeys;
try (Connection c = boneCp.getConnection()) {
newKeys = new NexusDao(c).selectServer();
}
serverManager.update(newKeys);
} catch (SQLException e) {
e.printStackTrace(System.out);
}
synchronized (shouldRun) {
shouldRun.wait(60 * 1000);
}
}
serverManager.close();
}
@Override
protected <T, E> Predicate<T> createPredicate(final EntityCache<T> cache) {
if (column == null && this.nodes == null) return null;
final EntityCache<E> joinCache = this.joinEntity.getCache();
final AtomicBoolean more = new AtomicBoolean();
Predicate<E> filter = createJoinPredicate(more);
Predicate<T> rs = null;
if (filter == null && !more.get()) return rs;
if (filter != null) {
final Predicate<E> inner = filter;
rs =
new Predicate<T>() {
@Override
public boolean test(final T t) {
Predicate<E> joinPredicate = null;
for (String joinColumn : joinColumns) {
final Serializable key = cache.getAttribute(joinColumn).get(t);
final Attribute<E, Serializable> joinAttr = joinCache.getAttribute(joinColumn);
Predicate<E> p = (E e) -> key.equals(joinAttr.get(e));
joinPredicate = joinPredicate == null ? p : joinPredicate.and(p);
}
return joinCache.exists(inner.and(joinPredicate));
}
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
sb.append(" #-- ON ")
.append(joinColumns[0])
.append("=")
.append(joinClass == null ? "null" : joinClass.getSimpleName())
.append(".")
.append(joinColumns[0]);
for (int i = 1; i < joinColumns.length; i++) {
sb.append(" AND ")
.append(joinColumns[i])
.append("=")
.append(joinClass == null ? "null" : joinClass.getSimpleName())
.append(".")
.append(joinColumns[i]);
}
sb.append(" --# ").append(inner.toString());
return sb.toString();
}
};
}
if (more.get()) { // 存在不同Class的关联表
if (this.nodes != null) {
for (FilterNode node : this.nodes) {
if (((FilterJoinNode) node).joinClass == this.joinClass) continue;
Predicate<T> f = node.createPredicate(cache);
if (f == null) continue;
final Predicate<T> one = rs;
final Predicate<T> two = f;
rs =
(rs == null)
? f
: (or
? new Predicate<T>() {
@Override
public boolean test(T t) {
return one.test(t) || two.test(t);
}
@Override
public String toString() {
return "(" + one + " OR " + two + ")";
}
}
: new Predicate<T>() {
@Override
public boolean test(T t) {
return one.test(t) && two.test(t);
}
@Override
public String toString() {
return "(" + one + " AND " + two + ")";
}
});
}
}
}
return rs;
}
@Override
public boolean isWaitingForPairing() {
return waitingForPairing.get();
}
private void runTestUnfinishedBlockCRCError(
final boolean transferToAllowed,
final SyncType syncType,
final int writeSize,
Configuration conf)
throws IOException {
conf.setBoolean("dfs.support.append", syncType == SyncType.APPEND);
conf.setBoolean("dfs.datanode.transferTo.allowed", transferToAllowed);
init(conf);
final Path file = new Path("/block-being-written-to");
final int numWrites = 2000;
final AtomicBoolean writerDone = new AtomicBoolean(false);
final AtomicBoolean writerStarted = new AtomicBoolean(false);
final AtomicBoolean error = new AtomicBoolean(false);
final FSDataOutputStream initialOutputStream = fileSystem.create(file);
final Thread writer =
new Thread(
new Runnable() {
private FSDataOutputStream outputStream = initialOutputStream;
@Override
public void run() {
try {
for (int i = 0; !error.get() && i < numWrites; i++) {
try {
final byte[] writeBuf =
DFSTestUtil.generateSequentialBytes(i * writeSize, writeSize);
outputStream.write(writeBuf);
if (syncType == SyncType.SYNC) {
outputStream.hflush();
} else { // append
outputStream.close();
outputStream = fileSystem.append(file);
}
writerStarted.set(true);
} catch (IOException e) {
error.set(true);
LOG.error("error writing to file", e);
}
}
writerDone.set(true);
outputStream.close();
} catch (Exception e) {
LOG.error("error in writer", e);
throw new RuntimeException(e);
}
}
});
Thread tailer =
new Thread(
new Runnable() {
@Override
public void run() {
try {
long startPos = 0;
while (!writerDone.get() && !error.get()) {
if (writerStarted.get()) {
try {
startPos = tailFile(file, startPos);
} catch (IOException e) {
LOG.error(String.format("error tailing file %s", file), e);
throw new RuntimeException(e);
}
}
}
} catch (RuntimeException e) {
if (e.getCause() instanceof ChecksumException) {
error.set(true);
}
writer.interrupt();
LOG.error("error in tailer", e);
throw e;
}
}
});
writer.start();
tailer.start();
try {
writer.join();
tailer.join();
assertFalse("error occurred, see log above", error.get());
} catch (InterruptedException e) {
LOG.info("interrupted waiting for writer or tailer to complete");
Thread.currentThread().interrupt();
}
initialOutputStream.close();
}
/**
* Grabs local events and detects if events was lost since last poll.
*
* @param ignite Target grid.
* @param evtOrderKey Unique key to take last order key from node local map.
* @param evtThrottleCntrKey Unique key to take throttle count from node local map.
* @param evtTypes Event types to collect.
* @param evtMapper Closure to map grid events to Visor data transfer objects.
* @return Collections of node events
*/
public static Collection<VisorGridEvent> collectEvents(
Ignite ignite,
String evtOrderKey,
String evtThrottleCntrKey,
final int[] evtTypes,
IgniteClosure<Event, VisorGridEvent> evtMapper) {
assert ignite != null;
assert evtTypes != null && evtTypes.length > 0;
ConcurrentMap<String, Long> nl = ignite.cluster().nodeLocalMap();
final long lastOrder = getOrElse(nl, evtOrderKey, -1L);
final long throttle = getOrElse(nl, evtThrottleCntrKey, 0L);
// When we first time arrive onto a node to get its local events,
// we'll grab only last those events that not older than given period to make sure we are
// not grabbing GBs of data accidentally.
final long notOlderThan = System.currentTimeMillis() - EVENTS_COLLECT_TIME_WINDOW;
// Flag for detecting gaps between events.
final AtomicBoolean lastFound = new AtomicBoolean(lastOrder < 0);
IgnitePredicate<Event> p =
new IgnitePredicate<Event>() {
/** */
private static final long serialVersionUID = 0L;
@Override
public boolean apply(Event e) {
// Detects that events were lost.
if (!lastFound.get() && (lastOrder == e.localOrder())) lastFound.set(true);
// Retains events by lastOrder, period and type.
return e.localOrder() > lastOrder
&& e.timestamp() > notOlderThan
&& F.contains(evtTypes, e.type());
}
};
Collection<Event> evts = ignite.events().localQuery(p);
// Update latest order in node local, if not empty.
if (!evts.isEmpty()) {
Event maxEvt = Collections.max(evts, EVTS_ORDER_COMPARATOR);
nl.put(evtOrderKey, maxEvt.localOrder());
}
// Update throttle counter.
if (!lastFound.get())
nl.put(evtThrottleCntrKey, throttle == 0 ? EVENTS_LOST_THROTTLE : throttle - 1);
boolean lost = !lastFound.get() && throttle == 0;
Collection<VisorGridEvent> res = new ArrayList<>(evts.size() + (lost ? 1 : 0));
if (lost) res.add(new VisorGridEventsLost(ignite.cluster().localNode().id()));
for (Event e : evts) {
VisorGridEvent visorEvt = evtMapper.apply(e);
if (visorEvt != null) res.add(visorEvt);
}
return res;
}
/** {@inheritDoc} */
@Override
public boolean isClosed() {
return closed.get();
}
