@Override
public void configure(Map<String, Object> params) throws ConfigurationException {
super.configure(params);
// Set the domr scripts directory
params.put("domr.scripts.dir", "scripts/network/domr/kvm");
String networkScriptsDir = (String) params.get("network.scripts.dir");
if (networkScriptsDir == null) {
networkScriptsDir = "scripts/vm/network/vnet";
}
bridgeNameSchema = (String) params.get("network.bridge.name.schema");
String value = (String) params.get("scripts.timeout");
_timeout = NumbersUtil.parseInt(value, 30 * 60) * 1000;
_modifyVlanPath = Script.findScript(networkScriptsDir, "modifyvlan.sh");
if (_modifyVlanPath == null) {
throw new ConfigurationException("Unable to find modifyvlan.sh");
}
_modifyVxlanPath = Script.findScript(networkScriptsDir, "modifyvxlan.sh");
if (_modifyVxlanPath == null) {
throw new ConfigurationException("Unable to find modifyvxlan.sh");
}
libvirtVersion = (Long) params.get("libvirtVersion");
if (libvirtVersion == null) {
libvirtVersion = 0L;
}
try {
createControlNetwork();
} catch (LibvirtException e) {
throw new ConfigurationException(e.getMessage());
}
}
@Override
protected void runInContext() {
synchronized (_storagePool) {
Set<String> removedPools = new HashSet<String>();
for (String uuid : _storagePool.keySet()) {
NfsStoragePool primaryStoragePool = _storagePool.get(uuid);
// check for any that have been deregistered with libvirt and
// skip,remove them
StoragePool storage = null;
try {
Connect conn = LibvirtConnection.getConnection();
storage = conn.storagePoolLookupByUUIDString(uuid);
if (storage == null) {
s_logger.debug("Libvirt storage pool " + uuid + " not found, removing from HA list");
removedPools.add(uuid);
continue;
} else if (storage.getInfo().state != StoragePoolState.VIR_STORAGE_POOL_RUNNING) {
s_logger.debug(
"Libvirt storage pool "
+ uuid
+ " found, but not running, removing from HA list");
removedPools.add(uuid);
continue;
}
s_logger.debug("Found NFS storage pool " + uuid + " in libvirt, continuing");
} catch (LibvirtException e) {
s_logger.debug("Failed to lookup libvirt storage pool " + uuid + " due to: " + e);
// we only want to remove pool if it's not found, not if libvirt
// connection fails
if (e.toString().contains("pool not found")) {
s_logger.debug("removing pool from HA monitor since it was deleted");
removedPools.add(uuid);
continue;
}
}
String result = null;
for (int i = 0; i < 5; i++) {
Script cmd = new Script(s_heartBeatPath, _heartBeatUpdateTimeout, s_logger);
cmd.add("-i", primaryStoragePool._poolIp);
cmd.add("-p", primaryStoragePool._poolMountSourcePath);
cmd.add("-m", primaryStoragePool._mountDestPath);
cmd.add("-h", _hostIP);
result = cmd.execute();
if (result != null) {
s_logger.warn("write heartbeat failed: " + result + ", retry: " + i);
} else {
break;
}
}
if (result != null) {
s_logger.warn("write heartbeat failed: " + result + "; reboot the host");
Script cmd = new Script(s_heartBeatPath, _heartBeatUpdateTimeout, s_logger);
cmd.add("-i", primaryStoragePool._poolIp);
cmd.add("-p", primaryStoragePool._poolMountSourcePath);
cmd.add("-m", primaryStoragePool._mountDestPath);
cmd.add("-c");
result = cmd.execute();
}
}
if (!removedPools.isEmpty()) {
for (String uuid : removedPools) {
removeStoragePool(uuid);
}
}
}
}
@Override
public Answer execute(
final BackupSnapshotCommand command,
final LibvirtComputingResource libvirtComputingResource) {
final Long dcId = command.getDataCenterId();
final Long accountId = command.getAccountId();
final Long volumeId = command.getVolumeId();
final String secondaryStoragePoolUrl = command.getSecondaryStorageUrl();
final String snapshotName = command.getSnapshotName();
String snapshotDestPath = null;
String snapshotRelPath = null;
final String vmName = command.getVmName();
KVMStoragePool secondaryStoragePool = null;
final KVMStoragePoolManager storagePoolMgr = libvirtComputingResource.getStoragePoolMgr();
try {
final LibvirtUtilitiesHelper libvirtUtilitiesHelper =
libvirtComputingResource.getLibvirtUtilitiesHelper();
final Connect conn = libvirtUtilitiesHelper.getConnectionByVmName(vmName);
secondaryStoragePool = storagePoolMgr.getStoragePoolByURI(secondaryStoragePoolUrl);
final String ssPmountPath = secondaryStoragePool.getLocalPath();
snapshotRelPath =
File.separator
+ "snapshots"
+ File.separator
+ dcId
+ File.separator
+ accountId
+ File.separator
+ volumeId;
snapshotDestPath =
ssPmountPath
+ File.separator
+ "snapshots"
+ File.separator
+ dcId
+ File.separator
+ accountId
+ File.separator
+ volumeId;
final KVMStoragePool primaryPool =
storagePoolMgr.getStoragePool(
command.getPool().getType(), command.getPrimaryStoragePoolNameLabel());
final KVMPhysicalDisk snapshotDisk = primaryPool.getPhysicalDisk(command.getVolumePath());
final String manageSnapshotPath = libvirtComputingResource.manageSnapshotPath();
final int cmdsTimeout = libvirtComputingResource.getCmdsTimeout();
/**
* RBD snapshots can't be copied using qemu-img, so we have to use the Java bindings for
* librbd here.
*
* <p>These bindings will read the snapshot and write the contents to the secondary storage
* directly
*
* <p>It will stop doing so if the amount of time spend is longer then cmds.timeout
*/
if (primaryPool.getType() == StoragePoolType.RBD) {
try {
final Rados r = new Rados(primaryPool.getAuthUserName());
r.confSet("mon_host", primaryPool.getSourceHost() + ":" + primaryPool.getSourcePort());
r.confSet("key", primaryPool.getAuthSecret());
r.confSet("client_mount_timeout", "30");
r.connect();
s_logger.debug("Succesfully connected to Ceph cluster at " + r.confGet("mon_host"));
final IoCTX io = r.ioCtxCreate(primaryPool.getSourceDir());
final Rbd rbd = new Rbd(io);
final RbdImage image = rbd.open(snapshotDisk.getName(), snapshotName);
final File fh = new File(snapshotDestPath);
try (BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(fh)); ) {
final int chunkSize = 4194304;
long offset = 0;
s_logger.debug(
"Backuping up RBD snapshot " + snapshotName + " to " + snapshotDestPath);
while (true) {
final byte[] buf = new byte[chunkSize];
final int bytes = image.read(offset, buf, chunkSize);
if (bytes <= 0) {
break;
}
bos.write(buf, 0, bytes);
offset += bytes;
}
s_logger.debug(
"Completed backing up RBD snapshot "
+ snapshotName
+ " to "
+ snapshotDestPath
+ ". Bytes written: "
+ offset);
} catch (final IOException ex) {
s_logger.error("BackupSnapshotAnswer:Exception:" + ex.getMessage());
}
r.ioCtxDestroy(io);
} catch (final RadosException e) {
s_logger.error("A RADOS operation failed. The error was: " + e.getMessage());
return new BackupSnapshotAnswer(command, false, e.toString(), null, true);
} catch (final RbdException e) {
s_logger.error(
"A RBD operation on "
+ snapshotDisk.getName()
+ " failed. The error was: "
+ e.getMessage());
return new BackupSnapshotAnswer(command, false, e.toString(), null, true);
}
} else {
final Script scriptCommand = new Script(manageSnapshotPath, cmdsTimeout, s_logger);
scriptCommand.add("-b", snapshotDisk.getPath());
scriptCommand.add("-n", snapshotName);
scriptCommand.add("-p", snapshotDestPath);
scriptCommand.add("-t", snapshotName);
final String result = scriptCommand.execute();
if (result != null) {
s_logger.debug("Failed to backup snaptshot: " + result);
return new BackupSnapshotAnswer(command, false, result, null, true);
}
}
/* Delete the snapshot on primary */
DomainState state = null;
Domain vm = null;
if (vmName != null) {
try {
vm = libvirtComputingResource.getDomain(conn, command.getVmName());
state = vm.getInfo().state;
} catch (final LibvirtException e) {
s_logger.trace("Ignoring libvirt error.", e);
}
}
final KVMStoragePool primaryStorage =
storagePoolMgr.getStoragePool(command.getPool().getType(), command.getPool().getUuid());
if (state == DomainState.VIR_DOMAIN_RUNNING && !primaryStorage.isExternalSnapshot()) {
final MessageFormat snapshotXML =
new MessageFormat(
" <domainsnapshot>"
+ " <name>{0}</name>"
+ " <domain>"
+ " <uuid>{1}</uuid>"
+ " </domain>"
+ " </domainsnapshot>");
final String vmUuid = vm.getUUIDString();
final Object[] args = new Object[] {snapshotName, vmUuid};
final String snapshot = snapshotXML.format(args);
s_logger.debug(snapshot);
final DomainSnapshot snap = vm.snapshotLookupByName(snapshotName);
if (snap != null) {
snap.delete(0);
} else {
throw new CloudRuntimeException("Unable to find vm snapshot with name -" + snapshotName);
}
/*
* libvirt on RHEL6 doesn't handle resume event emitted from
* qemu
*/
vm = libvirtComputingResource.getDomain(conn, command.getVmName());
state = vm.getInfo().state;
if (state == DomainState.VIR_DOMAIN_PAUSED) {
vm.resume();
}
} else {
final Script scriptCommand = new Script(manageSnapshotPath, cmdsTimeout, s_logger);
scriptCommand.add("-d", snapshotDisk.getPath());
scriptCommand.add("-n", snapshotName);
final String result = scriptCommand.execute();
if (result != null) {
s_logger.debug("Failed to backup snapshot: " + result);
return new BackupSnapshotAnswer(
command, false, "Failed to backup snapshot: " + result, null, true);
}
}
} catch (final LibvirtException e) {
return new BackupSnapshotAnswer(command, false, e.toString(), null, true);
} catch (final CloudRuntimeException e) {
return new BackupSnapshotAnswer(command, false, e.toString(), null, true);
} finally {
if (secondaryStoragePool != null) {
storagePoolMgr.deleteStoragePool(
secondaryStoragePool.getType(), secondaryStoragePool.getUuid());
}
}
return new BackupSnapshotAnswer(
command, true, null, snapshotRelPath + File.separator + snapshotName, true);
}