public boolean containsWord(final String word) {
if (status != STATUS.OPEN) {
return false;
}
final String testWord = normalizeWord(word);
if (testWord == null) {
return false;
}
// final long startTime = System.currentTimeMillis();
boolean result = false;
try {
final String hashedWord = hashWord(testWord);
final boolean inDB = localDB.contains(WORDS_DB, hashedWord);
if (inDB) {
final long timeStamp = Long.parseLong(localDB.get(WORDS_DB, hashedWord));
final long entryAge = System.currentTimeMillis() - timeStamp;
if (entryAge < settings.maxAgeMs) {
result = true;
}
}
} catch (Exception e) {
LOGGER.warn("error checking global history list: " + e.getMessage());
}
// LOGGER.trace(pwmSession, "successfully checked word, result=" + result + ", duration=" + new
// TimeDuration(System.currentTimeMillis(), startTime).asCompactString());
return result;
}
public synchronized void addWord(final PwmSession pwmSession, final String word) {
if (status != STATUS.OPEN) {
return;
}
final String addWord = normalizeWord(word);
if (addWord == null) {
return;
}
final long startTime = System.currentTimeMillis();
try {
final String hashedWord = hashWord(addWord);
final boolean preExisting = localDB.contains(WORDS_DB, hashedWord);
localDB.put(WORDS_DB, hashedWord, Long.toString(System.currentTimeMillis()));
{
final StringBuilder logOutput = new StringBuilder();
logOutput.append(preExisting ? "updated" : "added").append(" word");
logOutput
.append(" (")
.append(new TimeDuration(System.currentTimeMillis(), startTime).asCompactString())
.append(")");
logOutput.append(" (").append(this.size()).append(" total words)");
LOGGER.trace(logOutput.toString());
}
} catch (Exception e) {
LOGGER.warn(pwmSession, "error adding word to global history list: " + e.getMessage());
}
}
private boolean checkDbVersion() throws Exception {
LOGGER.trace("checking version number stored in LocalDB");
final Object versionInDB = localDB.get(META_DB, KEY_VERSION);
final String currentVersion = "version=" + settings.version;
final boolean result = currentVersion.equals(versionInDB);
if (!result) {
LOGGER.info(
"existing db version does not match current db version db=("
+ versionInDB
+ ") current=("
+ currentVersion
+ "), clearing db");
localDB.truncate(WORDS_DB);
localDB.put(META_DB, KEY_VERSION, currentVersion);
localDB.remove(META_DB, KEY_OLDEST_ENTRY);
} else {
LOGGER.trace(
"existing db version matches current db version db=("
+ versionInDB
+ ") current=("
+ currentVersion
+ ")");
}
return result;
}
@Override
protected void setUp() throws Exception {
super.setUp(); // To change body of overridden methods use File | Settings | File Templates.
TestHelper.setupLogging();
final File fileLocation = new File(TestHelper.getParameter("pwmDBlocation"));
pwmDB = LocalDBFactory.getInstance(fileLocation, false, null, null);
pwmDB.truncate(TEST_DB);
Assert.assertEquals(0, pwmDB.size(TEST_DB));
}
private UserCacheRecord read(StorageKey key) throws LocalDBException {
final String jsonValue = localDB.get(DB, key.getKey());
if (jsonValue != null && !jsonValue.isEmpty()) {
try {
return JsonUtil.deserialize(jsonValue, UserCacheRecord.class);
} catch (JsonSyntaxException e) {
LOGGER.error(
"error reading record from cache store for key="
+ key.getKey()
+ ", error: "
+ e.getMessage());
localDB.remove(DB, key.getKey());
}
}
return null;
}
public void testSize() throws LocalDBException {
final long startTime = System.currentTimeMillis();
for (final LocalDB.DB loopDB : LocalDB.DB.values()) {
final int size = pwmDB.size(loopDB);
System.out.println(loopDB + " size=" + size);
}
System.out.println("total duration: " + TimeDuration.fromCurrent(startTime).asLongString());
}
@Override
protected void tearDown() throws Exception {
super.tearDown();
if (pwmDB != null) {
pwmDB.close();
pwmDB = null;
}
}
public void shutdown() {
LOGGER.warn("shutting down");
{
// send system audit event
final SystemAuditRecord auditRecord =
SystemAuditRecord.create(AuditEvent.SHUTDOWN, null, getInstanceID());
try {
getAuditManager().submit(auditRecord);
} catch (PwmException e) {
LOGGER.warn("unable to submit alert event " + JsonUtil.serialize(auditRecord));
}
}
{
final List<Class> reverseServiceList = new ArrayList<Class>(PWM_SERVICE_CLASSES);
Collections.reverse(reverseServiceList);
for (final Class serviceClass : reverseServiceList) {
if (pwmServices.containsKey(serviceClass)) {
LOGGER.trace("closing service " + serviceClass.getName());
final PwmService loopService = pwmServices.get(serviceClass);
LOGGER.trace("successfully closed service " + serviceClass.getName());
try {
loopService.close();
} catch (Exception e) {
LOGGER.error(
"error closing " + loopService.getClass().getSimpleName() + ": " + e.getMessage(),
e);
}
}
}
}
if (localDBLogger != null) {
try {
localDBLogger.close();
} catch (Exception e) {
LOGGER.error("error closing localDBLogger: " + e.getMessage(), e);
}
localDBLogger = null;
}
if (localDB != null) {
try {
localDB.close();
} catch (Exception e) {
LOGGER.fatal("error closing localDB: " + e, e);
}
localDB = null;
}
LOGGER.info(
PwmConstants.PWM_APP_NAME
+ " "
+ PwmConstants.SERVLET_VERSION
+ " closed for bidness, cya!");
}
private void writeDbValues() {
if (localDB != null) {
try {
localDB.put(LocalDB.DB.PWM_STATS, DB_KEY_CUMULATIVE, statsCummulative.output());
localDB.put(LocalDB.DB.PWM_STATS, currentDailyKey.toString(), statsDaily.output());
for (final Statistic.EpsType loopEpsType : Statistic.EpsType.values()) {
for (final Statistic.EpsDuration loopEpsDuration : Statistic.EpsDuration.values()) {
final String key = "EPS-" + loopEpsType.toString();
final String mapKey = loopEpsType.toString() + loopEpsDuration.toString();
final String value = JsonUtil.serialize(this.epsMeterMap.get(mapKey));
localDB.put(LocalDB.DB.PWM_STATS, key, value);
}
}
} catch (LocalDBException e) {
LOGGER.error("error outputting pwm statistics: " + e.getMessage());
}
}
}
public int size() {
if (localDB != null) {
try {
return localDB.size(WORDS_DB);
} catch (Exception e) {
LOGGER.error("error checking wordlist size: " + e.getMessage());
return 0;
}
} else {
return 0;
}
}
public String readAppAttribute(final AppAttribute appAttribute) {
if (localDB == null || localDB.status() != LocalDB.Status.OPEN) {
LOGGER.error("error retrieving key '" + appAttribute.getKey() + "', localDB unavailable: ");
return null;
}
if (appAttribute == null) {
return null;
}
try {
return localDB.get(LocalDB.DB.PWM_META, appAttribute.getKey());
} catch (Exception e) {
LOGGER.error(
"error retrieving key '"
+ appAttribute.getKey()
+ "' installation date from localDB: "
+ e.getMessage());
}
return null;
}
public void writeAppAttribute(final AppAttribute appAttribute, final String value) {
if (localDB == null || localDB.status() != LocalDB.Status.OPEN) {
LOGGER.error("error writing key '" + appAttribute.getKey() + "', localDB unavailable: ");
return;
}
if (appAttribute == null) {
return;
}
try {
if (value == null) {
localDB.remove(LocalDB.DB.PWM_META, appAttribute.getKey());
} else {
localDB.put(LocalDB.DB.PWM_META, appAttribute.getKey(), value);
}
} catch (Exception e) {
LOGGER.error(
"error retrieving key '"
+ appAttribute.getKey()
+ "' installation date from localDB: "
+ e.getMessage());
}
}
public StatisticsBundle getStatBundleForKey(final String key) {
if (key == null || key.length() < 1 || KEY_CUMULATIVE.equals(key)) {
return statsCummulative;
}
if (KEY_CURRENT.equals(key)) {
return statsCurrent;
}
if (currentDailyKey.toString().equals(key)) {
return statsDaily;
}
if (cachedStoredStats.containsKey(key)) {
return cachedStoredStats.get(key);
}
if (localDB == null) {
return null;
}
try {
final String storedStat = localDB.get(LocalDB.DB.PWM_STATS, key);
final StatisticsBundle returnBundle;
if (storedStat != null && storedStat.length() > 0) {
returnBundle = StatisticsBundle.input(storedStat);
} else {
returnBundle = new StatisticsBundle();
}
cachedStoredStats.put(key, returnBundle);
return returnBundle;
} catch (LocalDBException e) {
LOGGER.error("error retrieving stored stat for " + key + ": " + e.getMessage());
}
return null;
}
private boolean remove(StorageKey key) throws LocalDBException {
return localDB.remove(DB, key.getKey());
}
private void init(final PwmApplication pwmApplication, final long maxAgeMs) {
status = STATUS.OPENING;
final long startTime = System.currentTimeMillis();
try {
checkDbVersion();
} catch (Exception e) {
LOGGER.error("error checking db version", e);
status = STATUS.CLOSED;
return;
}
try {
final String oldestEntryStr = localDB.get(META_DB, KEY_OLDEST_ENTRY);
if (oldestEntryStr == null || oldestEntryStr.length() < 1) {
oldestEntry = 0;
LOGGER.trace("no oldestEntry timestamp stored, will rescan");
} else {
oldestEntry = Long.parseLong(oldestEntryStr);
LOGGER.trace(
"oldest timestamp loaded from localDB, age is "
+ TimeDuration.fromCurrent(oldestEntry).asCompactString());
}
} catch (LocalDBException e) {
LOGGER.error(
"unexpected error loading oldest-entry meta record, will remain closed: "
+ e.getMessage(),
e);
status = STATUS.CLOSED;
return;
}
try {
final int size = localDB.size(WORDS_DB);
final StringBuilder sb = new StringBuilder();
sb.append("open with ").append(size).append(" words (");
sb.append(new TimeDuration(System.currentTimeMillis(), startTime).asCompactString())
.append(")");
sb.append(", maxAgeMs=").append(new TimeDuration(maxAgeMs).asCompactString());
sb.append(", oldestEntry=")
.append(new TimeDuration(System.currentTimeMillis(), oldestEntry).asCompactString());
LOGGER.info(sb.toString());
} catch (LocalDBException e) {
LOGGER.error(
"unexpected error examining size of DB, will remain closed: " + e.getMessage(), e);
status = STATUS.CLOSED;
return;
}
status = STATUS.OPEN;
// populateFromWordlist(); //only used for debugging!!!
if (pwmApplication.getApplicationMode() == PwmApplication.MODE.RUNNING
|| pwmApplication.getApplicationMode() == PwmApplication.MODE.CONFIGURATION) {
long frequencyMs = maxAgeMs > MAX_CLEANER_FREQUENCY ? MAX_CLEANER_FREQUENCY : maxAgeMs;
frequencyMs = frequencyMs < MIN_CLEANER_FREQUENCY ? MIN_CLEANER_FREQUENCY : frequencyMs;
LOGGER.debug(
"scheduling cleaner task to run once every "
+ new TimeDuration(frequencyMs).asCompactString());
final String threadName = Helper.makeThreadName(pwmApplication, this.getClass()) + " timer";
cleanerTimer = new Timer(threadName, true);
cleanerTimer.schedule(new CleanerTask(), 1000, frequencyMs);
}
}
private void write(StorageKey key, UserCacheRecord cacheBean) throws LocalDBException {
final String jsonValue = JsonUtil.serialize(cacheBean);
localDB.put(DB, key.getKey(), jsonValue);
}
public void init(final PwmApplication pwmApplication) throws PwmException {
settings.maxAgeMs =
1000
* pwmApplication
.getConfig()
.readSettingAsLong(PwmSetting.PASSWORD_SHAREDHISTORY_MAX_AGE); // convert to MS;
settings.caseInsensitive =
Boolean.parseBoolean(
pwmApplication
.getConfig()
.readAppProperty(AppProperty.SECURITY_SHAREDHISTORY_CASE_INSENSITIVE));
settings.hashName =
pwmApplication.getConfig().readAppProperty(AppProperty.SECURITY_SHAREDHISTORY_HASH_NAME);
settings.hashIterations =
Integer.parseInt(
pwmApplication
.getConfig()
.readAppProperty(AppProperty.SECURITY_SHAREDHISTORY_HASH_ITERATIONS));
settings.version =
"2"
+ "_"
+ settings.hashName
+ "_"
+ settings.hashIterations
+ "_"
+ settings.caseInsensitive;
final int SALT_LENGTH =
Integer.parseInt(
pwmApplication
.getConfig()
.readAppProperty(AppProperty.SECURITY_SHAREDHISTORY_SALT_LENGTH));
this.localDB = pwmApplication.getLocalDB();
boolean needsClearing = false;
if (localDB == null) {
LOGGER.info("LocalDB is not available, will remain closed");
status = STATUS.CLOSED;
return;
}
if (settings.maxAgeMs < 1) {
LOGGER.debug("max age=" + settings.maxAgeMs + ", will remain closed");
needsClearing = true;
}
{
this.salt = localDB.get(META_DB, KEY_SALT);
if (salt == null || salt.length() < SALT_LENGTH) {
LOGGER.warn("stored global salt value is not present, creating new salt");
this.salt = PwmRandom.getInstance().alphaNumericString(SALT_LENGTH);
localDB.put(META_DB, KEY_SALT, this.salt);
needsClearing = true;
}
}
if (needsClearing) {
LOGGER.trace("clearing wordlist");
try {
localDB.truncate(WORDS_DB);
} catch (Exception e) {
LOGGER.error("error during wordlist truncate", e);
}
}
new Thread(
new Runnable() {
public void run() {
LOGGER.debug("starting up in background thread");
init(pwmApplication, settings.maxAgeMs);
}
},
Helper.makeThreadName(pwmApplication, this.getClass()) + " initializer")
.start();
}
public void init(PwmApplication pwmApplication) throws PwmException {
for (final Statistic.EpsType type : Statistic.EpsType.values()) {
for (final Statistic.EpsDuration duration : Statistic.EpsDuration.values()) {
epsMeterMap.put(
type.toString() + duration.toString(), new EventRateMeter(duration.getTimeDuration()));
}
}
status = STATUS.OPENING;
this.localDB = pwmApplication.getLocalDB();
this.pwmApplication = pwmApplication;
if (localDB == null) {
LOGGER.error("LocalDB is not available, will remain closed");
status = STATUS.CLOSED;
return;
}
{
final String storedCummulativeBundleStr =
localDB.get(LocalDB.DB.PWM_STATS, DB_KEY_CUMULATIVE);
if (storedCummulativeBundleStr != null && storedCummulativeBundleStr.length() > 0) {
statsCummulative = StatisticsBundle.input(storedCummulativeBundleStr);
}
}
{
for (final Statistic.EpsType loopEpsType : Statistic.EpsType.values()) {
for (final Statistic.EpsType loopEpsDuration : Statistic.EpsType.values()) {
final String key = "EPS-" + loopEpsType.toString() + loopEpsDuration.toString();
final String storedValue = localDB.get(LocalDB.DB.PWM_STATS, key);
if (storedValue != null && storedValue.length() > 0) {
try {
final EventRateMeter eventRateMeter =
JsonUtil.deserialize(storedValue, EventRateMeter.class);
epsMeterMap.put(loopEpsType.toString() + loopEpsDuration.toString(), eventRateMeter);
} catch (Exception e) {
LOGGER.error(
"unexpected error reading last EPS rate for "
+ loopEpsType
+ " from LocalDB: "
+ e.getMessage());
}
}
}
}
}
{
final String storedInitialString =
localDB.get(LocalDB.DB.PWM_STATS, DB_KEY_INITIAL_DAILY_KEY);
if (storedInitialString != null && storedInitialString.length() > 0) {
initialDailyKey = new DailyKey(storedInitialString);
}
}
{
currentDailyKey = new DailyKey(new Date());
final String storedDailyStr = localDB.get(LocalDB.DB.PWM_STATS, currentDailyKey.toString());
if (storedDailyStr != null && storedDailyStr.length() > 0) {
statsDaily = StatisticsBundle.input(storedDailyStr);
}
}
try {
localDB.put(
LocalDB.DB.PWM_STATS,
DB_KEY_TEMP,
PwmConstants.DEFAULT_DATETIME_FORMAT.format(new Date()));
} catch (IllegalStateException e) {
LOGGER.error("unable to write to localDB, will remain closed, error: " + e.getMessage());
status = STATUS.CLOSED;
return;
}
localDB.put(LocalDB.DB.PWM_STATS, DB_KEY_VERSION, DB_VALUE_VERSION);
localDB.put(LocalDB.DB.PWM_STATS, DB_KEY_INITIAL_DAILY_KEY, initialDailyKey.toString());
{ // setup a timer to roll over at 0 Zula and one to write current stats every 10 seconds
final String threadName = Helper.makeThreadName(pwmApplication, this.getClass()) + " timer";
daemonTimer = new Timer(threadName, true);
daemonTimer.schedule(new FlushTask(), 10 * 1000, DB_WRITE_FREQUENCY_MS);
daemonTimer.schedule(new NightlyTask(), Helper.nextZuluZeroTime());
}
if (pwmApplication.getApplicationMode() == PwmApplication.MODE.RUNNING) {
if (pwmApplication.getConfig().readSettingAsBoolean(PwmSetting.PUBLISH_STATS_ENABLE)) {
long lastPublishTimestamp = pwmApplication.getInstallTime().getTime();
{
final String lastPublishDateStr =
localDB.get(LocalDB.DB.PWM_STATS, KEY_CLOUD_PUBLISH_TIMESTAMP);
if (lastPublishDateStr != null && lastPublishDateStr.length() > 0) {
try {
lastPublishTimestamp = Long.parseLong(lastPublishDateStr);
} catch (Exception e) {
LOGGER.error(
"unexpected error reading last publish timestamp from PwmDB: " + e.getMessage());
}
}
}
final Date nextPublishTime =
new Date(
lastPublishTimestamp
+ PwmConstants.STATISTICS_PUBLISH_FREQUENCY_MS
+ (long) PwmRandom.getInstance().nextInt(3600 * 1000));
daemonTimer.schedule(
new PublishTask(), nextPublishTime, PwmConstants.STATISTICS_PUBLISH_FREQUENCY_MS);
}
}
status = STATUS.OPEN;
}
private void publishStatisticsToCloud()
throws URISyntaxException, IOException, PwmUnrecoverableException {
final StatsPublishBean statsPublishData;
{
final StatisticsBundle bundle = getStatBundleForKey(KEY_CUMULATIVE);
final Map<String, String> statData = new HashMap<>();
for (final Statistic loopStat : Statistic.values()) {
statData.put(loopStat.getKey(), bundle.getStatistic(loopStat));
}
final Configuration config = pwmApplication.getConfig();
final List<String> configuredSettings = new ArrayList<>();
for (final PwmSetting pwmSetting : config.nonDefaultSettings()) {
if (!pwmSetting.getCategory().hasProfiles() && !config.isDefaultValue(pwmSetting)) {
configuredSettings.add(pwmSetting.getKey());
}
}
final Map<String, String> otherData = new HashMap<>();
otherData.put(
StatsPublishBean.KEYS.SITE_URL.toString(),
config.readSettingAsString(PwmSetting.PWM_SITE_URL));
otherData.put(
StatsPublishBean.KEYS.SITE_DESCRIPTION.toString(),
config.readSettingAsString(PwmSetting.PUBLISH_STATS_SITE_DESCRIPTION));
otherData.put(
StatsPublishBean.KEYS.INSTALL_DATE.toString(),
PwmConstants.DEFAULT_DATETIME_FORMAT.format(pwmApplication.getInstallTime()));
try {
otherData.put(
StatsPublishBean.KEYS.LDAP_VENDOR.toString(),
pwmApplication
.getProxyChaiProvider(config.getDefaultLdapProfile().getIdentifier())
.getDirectoryVendor()
.toString());
} catch (Exception e) {
LOGGER.trace("unable to read ldap vendor type for stats publication: " + e.getMessage());
}
statsPublishData =
new StatsPublishBean(
pwmApplication.getInstanceID(),
new Date(),
statData,
configuredSettings,
PwmConstants.BUILD_NUMBER,
PwmConstants.BUILD_VERSION,
otherData);
}
final URI requestURI = new URI(PwmConstants.PWM_URL_CLOUD + "/rest/pwm/statistics");
final HttpPost httpPost = new HttpPost(requestURI.toString());
final String jsonDataString = JsonUtil.serialize(statsPublishData);
httpPost.setEntity(new StringEntity(jsonDataString));
httpPost.setHeader("Accept", PwmConstants.AcceptValue.json.getHeaderValue());
httpPost.setHeader("Content-Type", PwmConstants.ContentTypeValue.json.getHeaderValue());
LOGGER.debug(
"preparing to send anonymous statistics to "
+ requestURI.toString()
+ ", data to send: "
+ jsonDataString);
final HttpResponse httpResponse =
PwmHttpClient.getHttpClient(pwmApplication.getConfig()).execute(httpPost);
if (httpResponse.getStatusLine().getStatusCode() != HttpStatus.SC_OK) {
throw new IOException(
"http response error code: " + httpResponse.getStatusLine().getStatusCode());
}
LOGGER.info("published anonymous statistics to " + requestURI.toString());
try {
localDB.put(
LocalDB.DB.PWM_STATS,
KEY_CLOUD_PUBLISH_TIMESTAMP,
String.valueOf(System.currentTimeMillis()));
} catch (LocalDBException e) {
LOGGER.error(
"unexpected error trying to save last statistics published time to LocalDB: "
+ e.getMessage());
}
}
private void clear() throws LocalDBException {
localDB.truncate(DB);
}
private int size() throws LocalDBException {
return localDB.size(DB);
}
public void testPut() throws LocalDBException {
Assert.assertNull(pwmDB.get(TEST_DB, "testKey1"));
pwmDB.put(TEST_DB, "testKey1", "testValue1");
Assert.assertEquals(pwmDB.get(TEST_DB, "testKey1"), "testValue1");
}
