private static long getLongOrTimestamp(Object value, DateTimeZone hiveTimeZone) {
if (value instanceof Date) {
long storageTime = ((Date) value).getTime();
// convert date from VM current time zone to UTC
long utcMillis = storageTime + DateTimeZone.getDefault().getOffset(storageTime);
return TimeUnit.MILLISECONDS.toDays(utcMillis);
}
if (value instanceof Timestamp) {
// The Hive SerDe parses timestamps using the default time zone of
// this JVM, but the data might have been written using a different
// time zone. We need to convert it to the configured time zone.
// the timestamp that Hive parsed using the JVM time zone
long parsedJvmMillis = ((Timestamp) value).getTime();
// remove the JVM time zone correction from the timestamp
DateTimeZone jvmTimeZone = DateTimeZone.getDefault();
long hiveMillis = jvmTimeZone.convertUTCToLocal(parsedJvmMillis);
// convert to UTC using the real time zone for the underlying data
long utcMillis = hiveTimeZone.convertLocalToUTC(hiveMillis, false);
return utcMillis;
}
return ((Number) value).longValue();
}
private DateTimeZone getTimeZone(HttpServletRequest request) {
Integer millisOffset = getMillisOffset(request);
if (millisOffset != null) {
try {
return DateTimeZone.forOffsetMillis(millisOffset);
} catch (IllegalArgumentException e) {
return DateTimeZone.getDefault();
}
} else {
return DateTimeZone.getDefault();
}
}
@Override
public void serialize(DateTime value, JsonGenerator jgen, SerializerProvider provider)
throws IOException {
DateTimeSerializer serializer =
new DateTimeSerializer(formatterWithTimeZone(DateTimeZone.getDefault()));
serializer.serialize(value, jgen, provider);
}
private void setDefaultTimeZone() {
String dtzStr = pc.getProperties().getProperty("pig.datetime.default.tz");
if (dtzStr != null && dtzStr.length() > 0) {
currentDTZ = DateTimeZone.getDefault();
DateTimeZone.setDefault(DateTimeZone.forID(dtzStr));
}
}
public HiveMetadata(
HiveConnectorId connectorId,
HiveMetastore metastore,
HdfsEnvironment hdfsEnvironment,
DateTimeZone timeZone,
boolean allowDropTable,
boolean allowRenameTable,
boolean allowCorruptWritesForTesting,
HiveStorageFormat hiveStorageFormat,
TypeManager typeManager) {
this.connectorId = checkNotNull(connectorId, "connectorId is null").toString();
this.allowDropTable = allowDropTable;
this.allowRenameTable = allowRenameTable;
this.allowCorruptWritesForTesting = allowCorruptWritesForTesting;
this.metastore = checkNotNull(metastore, "metastore is null");
this.hdfsEnvironment = checkNotNull(hdfsEnvironment, "hdfsEnvironment is null");
this.timeZone = checkNotNull(timeZone, "timeZone is null");
this.hiveStorageFormat = hiveStorageFormat;
this.typeManager = checkNotNull(typeManager, "typeManager is null");
if (!allowCorruptWritesForTesting && !timeZone.equals(DateTimeZone.getDefault())) {
log.warn(
"Hive writes are disabled. "
+ "To write data to Hive, your JVM timezone must match the Hive storage timezone. "
+ "Add -Duser.timezone=%s to your JVM arguments",
timeZone.getID());
}
}
public static String formatUTCToLocal(String datetime) {
String returnTimeDate = "";
DateTime dtUTC = null;
DateTimeZone timezone = DateTimeZone.getDefault();
DateTimeFormatter formatDT = DateTimeFormat.forPattern("MMM dd, yyyy hh:mma");
try {
DateTime dateDateTime1 = formatDT.parseDateTime(datetime);
DateTime now = new DateTime();
DateTime nowUTC = new LocalDateTime(now).toDateTime(DateTimeZone.UTC);
long instant = now.getMillis();
long instantUTC = nowUTC.getMillis();
long offset = instantUTC - instant;
// convert to local time
dtUTC = dateDateTime1.withZoneRetainFields(DateTimeZone.UTC);
// dtUTC = dateDateTime1.toDateTime(timezone);
dtUTC = dtUTC.plusMillis((int) offset);
returnTimeDate = dtUTC.toString(formatDT);
} catch (Exception e) {
returnTimeDate = "null";
e.printStackTrace();
}
return returnTimeDate;
}
/**
* Gets an instance of the IslamicChronology in the given time zone.
*
* @param zone the time zone to get the chronology in, null is default
* @param leapYears the type defining the leap year pattern
* @return a chronology in the specified time zone
*/
public static IslamicChronology getInstance(DateTimeZone zone, LeapYearPatternType leapYears) {
if (zone == null) {
zone = DateTimeZone.getDefault();
}
IslamicChronology chrono;
synchronized (cCache) {
IslamicChronology[] chronos = (IslamicChronology[]) cCache.get(zone);
if (chronos == null) {
chronos = new IslamicChronology[4];
cCache.put(zone, chronos);
}
chrono = chronos[leapYears.index];
if (chrono == null) {
if (zone == DateTimeZone.UTC) {
// First create without a lower limit.
chrono = new IslamicChronology(null, null, leapYears);
// Impose lower limit and make another IslamicChronology.
DateTime lowerLimit = new DateTime(1, 1, 1, 0, 0, 0, 0, chrono);
chrono =
new IslamicChronology(
LimitChronology.getInstance(chrono, lowerLimit, null), null, leapYears);
} else {
chrono = getInstance(DateTimeZone.UTC, leapYears);
chrono =
new IslamicChronology(ZonedChronology.getInstance(chrono, zone), null, leapYears);
}
chronos[leapYears.index] = chrono;
}
}
return chrono;
}
/**
* Gets the Chronology in a specific time zone.
*
* @param zone the zone to get the chronology in, null is default
* @return the chronology
*/
public Chronology withZone(DateTimeZone zone) {
if (zone == null) {
zone = DateTimeZone.getDefault();
}
if (zone == getZone()) {
return this;
}
return getInstance(zone);
}
/**
* @param when The instant
* @param locale The required locale
* @return The instant formatted as "yyyyMMdd" in the system timezone
*/
public static String formatCompactDateLocal(ReadableInstant when, Locale locale) {
if (when == null) {
return "";
}
return ISODateTimeFormat.basicDate()
.withZone(DateTimeZone.getDefault())
.withLocale(locale)
.print(when);
}
@Override
public BeerViewModel from(Beer beer) {
return new BeerViewModel(
beer.getId(),
beer.getName(),
beer.getDesc(),
beer.getOrigin(),
new DateTime(beer.getCreatedAt(), DateTimeZone.getDefault()).toString("yyy-MM-dd"));
}
public static DateTimeZone getLocalTimeZone(Ruby runtime) {
IRubyObject tz = getEnvTimeZone(runtime);
if (tz == null || !(tz instanceof RubyString)) {
return DateTimeZone.getDefault();
} else {
return getTimeZone(runtime, tz.toString());
}
}
/**
* @param when The instant
* @param locale The required locale
* @return The instant formatted as "ddd, MMM dd" (Saturday, January 01) in the system timezone
*/
public static String formatTransactionDateLocal(ReadableInstant when, Locale locale) {
if (when == null) {
return "";
}
return utcTransactionDateFormatter
.withZone(DateTimeZone.getDefault())
.withLocale(locale)
.print(when);
}
private void verifyJvmTimeZone() {
if (!allowCorruptWritesForTesting && !timeZone.equals(DateTimeZone.getDefault())) {
throw new PrestoException(
HIVE_TIMEZONE_MISMATCH,
format(
"To write Hive data, your JVM timezone must match the Hive storage timezone. Add -Duser.timezone=%s to your JVM arguments.",
timeZone.getID()));
}
}
protected void setUp() throws Exception {
DateTimeUtils.setCurrentMillisFixed(TEST_TIME_NOW);
originalDateTimeZone = DateTimeZone.getDefault();
originalTimeZone = TimeZone.getDefault();
originalLocale = Locale.getDefault();
DateTimeZone.setDefault(LONDON);
TimeZone.setDefault(TimeZone.getTimeZone("Europe/London"));
Locale.setDefault(Locale.UK);
}
public static void init() {
InputStream input = null;
try {
File file = new File(PROPERTYFILE);
input = new FileInputStream(file);
} catch (Exception e) {
LOGGER.info("" + e);
input = null;
}
try {
if (input == null) {
input = Environment.class.getClassLoader().getResourceAsStream(PROPERTYFILE);
}
if (input == null) {
throw new ResourceNotFoundException(PROPERTYFILE);
}
props.load(input);
Enumeration<?> e = props.propertyNames();
while (e.hasMoreElements()) {
String key = (String) e.nextElement();
String value = props.getProperty(key);
config.put(key, value);
}
String language = getProperty("locale.language", Locale.getDefault().getLanguage());
String country = getProperty("locale.country", Locale.getDefault().getCountry());
locale = new Locale(language, country);
String timeZoneId = getProperty("datetime.timeZone");
timeZone = (timeZoneId == null) ? DateTimeZone.getDefault() : DateTimeZone.forID(timeZoneId);
appHome = getProperty("app.home", APPHOME);
if (appHome.startsWith("~")) {
appHome = appHome.replaceFirst("~", System.getProperty("user.home"));
}
new File(getDataFolder()).mkdirs();
new File(getRptFolder()).mkdirs();
new File(getTmpFolder()).mkdirs();
new File(getCacheFolder()).mkdirs();
new File(getLogFolder()).mkdirs();
} catch (IOException e) {
throw new ResourceIOException(PROPERTYFILE, e);
} finally {
if (input != null) {
try {
input.close();
} catch (IOException e) {
throw new ResourceIOException(PROPERTYFILE, e);
}
}
}
}
@Test
public void testTimeZones() {
DateTimeZone aDTZ = PDTConfig.getDefaultDateTimeZone();
assertNotNull(aDTZ);
aDTZ = PDTConfig.getDateTimeZoneUTC();
assertNotNull(aDTZ);
assertEquals("UTC", aDTZ.getID());
assertNotNull(PDTConfig.getDefaultChronologyUTC());
assertEquals(PDTConfig.getDateTimeZoneUTC(), PDTConfig.getDefaultChronologyUTC().getZone());
assertNotNull(PDTConfig.getDefaultChronologyWithoutDateTimeZone());
assertEquals(
DateTimeZone.getDefault(), PDTConfig.getDefaultChronologyWithoutDateTimeZone().getZone());
try {
// Invalid
assertFalse(PDTConfig.setDefaultDateTimeZoneID("does not exist").isSuccess());
// Regular
assertTrue(PDTConfig.setDefaultDateTimeZoneID("Europe/Berlin").isSuccess());
assertEquals("Europe/Berlin", PDTConfig.getDefaultDateTimeZone().getID());
// I hope this is not the standard time zone anywhere
assertTrue(PDTConfig.setDefaultDateTimeZoneID("UTC").isSuccess());
assertEquals("UTC", PDTConfig.getDefaultDateTimeZone().getID());
// The default date time zone was not modified
assertEquals(
DateTimeZone.getDefault(), PDTConfig.getDefaultChronologyWithoutDateTimeZone().getZone());
if (!DateTimeZone.getDefault().getID().equals("UTC")) {
// And therefore this must not be equal
assertTrue(
!PDTConfig.getDefaultDateTimeZone()
.equals(PDTConfig.getDefaultChronologyWithoutDateTimeZone().getZone()));
}
} finally {
assertTrue(PDTConfig.setDefaultDateTimeZoneID(PDTConfig.DEFAULT_DATETIMEZONEID).isSuccess());
}
}
public Chronology withZone(DateTimeZone zone) {
if (zone == null) {
zone = DateTimeZone.getDefault();
}
if (zone == getParam()) {
return this;
}
if (zone == DateTimeZone.UTC) {
return getBase();
}
return new ZonedChronology(getBase(), zone);
}
protected void setUp() throws Exception {
DateTimeUtils.setCurrentMillisFixed(TEST_TIME_NOW);
originalDateTimeZone = DateTimeZone.getDefault();
originalTimeZone = TimeZone.getDefault();
originalLocale = Locale.getDefault();
DateTimeZone.setDefault(DateTimeZone.forID("Europe/London"));
TimeZone.setDefault(TimeZone.getTimeZone("Europe/London"));
Locale.setDefault(Locale.UK);
ISO = ISOChronology.getInstance();
JULIAN = JulianChronology.getInstance();
}
public MaterializedResult execute(
Session session, @Language("SQL") String sql, List<? extends Type> resultTypes) {
MaterializedResult materializedRows =
new MaterializedResult(
handle.createQuery(sql).map(rowMapper(resultTypes)).list(), resultTypes);
// H2 produces dates in the JVM time zone instead of the session timezone
materializedRows =
materializedRows.toTimeZone(
DateTimeZone.getDefault(), getDateTimeZone(session.getTimeZoneKey()));
return materializedRows;
}
private static void insertRows(
ConnectorTableMetadata tableMetadata, Handle handle, RecordSet data) {
List<ColumnMetadata> columns =
ImmutableList.copyOf(
Iterables.filter(
tableMetadata.getColumns(),
new Predicate<ColumnMetadata>() {
@Override
public boolean apply(ColumnMetadata columnMetadata) {
return !columnMetadata.isHidden();
}
}));
String vars = Joiner.on(',').join(nCopies(columns.size(), "?"));
String sql =
format("INSERT INTO %s VALUES (%s)", tableMetadata.getTable().getTableName(), vars);
RecordCursor cursor = data.cursor();
while (true) {
// insert 1000 rows at a time
PreparedBatch batch = handle.prepareBatch(sql);
for (int row = 0; row < 1000; row++) {
if (!cursor.advanceNextPosition()) {
batch.execute();
return;
}
PreparedBatchPart part = batch.add();
for (int column = 0; column < columns.size(); column++) {
Type type = columns.get(column).getType();
if (BOOLEAN.equals(type)) {
part.bind(column, cursor.getBoolean(column));
} else if (BIGINT.equals(type)) {
part.bind(column, cursor.getLong(column));
} else if (DOUBLE.equals(type)) {
part.bind(column, cursor.getDouble(column));
} else if (VARCHAR.equals(type)) {
part.bind(column, cursor.getSlice(column).toStringUtf8());
} else if (DATE.equals(type)) {
long millisUtc = TimeUnit.DAYS.toMillis(cursor.getLong(column));
// H2 expects dates in to be millis at midnight in the JVM timezone
long localMillis =
DateTimeZone.UTC.getMillisKeepLocal(DateTimeZone.getDefault(), millisUtc);
part.bind(column, new Date(localMillis));
} else {
throw new IllegalArgumentException("Unsupported type " + type);
}
}
}
batch.execute();
}
}
/**
* Standard instance of a Buddhist Chronology, that matches Sun's BuddhistCalendar class. This
* means that it follows the GregorianJulian calendar rules with a cutover date.
*
* @param zone the time zone to use, null is default
*/
public static synchronized BuddhistChronology getInstance(DateTimeZone zone) {
if (zone == null) {
zone = DateTimeZone.getDefault();
}
BuddhistChronology chrono = (BuddhistChronology) cCache.get(zone);
if (chrono == null) {
// First create without a lower limit.
chrono = new BuddhistChronology(GJChronology.getInstance(zone, null), null);
// Impose lower limit and make another BuddhistChronology.
DateTime lowerLimit = new DateTime(1, 1, 1, 0, 0, 0, 0, chrono);
chrono = new BuddhistChronology(LimitChronology.getInstance(chrono, lowerLimit, null), "");
cCache.put(zone, chrono);
}
return chrono;
}
private static long formatDateAsLong(Object object, DateObjectInspector inspector) {
if (object instanceof LazyDate) {
return ((LazyDate) object).getWritableObject().getDays();
}
if (object instanceof DateWritable) {
return ((DateWritable) object).getDays();
}
// Hive will return java.sql.Date at midnight in JVM time zone
long millisLocal = inspector.getPrimitiveJavaObject(object).getTime();
// Convert it to midnight in UTC
long millisUtc = DateTimeZone.getDefault().getMillisKeepLocal(DateTimeZone.UTC, millisLocal);
// Convert midnight UTC to days
return TimeUnit.MILLISECONDS.toDays(millisUtc);
}
private static String getErrMsgTimeZone(final String timeStamp, final BuildDetail detail) {
return "This test failure could be a JVM TimeZone issue: "
+ "\ntimestamp: "
+ timeStamp
+ "; \nBuildDetail.getTimeStamp(): "
+ detail.getTimeStamp()
+ "; \nBuildDetail.getBuildDate(): "
+ detail
.getBuildDate()
.toString(org.joda.time.format.DateTimeFormat.forPattern("yyyyMMddHHmmss"))
+ "; \nDo you see the extra/missing '5' above?"
+ "; \nJoda TimeZone: "
+ org.joda.time.DateTimeZone.getDefault()
+ "; \nIs Joda TimeZone above what you'd expect? Does it match your OS setting?"
+ "; \nsee: http://www.mail-archive.com/[email protected]/msg01073.html\n";
}
/**
* Gets an instance of the ISOChronology in the given time zone.
*
* @param zone the time zone to get the chronology in, null is default
* @return a chronology in the specified time zone
*/
public static ISOChronology getInstance(DateTimeZone zone) {
if (zone == null) {
zone = DateTimeZone.getDefault();
}
int index = System.identityHashCode(zone) & (FAST_CACHE_SIZE - 1);
ISOChronology chrono = cFastCache[index];
if (chrono != null && chrono.getZone() == zone) {
return chrono;
}
synchronized (cCache) {
chrono = (ISOChronology) cCache.get(zone);
if (chrono == null) {
chrono = new ISOChronology(ZonedChronology.getInstance(INSTANCE_UTC, zone));
cCache.put(zone, chrono);
}
}
cFastCache[index] = chrono;
return chrono;
}
/**
* @param when The instant
* @return The instant formatted as "yyyy-MM-dd" in the system timezone
*/
public static String formatCompactDateWithHyphensLocal(ReadableInstant when) {
if (when == null) {
return "";
}
return utcShortDateWithHyphensFormatter.withZone(DateTimeZone.getDefault()).print(when);
}
/**
* @param when The instant
* @return The instant formatted for HTTP as defined in RFC 1123 e.g. "Sat, 01 Jan 2000 23:59:59
* GMT" in UTC
*/
public static String formatHttpDateHeaderLocal(ReadableInstant when) {
if (when == null) {
return "";
}
return utcHttpDateFormatter.withZone(DateTimeZone.getDefault()).print(when);
}
/**
* @param when The instant
* @param locale The required locale
* @return The instant formatted as ISO8601 e.g. "2000-01-02T03:04:05" in the system timezone
*/
public static String formatIso8601Local(ReadableInstant when, Locale locale) {
if (when == null) {
return "";
}
return utcIso8601.withZone(DateTimeZone.getDefault()).withLocale(locale).print(when);
}
/** @return The current midnight in the system timezone */
public static DateTime midnightLocal() {
return nowUtc().withZone(DateTimeZone.getDefault()).toDateMidnight().toDateTime();
}
public void testSimpleStreams() throws Exception {
assumeTrue("requires a 64-bit JRE ... ?!", Constants.JRE_IS_64BIT);
BytesStreamOutput out = new BytesStreamOutput();
out.writeBoolean(false);
out.writeByte((byte) 1);
out.writeShort((short) -1);
out.writeInt(-1);
out.writeVInt(2);
out.writeLong(-3);
out.writeVLong(4);
out.writeOptionalLong(11234234L);
out.writeFloat(1.1f);
out.writeDouble(2.2);
int[] intArray = {1, 2, 3};
out.writeGenericValue(intArray);
int[] vIntArray = {4, 5, 6};
out.writeVIntArray(vIntArray);
long[] longArray = {1, 2, 3};
out.writeGenericValue(longArray);
long[] vLongArray = {4, 5, 6};
out.writeVLongArray(vLongArray);
float[] floatArray = {1.1f, 2.2f, 3.3f};
out.writeGenericValue(floatArray);
double[] doubleArray = {1.1, 2.2, 3.3};
out.writeGenericValue(doubleArray);
out.writeString("hello");
out.writeString("goodbye");
out.writeGenericValue(BytesRefs.toBytesRef("bytesref"));
out.writeStringArray(new String[] {"a", "b", "cat"});
out.writeBytesReference(new BytesArray("test"));
out.writeOptionalBytesReference(new BytesArray("test"));
out.writeOptionalDouble(null);
out.writeOptionalDouble(1.2);
out.writeTimeZone(DateTimeZone.forID("CET"));
out.writeOptionalTimeZone(DateTimeZone.getDefault());
out.writeOptionalTimeZone(null);
final byte[] bytes = BytesReference.toBytes(out.bytes());
StreamInput in = StreamInput.wrap(BytesReference.toBytes(out.bytes()));
assertEquals(in.available(), bytes.length);
assertThat(in.readBoolean(), equalTo(false));
assertThat(in.readByte(), equalTo((byte) 1));
assertThat(in.readShort(), equalTo((short) -1));
assertThat(in.readInt(), equalTo(-1));
assertThat(in.readVInt(), equalTo(2));
assertThat(in.readLong(), equalTo(-3L));
assertThat(in.readVLong(), equalTo(4L));
assertThat(in.readOptionalLong(), equalTo(11234234L));
assertThat((double) in.readFloat(), closeTo(1.1, 0.0001));
assertThat(in.readDouble(), closeTo(2.2, 0.0001));
assertThat(in.readGenericValue(), equalTo((Object) intArray));
assertThat(in.readVIntArray(), equalTo(vIntArray));
assertThat(in.readGenericValue(), equalTo((Object) longArray));
assertThat(in.readVLongArray(), equalTo(vLongArray));
assertThat(in.readGenericValue(), equalTo((Object) floatArray));
assertThat(in.readGenericValue(), equalTo((Object) doubleArray));
assertThat(in.readString(), equalTo("hello"));
assertThat(in.readString(), equalTo("goodbye"));
assertThat(in.readGenericValue(), equalTo((Object) BytesRefs.toBytesRef("bytesref")));
assertThat(in.readStringArray(), equalTo(new String[] {"a", "b", "cat"}));
assertThat(in.readBytesReference(), equalTo(new BytesArray("test")));
assertThat(in.readOptionalBytesReference(), equalTo(new BytesArray("test")));
assertNull(in.readOptionalDouble());
assertThat(in.readOptionalDouble(), closeTo(1.2, 0.0001));
assertEquals(DateTimeZone.forID("CET"), in.readTimeZone());
assertEquals(DateTimeZone.getDefault(), in.readOptionalTimeZone());
assertNull(in.readOptionalTimeZone());
assertEquals(0, in.available());
in.close();
out.close();
}
/**
* Adjust milliseconds since 1970-01-01 00:00:00-UTC to seconds since 2000-01-01 00:00:00
* timezoneless. A conversion from local time to UTC involves an offset that varies for Summer
* time. A conversion from local time to timezoneless just removes the zone as though all days
* were the same length.
*/
private static long seconds2000NoTZ(long millis) {
DateTimeZone dtz = DateTimeZone.getDefault();
millis += dtz.getOffset(millis);
return millis / 1000 - 946684800; // 2000-01-01 00:00:00-UTC.
}
