// Writes a C string and expands the frame if necessary
// FIXME: Property strings containing nul ('\0') characters will corrupt the frame when they are
// written.
// Figure out how to throw an exception here if any nul chars are encountered
private static ByteBuffer writeCString(ByteBuffer frame, String string) {
Byte b = propertyAbbreviations.get(string);
if (b != null) {
if (frame.remaining() < 2)
frame = ByteBuffer.allocate(frame.capacity() << 1).put((ByteBuffer) frame.rewind());
frame.put(b);
frame.put((byte) 0);
} else {
CharsetEncoder cStringEncoder = cStringCharset.newEncoder();
CharBuffer chars = CharBuffer.wrap(string);
for (int size = frame.capacity(); ; cStringEncoder.flush(frame)) {
cStringEncoder.reset();
if (cStringEncoder.encode(chars, frame, true) == CoderResult.OVERFLOW) {
// debug output
// System.out.println("overflow, reallocating to size " + (size << 1) + " (printing \"" +
// string + "\")");
frame = ByteBuffer.allocate(size = (size << 1)).put((ByteBuffer) frame.rewind());
} else break;
}
cStringEncoder.flush(frame);
frame.put((byte) 0);
}
return frame;
}
public SVNCharsetConvertor reset() {
myEncoder = myEncoder.reset();
myDecoder = myDecoder.reset();
myCharBuffer = null;
myInputByteBuffer = null;
return this;
}
static byte[] encode(char[] cc, Charset cs, boolean testDirect, Time t) throws Exception {
ByteBuffer bbf;
CharBuffer cbf;
CharsetEncoder enc = cs.newEncoder();
String csn = cs.name();
if (testDirect) {
bbf = ByteBuffer.allocateDirect(cc.length * 4);
cbf = ByteBuffer.allocateDirect(cc.length * 2).asCharBuffer();
cbf.put(cc).flip();
} else {
bbf = ByteBuffer.allocate(cc.length * 4);
cbf = CharBuffer.wrap(cc);
}
CoderResult cr = null;
long t1 = System.nanoTime() / 1000;
for (int i = 0; i < iteration; i++) {
cbf.rewind();
bbf.clear();
enc.reset();
cr = enc.encode(cbf, bbf, true);
}
long t2 = System.nanoTime() / 1000;
t.t = (t2 - t1) / iteration;
if (cr != CoderResult.UNDERFLOW) {
System.out.println("ENC-----------------");
int pos = cbf.position();
System.out.printf(" cr=%s, cbf.pos=%d, cc[pos]=%x%n", cr.toString(), pos, cc[pos] & 0xffff);
throw new RuntimeException("Encoding err: " + csn);
}
byte[] bb = new byte[bbf.position()];
bbf.flip();
bbf.get(bb);
return bb;
}
public static int encode(
final CharsetEncoder ce,
final char[] ca,
final int off,
final int len,
final byte[] targetArray) {
if (len == 0) {
return 0;
}
if (ce instanceof ArrayEncoder) {
return ((ArrayEncoder) ce).encode(ca, off, len, targetArray);
} else {
ce.reset();
ByteBuffer bb = ByteBuffer.wrap(targetArray);
CharBuffer cb = CharBuffer.wrap(ca, off, len);
try {
CoderResult cr = ce.encode(cb, bb, true);
if (!cr.isUnderflow()) {
cr.throwException();
}
cr = ce.flush(bb);
if (!cr.isUnderflow()) {
cr.throwException();
}
} catch (CharacterCodingException x) {
throw new Error(x);
}
return bb.position();
}
}
public static void write(ICharsetCodec cc, CharBuffer cb, IUnitChain unitChain) {
final CharsetEncoder encoder = cc.getEncoder();
try {
IUnit unit = Util.lastUnit(unitChain);
ByteBuffer bb = unit.getByteBufferForWrite();
boolean flush = false;
encoder.reset();
for (; ; ) {
CoderResult cr = flush ? encoder.flush(bb) : encoder.encode(cb, bb, true);
unit.size(bb.position() - unit.start());
if (cr.isOverflow()) {
unit = Util.appendNewUnit(unitChain);
bb = unit.getByteBufferForWrite();
continue;
}
if (!cr.isUnderflow()) cr.throwException();
if (flush) break;
else flush = true;
}
} catch (CharacterCodingException e) {
throw new RuntimeException(e);
} finally {
cc.releaseEncoder(encoder);
}
}
byte[] encode(char[] ca, int off, int len) {
int en = (int) (ce.maxBytesPerChar() * len);
byte[] ba = new byte[en];
if (len == 0) return ba;
ce.reset();
ByteBuffer bb = ByteBuffer.wrap(ba);
CharBuffer cb = CharBuffer.wrap(ca, off, len);
try {
CoderResult cr = ce.encode(cb, bb, true);
if (!cr.isUnderflow()) cr.throwException();
cr = ce.flush(bb);
if (!cr.isUnderflow()) cr.throwException();
} catch (CharacterCodingException x) {
// Substitution is always enabled,
// so this shouldn't happen
throw new Error(x);
}
return trim(ba, bb.position());
}
private static void writeEntry(
OutputStream os, Attributes.Name name, String value, CharsetEncoder encoder, ByteBuffer bBuf)
throws IOException {
byte[] out = name.getBytes();
if (out.length > LINE_LENGTH_LIMIT) {
throw new IOException(
Messages.getString(
"archive.33", name, Integer.valueOf(LINE_LENGTH_LIMIT))); // $NON-NLS-1$
}
os.write(out);
os.write(VALUE_SEPARATOR);
encoder.reset();
bBuf.clear().limit(LINE_LENGTH_LIMIT - out.length - 2);
CharBuffer cBuf = CharBuffer.wrap(value);
CoderResult r;
while (true) {
r = encoder.encode(cBuf, bBuf, true);
if (CoderResult.UNDERFLOW == r) {
r = encoder.flush(bBuf);
}
os.write(bBuf.array(), bBuf.arrayOffset(), bBuf.position());
os.write(LINE_SEPARATOR);
if (CoderResult.UNDERFLOW == r) {
break;
}
os.write(' ');
bBuf.clear().limit(LINE_LENGTH_LIMIT - 1);
}
}
@JRubyMethod
public IRubyObject convert(ThreadContext context, IRubyObject srcBuffer) {
if (!(srcBuffer instanceof RubyString)) {
throw context.runtime.newTypeError(srcBuffer, context.runtime.getString());
}
RubyString srcString = (RubyString) srcBuffer;
ByteList srcBL = srcString.getByteList();
if (srcBL.getRealSize() == 0) return context.runtime.newSymbol("source_buffer_empty");
ByteBuffer srcBB = ByteBuffer.wrap(srcBL.getUnsafeBytes(), srcBL.begin(), srcBL.getRealSize());
try {
CharBuffer srcCB =
CharBuffer.allocate((int) (srcDecoder.maxCharsPerByte() * srcBL.getRealSize()) + 1);
CoderResult decodeResult = srcDecoder.decode(srcBB, srcCB, true);
srcCB.flip();
ByteBuffer destBB =
ByteBuffer.allocate((int) (destEncoder.maxBytesPerChar() * srcCB.limit()) + 1);
CoderResult encodeResult = destEncoder.encode(srcCB, destBB, true);
destBB.flip();
byte[] destBytes = new byte[destBB.limit()];
destBB.get(destBytes);
srcDecoder.reset();
destEncoder.reset();
return context.runtime.newString(new ByteList(destBytes, destEncoding.getEncoding(), false));
} catch (Exception e) {
throw context.runtime.newRuntimeError(e.getLocalizedMessage());
}
}
private IRubyObject iconv(IRubyObject str, int start, int end) {
if (str.isNil()) {
fromEncoding.reset();
toEncoding.reset();
return RubyString.newEmptyString(getRuntime());
}
return _iconv(str.convertToString(), start, end);
}
public byte[] encode(char[] chars, int off, int len) {
if (len == 0) {
return new byte[0];
}
encoder.reset();
int bytesLength = scale(len, encoder.maxBytesPerChar());
byte[] bytes = ThreadLocalCache.getBytes(bytesLength);
return encode(chars, off, len, bytes);
}
/** Encode a string into a ByteBuffer : on return position is the end of the encoding */
public static int toByteBuffer(CharSequence s, ByteBuffer bb, CharsetEncoder enc) {
int start = bb.position();
CharBuffer cBuff = CharBuffer.wrap(s);
enc.reset();
CoderResult r = enc.encode(cBuff, bb, true);
if (r.isOverflow()) throw new InternalErrorException("Bytes.toByteBuffer: encode overflow (1)");
r = enc.flush(bb);
if (r.isOverflow()) throw new InternalErrorException("Bytes.toByteBuffer: encode overflow (2)");
// if ( r.isUnderflow() )
// throw new InternalErrorException("Bytes.toByteBuffer: encode underflow") ;
int finish = bb.position();
return finish - start;
}
static void testMixed(Charset cs) throws Throwable {
CharsetDecoder dec =
cs.newDecoder()
.onMalformedInput(CodingErrorAction.REPLACE)
.onUnmappableCharacter(CodingErrorAction.REPLACE);
CharsetEncoder enc =
cs.newEncoder()
.onMalformedInput(CodingErrorAction.REPLACE)
.onUnmappableCharacter(CodingErrorAction.REPLACE);
List<Integer> cps = new ArrayList<>(0x10000);
int off = 0;
int cp = 0;
while (cp < 0x10000) {
if (enc.canEncode((char) cp)) {
cps.add(cp);
}
cp++;
}
Collections.shuffle(cps);
char[] bmpCA = new char[cps.size()];
for (int i = 0; i < cps.size(); i++) bmpCA[i] = (char) (int) cps.get(i);
String bmpStr = new String(bmpCA);
// getBytes(csn);
byte[] bmpBA = bmpStr.getBytes(cs.name());
ByteBuffer bf = enc.reset().encode(CharBuffer.wrap(bmpCA));
byte[] baNIO = new byte[bf.limit()];
bf.get(baNIO, 0, baNIO.length);
if (!Arrays.equals(bmpBA, baNIO)) {
throw new RuntimeException("getBytes(csn) failed -> " + cs.name());
}
// getBytes(cs);
bmpBA = bmpStr.getBytes(cs);
if (!Arrays.equals(bmpBA, baNIO))
throw new RuntimeException("getBytes(cs) failed -> " + cs.name());
// new String(csn);
String strSC = new String(bmpBA, cs.name());
String strNIO = dec.reset().decode(ByteBuffer.wrap(bmpBA)).toString();
if (!strNIO.equals(strSC)) {
throw new RuntimeException("new String(csn) failed -> " + cs.name());
}
// new String(cs);
strSC = new String(bmpBA, cs);
if (!strNIO.equals(strSC)) throw new RuntimeException("new String(cs) failed -> " + cs.name());
}
/**
* Encodes any String into a buffer to send it with a message
*
* @param content the String to encode into a buffer
* @return a buffer containing the (encoded) String.
*/
public static Buffer encodeString(String content) {
if (content == null) {
return null;
}
ByteBuffer byteBuffer;
synchronized (encoder) {
encoder.reset();
try {
byteBuffer = encoder.encode(CharBuffer.wrap(content));
} catch (CharacterCodingException e) {
return null;
}
encoder.flush(byteBuffer);
}
ByteBuf wrappedBuffer = Unpooled.wrappedBuffer(byteBuffer);
return new Buffer(wrappedBuffer);
}
private void fillBuffer() throws IOException {
_byteBuf.clear();
_charBuf.clear();
while (_byteBuf.position() == 0) {
fillCharBuf();
if (_charBuf.limit() == 0) {
_byteBuf.limit(0);
return;
}
_encoder.reset();
CoderResult rslt = _encoder.encode(_charBuf, _byteBuf, true);
// FIXME - optionally throw on malformed input
_encoder.flush(_byteBuf);
}
_byteBuf.limit(_byteBuf.position());
_byteBuf.position(0);
}
static CoderResult encodeCR(char[] cc, Charset cs, boolean testDirect) throws Exception {
ByteBuffer bbf;
CharBuffer cbf;
CharsetEncoder enc = cs.newEncoder();
if (testDirect) {
bbf = ByteBuffer.allocateDirect(cc.length * 4);
cbf = ByteBuffer.allocateDirect(cc.length * 2).asCharBuffer();
cbf.put(cc).flip();
} else {
bbf = ByteBuffer.allocate(cc.length * 4);
cbf = CharBuffer.wrap(cc);
}
CoderResult cr = null;
for (int i = 0; i < iteration; i++) {
cbf.rewind();
bbf.clear();
enc.reset();
cr = enc.encode(cbf, bbf, true);
}
return cr;
}
/**
* Write the UTF-8 representation of a single Unicode code point to the terminal output. The
* written data will be consumed by the emulation client as input.
*
* <p>This implementation encodes the code point and then calls {@link #write(byte[], int, int)}
* to do the actual writing. It should therefore usually be unnecessary to override this method;
* override {@link #write(byte[], int, int)} instead.
*
* @param codePoint The Unicode code point to write to the terminal.
*/
public void write(int codePoint) {
ByteBuffer byteBuf = mWriteByteBuffer;
if (codePoint < 128) {
// Fast path for ASCII characters
byte[] buf = byteBuf.array();
buf[0] = (byte) codePoint;
write(buf, 0, 1);
return;
}
CharBuffer charBuf = mWriteCharBuffer;
CharsetEncoder encoder = mUTF8Encoder;
charBuf.clear();
byteBuf.clear();
Character.toChars(codePoint, charBuf.array(), 0);
encoder.reset();
encoder.encode(charBuf, byteBuf, true);
encoder.flush(byteBuf);
write(byteBuf.array(), 0, byteBuf.position() - 1);
}
byte[] encode(char[] ca, int off, int len) {
int en = scale(len, ce.maxBytesPerChar());
byte[] ba = new byte[en];
if (len == 0) return ba;
if (ce instanceof ArrayEncoder) {
int blen = ((ArrayEncoder) ce).encode(ca, off, len, ba);
return safeTrim(ba, blen, cs, isTrusted);
} else {
ce.reset();
ByteBuffer bb = ByteBuffer.wrap(ba);
CharBuffer cb = CharBuffer.wrap(ca, off, len);
try {
CoderResult cr = ce.encode(cb, bb, true);
if (!cr.isUnderflow()) cr.throwException();
cr = ce.flush(bb);
if (!cr.isUnderflow()) cr.throwException();
} catch (CharacterCodingException x) {
// Substitution is always enabled,
// so this shouldn't happen
throw new Error(x);
}
return safeTrim(ba, bb.position(), cs, isTrusted);
}
}
/**
* Write out as many pages as possible.
*
* @param force Should we write non-full pages?
* @throws ShutdownException
*/
private void writeOutPages(boolean force) throws ShutdownException {
int sz = 0;
int size = EMPTY_SIZE + 3 * current_offset;
int currIdx = 0;
short totalStringsLength = 0;
string2index.clear();
boolean stopped = false;
for (int i = 0; i < current_offset; i++) {
String s = strings[i];
if (!string2index.containsKey(s)) {
string2index.put(s, null);
encoder.reset();
size += 2;
if (s != null) {
try {
int len = encoder.encode(CharBuffer.wrap(s)).limit();
if (len > Short.MAX_VALUE)
throw new ShutdownException("Cannot store strings larger than " + Short.MAX_VALUE);
string_lengths[currIdx++] = (short) s.length();
size += len;
totalStringsLength += len;
} catch (CharacterCodingException cce) {
throw new ShutdownException("Cannot encode string " + s + " in UTF-8");
}
} else string_lengths[currIdx++] = 0;
if (size >= PAGE_SIZE) {
sz = i;
stopped = true;
break;
}
}
}
if (!stopped) sz = current_offset;
if (!force && sz == current_offset && current_offset < MAX_EVENTS) return;
string2index.clear();
index2string.clear();
// Index of the next string we'll add to the buffer
short current_index = 0;
// Number of events we can write out
short writableEvents;
for (writableEvents = 0; writableEvents < sz; writableEvents++) {
// An event costs at least 3 bytes
String s = strings[writableEvents];
boolean newString = !string2index.containsKey(s);
if (newString) {
string2index.put(s, new Short(current_index));
index2string.add(s);
current_index++;
}
}
assert writableEvents > 0;
// Write the page at last
// Number of events
buffer.putShort((short) (writableEvents));
// The event type array
buffer.put(types, 0, writableEvents);
// The event string array
for (int i = 0; i < writableEvents; i++)
buffer.putShort(((Short) string2index.get(strings[i])).shortValue());
// The number of strings
buffer.putShort((short) (current_index));
buffer.putShort(totalStringsLength);
// The string contents
for (int i = 0; i < current_index; i++) {
String s = (String) index2string.get(i);
if (s == null || s.length() == 0) continue;
encoder.encode(CharBuffer.wrap(s), sbuf, true);
encoder.reset();
sbuf.flip();
buffer.put(sbuf);
sbuf.clear();
}
// The string lengths
buffer.asShortBuffer().put(string_lengths, 0, current_index);
buffer.position(buffer.position() + 2 * current_index);
try {
buffer.flip();
channel.write(buffer);
buffer.clear();
} catch (IOException ioe) {
throw new ShutdownException("IO Exception while storing SAXDOM data: " + ioe.getMessage());
}
for (int i = writableEvents; i < current_offset; i++) {
types[i - writableEvents] = types[i];
strings[i - writableEvents] = strings[i];
}
current_offset -= writableEvents;
if (current_offset > 0) writeOutPages(force);
}
public void startDocument() throws SAXException {
charBuff.clear();
byteBuff.clear();
charEncoder.reset();
}
/**
* Constructs a writer that encodes characters using the given encoder and writes the resulting
* bytes to the given channel.
*
* <p>The resulting stream will contain an internal output buffer of at least {@code minBufferCap}
* bytes. The stream's {@code write} methods will, as needed, flush the buffer by writing bytes to
* the underlying channel; if the channel is in non-blocking mode when bytes are to be written
* then an {@link IllegalBlockingModeException} will be thrown. The resulting stream will not
* otherwise be buffered. Closing the stream will in turn cause the channel to be closed.
*
* @param ch The channel to which bytes will be written
* @param enc The charset encoder to be used
* @param minBufferCap The minimum capacity of the internal byte buffer, or {@code -1} if an
* implementation-dependent default capacity is to be used
* @return A new writer
*/
public static Writer newWriter(WritableByteChannel ch, CharsetEncoder enc, int minBufferCap) {
Objects.requireNonNull(ch, "ch");
return StreamEncoder.forEncoder(ch, enc.reset(), minBufferCap);
}
protected long buildResponseStringBuf(String s, long r, final long preChain) {
if (s == null) {
return 0;
}
if (s.length() == 0) {
return -NGX_HTTP_NO_CONTENT;
}
CharsetEncoder charsetEncoder =
ThreadLocalCoders.encoderFor(DEFAULT_ENCODING)
.onMalformedInput(CodingErrorAction.REPLACE)
.onUnmappableCharacter(CodingErrorAction.REPLACE);
ByteBuffer bb = pickByteBuffer();
CharBuffer cb = CharBuffer.wrap((char[]) UNSAFE.getObject(s, STRING_CHAR_ARRAY_OFFSET));
charsetEncoder.reset();
CoderResult result = CoderResult.UNDERFLOW;
long first = 0;
long chain = preChain;
do {
result = charsetEncoder.encode(cb, bb, true);
if (result == CoderResult.OVERFLOW) {
bb.flip();
chain =
ngx_http_clojure_mem_build_temp_chain(
r, chain, bb.array(), BYTE_ARRAY_OFFSET, bb.remaining());
if (chain <= 0) {
return chain;
}
bb.clear();
if (first == 0) {
first = chain;
}
} else if (result == CoderResult.UNDERFLOW) {
break;
} else {
log.error("%s can not decode string : %s", result.toString(), s);
return -NGX_HTTP_INTERNAL_SERVER_ERROR;
}
} while (true);
while (charsetEncoder.flush(bb) == CoderResult.OVERFLOW) {
bb.flip();
chain =
ngx_http_clojure_mem_build_temp_chain(
r, chain, bb.array(), BYTE_ARRAY_OFFSET, bb.remaining());
if (chain <= 0) {
return chain;
}
if (first == 0) {
first = chain;
}
bb.clear();
}
if (bb.hasRemaining()) {
bb.flip();
chain =
ngx_http_clojure_mem_build_temp_chain(
r, chain, bb.array(), BYTE_ARRAY_OFFSET, bb.remaining());
if (chain <= 0) {
return chain;
}
if (first == 0) {
first = chain;
}
bb.clear();
}
return preChain == 0 ? first : chain;
}
static void test(Charset cs, char[] bmpCA, byte[] sbBA) throws Throwable {
String bmpStr = new String(bmpCA);
CharsetDecoder dec =
cs.newDecoder()
.onMalformedInput(CodingErrorAction.REPLACE)
.onUnmappableCharacter(CodingErrorAction.REPLACE);
CharsetEncoder enc =
cs.newEncoder()
.onMalformedInput(CodingErrorAction.REPLACE)
.onUnmappableCharacter(CodingErrorAction.REPLACE);
// getBytes(csn);
byte[] baSC = bmpStr.getBytes(cs.name());
ByteBuffer bf = enc.reset().encode(CharBuffer.wrap(bmpCA));
byte[] baNIO = new byte[bf.limit()];
bf.get(baNIO, 0, baNIO.length);
if (!Arrays.equals(baSC, baNIO))
throw new RuntimeException("getBytes(csn) failed -> " + cs.name());
// getBytes(cs);
baSC = bmpStr.getBytes(cs);
if (!Arrays.equals(baSC, baNIO))
throw new RuntimeException("getBytes(cs) failed -> " + cs.name());
// new String(csn);
String strSC = new String(sbBA, cs.name());
String strNIO = dec.reset().decode(ByteBuffer.wrap(sbBA)).toString();
if (!strNIO.equals(strSC))
throw new RuntimeException("new String(csn) failed -> " + cs.name());
// new String(cs);
strSC = new String(sbBA, cs);
if (!strNIO.equals(strSC)) throw new RuntimeException("new String(cs) failed -> " + cs.name());
// encode unmappable surrogates
if (enc instanceof sun.nio.cs.ArrayEncoder && cs.contains(Charset.forName("ASCII"))) {
if (cs.name().equals("UTF-8")
|| // utf8 handles surrogates
cs.name().equals("CESU-8")) // utf8 handles surrogates
return;
enc.replaceWith(new byte[] {(byte) 'A'});
sun.nio.cs.ArrayEncoder cae = (sun.nio.cs.ArrayEncoder) enc;
String str = "ab\uD800\uDC00\uD800\uDC00cd";
byte[] ba = new byte[str.length() - 2];
int n = cae.encode(str.toCharArray(), 0, str.length(), ba);
if (n != 6 || !"abAAcd".equals(new String(ba, cs.name())))
throw new RuntimeException("encode1(surrogates) failed -> " + cs.name());
ba = new byte[str.length()];
n = cae.encode(str.toCharArray(), 0, str.length(), ba);
if (n != 6 || !"abAAcd".equals(new String(ba, 0, n, cs.name())))
throw new RuntimeException("encode2(surrogates) failed -> " + cs.name());
str = "ab\uD800B\uDC00Bcd";
ba = new byte[str.length()];
n = cae.encode(str.toCharArray(), 0, str.length(), ba);
if (n != 8 || !"abABABcd".equals(new String(ba, 0, n, cs.name())))
throw new RuntimeException("encode3(surrogates) failed -> " + cs.name());
/* sun.nio.cs.ArrayDeEncoder works on the assumption that the
invoker (StringCoder) allocates enough output buf, utf8
and double-byte coder does not check the output buffer limit.
ba = new byte[str.length() - 1];
n = cae.encode(str.toCharArray(), 0, str.length(), ba);
if (n != 7 || !"abABABc".equals(new String(ba, 0, n, cs.name()))) {
throw new RuntimeException("encode4(surrogates) failed -> "
+ cs.name());
}
*/
}
}
