Skip to content

FileStore message length fix #651

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
chrjohn merged 3 commits into from
Dec 7, 2023
Merged

FileStore message length fix #651

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
8bae003
650 - filestore message length fix
the-thing Jul 25, 2023
88ddb3d
650 - filestore message length fix
the-thing Oct 8, 2023
ddeb9bb
650 - filestore message length fix
the-thing Dec 7, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions quickfixj-base/src/main/java/org/quickfixj/CharsetSupport.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,10 @@ public static void setCharset(String charset) throws UnsupportedEncodingExceptio
CharsetSupport.isStringEquivalent = isStringEquivalent(charsetInstance);
}

public static void setDefaultCharset() throws UnsupportedEncodingException {
setCharset(getDefaultCharset());
}

public static String getCharset() {
return charset;
}
Expand Down
27 changes: 13 additions & 14 deletions quickfixj-core/src/main/java/quickfix/CachedFileStore.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import java.io.BufferedOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.EOFException;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
Expand Down Expand Up @@ -86,8 +87,6 @@ public class CachedFileStore implements MessageStore {

private FileOutputStream headerFileOutputStream;

private final String charsetEncoding = CharsetSupport.getCharset();

CachedFileStore(String path, SessionID sessionID, boolean syncWrites) throws IOException {
this.syncWrites = syncWrites;

Expand Down Expand Up @@ -308,16 +307,15 @@ public boolean get(int sequence, String message) {
throw new UnsupportedOperationException("not supported");
}

private String read(long offset, long size) throws IOException {
final byte[] data = new byte[(int) size];

messageFileReader.seek(offset);
if (messageFileReader.read(data) != size) {
throw new IOException("Truncated input while reading message: "
+ new String(data, charsetEncoding));
private String read(long offset, int size) throws IOException {
try {
final byte[] data = new byte[size];
messageFileReader.seek(offset);
messageFileReader.readFully(data);
return new String(data, CharsetSupport.getCharset());
} catch (EOFException eofe) { // can't read fully
throw new IOException("Truncated input while reading message: offset=" + offset + ", expected size=" + size, eofe);
}

return new String(data, charsetEncoding);
}

private Collection<String> getMessage(long startSequence, long endSequence) throws IOException {
Expand All @@ -326,7 +324,7 @@ private Collection<String> getMessage(long startSequence, long endSequence) thro
final List<long[]> offsetAndSizes = messageIndex.get(startSequence, endSequence);
for (final long[] offsetAndSize : offsetAndSizes) {
if (offsetAndSize != null) {
final String message = read(offsetAndSize[0], offsetAndSize[1]);
final String message = read(offsetAndSize[0], (int) offsetAndSize[1]);
messages.add(message);
}
}
Expand All @@ -341,7 +339,8 @@ private Collection<String> getMessage(long startSequence, long endSequence) thro
*/
public boolean set(int sequence, String message) throws IOException {
final long offset = messageFileWriter.getFilePointer();
final int size = message.length();
final byte[] messageBytes = message.getBytes(CharsetSupport.getCharset());
final int size = messageBytes.length;
messageIndex.put((long) sequence, new long[] { offset, size });
headerDataOutputStream.writeInt(sequence);
headerDataOutputStream.writeLong(offset);
Expand All @@ -350,7 +349,7 @@ public boolean set(int sequence, String message) throws IOException {
if (syncWrites) {
headerFileOutputStream.getFD().sync();
}
messageFileWriter.write(message.getBytes(CharsetSupport.getCharset()));
messageFileWriter.write(messageBytes);
return true;
}

Expand Down
8 changes: 4 additions & 4 deletions quickfixj-core/src/main/java/quickfix/FileStore.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,6 @@ public class FileStore implements MessageStore, Closeable {
private final String sessionFileName;
private final boolean syncWrites;
private final int maxCachedMsgs;
private final String charsetEncoding = CharsetSupport.getCharset();
private RandomAccessFile messageFileReader;
private RandomAccessFile messageFileWriter;
private DataOutputStream headerDataOutputStream;
Expand Down Expand Up @@ -350,7 +349,7 @@ private String getMessage(long offset, int size, int i) throws IOException {
final byte[] data = new byte[size];
messageFileReader.seek(offset);
messageFileReader.readFully(data);
return new String(data, charsetEncoding);
return new String(data, CharsetSupport.getCharset());
} catch (EOFException eofe) { // can't read fully
throw new IOException("Truncated input while reading message: messageIndex=" + i
+ ", offset=" + offset + ", expected size=" + size, eofe);
Expand All @@ -363,7 +362,8 @@ private String getMessage(long offset, int size, int i) throws IOException {
@Override
public boolean set(int sequence, String message) throws IOException {
final long offset = messageFileWriter.getFilePointer();
final int size = message.length();
final byte[] messageBytes = message.getBytes(CharsetSupport.getCharset());
final int size = messageBytes.length;
if (messageIndex != null) {
updateMessageIndex(sequence, offset, size);
}
Expand All @@ -374,7 +374,7 @@ public boolean set(int sequence, String message) throws IOException {
if (syncWrites) {
headerFileOutputStream.getFD().sync();
}
messageFileWriter.write(message.getBytes(CharsetSupport.getCharset()));
messageFileWriter.write(messageBytes);
return true;
}

Expand Down
41 changes: 41 additions & 0 deletions quickfixj-core/src/test/java/quickfix/AbstractMessageStoreTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,14 @@
package quickfix;

import junit.framework.TestCase;
import org.quickfixj.CharsetSupport;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

public abstract class AbstractMessageStoreTest extends TestCase {

private SessionID sessionID;
private MessageStore store;

Expand Down Expand Up @@ -167,6 +170,44 @@ public void testRefreshMessageStore() throws Exception {
}
}

public void testSetAndGetMessageWithAsciiCharacters() throws IOException {
MessageStore underTest = getStore();

if (underTest instanceof SleepycatStore) {
return;
}

underTest.set(1, "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVXYZ0123456789");

List<String> messages = new ArrayList<>();
underTest.get(1, 1, messages);

assertEquals(1, messages.size());
assertEquals("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVXYZ0123456789", messages.get(0));
}

public void testSetAndGetMessageWithUnicodeCharacters() throws IOException {
MessageStore underTest = getStore();

if (underTest instanceof SleepycatStore) {
return;
}

CharsetSupport.setCharset("UTF-8");

try {
underTest.set(1, "a \u00A9 \u2603 \uD834\uDF06");

List<String> messages = new ArrayList<>();
underTest.get(1, 1, messages);

assertEquals(1, messages.size());
assertEquals("a \u00A9 \u2603 \uD834\uDF06", messages.get(0));
} finally {
CharsetSupport.setDefaultCharset();
}
}

protected void closeMessageStore(MessageStore store) throws IOException {
// does nothing, by default
}
Expand Down
191 changes: 188 additions & 3 deletions quickfixj-core/src/test/java/quickfix/FieldTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,13 @@

package quickfix;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;

import java.io.UnsupportedEncodingException;
import java.math.BigDecimal;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.LocalTime;
import java.util.Arrays;
import java.util.Date;

import org.junit.Test;
import org.quickfixj.CharsetSupport;
Expand All @@ -39,6 +41,10 @@
import quickfix.fix50.MarketDataIncrementalRefresh;
import quickfix.fix50.NewOrderSingle;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;

public class FieldTest {

@Test
Expand All @@ -52,6 +58,185 @@ public void testMessageSetGetString() {
assertEquals(side1, side2);
}

private void testFieldCalcuations(String value, int checksum, int length) {
Field<String> field = new Field<>(12, value);
field.setObject(value);
assertEquals("12=" + value, field.toString());
assertEquals(checksum, field.getChecksum());
assertEquals(length, field.getLength());

value = value.substring(0, value.length() - 1) + (char)(value.charAt(value.length() - 1) + 1);
checksum = (checksum + 1) & 0xFF;
field.setObject(value);
assertEquals("12=" + value, field.toString());
assertEquals(checksum, field.getChecksum());
assertEquals(length, field.getLength());

field.setTag(13);
checksum = (checksum + 1) & 0xFF;
assertEquals("13=" + value, field.toString());
assertEquals(checksum, field.getChecksum());
assertEquals(length, field.getLength());
}

@Test
public void testFieldCalculationsWithDefaultCharset() {
testFieldCalcuations("VALUE", 30, 9);
}

@Test
public void testFieldCalculationsWithUTF8Charset() throws UnsupportedEncodingException {
CharsetSupport.setCharset("UTF-8");
try {
testFieldCalcuations("\u6D4B\u9A8C\u6570\u636E", 50, 16);
} finally {
CharsetSupport.setDefaultCharset();
}
}

@Test
public void testDateField() {
DateField field = new DateField(11);
Date date = new Date();
field.setValue(date);
assertEquals(11, field.getTag());
assertEquals(date, field.getValue());
field = new DateField(11, date);
assertEquals(11, field.getTag());
assertEquals(date, field.getValue());
}

@Test
public void testUtcDateOnlyField() {
UtcDateOnlyField field = new UtcDateOnlyField(11);
LocalDate date = LocalDate.now();
field.setValue(date);
assertEquals(11, field.getTag());
assertEquals(date, field.getValue());
field = new UtcDateOnlyField(11, date);
assertEquals(11, field.getTag());
assertEquals(date, field.getValue());
}

@Test
public void testUtcTimeOnlyField() {
UtcTimeOnlyField field = new UtcTimeOnlyField(11);
LocalTime date = LocalTime.now();
field.setValue(date);
assertEquals(11, field.getTag());
assertEquals(date, field.getValue());
field = new UtcTimeOnlyField(11, date);
assertEquals(11, field.getTag());
assertEquals(date, field.getValue());
}

@Test
public void testUtcTimeStampField() {
UtcTimeStampField field = new UtcTimeStampField(11);
LocalDateTime date = LocalDateTime.now();
field.setValue(date);
assertEquals(11, field.getTag());
assertEquals(date, field.getValue());
field = new UtcTimeStampField(11, date);
assertEquals(11, field.getTag());
assertEquals(date, field.getValue());
}

@Test
public void testBooleanField() {
BooleanField field = new BooleanField(11);
field.setValue(true);
assertEquals(11, field.getTag());
assertTrue(field.getValue());
field.setValue(Boolean.FALSE);
assertEquals(11, field.getTag());
assertFalse(field.getValue());
field = new BooleanField(22, true);
assertEquals(22, field.getTag());
assertTrue(field.getValue());
field = new BooleanField(33, Boolean.TRUE);
assertEquals(33, field.getTag());
assertTrue(field.getValue());
}

@Test
public void testDoubleField() {
DoubleField field = new DoubleField(11);
field.setValue(12.3);
assertEquals(11, field.getTag());
assertEquals(12.3, field.getValue(), 0);
field.setValue(new Double(23.4));
assertEquals(11, field.getTag());
assertEquals(23.4, field.getValue(), 0);
field = new DoubleField(22, 34.5);
assertEquals(22, field.getTag());
assertEquals(34.5, field.getValue(), 0);
field = new DoubleField(33, new Double(45.6));
assertEquals(33, field.getTag());
assertEquals(45.6, field.getValue(), 0);
}

@Test(expected = NumberFormatException.class)
public void testDoubleFieldException() {
DoubleField field = new DoubleField(11, Double.NaN);
}

@Test
public void testDecimalField() {
DecimalField field = new DecimalField(11);
field.setValue(12.3);
assertEquals(11, field.getTag());
assertEquals(BigDecimal.valueOf(12.3), field.getValue());
field.setValue(23.4);
assertEquals(11, field.getTag());
assertEquals(BigDecimal.valueOf(23.4), field.getValue());
field = new DecimalField(22, 34.5);
assertEquals(22, field.getTag());
assertEquals(BigDecimal.valueOf(34.5), field.getValue());
field = new DecimalField(33, 45.6);
assertEquals(33, field.getTag());
assertEquals(BigDecimal.valueOf(45.6), field.getValue());
}

@Test(expected = NumberFormatException.class)
public void testDecimalFieldException() {
DecimalField field = new DecimalField(11, Double.POSITIVE_INFINITY);
}

@Test
public void testCharField() {
CharField field = new CharField(11);
field.setValue('x');
assertEquals(11, field.getTag());
assertEquals('x', field.getValue());
field.setValue(Character.valueOf('X'));
assertEquals(11, field.getTag());
assertEquals('X', field.getValue());
field = new CharField(22, 'a');
assertEquals(22, field.getTag());
assertEquals('a', field.getValue());
field = new CharField(33, Character.valueOf('A'));
assertEquals(33, field.getTag());
assertEquals('A', field.getValue());
}

@Test
public void testIntField() {
IntField field = new IntField(11);
field.setValue(12);
assertEquals(11, field.getTag());
assertEquals(12, field.getValue());
field.setValue(Integer.valueOf(23));
assertEquals(11, field.getTag());
assertEquals(23, field.getValue());
field = new IntField(22, 23);
assertEquals(22, field.getTag());
assertEquals(23, field.getValue());
field = new IntField(33, Integer.valueOf(44));
assertEquals(33, field.getTag());
assertEquals(44, field.getValue());
}

@Test
public void testBytesField() {
byte[] data = "rawdata".getBytes();
Expand Down
Loading