Skip to content

Commit cd9b60d

Browse files
chernserchernser
authored
Merge pull request #2814 from ClickHouse/03/31/26/fix_decimal_truncate
[client-v2] Fixed converting integers thru decimal
2 parents be67860 + e1a0be6 commit cd9b60d

11 files changed

Lines changed: 709 additions & 38 deletions

File tree

‎client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/NumberConverter.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -142,8 +142,10 @@ public static BigDecimal toBigDecimal(Object value) {
142142
return (BigDecimal) value;
143143
} else if (value instanceof BigInteger) {
144144
return new BigDecimal((BigInteger) value);
145+
} else if (value instanceof Byte || value instanceof Short || value instanceof Integer || value instanceof Long) {
146+
return BigDecimal.valueOf(((Number) value).longValue());
145147
} else if (value instanceof Number) {
146-
return BigDecimal.valueOf(((Number) value).doubleValue());
148+
return new BigDecimal(value.toString());
147149
} else if (value instanceof String) {
148150
return new BigDecimal((String) value);
149151
} else if (value instanceof Boolean) {

‎client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/SerializerUtils.java‎

Lines changed: 7 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@
5757
import static org.objectweb.asm.Opcodes.INVOKEVIRTUAL;
5858
import static org.objectweb.asm.Opcodes.RETURN;
5959

60+
@SuppressWarnings("deprecation")
6061
public class SerializerUtils {
6162

6263
public static void serializeData(OutputStream stream, Object value, ClickHouseColumn column) throws IOException {
@@ -505,10 +506,10 @@ private static void serializePrimitiveData(OutputStream stream, Object value, Cl
505506
BinaryStreamUtils.writeInt64(stream, convertToLong(value));
506507
break;
507508
case Int128:
508-
BinaryStreamUtils.writeInt128(stream, convertToBigInteger(value));
509+
BinaryStreamUtils.writeInt128(stream, NumberConverter.toBigInteger(value));
509510
break;
510511
case Int256:
511-
BinaryStreamUtils.writeInt256(stream, convertToBigInteger(value));
512+
BinaryStreamUtils.writeInt256(stream, NumberConverter.toBigInteger(value));
512513
break;
513514
case UInt8:
514515
BinaryStreamUtils.writeUnsignedInt8(stream, convertToInteger(value));
@@ -520,13 +521,13 @@ private static void serializePrimitiveData(OutputStream stream, Object value, Cl
520521
BinaryStreamUtils.writeUnsignedInt32(stream, convertToLong(value));
521522
break;
522523
case UInt64:
523-
BinaryStreamUtils.writeUnsignedInt64(stream, convertToLong(value));
524+
BinaryStreamUtils.writeUnsignedInt64(stream, NumberConverter.toBigInteger(value));
524525
break;
525526
case UInt128:
526-
BinaryStreamUtils.writeUnsignedInt128(stream, convertToBigInteger(value));
527+
BinaryStreamUtils.writeUnsignedInt128(stream, NumberConverter.toBigInteger(value));
527528
break;
528529
case UInt256:
529-
BinaryStreamUtils.writeUnsignedInt256(stream, convertToBigInteger(value));
530+
BinaryStreamUtils.writeUnsignedInt256(stream, NumberConverter.toBigInteger(value));
530531
break;
531532
case Float32:
532533
BinaryStreamUtils.writeFloat32(stream, (float) value);
@@ -539,7 +540,7 @@ private static void serializePrimitiveData(OutputStream stream, Object value, Cl
539540
case Decimal64:
540541
case Decimal128:
541542
case Decimal256:
542-
BinaryStreamUtils.writeDecimal(stream, convertToBigDecimal(value), column.getPrecision(), column.getScale());
543+
BinaryStreamUtils.writeDecimal(stream, NumberConverter.toBigDecimal(value), column.getPrecision(), column.getScale());
543544
break;
544545
case Bool:
545546
BinaryStreamUtils.writeBoolean(stream, (Boolean) value);
@@ -765,32 +766,6 @@ public static Long convertToLong(Object value) {
765766
}
766767
}
767768

768-
public static BigInteger convertToBigInteger(Object value) {
769-
if (value instanceof BigInteger) {
770-
return (BigInteger) value;
771-
} else if (value instanceof Number) {
772-
return BigInteger.valueOf(((Number) value).longValue());
773-
} else if (value instanceof String) {
774-
return new BigInteger((String) value);
775-
} else {
776-
throw new IllegalArgumentException("Cannot convert " + value + " to BigInteger");
777-
}
778-
}
779-
780-
public static BigDecimal convertToBigDecimal(Object value) {
781-
if (value instanceof BigDecimal) {
782-
return (BigDecimal) value;
783-
} else if (value instanceof BigInteger) {
784-
return new BigDecimal((BigInteger) value);
785-
} else if (value instanceof Number) {
786-
return BigDecimal.valueOf(((Number) value).doubleValue());
787-
} else if (value instanceof String) {
788-
return new BigDecimal((String) value);
789-
} else {
790-
throw new IllegalArgumentException("Cannot convert " + value + " to BigDecimal");
791-
}
792-
}
793-
794769
public static String convertToString(Object value) {
795770
return java.lang.String.valueOf(value);
796771
}

‎client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/ValueConverters.java‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,13 +6,16 @@
66
import java.math.BigInteger;
77
import java.net.MalformedURLException;
88
import java.net.URL;
9-
import java.sql.Date;
109
import java.sql.Time;
1110
import java.sql.Timestamp;
1211
import java.util.Collections;
1312
import java.util.Map;
1413
import java.util.function.Function;
1514

15+
/**
16+
* This class should be used to convert non-null value with known type.
17+
* All methods should be kept minimal without null or instanceOf checks.
18+
*/
1619
public final class ValueConverters {
1720

1821

@@ -210,7 +213,10 @@ public BigInteger convertNumberToBigInteger(Object value) {
210213
}
211214

212215
public BigDecimal convertNumberToBigDecimal(Object value) {
213-
return BigDecimal.valueOf(((Number) value).doubleValue());
216+
if (value instanceof Byte || value instanceof Short || value instanceof Integer || value instanceof Long) {
217+
return BigDecimal.valueOf(((Number)value).longValue());
218+
}
219+
return new BigDecimal(value.toString());
214220
}
215221

216222
// Date & Time converters

‎client-v2/src/test/java/com/clickhouse/client/api/data_formats/ClickHouseBinaryFormatReaderTest.java‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
import java.util.TimeZone;
2020
import java.util.function.Consumer;
2121

22+
@SuppressWarnings("deprecation")
2223
public class ClickHouseBinaryFormatReaderTest {
2324

2425
@Test
@@ -83,6 +84,66 @@ public void testReadingNumbers() throws IOException {
8384
}
8485
}
8586

87+
@Test
88+
public void testGetBigDecimalForIntegerWidths8To256() throws IOException {
89+
ByteArrayOutputStream out = new ByteArrayOutputStream();
90+
91+
String[] names = new String[] {
92+
"i8", "u8", "i16", "u16", "i32", "u32", "i64", "u64", "i128", "u128", "i256", "u256"
93+
};
94+
String[] types = new String[] {
95+
"Int8", "UInt8", "Int16", "UInt16", "Int32", "UInt32", "Int64", "UInt64",
96+
"Int128", "UInt128", "Int256", "UInt256"
97+
};
98+
99+
BinaryStreamUtils.writeVarInt(out, names.length);
100+
for (String name : names) {
101+
BinaryStreamUtils.writeString(out, name);
102+
}
103+
for (String type : types) {
104+
BinaryStreamUtils.writeString(out, type);
105+
}
106+
107+
BigInteger u64 = new BigInteger("18446744073709551615");
108+
BigInteger i128 = new BigInteger("-170141183460469231731687303715884105728");
109+
BigInteger u128 = new BigInteger("340282366920938463463374607431768211455");
110+
BigInteger i256 = new BigInteger("-57896044618658097711785492504343953926634992332820282019728792003956564819968");
111+
BigInteger u256 = new BigInteger("115792089237316195423570985008687907853269984665640564039457584007913129639935");
112+
113+
BinaryStreamUtils.writeInt8(out, -128);
114+
BinaryStreamUtils.writeUnsignedInt8(out, 255);
115+
BinaryStreamUtils.writeInt16(out, -32768);
116+
BinaryStreamUtils.writeUnsignedInt16(out, 65535);
117+
BinaryStreamUtils.writeInt32(out, Integer.MIN_VALUE);
118+
BinaryStreamUtils.writeUnsignedInt32(out, 4294967295L);
119+
BinaryStreamUtils.writeInt64(out, Long.MAX_VALUE);
120+
BinaryStreamUtils.writeUnsignedInt64(out, u64);
121+
BinaryStreamUtils.writeInt128(out, i128);
122+
BinaryStreamUtils.writeUnsignedInt128(out, u128);
123+
BinaryStreamUtils.writeInt256(out, i256);
124+
BinaryStreamUtils.writeUnsignedInt256(out, u256);
125+
126+
InputStream in = new ByteArrayInputStream(out.toByteArray());
127+
QuerySettings querySettings = new QuerySettings().setUseTimeZone(TimeZone.getTimeZone("UTC").toZoneId().getId());
128+
RowBinaryWithNamesAndTypesFormatReader reader =
129+
new RowBinaryWithNamesAndTypesFormatReader(in, querySettings, new BinaryStreamReader.CachingByteBufferAllocator());
130+
131+
reader.next();
132+
133+
Assert.assertEquals(reader.getBigDecimal("i8"), BigDecimal.valueOf(-128));
134+
Assert.assertEquals(reader.getBigDecimal("u8"), BigDecimal.valueOf(255));
135+
Assert.assertEquals(reader.getBigDecimal("i16"), BigDecimal.valueOf(-32768));
136+
Assert.assertEquals(reader.getBigDecimal("u16"), BigDecimal.valueOf(65535));
137+
Assert.assertEquals(reader.getBigDecimal("i32"), BigDecimal.valueOf(Integer.MIN_VALUE));
138+
Assert.assertEquals(reader.getBigDecimal("u32"), BigDecimal.valueOf(4294967295L));
139+
Assert.assertEquals(reader.getBigDecimal("i64"), BigDecimal.valueOf(Long.MAX_VALUE));
140+
Assert.assertEquals(reader.getBigDecimal("u64"), new BigDecimal(u64));
141+
Assert.assertEquals(reader.getBigDecimal("i128"), new BigDecimal(i128));
142+
Assert.assertEquals(reader.getBigDecimal("u128"), new BigDecimal(u128));
143+
Assert.assertEquals(reader.getBigDecimal("i256"), new BigDecimal(i256));
144+
Assert.assertEquals(reader.getBigDecimal("u256"), new BigDecimal(u256));
145+
}
146+
86147
@Test
87148
public void testReadingNumbersWithOverflow() throws IOException {
88149
ByteArrayOutputStream out = new ByteArrayOutputStream();

‎client-v2/src/test/java/com/clickhouse/client/api/data_formats/internal/NumberConverterTests.java‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
package com.clickhouse.client.api.data_formats.internal;
2+
3+
import org.testng.Assert;
4+
import org.testng.annotations.Test;
5+
6+
import java.math.BigDecimal;
7+
8+
public class NumberConverterTests {
9+
10+
@Test
11+
public void testToBigDecimalSupportsGenericNumberValues() {
12+
Assert.assertEquals(NumberConverter.toBigDecimal(new CustomNumber("1234567890.123456789")),
13+
new BigDecimal("1234567890.123456789"));
14+
}
15+
16+
@Test
17+
public void testToBigDecimalSupportsStringValues() {
18+
Assert.assertEquals(NumberConverter.toBigDecimal("98765.4321"), new BigDecimal("98765.4321"));
19+
}
20+
21+
@Test
22+
public void testToBigDecimalPreservesFractionalFloatBoundaries() {
23+
Assert.assertEquals(NumberConverter.toBigDecimal(0.0001f).compareTo(new BigDecimal("0.0001")), 0);
24+
Assert.assertEquals(NumberConverter.toBigDecimal(0.0256f).compareTo(new BigDecimal("0.0256")), 0);
25+
Assert.assertEquals(NumberConverter.toBigDecimal(6.5536f).compareTo(new BigDecimal("6.5536")), 0);
26+
Assert.assertEquals(NumberConverter.toBigDecimal(838.8608f).compareTo(new BigDecimal("838.8608")), 0);
27+
}
28+
29+
private static final class CustomNumber extends Number {
30+
private final BigDecimal value;
31+
32+
private CustomNumber(String value) {
33+
this.value = new BigDecimal(value);
34+
}
35+
36+
@Override
37+
public int intValue() {
38+
return value.intValue();
39+
}
40+
41+
@Override
42+
public long longValue() {
43+
return value.longValue();
44+
}
45+
46+
@Override
47+
public float floatValue() {
48+
return value.floatValue();
49+
}
50+
51+
@Override
52+
public double doubleValue() {
53+
return value.doubleValue();
54+
}
55+
56+
@Override
57+
public String toString() {
58+
return value.toPlainString();
59+
}
60+
}
61+
}

‎client-v2/src/test/java/com/clickhouse/client/api/data_formats/internal/SerializerUtilsPrimitiveSerializationTests.java‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
package com.clickhouse.client.api.data_formats.internal;
2+
3+
import com.clickhouse.client.api.data_formats.RowBinaryWithNamesAndTypesFormatReader;
4+
import com.clickhouse.client.api.query.QuerySettings;
5+
import com.clickhouse.data.ClickHouseColumn;
6+
import com.clickhouse.data.format.BinaryStreamUtils;
7+
import org.testng.Assert;
8+
import org.testng.annotations.Test;
9+
10+
import java.io.ByteArrayInputStream;
11+
import java.io.ByteArrayOutputStream;
12+
import java.io.IOException;
13+
import java.math.BigDecimal;
14+
import java.util.TimeZone;
15+
16+
public class SerializerUtilsPrimitiveSerializationTests {
17+
18+
@Test
19+
public void testSerializeIntegerTargetFromStringValue() throws IOException {
20+
RowBinaryWithNamesAndTypesFormatReader reader =
21+
serializeSingleValue("Int32", "123456");
22+
23+
reader.next();
24+
25+
Assert.assertEquals(reader.getInteger("value"), Integer.valueOf(123456));
26+
}
27+
28+
@Test
29+
public void testSerializeDecimalTargetFromStringValue() throws IOException {
30+
RowBinaryWithNamesAndTypesFormatReader reader =
31+
serializeSingleValue("Decimal64(3)", "123456.789");
32+
33+
reader.next();
34+
35+
Assert.assertEquals(reader.getBigDecimal("value"), new BigDecimal("123456.789"));
36+
}
37+
38+
@Test
39+
public void testSerializeDecimalTargetFromGenericNumberValue() throws IOException {
40+
RowBinaryWithNamesAndTypesFormatReader reader =
41+
serializeSingleValue("Decimal64(3)", new CustomNumber("987654.321"));
42+
43+
reader.next();
44+
45+
Assert.assertEquals(reader.getBigDecimal("value"), new BigDecimal("987654.321"));
46+
}
47+
48+
private RowBinaryWithNamesAndTypesFormatReader serializeSingleValue(String type, Object value)
49+
throws IOException {
50+
ByteArrayOutputStream out = new ByteArrayOutputStream();
51+
BinaryStreamUtils.writeVarInt(out, 1);
52+
BinaryStreamUtils.writeString(out, "value");
53+
BinaryStreamUtils.writeString(out, type);
54+
SerializerUtils.serializeData(out, value, ClickHouseColumn.of("value", type));
55+
56+
return new RowBinaryWithNamesAndTypesFormatReader(
57+
new ByteArrayInputStream(out.toByteArray()),
58+
new QuerySettings().setUseTimeZone(TimeZone.getTimeZone("UTC").toZoneId().getId()),
59+
new BinaryStreamReader.CachingByteBufferAllocator());
60+
}
61+
62+
private static final class CustomNumber extends Number {
63+
private final BigDecimal value;
64+
65+
private CustomNumber(String value) {
66+
this.value = new BigDecimal(value);
67+
}
68+
69+
@Override
70+
public int intValue() {
71+
return value.intValue();
72+
}
73+
74+
@Override
75+
public long longValue() {
76+
return value.longValue();
77+
}
78+
79+
@Override
80+
public float floatValue() {
81+
return value.floatValue();
82+
}
83+
84+
@Override
85+
public double doubleValue() {
86+
return value.doubleValue();
87+
}
88+
89+
@Override
90+
public String toString() {
91+
return value.toPlainString();
92+
}
93+
}
94+
}

0 commit comments

Comments
 (0)