From 22c667c003e705613c202355a8791978d790591e Mon Sep 17 00:00:00 2001 From: chutium Date: Mon, 1 Sep 2014 18:25:23 +0200 Subject: [PATCH 1/3] add PrimitiveTypes support for CSVSerDe --- .../com/bizo/hive/serde/csv/CSVSerde.java | 124 ++++++++++++------ .../com/bizo/hive/serde/csv/CSVSerdeTest.java | 57 ++++---- 2 files changed, 114 insertions(+), 67 deletions(-) diff --git a/src/main/java/com/bizo/hive/serde/csv/CSVSerde.java b/src/main/java/com/bizo/hive/serde/csv/CSVSerde.java index 82fceb9..379b9dc 100644 --- a/src/main/java/com/bizo/hive/serde/csv/CSVSerde.java +++ b/src/main/java/com/bizo/hive/serde/csv/CSVSerde.java @@ -23,71 +23,71 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; +import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Writable; import au.com.bytecode.opencsv.CSVReader; import au.com.bytecode.opencsv.CSVWriter; - /** * CSVSerde uses opencsv (http://opencsv.sourceforge.net/) to serialize/deserialize columns as CSV. - * + * * @author Larry Ogrodnek */ public final class CSVSerde implements SerDe { - + private ObjectInspector inspector; private String[] outputFields; private int numCols; - private List row; - + private List row; + private List columnTypes; + private char separatorChar; private char quoteChar; private char escapeChar; - - + @Override public void initialize(final Configuration conf, final Properties tbl) throws SerDeException { final List columnNames = Arrays.asList(tbl.getProperty(Constants.LIST_COLUMNS).split(",")); - final List columnTypes = TypeInfoUtils.getTypeInfosFromTypeString(tbl.getProperty(Constants.LIST_COLUMN_TYPES)); - + columnTypes = TypeInfoUtils.getTypeInfosFromTypeString(tbl.getProperty(Constants.LIST_COLUMN_TYPES)); + numCols = columnNames.size(); - + final List columnOIs = new ArrayList(numCols); - + for (int i=0; i< numCols; i++) { - columnOIs.add(PrimitiveObjectInspectorFactory.javaStringObjectInspector); + columnOIs.add(TypeInfoUtils.getStandardJavaObjectInspectorFromTypeInfo(columnTypes.get(i))); } - - this.inspector = ObjectInspectorFactory.getStandardStructObjectInspector(columnNames, columnOIs); - this.outputFields = new String[numCols]; - row = new ArrayList(numCols); - + + inspector = ObjectInspectorFactory.getStandardStructObjectInspector(columnNames, columnOIs); + outputFields = new String[numCols]; + row = new ArrayList(numCols); + for (int i=0; i< numCols; i++) { row.add(null); } - + separatorChar = getProperty(tbl, "separatorChar", CSVWriter.DEFAULT_SEPARATOR); quoteChar = getProperty(tbl, "quoteChar", CSVWriter.DEFAULT_QUOTE_CHARACTER); escapeChar = getProperty(tbl, "escapeChar", CSVWriter.DEFAULT_ESCAPE_CHARACTER); } - + private final char getProperty(final Properties tbl, final String property, final char def) { final String val = tbl.getProperty(property); - + if (val != null) { return val.charAt(0); } - + return def; } - + @Override public Writable serialize(Object obj, ObjectInspector objInspector) throws SerDeException { final StructObjectInspector outputRowOI = (StructObjectInspector) objInspector; final List outputFieldRefs = outputRowOI.getAllStructFieldRefs(); - + if (outputFieldRefs.size() != numCols) { throw new SerDeException("Cannot serialize the object because there are " + outputFieldRefs.size() + " fields but the table has " + numCols + " columns."); @@ -97,45 +97,83 @@ public Writable serialize(Object obj, ObjectInspector objInspector) throws SerDe for (int c = 0; c < numCols; c++) { final Object field = outputRowOI.getStructFieldData(obj, outputFieldRefs.get(c)); final ObjectInspector fieldOI = outputFieldRefs.get(c).getFieldObjectInspector(); - + // The data must be of type String + // TODO: need to add switch case too final StringObjectInspector fieldStringOI = (StringObjectInspector) fieldOI; - + // Convert the field to Java class String, because objects of String type // can be stored in String, Text, or some other classes. outputFields[c] = fieldStringOI.getPrimitiveJavaObject(field); } - + final StringWriter writer = new StringWriter(); final CSVWriter csv = newWriter(writer, separatorChar, quoteChar, escapeChar); - + try { csv.writeNext(outputFields); csv.close(); - + return new Text(writer.toString()); } catch (final IOException ioe) { throw new SerDeException(ioe); } - } + } @Override public Object deserialize(final Writable blob) throws SerDeException { Text rowText = (Text) blob; - + CSVReader csv = null; try { - csv = newReader(new CharArrayReader(rowText.toString().toCharArray()), separatorChar, quoteChar, escapeChar); + csv = newReader(new CharArrayReader(rowText.toString().toCharArray()), separatorChar, quoteChar, escapeChar); final String[] read = csv.readNext(); - + for (int i=0; i< numCols; i++) { - if (read != null && i < read.length) { - row.set(i, read[i]); - } else { - row.set(i, null); + if (columnTypes.get(i).getCategory() == ObjectInspector.Category.PRIMITIVE) { + PrimitiveTypeInfo columnType = (PrimitiveTypeInfo) columnTypes.get(i); + if (read != null && i < read.length) { + switch (columnType.getPrimitiveCategory()) { + case BOOLEAN: + row.set(i, Boolean.valueOf(read[i])); + break; + case BYTE: + row.set(i, Byte.valueOf(read[i])); + break; + case DOUBLE: + row.set(i, Double.valueOf(read[i])); + break; + case FLOAT: + row.set(i, Float.valueOf(read[i])); + break; + case INT: + row.set(i, Integer.valueOf(read[i])); + break; + case LONG: + row.set(i, Long.valueOf(read[i])); + break; + case SHORT: + row.set(i, Short.valueOf(read[i])); + break; + case TIMESTAMP: + row.set(i, java.sql.Timestamp.valueOf(read[i])); + break; + case STRING: + row.set(i, read[i]); + break; + case VOID: + row.set(i, null); + break; + case UNKNOWN: + default: + throw new RuntimeException("Unknown PrimitiveType"); + } + } else { + row.set(i, null); + } } } - + return row; } catch (final Exception e) { throw new SerDeException(e); @@ -149,22 +187,22 @@ public Object deserialize(final Writable blob) throws SerDeException { } } } - + private CSVReader newReader(final Reader reader, char separator, char quote, char escape) { - // CSVReader will throw an exception if any of separator, quote, or escape is the same, but + // CSVReader will throw an exception if any of separator, quote, or escape is the same, but // the CSV format specifies that the escape character and quote char are the same... very weird if (CSVWriter.DEFAULT_ESCAPE_CHARACTER == escape) { return new CSVReader(reader, separator, quote); } else { - return new CSVReader(reader, separator, quote, escape); + return new CSVReader(reader, separator, quote, escape); } } - + private CSVWriter newWriter(final Writer writer, char separator, char quote, char escape) { if (CSVWriter.DEFAULT_ESCAPE_CHARACTER == escape) { return new CSVWriter(writer, separator, quote, ""); } else { - return new CSVWriter(writer, separator, quote, escape, ""); + return new CSVWriter(writer, separator, quote, escape, ""); } } @@ -177,7 +215,7 @@ public ObjectInspector getObjectInspector() throws SerDeException { public Class getSerializedClass() { return Text.class; } - + public SerDeStats getSerDeStats() { return null; } diff --git a/src/test/java/com/bizo/hive/serde/csv/CSVSerdeTest.java b/src/test/java/com/bizo/hive/serde/csv/CSVSerdeTest.java index 9e247dc..a4a0fc7 100644 --- a/src/test/java/com/bizo/hive/serde/csv/CSVSerdeTest.java +++ b/src/test/java/com/bizo/hive/serde/csv/CSVSerdeTest.java @@ -12,54 +12,63 @@ public final class CSVSerdeTest { private final CSVSerde csv = new CSVSerde(); - final Properties props = new Properties(); - + + final Properties new_props = new Properties(); + final Properties props = new Properties(); + @Before public void setup() throws Exception { + new_props.put(Constants.LIST_COLUMNS, "a,b,c,d,e,f,g,h,i"); + new_props.put(Constants.LIST_COLUMN_TYPES, "string,string,int,bigint,float,double,timestamp,timestamp,void"); props.put(Constants.LIST_COLUMNS, "a,b,c"); - props.put(Constants.LIST_COLUMN_TYPES, "string,string,string"); + props.put(Constants.LIST_COLUMN_TYPES, "string,string,int"); } - + @Test public void testDeserialize() throws Exception { - csv.initialize(null, props); - final Text in = new Text("hello,\"yes, okay\",1"); - - final List row = (List) csv.deserialize(in); + csv.initialize(null, new_props); + final Text in = new Text("hello,\"yes, okay\",1,\"1409576461919\",100.05,\"1024.19002010\",\"2014-04-14 00:03:49\",\"2014-04-14 00:09:42.228000\","); + + final List row = (List) csv.deserialize(in); assertEquals("hello", row.get(0)); assertEquals("yes, okay", row.get(1)); - assertEquals("1", row.get(2)); + assertEquals(1, row.get(2)); + assertEquals(1409576461919L, row.get(3)); + assertEquals(Float.valueOf("100.05"), row.get(4)); + assertEquals((Double) 1024.19002010, row.get(5)); + assertEquals(java.sql.Timestamp.valueOf("2014-04-14 00:03:49"), row.get(6)); + assertEquals(java.sql.Timestamp.valueOf("2014-04-14 00:09:42.228"), row.get(7)); + assertEquals(null, row.get(8)); } - - + @Test public void testDeserializeCustomSeparators() throws Exception { props.put("separatorChar", "\t"); props.put("quoteChar", "'"); - + csv.initialize(null, props); - + final Text in = new Text("hello\t'yes\tokay'\t1"); - final List row = (List) csv.deserialize(in); - + final List row = (List) csv.deserialize(in); + assertEquals("hello", row.get(0)); - assertEquals("yes\tokay", row.get(1)); - assertEquals("1", row.get(2)); + assertEquals("yes\tokay", row.get(1)); + assertEquals(1, row.get(2)); } - + @Test public void testDeserializeCustomEscape() throws Exception { props.put("quoteChar", "'"); props.put("escapeChar", "\\"); - + csv.initialize(null, props); - + final Text in = new Text("hello,'yes\\'okay',1"); - final List row = (List) csv.deserialize(in); - + final List row = (List) csv.deserialize(in); + assertEquals("hello", row.get(0)); assertEquals("yes'okay", row.get(1)); - assertEquals("1", row.get(2)); - } + assertEquals(1, row.get(2)); + } } From 4186676cb3492337e44c5fcb773c7604d3d8970b Mon Sep 17 00:00:00 2001 From: chutium Date: Mon, 6 Oct 2014 20:19:49 +0200 Subject: [PATCH 2/3] check empty string before parsing numbers --- .../com/bizo/hive/serde/csv/CSVSerde.java | 43 +++-- .../com/bizo/hive/serde/csv/CSVSerdeTest.java | 149 +++++++++--------- 2 files changed, 109 insertions(+), 83 deletions(-) diff --git a/src/main/java/com/bizo/hive/serde/csv/CSVSerde.java b/src/main/java/com/bizo/hive/serde/csv/CSVSerde.java index 379b9dc..3fc0b01 100644 --- a/src/main/java/com/bizo/hive/serde/csv/CSVSerde.java +++ b/src/main/java/com/bizo/hive/serde/csv/CSVSerde.java @@ -133,33 +133,58 @@ public Object deserialize(final Writable blob) throws SerDeException { if (columnTypes.get(i).getCategory() == ObjectInspector.Category.PRIMITIVE) { PrimitiveTypeInfo columnType = (PrimitiveTypeInfo) columnTypes.get(i); if (read != null && i < read.length) { + String val = read[i].trim(); switch (columnType.getPrimitiveCategory()) { case BOOLEAN: - row.set(i, Boolean.valueOf(read[i])); + if (val.length() == 0) + row.set(i, null); + else + row.set(i, Boolean.valueOf(val)); break; case BYTE: - row.set(i, Byte.valueOf(read[i])); + if (val.length() == 0) + row.set(i, null); + else + row.set(i, Byte.valueOf(val)); break; case DOUBLE: - row.set(i, Double.valueOf(read[i])); + if (val.length() == 0) + row.set(i, 0); + else + row.set(i, Double.valueOf(val)); break; case FLOAT: - row.set(i, Float.valueOf(read[i])); + if (val.length() == 0) + row.set(i, 0); + else + row.set(i, Float.valueOf(val)); break; case INT: - row.set(i, Integer.valueOf(read[i])); + if (val.length() == 0) + row.set(i, 0); + else + row.set(i, Integer.valueOf(val)); break; case LONG: - row.set(i, Long.valueOf(read[i])); + if (val.length() == 0) + row.set(i, 0); + else + row.set(i, Long.valueOf(val)); break; case SHORT: - row.set(i, Short.valueOf(read[i])); + if (val.length() == 0) + row.set(i, 0); + else + row.set(i, Short.valueOf(val)); break; case TIMESTAMP: - row.set(i, java.sql.Timestamp.valueOf(read[i])); + if (val.length() == 0) + row.set(i, null); + else + row.set(i, java.sql.Timestamp.valueOf(val)); break; case STRING: - row.set(i, read[i]); + row.set(i, val); break; case VOID: row.set(i, null); diff --git a/src/test/java/com/bizo/hive/serde/csv/CSVSerdeTest.java b/src/test/java/com/bizo/hive/serde/csv/CSVSerdeTest.java index a4a0fc7..d256ffb 100644 --- a/src/test/java/com/bizo/hive/serde/csv/CSVSerdeTest.java +++ b/src/test/java/com/bizo/hive/serde/csv/CSVSerdeTest.java @@ -1,74 +1,75 @@ -package com.bizo.hive.serde.csv; - -import java.util.List; -import java.util.Properties; - -import org.apache.hadoop.hive.serde.Constants; -import org.apache.hadoop.io.Text; -import org.junit.Before; -import org.junit.Test; -import static org.junit.Assert.assertEquals; - - -public final class CSVSerdeTest { - private final CSVSerde csv = new CSVSerde(); - - final Properties new_props = new Properties(); - final Properties props = new Properties(); - - @Before - public void setup() throws Exception { - new_props.put(Constants.LIST_COLUMNS, "a,b,c,d,e,f,g,h,i"); - new_props.put(Constants.LIST_COLUMN_TYPES, "string,string,int,bigint,float,double,timestamp,timestamp,void"); - props.put(Constants.LIST_COLUMNS, "a,b,c"); - props.put(Constants.LIST_COLUMN_TYPES, "string,string,int"); - } - - @Test - public void testDeserialize() throws Exception { - csv.initialize(null, new_props); - final Text in = new Text("hello,\"yes, okay\",1,\"1409576461919\",100.05,\"1024.19002010\",\"2014-04-14 00:03:49\",\"2014-04-14 00:09:42.228000\","); - - final List row = (List) csv.deserialize(in); - - assertEquals("hello", row.get(0)); - assertEquals("yes, okay", row.get(1)); - assertEquals(1, row.get(2)); - assertEquals(1409576461919L, row.get(3)); - assertEquals(Float.valueOf("100.05"), row.get(4)); - assertEquals((Double) 1024.19002010, row.get(5)); - assertEquals(java.sql.Timestamp.valueOf("2014-04-14 00:03:49"), row.get(6)); - assertEquals(java.sql.Timestamp.valueOf("2014-04-14 00:09:42.228"), row.get(7)); - assertEquals(null, row.get(8)); - } - - @Test - public void testDeserializeCustomSeparators() throws Exception { - props.put("separatorChar", "\t"); - props.put("quoteChar", "'"); - - csv.initialize(null, props); - - final Text in = new Text("hello\t'yes\tokay'\t1"); - final List row = (List) csv.deserialize(in); - - assertEquals("hello", row.get(0)); - assertEquals("yes\tokay", row.get(1)); - assertEquals(1, row.get(2)); - } - - @Test - public void testDeserializeCustomEscape() throws Exception { - props.put("quoteChar", "'"); - props.put("escapeChar", "\\"); - - csv.initialize(null, props); - - final Text in = new Text("hello,'yes\\'okay',1"); - final List row = (List) csv.deserialize(in); - - assertEquals("hello", row.get(0)); - assertEquals("yes'okay", row.get(1)); - assertEquals(1, row.get(2)); - } -} +package com.bizo.hive.serde.csv; + +import java.util.List; +import java.util.Properties; + +import org.apache.hadoop.hive.serde.Constants; +import org.apache.hadoop.io.Text; +import org.junit.Before; +import org.junit.Test; +import static org.junit.Assert.assertEquals; + + +public final class CSVSerdeTest { + private final CSVSerde csv = new CSVSerde(); + + final Properties new_props = new Properties(); + final Properties props = new Properties(); + + @Before + public void setup() throws Exception { + new_props.put(Constants.LIST_COLUMNS, "a,b,c,d,e,f,g,h,i,j"); + new_props.put(Constants.LIST_COLUMN_TYPES, "string,string,int,bigint,float,double,timestamp,timestamp,void,int"); + props.put(Constants.LIST_COLUMNS, "a,b,c"); + props.put(Constants.LIST_COLUMN_TYPES, "string,string,int"); + } + + @Test + public void testDeserialize() throws Exception { + csv.initialize(null, new_props); + final Text in = new Text("hello,\"yes, okay\",1,\"1409576461919\",100.05,\"1024.19002010\",\"2014-04-14 00:03:49\",\"2014-04-14 00:09:42.228000\",,"); + + final List row = (List) csv.deserialize(in); + + assertEquals("hello", row.get(0)); + assertEquals("yes, okay", row.get(1)); + assertEquals(1, row.get(2)); + assertEquals(1409576461919L, row.get(3)); + assertEquals(Float.valueOf("100.05"), row.get(4)); + assertEquals((Double) 1024.19002010, row.get(5)); + assertEquals(java.sql.Timestamp.valueOf("2014-04-14 00:03:49"), row.get(6)); + assertEquals(java.sql.Timestamp.valueOf("2014-04-14 00:09:42.228"), row.get(7)); + assertEquals(null, row.get(8)); + assertEquals(0, row.get(9)); + } + + @Test + public void testDeserializeCustomSeparators() throws Exception { + props.put("separatorChar", "\t"); + props.put("quoteChar", "'"); + + csv.initialize(null, props); + + final Text in = new Text("hello\t'yes\tokay'\t1"); + final List row = (List) csv.deserialize(in); + + assertEquals("hello", row.get(0)); + assertEquals("yes\tokay", row.get(1)); + assertEquals(1, row.get(2)); + } + + @Test + public void testDeserializeCustomEscape() throws Exception { + props.put("quoteChar", "'"); + props.put("escapeChar", "\\"); + + csv.initialize(null, props); + + final Text in = new Text("hello,'yes\\'okay',1"); + final List row = (List) csv.deserialize(in); + + assertEquals("hello", row.get(0)); + assertEquals("yes'okay", row.get(1)); + assertEquals(1, row.get(2)); + } +} From 2bea36bd958639bf5b99859abef52ddf337649c5 Mon Sep 17 00:00:00 2001 From: chutium Date: Mon, 6 Oct 2014 21:04:34 +0200 Subject: [PATCH 3/3] fix EOL... --- .../com/bizo/hive/serde/csv/CSVSerdeTest.java | 150 +++++++++--------- 1 file changed, 75 insertions(+), 75 deletions(-) diff --git a/src/test/java/com/bizo/hive/serde/csv/CSVSerdeTest.java b/src/test/java/com/bizo/hive/serde/csv/CSVSerdeTest.java index d256ffb..857a001 100644 --- a/src/test/java/com/bizo/hive/serde/csv/CSVSerdeTest.java +++ b/src/test/java/com/bizo/hive/serde/csv/CSVSerdeTest.java @@ -1,75 +1,75 @@ -package com.bizo.hive.serde.csv; - -import java.util.List; -import java.util.Properties; - -import org.apache.hadoop.hive.serde.Constants; -import org.apache.hadoop.io.Text; -import org.junit.Before; -import org.junit.Test; -import static org.junit.Assert.assertEquals; - - -public final class CSVSerdeTest { - private final CSVSerde csv = new CSVSerde(); - - final Properties new_props = new Properties(); - final Properties props = new Properties(); - - @Before - public void setup() throws Exception { - new_props.put(Constants.LIST_COLUMNS, "a,b,c,d,e,f,g,h,i,j"); - new_props.put(Constants.LIST_COLUMN_TYPES, "string,string,int,bigint,float,double,timestamp,timestamp,void,int"); - props.put(Constants.LIST_COLUMNS, "a,b,c"); - props.put(Constants.LIST_COLUMN_TYPES, "string,string,int"); - } - - @Test - public void testDeserialize() throws Exception { - csv.initialize(null, new_props); - final Text in = new Text("hello,\"yes, okay\",1,\"1409576461919\",100.05,\"1024.19002010\",\"2014-04-14 00:03:49\",\"2014-04-14 00:09:42.228000\",,"); - - final List row = (List) csv.deserialize(in); - - assertEquals("hello", row.get(0)); - assertEquals("yes, okay", row.get(1)); - assertEquals(1, row.get(2)); - assertEquals(1409576461919L, row.get(3)); - assertEquals(Float.valueOf("100.05"), row.get(4)); - assertEquals((Double) 1024.19002010, row.get(5)); - assertEquals(java.sql.Timestamp.valueOf("2014-04-14 00:03:49"), row.get(6)); - assertEquals(java.sql.Timestamp.valueOf("2014-04-14 00:09:42.228"), row.get(7)); - assertEquals(null, row.get(8)); - assertEquals(0, row.get(9)); - } - - @Test - public void testDeserializeCustomSeparators() throws Exception { - props.put("separatorChar", "\t"); - props.put("quoteChar", "'"); - - csv.initialize(null, props); - - final Text in = new Text("hello\t'yes\tokay'\t1"); - final List row = (List) csv.deserialize(in); - - assertEquals("hello", row.get(0)); - assertEquals("yes\tokay", row.get(1)); - assertEquals(1, row.get(2)); - } - - @Test - public void testDeserializeCustomEscape() throws Exception { - props.put("quoteChar", "'"); - props.put("escapeChar", "\\"); - - csv.initialize(null, props); - - final Text in = new Text("hello,'yes\\'okay',1"); - final List row = (List) csv.deserialize(in); - - assertEquals("hello", row.get(0)); - assertEquals("yes'okay", row.get(1)); - assertEquals(1, row.get(2)); - } -} +package com.bizo.hive.serde.csv; + +import java.util.List; +import java.util.Properties; + +import org.apache.hadoop.hive.serde.Constants; +import org.apache.hadoop.io.Text; +import org.junit.Before; +import org.junit.Test; +import static org.junit.Assert.assertEquals; + + +public final class CSVSerdeTest { + private final CSVSerde csv = new CSVSerde(); + + final Properties new_props = new Properties(); + final Properties props = new Properties(); + + @Before + public void setup() throws Exception { + new_props.put(Constants.LIST_COLUMNS, "a,b,c,d,e,f,g,h,i,j"); + new_props.put(Constants.LIST_COLUMN_TYPES, "string,string,int,bigint,float,double,timestamp,timestamp,void,int"); + props.put(Constants.LIST_COLUMNS, "a,b,c"); + props.put(Constants.LIST_COLUMN_TYPES, "string,string,int"); + } + + @Test + public void testDeserialize() throws Exception { + csv.initialize(null, new_props); + final Text in = new Text("hello,\"yes, okay\",1,\"1409576461919\",100.05,\"1024.19002010\",\"2014-04-14 00:03:49\",\"2014-04-14 00:09:42.228000\",,"); + + final List row = (List) csv.deserialize(in); + + assertEquals("hello", row.get(0)); + assertEquals("yes, okay", row.get(1)); + assertEquals(1, row.get(2)); + assertEquals(1409576461919L, row.get(3)); + assertEquals(Float.valueOf("100.05"), row.get(4)); + assertEquals((Double) 1024.19002010, row.get(5)); + assertEquals(java.sql.Timestamp.valueOf("2014-04-14 00:03:49"), row.get(6)); + assertEquals(java.sql.Timestamp.valueOf("2014-04-14 00:09:42.228"), row.get(7)); + assertEquals(null, row.get(8)); + assertEquals(0, row.get(9)); + } + + @Test + public void testDeserializeCustomSeparators() throws Exception { + props.put("separatorChar", "\t"); + props.put("quoteChar", "'"); + + csv.initialize(null, props); + + final Text in = new Text("hello\t'yes\tokay'\t1"); + final List row = (List) csv.deserialize(in); + + assertEquals("hello", row.get(0)); + assertEquals("yes\tokay", row.get(1)); + assertEquals(1, row.get(2)); + } + + @Test + public void testDeserializeCustomEscape() throws Exception { + props.put("quoteChar", "'"); + props.put("escapeChar", "\\"); + + csv.initialize(null, props); + + final Text in = new Text("hello,'yes\\'okay',1"); + final List row = (List) csv.deserialize(in); + + assertEquals("hello", row.get(0)); + assertEquals("yes'okay", row.get(1)); + assertEquals(1, row.get(2)); + } +}