From f54bb9709836ab5a652cf130d0d375f14aafde8a Mon Sep 17 00:00:00 2001
From: Andrew Ash <andrew@andrewash.com>
Date: Fri, 6 Dec 2013 00:20:37 -0800
Subject: [PATCH] Don't create as many garbage objects during reads

The old method of reading a row by creating a new reader for every row is
quite inefficient because it leaves a lot of objects laying around for the
garbage collector.  This change creates a CSVParser once and then re-uses it
for every row.

There's a chance this change causes a regression, because CSVSerde is no
longer serializable (CSVParser isn't serializable) but this doesn't seem to be
an issue for me in my testing.
---
 .../com/bizo/hive/serde/csv/CSVSerde.java     | 28 ++++++++-----------
 1 file changed, 12 insertions(+), 16 deletions(-)

diff --git a/src/main/java/com/bizo/hive/serde/csv/CSVSerde.java b/src/main/java/com/bizo/hive/serde/csv/CSVSerde.java
index 82fceb9..246e817 100644
--- a/src/main/java/com/bizo/hive/serde/csv/CSVSerde.java
+++ b/src/main/java/com/bizo/hive/serde/csv/CSVSerde.java
@@ -26,6 +26,7 @@
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.io.Writable;
 
+import au.com.bytecode.opencsv.CSVParser;
 import au.com.bytecode.opencsv.CSVReader;
 import au.com.bytecode.opencsv.CSVWriter;
 
@@ -45,7 +46,7 @@ public final class CSVSerde implements SerDe {
   private char separatorChar;
   private char quoteChar;
   private char escapeChar;
-  
+  private CSVParser csvParser;
     
   @Override
   public void initialize(final Configuration conf, final Properties tbl) throws SerDeException {
@@ -71,6 +72,11 @@ public void initialize(final Configuration conf, final Properties tbl) throws Se
     separatorChar = getProperty(tbl, "separatorChar", CSVWriter.DEFAULT_SEPARATOR);
     quoteChar = getProperty(tbl, "quoteChar", CSVWriter.DEFAULT_QUOTE_CHARACTER);
     escapeChar = getProperty(tbl, "escapeChar", CSVWriter.DEFAULT_ESCAPE_CHARACTER);
+    if (CSVWriter.DEFAULT_ESCAPE_CHARACTER == escapeChar) {
+      this.csvParser = new CSVParser(separatorChar, quoteChar);
+    } else {
+      this.csvParser = new CSVParser(separatorChar, quoteChar, escapeChar);
+    }
   }
   
   private final char getProperty(final Properties tbl, final String property, final char def) {
@@ -122,15 +128,13 @@ public Writable serialize(Object obj, ObjectInspector objInspector) throws SerDe
   @Override
   public Object deserialize(final Writable blob) throws SerDeException {
     Text rowText = (Text) blob;
-    
-    CSVReader csv = null;
+
     try {
-      csv = newReader(new CharArrayReader(rowText.toString().toCharArray()), separatorChar, quoteChar, escapeChar);      
-      final String[] read = csv.readNext();
+      final String[] strings = this.csvParser.parseLine(rowText.toString());
       
-      for (int i=0; i< numCols; i++) {
-        if (read != null && i < read.length) {
-          row.set(i, read[i]);
+      for (int i=0; i < numCols; i++) {
+        if (strings != null && i < strings.length) {
+          row.set(i, strings[i]);
         } else {
           row.set(i, null);
         }
@@ -139,14 +143,6 @@ public Object deserialize(final Writable blob) throws SerDeException {
       return row;
     } catch (final Exception e) {
       throw new SerDeException(e);
-    } finally {
-      if (csv != null) {
-        try {
-          csv.close();
-        } catch (final Exception e) {
-          // ignore
-        }
-      }
     }
   }