Merge pull request #868 from tapdata/TAP-12724-excle-2.0.8

fix(TAP-12724): [to 2.0.8] Exclude cell type does not exist or unsupported type …

Author: jarad0628

connectors/excel-connector/src/main/java/io/tapdata/connector/excel/ExcelConnector.java

@@ -25,11 +25,8 @@
 import org.apache.poi.xssf.usermodel.XSSFWorkbookFactory;
 
 import java.io.IOException;
-import java.math.BigDecimal;
 import java.time.LocalDate;
-import java.time.LocalDateTime;
 import java.time.LocalTime;
-import java.time.format.DateTimeFormatter;
 import java.util.*;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.atomic.AtomicReference;
@@ -100,13 +97,15 @@ protected void readOneFile(FileOffset fileOffset, TapTable tapTable, int eventBa
                                 Cell cell = row.getCell(k);
                                 checkCellType(k, cell, cellTypeMap);
                                 String fieldName = (String) headers[k - excelConfig.getFirstColumn() + 1];
+                                String fieldType = CellValueConvert.getFieldDataType(tapTable, fieldName);
                                 Object val;
                                 if (excelConfig.getJustString()) {
                                     Object cellValue = ExcelUtil.getCellValue(cell, formulaEvaluator);
                                     Object displayValue = ExcelUtil.getCellDisplayValue(cell, formulaEvaluator, dataFormatter);
-                                    val = CellValueConvert.parseValue(cellValue, displayValue, CellValueConvert.getFieldDataType(tapTable, fieldName));
+                                    val = CellValueConvert.parseValue(cellValue, displayValue, fieldType);
                                 } else {
-                                    val = ExcelUtil.getCellValue(cell, formulaEvaluator);
+                                    Object cellValue = ExcelUtil.getCellValue(cell, formulaEvaluator);
+                                    val = CellValueConvert.parseOriginValue(cellValue, fieldType);
                                 }
                                 after.put(fieldName, val);
                             }
@@ -115,13 +114,15 @@ protected void readOneFile(FileOffset fileOffset, TapTable tapTable, int eventBa
                                 Cell cell = row.getCell(k);
                                 checkCellType(k, cell, cellTypeMap);
                                 String fieldName = (String) headers[k - excelConfig.getFirstColumn() + 1];
+                                String fieldType = CellValueConvert.getFieldDataType(tapTable, fieldName);
                                 Object val;
                                 if (excelConfig.getJustString()) {
                                     Object cellValue = ExcelUtil.getMergedCellValue(mergedList, mergedDataMap, cell, formulaEvaluator);
                                     Object displayValue = ExcelUtil.getMergedCellDisplayValue(mergedList, mergedDataMap, cell, formulaEvaluator, dataFormatter);
                                     val = CellValueConvert.parseValue(cellValue, displayValue, CellValueConvert.getFieldDataType(tapTable, fieldName));
                                 } else {
-                                    val = ExcelUtil.getMergedCellValue(mergedList, mergedDataMap, cell, formulaEvaluator);
+                                    Object cellValue = ExcelUtil.getMergedCellValue(mergedList, mergedDataMap, cell, formulaEvaluator);
+                                    val = CellValueConvert.parseOriginValue(cellValue, fieldType);
                                 }
                                 after.put(fieldName, val);
                             }

connectors/excel-connector/src/main/java/io/tapdata/connector/excel/util/CellValueConvert.java

@@ -3,6 +3,7 @@
 import io.tapdata.entity.schema.TapField;
 import io.tapdata.entity.schema.TapTable;
 import io.tapdata.kit.EmptyKit;
+import org.apache.commons.lang3.StringUtils;
 
 import java.math.BigDecimal;
 import java.time.LocalDate;
@@ -40,6 +41,24 @@ public static String toExcelDataType(Object val) {
         return val.getClass().getSimpleName().toUpperCase();
     }
 
+    static Object stringValue(Object val) {
+        String temporalValue = formatTemporalValue(val);
+        if (temporalValue != null) {
+            return temporalValue;
+        }
+        return parseValue(val);
+    }
+
+    public static Object parseOriginValue(Object val, String fieldType) {
+        if (isStringField(fieldType)) {
+            return stringValue(val);
+        }
+        if (val instanceof String && StringUtils.isBlank((String) val)) {
+            return null;
+        }
+        return val;
+    }
+
     public static Object parseValue(Object val) {
         if (val instanceof Double || val instanceof Float || val instanceof Long) {
             val = BigDecimal.valueOf(((Number) val).doubleValue()).stripTrailingZeros().toPlainString();
@@ -55,11 +74,10 @@ public static Object parseValue(Object val, String fieldDataType) {
 
     public static Object parseValue(Object val, Object displayValue, String fieldDataType) {
         if (isStringField(fieldDataType)) {
-            String temporalValue = formatTemporalValue(val);
-            if (temporalValue != null) {
-                return temporalValue;
-            }
-            return parseValue(displayValue);
+            return stringValue(val);
+        }
+        if (displayValue instanceof String && StringUtils.isBlank((String) displayValue)) {
+            return null;
         }
         return parseValue(val);
     }

connectors/excel-connector/src/main/java/io/tapdata/connector/excel/util/ExcelUtil.java

@@ -198,18 +198,20 @@ public static Object getCellValue(Cell cell, FormulaEvaluator formulaEvaluator)
                         case FORMULA:
                             return cell.getCellFormula();
                         case BLANK:
+                            return "";
                         case ERROR:
                         case _NONE:
                         default:
-                            return "";
+                            return null;
                     }
                 }
                 return cell.getCellFormula();
             case BLANK:
+                return "";
             case ERROR:
             case _NONE:
             default:
-                return "";
+                return null;
         }
     }