Skip to content

fix(TAP-12577): 【to main】Excle Need to distinguish between date/time #864

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jarad0628 merged 2 commits into from
Aug 24, 2026
Merged

fix(TAP-12577): 【to main】Excle Need to distinguish between date/time #864

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7ffd813
fix(TAP-12577): Need to distinguish between date/time
11000100111010101100111 Aug 21, 2026
273869d
fix(TAP-12477): Paimon encountered an illegal date, which is null com…
11000100111010101100111 Aug 21, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
58 changes: 35 additions & 23 deletions connectors/excel-connector/src/main/java/io/tapdata/connector/excel/ExcelConnector.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import io.tapdata.common.FileConnector;
import io.tapdata.common.FileOffset;
import io.tapdata.connector.excel.config.ExcelConfig;
import io.tapdata.connector.excel.util.CellValueConvert;
import io.tapdata.connector.excel.util.ExcelUtil;
import io.tapdata.entity.codec.TapCodecsRegistry;
import io.tapdata.entity.event.TapEvent;
Expand All @@ -24,8 +25,11 @@
import org.apache.poi.xssf.usermodel.XSSFWorkbookFactory;

import java.io.IOException;
import java.time.LocalDateTime;
import java.math.BigDecimal;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.LocalTime;
import java.time.format.DateTimeFormatter;
import java.util.*;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.atomic.AtomicReference;
Expand All @@ -37,7 +41,6 @@
public class ExcelConnector extends FileConnector {

private static final String TAG = ExcelConnector.class.getSimpleName();

static {
try {
XSSFWorkbookFactory factory = new XSSFWorkbookFactory();
Expand Down Expand Up @@ -93,19 +96,33 @@ protected void readOneFile(FileOffset fileOffset, TapTable tapTable, int eventBa
Map<String, Object> after = new HashMap<>();
if (j > lastMergedRow) {
for (int k = excelConfig.getFirstColumn() - 1; k < excelConfig.getLastColumn(); k++) {
checkCellType(k, row.getCell(k), cellTypeMap);
Object val = excelConfig.getJustString()
? ExcelUtil.getCellDisplayValue(row.getCell(k), formulaEvaluator, dataFormatter)
: ExcelUtil.getCellValue(row.getCell(k), formulaEvaluator);
after.put((String) headers[k - excelConfig.getFirstColumn() + 1], excelConfig.getJustString() ? parseValue(val) : val);
Cell cell = row.getCell(k);
checkCellType(k, cell, cellTypeMap);
String fieldName = (String) headers[k - excelConfig.getFirstColumn() + 1];
Object val;
if (excelConfig.getJustString()) {
Object cellValue = ExcelUtil.getCellValue(cell, formulaEvaluator);
Object displayValue = ExcelUtil.getCellDisplayValue(cell, formulaEvaluator, dataFormatter);
val = CellValueConvert.parseValue(cellValue, displayValue, CellValueConvert.getFieldDataType(tapTable, fieldName));
} else {
val = ExcelUtil.getCellValue(cell, formulaEvaluator);
}
after.put(fieldName, val);
}
} else {
for (int k = excelConfig.getFirstColumn() - 1; k < excelConfig.getLastColumn(); k++) {
checkCellType(k, row.getCell(k), cellTypeMap);
Object val = excelConfig.getJustString()
? ExcelUtil.getMergedCellDisplayValue(mergedList, mergedDataMap, row.getCell(k), formulaEvaluator, dataFormatter)
: ExcelUtil.getMergedCellValue(mergedList, mergedDataMap, row.getCell(k), formulaEvaluator);
after.put((String) headers[k - excelConfig.getFirstColumn() + 1], excelConfig.getJustString() ? parseValue(val) : val);
Cell cell = row.getCell(k);
checkCellType(k, cell, cellTypeMap);
String fieldName = (String) headers[k - excelConfig.getFirstColumn() + 1];
Object val;
if (excelConfig.getJustString()) {
Object cellValue = ExcelUtil.getMergedCellValue(mergedList, mergedDataMap, cell, formulaEvaluator);
Object displayValue = ExcelUtil.getMergedCellDisplayValue(mergedList, mergedDataMap, cell, formulaEvaluator, dataFormatter);
val = CellValueConvert.parseValue(cellValue, displayValue, CellValueConvert.getFieldDataType(tapTable, fieldName));
} else {
val = ExcelUtil.getMergedCellValue(mergedList, mergedDataMap, cell, formulaEvaluator);
}
after.put(fieldName, val);
}
}
TapRecordEvent recordEvent = insertRecordEvent(after, tapTable.getId()).referenceTime(lastModified);
Expand Down Expand Up @@ -156,6 +173,10 @@ private int checkLevel(CellType type) {

@Override
public void registerCapabilities(ConnectorFunctions connectorFunctions, TapCodecsRegistry codecRegistry) {
codecRegistry.registerToTapValue(LocalDate.class, (value, tapType) ->
new TapDateValue(new DateTime(((LocalDate) value).atStartOfDay())));
codecRegistry.registerToTapValue(LocalTime.class, (value, tapType) ->
new TapTimeValue(DateTime.withTimeStr(value.toString())));
codecRegistry.registerFromTapValue(TapRawValue.class, "STRING", tapRawValue -> {
if (tapRawValue != null && tapRawValue.getValue() != null) return toJson(tapRawValue.getValue());
return "null";
Expand Down Expand Up @@ -211,7 +232,7 @@ protected void makeTapTable(TapTable tapTable, Map<String, Object> sample, boole
field.name(objectEntry.getKey().replaceAll("\n", ""));
Object val = objectEntry.getValue();
if (isJustString) {
val = parseValue(val);
val = CellValueConvert.parseValue(val, CellValueConvert.STRING_DATA_TYPE);
}
if (EmptyKit.isNull(val) || val instanceof String) {
if (EmptyKit.isNotEmpty((String) val) && ((String) val).length() > 200) {
Expand All @@ -222,18 +243,9 @@ protected void makeTapTable(TapTable tapTable, Map<String, Object> sample, boole
} else if (val instanceof LocalDateTime) {
field.dataType("DATE");
} else {
field.dataType(val.getClass().getSimpleName().toUpperCase());
field.dataType(CellValueConvert.toExcelDataType(val));
}
tapTable.add(field);
}
}

Object parseValue(Object val) {
if (val instanceof Double || val instanceof Float || val instanceof Long) {
val = BigDecimal.valueOf(((Number) val).doubleValue()).stripTrailingZeros().toPlainString();
} else {
val = EmptyKit.isNull(val) ? "null" : String.valueOf(val);
}
return val;
}
}
95 changes: 95 additions & 0 deletions ...ctors/excel-connector/src/main/java/io/tapdata/connector/excel/util/CellValueConvert.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
package io.tapdata.connector.excel.util;

import io.tapdata.entity.schema.TapField;
import io.tapdata.entity.schema.TapTable;
import io.tapdata.kit.EmptyKit;

import java.math.BigDecimal;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.LocalTime;
import java.time.format.DateTimeFormatter;

/**
* @author <a href="2749984520@qq.com">Gavin'Xiao</a>
* @author <a href="https://github.com/11000100111010101100111">Gavin'Xiao</a>
* @version v1.0 2026/8/21 15:13 Create
* @description
*/
public final class CellValueConvert {
public static final String STRING_DATA_TYPE = "STRING";
public static final String TEXT_DATA_TYPE = "TEXT";
public static final DateTimeFormatter LOCAL_DATE_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd");
public static final DateTimeFormatter LOCAL_TIME_FORMATTER = DateTimeFormatter.ofPattern("HH:mm:ss");
public static final DateTimeFormatter LOCAL_DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSSSSS");

private CellValueConvert() {

}

public static String toExcelDataType(Object val) {
if (val instanceof LocalDate) {
return "DATE";
}
if (val instanceof LocalTime) {
return "TIME";
}
if (val instanceof LocalDateTime) {
return "DATETIME";
}
return val.getClass().getSimpleName().toUpperCase();
}

public static Object parseValue(Object val) {
if (val instanceof Double || val instanceof Float || val instanceof Long) {
val = BigDecimal.valueOf(((Number) val).doubleValue()).stripTrailingZeros().toPlainString();
} else {
val = EmptyKit.isNull(val) ? "null" : String.valueOf(val);
}
return val;
}

public static Object parseValue(Object val, String fieldDataType) {
return parseValue(val, val, fieldDataType);
}

public static Object parseValue(Object val, Object displayValue, String fieldDataType) {
if (isStringField(fieldDataType)) {
String temporalValue = formatTemporalValue(val);
if (temporalValue != null) {
return temporalValue;
}
return parseValue(displayValue);
}
return parseValue(val);
}

public static String getFieldDataType(TapTable tapTable, String fieldName) {
if (tapTable == null || tapTable.getNameFieldMap() == null) {
return null;
}
TapField tapField = tapTable.getNameFieldMap().get(fieldName);
return tapField == null ? null : tapField.getDataType();
}

public static boolean isStringField(String fieldDataType) {
if (EmptyKit.isBlank(fieldDataType)) {
return false;
}
return STRING_DATA_TYPE.equalsIgnoreCase(fieldDataType)
|| TEXT_DATA_TYPE.equalsIgnoreCase(fieldDataType);
}

public static String formatTemporalValue(Object val) {
if (val instanceof LocalDate) {
return ((LocalDate) val).format(LOCAL_DATE_FORMATTER);
}
if (val instanceof LocalTime) {
return ((LocalTime) val).format(LOCAL_TIME_FORMATTER);
}
if (val instanceof LocalDateTime) {
return ((LocalDateTime) val).format(LOCAL_DATE_TIME_FORMATTER);
}
return null;
}
}
111 changes: 106 additions & 5 deletions connectors/excel-connector/src/main/java/io/tapdata/connector/excel/util/ExcelUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,12 @@
import org.apache.poi.ss.util.CellRangeAddress;

import java.math.BigDecimal;
import java.time.Instant;
import java.time.ZoneId;
import java.time.LocalDateTime;
import java.util.*;
import java.util.stream.Collectors;

public class ExcelUtil {
private static final double EXCEL_TIME_EPSILON = 1.0E-9;

//3,5~9,12
public static List<Integer> getSheetNumber(String reg) {
Expand Down Expand Up @@ -165,7 +165,7 @@ public static Object getCellValue(Cell cell, FormulaEvaluator formulaEvaluator)

case NUMERIC:
if (DateUtil.isCellDateFormatted(cell) || cell.getCellStyle().getDataFormat() == 58) {
return Instant.ofEpochMilli((cell.getDateCellValue()).getTime()).atZone(ZoneId.systemDefault()).toLocalDateTime();
return parseDateTimeValue(cell);
} else {
return parseNumberValue(cell);
}
Expand All @@ -190,8 +190,8 @@ public static Object getCellValue(Cell cell, FormulaEvaluator formulaEvaluator)
return cell.getRichStringCellValue().getString();

case NUMERIC:
if (DateUtil.isCellDateFormatted(cell)) {
return Instant.ofEpochMilli((cell.getDateCellValue()).getTime()).atZone(ZoneId.systemDefault()).toLocalDateTime();
if (DateUtil.isCellDateFormatted(cell) || cell.getCellStyle().getDataFormat() == 58) {
return parseDateTimeValue(cell);
} else {
return parseNumberValue(cell);
}
Expand All @@ -213,6 +213,107 @@ public static Object getCellValue(Cell cell, FormulaEvaluator formulaEvaluator)
}
}

private static Object parseDateTimeValue(Cell cell) {
LocalDateTime localDateTime = cell.getLocalDateTimeCellValue();
double numericValue = cell.getNumericCellValue();
boolean hasDatePart = hasDatePart(numericValue);
boolean hasTimePart = hasTimePart(numericValue);
TemporalFormat temporalFormat = parseTemporalFormat(cell.getCellStyle());
if (!hasDatePart && (hasTimePart || temporalFormat.hasTime)) {
return localDateTime.toLocalTime();
}
if (hasDatePart && hasTimePart) {
return localDateTime;
}
if (temporalFormat.hasDate && temporalFormat.hasTime) {
return localDateTime;
}
if (temporalFormat.hasTime && !temporalFormat.hasDate) {
return localDateTime.toLocalTime();
}
return localDateTime.toLocalDate();
}

private static boolean hasDatePart(double numericValue) {
return Math.floor(numericValue) >= 1D;
}

private static boolean hasTimePart(double numericValue) {
double fraction = Math.abs(numericValue - Math.floor(numericValue));
return fraction > EXCEL_TIME_EPSILON && Math.abs(1D - fraction) > EXCEL_TIME_EPSILON;
}

private static TemporalFormat parseTemporalFormat(CellStyle cellStyle) {
if (cellStyle == null) {
return new TemporalFormat(false, false);
}
String format = normalizeDateFormat(cellStyle.getDataFormatString());
boolean hasTime = format.contains("{time}")
|| format.contains(":")
|| format.contains("am/pm")
|| format.contains("a/p")
|| format.indexOf('h') >= 0
|| format.indexOf('s') >= 0;
boolean hasDate = cellStyle.getDataFormat() == 58
|| format.indexOf('y') >= 0
|| format.indexOf('d') >= 0
|| (!hasTime && format.indexOf('m') >= 0);
return new TemporalFormat(hasDate, hasTime);
}

private static String normalizeDateFormat(String format) {
if (EmptyKit.isBlank(format)) {
return "";
}
StringBuilder builder = new StringBuilder();
boolean inQuote = false;
for (int i = 0; i < format.length(); i++) {
char c = format.charAt(i);
if (inQuote) {
if (c == '"') {
inQuote = false;
}
continue;
}
if (c == '"') {
inQuote = true;
continue;
}
if (c == '\\' || c == '_' || c == '*') {
i++;
continue;
}
if (c == ';') {
break;
}
if (c == '[') {
int end = format.indexOf(']', i);
if (end > i) {
String token = format.substring(i + 1, end).toLowerCase(Locale.ROOT);
if ("h".equals(token) || "hh".equals(token)
|| "m".equals(token) || "mm".equals(token)
|| "s".equals(token) || "ss".equals(token)) {
builder.append("{time}");
}
i = end;
continue;
}
}
builder.append(Character.toLowerCase(c));
}
return builder.toString();
}

private static class TemporalFormat {
private final boolean hasDate;
private final boolean hasTime;

private TemporalFormat(boolean hasDate, boolean hasTime) {
this.hasDate = hasDate;
this.hasTime = hasTime;
}
}

public static void main(String[] args) {
System.out.println(getColumnNumber("BB"));
System.out.println(getSheetNumber("7~10,11~13,17,2,5"));
Expand Down
33 changes: 30 additions & 3 deletions connectors/excel-connector/src/main/resources/spec_excel.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -654,10 +654,37 @@
},
"DATE": {
"range": [
"1000-01-01 00:00:00",
"9999-12-31 23:59:59"
"1000-01-01",
"9999-12-31"
],
"pattern": "yyyy-MM-dd",
"priority": 2,
"to": "TapDate"
},
"TIME": {
"range": [
"00:00:00.000000000",
"23:59:59.999999999"
],
"pattern": "HH:mm:ss.SSSSSSSSS",
"fraction": [
0,
9
],
"defaultFraction": 0,
"priority": 2,
"to": "TapTime"
},
"DATETIME": {
"range": [
"1000-01-01 00:00:00.000000000",
"9999-12-31 23:59:59.999999999"
],
"pattern": "yyyy-MM-dd HH:mm:ss.SSSSSSSSS",
"fraction": [
0,
9
],
"pattern": "yyyy-MM-dd HH:mm:ss",
"defaultFraction": 3,
"withTimeZone": false,
"priority": 2,
Expand Down
Loading