Skip to content

Commit c8224b8

Browse files
sapienza88Selim Soufargibengbengbalabalabeng
authored
feat: add support for column-based reading of XLS files (#981)
* feature: number and sstlabel with column support * remove null checks for sheet holder and sheet * feat: adding filtering capability for columns for additional typeset: formula, bool, rk, blank etc * fix CI: missing import * fix CI : add missing imports * address reviewing comments : revert NOTE * fix: code style and restore the missing test --------- Co-authored-by: Selim Soufargi <ssoufargi.idealab.unical@gmail.com~> Co-authored-by: Bengbengbalabalabeng <baicaixiaozhan1512@163.com>
1 parent 30a0b74 commit c8224b8

9 files changed

Lines changed: 197 additions & 18 deletions

File tree

‎fesod-sheet/src/main/java/org/apache/fesod/sheet/analysis/v03/handlers/BlankRecordHandler.java‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525

2626
package org.apache.fesod.sheet.analysis.v03.handlers;
2727

28+
import java.util.List;
2829
import org.apache.fesod.sheet.analysis.v03.IgnorableXlsRecordHandler;
2930
import org.apache.fesod.sheet.context.xls.XlsReadContext;
3031
import org.apache.fesod.sheet.metadata.data.ReadCellData;
@@ -39,9 +40,21 @@ public class BlankRecordHandler extends AbstractXlsRecordHandler implements Igno
3940
@Override
4041
public void processRecord(XlsReadContext xlsReadContext, Record record) {
4142
BlankRecord br = (BlankRecord) record;
43+
int originalColumnIndex = br.getColumn();
44+
45+
List<Integer> includeColumnIndexes =
46+
xlsReadContext.readSheetHolder().getReadSheet().getColumnIndexes();
47+
48+
int targetColumnIndex = originalColumnIndex;
49+
if (includeColumnIndexes != null) {
50+
targetColumnIndex = includeColumnIndexes.indexOf(originalColumnIndex);
51+
if (targetColumnIndex < 0) {
52+
return;
53+
}
54+
}
4255
xlsReadContext
4356
.xlsReadSheetHolder()
4457
.getCellMap()
45-
.put((int) br.getColumn(), ReadCellData.newEmptyInstance(br.getRow(), (int) br.getColumn()));
58+
.put(targetColumnIndex, ReadCellData.newEmptyInstance(br.getRow(), targetColumnIndex));
4659
}
4760
}

‎fesod-sheet/src/main/java/org/apache/fesod/sheet/analysis/v03/handlers/BoolErrRecordHandler.java‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525

2626
package org.apache.fesod.sheet.analysis.v03.handlers;
2727

28+
import java.util.List;
2829
import org.apache.fesod.sheet.analysis.v03.IgnorableXlsRecordHandler;
2930
import org.apache.fesod.sheet.context.xls.XlsReadContext;
3031
import org.apache.fesod.sheet.enums.RowTypeEnum;
@@ -40,11 +41,22 @@ public class BoolErrRecordHandler extends AbstractXlsRecordHandler implements Ig
4041
@Override
4142
public void processRecord(XlsReadContext xlsReadContext, Record record) {
4243
BoolErrRecord ber = (BoolErrRecord) record;
44+
int originalColumnIndex = ber.getColumn();
45+
List<Integer> includeColumnIndexes =
46+
xlsReadContext.readSheetHolder().getReadSheet().getColumnIndexes();
47+
48+
int targetColumnIndex = originalColumnIndex;
49+
if (includeColumnIndexes != null) {
50+
targetColumnIndex = includeColumnIndexes.indexOf(originalColumnIndex);
51+
if (targetColumnIndex < 0) {
52+
return;
53+
}
54+
}
4355
xlsReadContext
4456
.xlsReadSheetHolder()
4557
.getCellMap()
46-
.put((int) ber.getColumn(), ReadCellData.newInstance(ber.getBooleanValue(), ber.getRow(), (int)
47-
ber.getColumn()));
58+
.put(targetColumnIndex, ReadCellData.newInstance(ber.getBooleanValue(), ber.getRow(), (int)
59+
targetColumnIndex));
4860
xlsReadContext.xlsReadSheetHolder().setTempRowType(RowTypeEnum.DATA);
4961
}
5062
}

‎fesod-sheet/src/main/java/org/apache/fesod/sheet/analysis/v03/handlers/DummyRecordHandler.java‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
package org.apache.fesod.sheet.analysis.v03.handlers;
2727

2828
import java.util.LinkedHashMap;
29+
import java.util.List;
2930
import org.apache.fesod.sheet.analysis.v03.IgnorableXlsRecordHandler;
3031
import org.apache.fesod.sheet.context.xls.XlsReadContext;
3132
import org.apache.fesod.sheet.enums.RowTypeEnum;
@@ -58,11 +59,22 @@ public void processRecord(XlsReadContext xlsReadContext, Record record) {
5859
xlsReadSheetHolder.setTempRowType(RowTypeEnum.EMPTY);
5960
} else if (record instanceof MissingCellDummyRecord) {
6061
MissingCellDummyRecord mcdr = (MissingCellDummyRecord) record;
62+
int originalColumnIndex = mcdr.getColumn();
63+
List<Integer> includeColumnIndexes =
64+
xlsReadContext.readSheetHolder().getReadSheet().getColumnIndexes();
65+
66+
int targetColumnIndex = originalColumnIndex;
67+
if (includeColumnIndexes != null) {
68+
targetColumnIndex = includeColumnIndexes.indexOf(originalColumnIndex);
69+
if (targetColumnIndex < 0) {
70+
return;
71+
}
72+
}
6173
// Some abnormal XLS, in the case of data already exist, or there will be a "MissingCellDummyRecord"
6274
// records, so if the existing data, empty data is ignored
6375
xlsReadSheetHolder
6476
.getCellMap()
65-
.putIfAbsent(mcdr.getColumn(), ReadCellData.newEmptyInstance(mcdr.getRow(), mcdr.getColumn()));
77+
.putIfAbsent(targetColumnIndex, ReadCellData.newEmptyInstance(mcdr.getRow(), targetColumnIndex));
6678
}
6779
}
6880
}

‎fesod-sheet/src/main/java/org/apache/fesod/sheet/analysis/v03/handlers/FormulaRecordHandler.java‎

Lines changed: 19 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
package org.apache.fesod.sheet.analysis.v03.handlers;
2727

2828
import java.math.BigDecimal;
29+
import java.util.List;
2930
import java.util.Map;
3031
import lombok.extern.slf4j.Slf4j;
3132
import org.apache.fesod.sheet.analysis.v03.IgnorableXlsRecordHandler;
@@ -53,10 +54,23 @@ public class FormulaRecordHandler extends AbstractXlsRecordHandler implements Ig
5354
@Override
5455
public void processRecord(XlsReadContext xlsReadContext, Record record) {
5556
FormulaRecord frec = (FormulaRecord) record;
57+
int originalColumnIndex = frec.getColumn();
58+
59+
List<Integer> includeColumnIndexes =
60+
xlsReadContext.readSheetHolder().getReadSheet().getColumnIndexes();
61+
62+
int targetColumnIndex = originalColumnIndex;
63+
if (includeColumnIndexes != null) {
64+
targetColumnIndex = includeColumnIndexes.indexOf(originalColumnIndex);
65+
if (targetColumnIndex < 0) {
66+
return;
67+
}
68+
}
69+
5670
Map<Integer, Cell> cellMap = xlsReadContext.xlsReadSheetHolder().getCellMap();
5771
ReadCellData<?> tempCellData = new ReadCellData<>();
5872
tempCellData.setRowIndex(frec.getRow());
59-
tempCellData.setColumnIndex((int) frec.getColumn());
73+
tempCellData.setColumnIndex(targetColumnIndex);
6074
CellType cellType = CellType.forInt(frec.getCachedResultType());
6175
String formulaValue = null;
6276
try {
@@ -98,21 +112,21 @@ public void processRecord(XlsReadContext xlsReadContext, Record record) {
98112
.getGlobalConfiguration()
99113
.getLocale()));
100114
tempCellData.setDataFormatData(dataFormatData);
101-
cellMap.put((int) frec.getColumn(), tempCellData);
115+
cellMap.put(targetColumnIndex, tempCellData);
102116
break;
103117
case ERROR:
104118
tempCellData.setType(CellDataTypeEnum.ERROR);
105119
tempCellData.setStringValue(ERROR);
106-
cellMap.put((int) frec.getColumn(), tempCellData);
120+
cellMap.put(targetColumnIndex, tempCellData);
107121
break;
108122
case BOOLEAN:
109123
tempCellData.setType(CellDataTypeEnum.BOOLEAN);
110124
tempCellData.setBooleanValue(frec.getCachedBooleanValue());
111-
cellMap.put((int) frec.getColumn(), tempCellData);
125+
cellMap.put(targetColumnIndex, tempCellData);
112126
break;
113127
default:
114128
tempCellData.setType(CellDataTypeEnum.EMPTY);
115-
cellMap.put((int) frec.getColumn(), tempCellData);
129+
cellMap.put(targetColumnIndex, tempCellData);
116130
break;
117131
}
118132
}

‎fesod-sheet/src/main/java/org/apache/fesod/sheet/analysis/v03/handlers/LabelRecordHandler.java‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525

2626
package org.apache.fesod.sheet.analysis.v03.handlers;
2727

28+
import java.util.List;
2829
import org.apache.fesod.common.util.StringUtils;
2930
import org.apache.fesod.sheet.analysis.v03.IgnorableXlsRecordHandler;
3031
import org.apache.fesod.sheet.context.xls.XlsReadContext;
@@ -41,6 +42,19 @@ public class LabelRecordHandler extends AbstractXlsRecordHandler implements Igno
4142
@Override
4243
public void processRecord(XlsReadContext xlsReadContext, Record record) {
4344
LabelRecord lrec = (LabelRecord) record;
45+
int originalColumnIndex = lrec.getColumn();
46+
47+
List<Integer> includeColumnIndexes =
48+
xlsReadContext.readSheetHolder().getReadSheet().getColumnIndexes();
49+
50+
int targetColumnIndex = originalColumnIndex;
51+
if (includeColumnIndexes != null) {
52+
targetColumnIndex = includeColumnIndexes.indexOf(originalColumnIndex);
53+
if (targetColumnIndex < 0) {
54+
return;
55+
}
56+
}
57+
4458
String data = lrec.getValue();
4559
if (data != null) {
4660
GlobalConfiguration globalConfiguration =
@@ -54,7 +68,7 @@ public void processRecord(XlsReadContext xlsReadContext, Record record) {
5468
xlsReadContext
5569
.xlsReadSheetHolder()
5670
.getCellMap()
57-
.put((int) lrec.getColumn(), ReadCellData.newInstance(data, lrec.getRow(), (int) lrec.getColumn()));
71+
.put(targetColumnIndex, ReadCellData.newInstance(data, lrec.getRow(), targetColumnIndex));
5872
xlsReadContext.xlsReadSheetHolder().setTempRowType(RowTypeEnum.DATA);
5973
}
6074
}

‎fesod-sheet/src/main/java/org/apache/fesod/sheet/analysis/v03/handlers/LabelSstRecordHandler.java‎

Lines changed: 17 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525

2626
package org.apache.fesod.sheet.analysis.v03.handlers;
2727

28+
import java.util.List;
2829
import java.util.Map;
2930
import org.apache.fesod.common.util.StringUtils;
3031
import org.apache.fesod.sheet.analysis.v03.IgnorableXlsRecordHandler;
@@ -45,17 +46,28 @@ public class LabelSstRecordHandler extends AbstractXlsRecordHandler implements I
4546
@Override
4647
public void processRecord(XlsReadContext xlsReadContext, Record record) {
4748
LabelSSTRecord lsrec = (LabelSSTRecord) record;
49+
int originalColumnIndex = lsrec.getColumn();
50+
51+
List<Integer> includeColumnIndexes =
52+
xlsReadContext.readSheetHolder().getReadSheet().getColumnIndexes();
53+
54+
int targetColumnIndex = originalColumnIndex;
55+
if (includeColumnIndexes != null) {
56+
targetColumnIndex = includeColumnIndexes.indexOf(originalColumnIndex);
57+
if (targetColumnIndex < 0) {
58+
return;
59+
}
60+
}
61+
4862
ReadCache readCache = xlsReadContext.readWorkbookHolder().getReadCache();
4963
Map<Integer, Cell> cellMap = xlsReadContext.xlsReadSheetHolder().getCellMap();
5064
if (readCache == null) {
51-
cellMap.put(
52-
(int) lsrec.getColumn(), ReadCellData.newEmptyInstance(lsrec.getRow(), (int) lsrec.getColumn()));
65+
cellMap.put(targetColumnIndex, ReadCellData.newEmptyInstance(lsrec.getRow(), targetColumnIndex));
5366
return;
5467
}
5568
String data = readCache.get(lsrec.getSSTIndex());
5669
if (data == null) {
57-
cellMap.put(
58-
(int) lsrec.getColumn(), ReadCellData.newEmptyInstance(lsrec.getRow(), (int) lsrec.getColumn()));
70+
cellMap.put(targetColumnIndex, ReadCellData.newEmptyInstance(lsrec.getRow(), targetColumnIndex));
5971
return;
6072
}
6173

@@ -66,7 +78,7 @@ public void processRecord(XlsReadContext xlsReadContext, Record record) {
6678
} else if (globalConfiguration.getAutoTrim()) {
6779
data = data.trim();
6880
}
69-
cellMap.put((int) lsrec.getColumn(), ReadCellData.newInstance(data, lsrec.getRow(), (int) lsrec.getColumn()));
81+
cellMap.put(targetColumnIndex, ReadCellData.newInstance(data, lsrec.getRow(), targetColumnIndex));
7082
xlsReadContext.xlsReadSheetHolder().setTempRowType(RowTypeEnum.DATA);
7183
}
7284
}

‎fesod-sheet/src/main/java/org/apache/fesod/sheet/analysis/v03/handlers/NumberRecordHandler.java‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
package org.apache.fesod.sheet.analysis.v03.handlers;
2727

2828
import java.math.BigDecimal;
29+
import java.util.List;
2930
import org.apache.fesod.sheet.analysis.v03.IgnorableXlsRecordHandler;
3031
import org.apache.fesod.sheet.constant.BuiltinFormats;
3132
import org.apache.fesod.sheet.context.xls.XlsReadContext;
@@ -43,8 +44,21 @@ public class NumberRecordHandler extends AbstractXlsRecordHandler implements Ign
4344
@Override
4445
public void processRecord(XlsReadContext xlsReadContext, Record record) {
4546
NumberRecord nr = (NumberRecord) record;
47+
int originalColumnIndex = nr.getColumn();
48+
49+
List<Integer> includeColumnIndexes =
50+
xlsReadContext.readSheetHolder().getReadSheet().getColumnIndexes();
51+
52+
int targetColumnIndex = originalColumnIndex;
53+
if (includeColumnIndexes != null) {
54+
targetColumnIndex = includeColumnIndexes.indexOf(originalColumnIndex);
55+
if (targetColumnIndex < 0) {
56+
return;
57+
}
58+
}
59+
4660
ReadCellData<?> cellData =
47-
ReadCellData.newInstanceOriginal(BigDecimal.valueOf(nr.getValue()), nr.getRow(), (int) nr.getColumn());
61+
ReadCellData.newInstanceOriginal(BigDecimal.valueOf(nr.getValue()), nr.getRow(), targetColumnIndex);
4862
short dataFormat = (short) xlsReadContext
4963
.xlsReadWorkbookHolder()
5064
.getFormatTrackingHSSFListener()
@@ -59,7 +73,7 @@ public void processRecord(XlsReadContext xlsReadContext, Record record) {
5973
.getFormatString(nr),
6074
xlsReadContext.readSheetHolder().getGlobalConfiguration().getLocale()));
6175
cellData.setDataFormatData(dataFormatData);
62-
xlsReadContext.xlsReadSheetHolder().getCellMap().put((int) nr.getColumn(), cellData);
76+
xlsReadContext.xlsReadSheetHolder().getCellMap().put(targetColumnIndex, cellData);
6377
xlsReadContext.xlsReadSheetHolder().setTempRowType(RowTypeEnum.DATA);
6478
}
6579
}

‎fesod-sheet/src/main/java/org/apache/fesod/sheet/analysis/v03/handlers/RkRecordHandler.java‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525

2626
package org.apache.fesod.sheet.analysis.v03.handlers;
2727

28+
import java.util.List;
2829
import org.apache.fesod.sheet.analysis.v03.IgnorableXlsRecordHandler;
2930
import org.apache.fesod.sheet.context.xls.XlsReadContext;
3031
import org.apache.fesod.sheet.metadata.data.ReadCellData;
@@ -39,9 +40,21 @@ public class RkRecordHandler extends AbstractXlsRecordHandler implements Ignorab
3940
@Override
4041
public void processRecord(XlsReadContext xlsReadContext, Record record) {
4142
RKRecord re = (RKRecord) record;
43+
int originalColumnIndex = re.getColumn();
44+
45+
List<Integer> includeColumnIndexes =
46+
xlsReadContext.readSheetHolder().getReadSheet().getColumnIndexes();
47+
48+
int targetColumnIndex = originalColumnIndex;
49+
if (includeColumnIndexes != null) {
50+
targetColumnIndex = includeColumnIndexes.indexOf(originalColumnIndex);
51+
if (targetColumnIndex < 0) {
52+
return;
53+
}
54+
}
4255
xlsReadContext
4356
.xlsReadSheetHolder()
4457
.getCellMap()
45-
.put((int) re.getColumn(), ReadCellData.newEmptyInstance(re.getRow(), (int) re.getColumn()));
58+
.put(targetColumnIndex, ReadCellData.newEmptyInstance(re.getRow(), targetColumnIndex));
4659
}
4760
}

‎fesod-sheet/src/test/java/org/apache/fesod/sheet/FesodSheetTest.java‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,8 @@
2727
import java.nio.file.Path;
2828
import java.util.ArrayList;
2929
import java.util.Arrays;
30+
import java.util.Collections;
31+
import java.util.Date;
3032
import java.util.List;
3133
import java.util.Map;
3234
import org.apache.commons.io.FileUtils;
@@ -316,4 +318,77 @@ void testReadSheet_withColumnIndexes_shouldConfigureAll() {
316318
Assertions.assertEquals("1", parsedRow.get(0));
317319
Assertions.assertEquals("30", parsedRow.get(1));
318320
}
321+
322+
@Test
323+
void testReadSheet_withColumnIndexes_xlsFormat() {
324+
File xlsFile = tempDir.resolve("test.xls").toFile();
325+
326+
List<List<String>> head = new ArrayList<>();
327+
head.add(Arrays.asList("ID"));
328+
head.add(Arrays.asList("Name"));
329+
head.add(Arrays.asList("Age"));
330+
head.add(Arrays.asList("Gender"));
331+
332+
List<List<Object>> dataList = new ArrayList<>();
333+
dataList.add(Arrays.asList("1", "Alice", "30", "Female"));
334+
335+
FesodSheet.write(xlsFile).head(head).sheet("Sheet1").doWrite(dataList);
336+
337+
List<Integer> targetColumns = Arrays.asList(0, 2);
338+
339+
List<Map<Integer, String>> readResults = FesodSheet.read(xlsFile)
340+
.sheet(0)
341+
.includeColumnIndexes(targetColumns)
342+
.doReadSync();
343+
344+
Assertions.assertNotNull(readResults);
345+
Map<Integer, String> parsedRow = readResults.get(0);
346+
Assertions.assertEquals(2, parsedRow.size(), "Should only contain 2 filtered columns");
347+
Assertions.assertEquals("1", parsedRow.get(0));
348+
Assertions.assertEquals("30", parsedRow.get(1));
349+
}
350+
351+
@Test
352+
void testReadSheet_withColumnIndexes_xlsFormat_allCellTypes() {
353+
File xlsFile = tempDir.resolve("test_all_types.xls").toFile();
354+
355+
List<List<String>> head = new ArrayList<>();
356+
head.add(Arrays.asList("StringCol"));
357+
head.add(Arrays.asList("NumberCol"));
358+
head.add(Arrays.asList("BooleanCol"));
359+
head.add(Arrays.asList("DateCol"));
360+
head.add(Arrays.asList("FormulaCol"));
361+
head.add(Arrays.asList("BlankCol"));
362+
363+
List<Object> row = new ArrayList<>();
364+
row.add("Hello Fesod");
365+
row.add(100.50);
366+
row.add(true);
367+
row.add(new Date());
368+
row.add("=SUM(10, 20)");
369+
row.add(null);
370+
371+
List<List<Object>> dataList = Collections.singletonList(row);
372+
373+
FesodSheet.write(xlsFile).head(head).sheet("Sheet1").doWrite(dataList);
374+
375+
List<Integer> targetColumns = Arrays.asList(0, 2, 4);
376+
377+
List<Map<Integer, String>> readResults = FesodSheet.read(xlsFile)
378+
.sheet(0)
379+
.includeColumnIndexes(targetColumns)
380+
.doReadSync();
381+
382+
Assertions.assertNotNull(readResults);
383+
Assertions.assertEquals(1, readResults.size());
384+
385+
Map<Integer, String> parsedRow = readResults.get(0);
386+
387+
Assertions.assertEquals(3, parsedRow.size(), "Should only contain the 3 requested target columns");
388+
389+
Assertions.assertEquals("Hello Fesod", parsedRow.get(0), "Target 0 should contain String from Col 0");
390+
Assertions.assertEquals("TRUE", parsedRow.get(1).toUpperCase(), "Target 1 should contain Boolean from Col 2");
391+
392+
Assertions.assertNotNull(parsedRow.get(2), "Target 2 should contain Formula result from Col 4");
393+
}
319394
}

0 commit comments

Comments
 (0)