Skip to content

Commit d0cec35

Browse files
goekaygoekay
committed
add impl for data export and imports
1 parent b0eab45 commit d0cec35

File tree

7 files changed

+519
-0
lines changed

7 files changed

+519
-0
lines changed

‎src/main/java/de/rwth/idsg/steve/repository/DataImportExportRepository.java‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
/*
2+
* SteVe - SteckdosenVerwaltung - https://github.com/steve-community/steve
3+
* Copyright (C) 2013-2025 SteVe Community Team
4+
* All Rights Reserved.
5+
*
6+
* This program is free software: you can redistribute it and/or modify
7+
* it under the terms of the GNU General Public License as published by
8+
* the Free Software Foundation, either version 3 of the License, or
9+
* (at your option) any later version.
10+
*
11+
* This program is distributed in the hope that it will be useful,
12+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
13+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14+
* GNU General Public License for more details.
15+
*
16+
* You should have received a copy of the GNU General Public License
17+
* along with this program. If not, see <https://www.gnu.org/licenses/>.
18+
*/
19+
package de.rwth.idsg.steve.repository;
20+
21+
import org.jooq.Table;
22+
23+
import java.io.IOException;
24+
import java.io.InputStream;
25+
import java.io.Writer;
26+
27+
/**
28+
* @author Sevket Goekay <sevketgokay@gmail.com>
29+
* @since 20.11.2025
30+
*/
31+
public interface DataImportExportRepository {
32+
33+
void exportCsv(Writer writer, Table<?> table) throws IOException;
34+
35+
void importCsv(InputStream in, Table<?> table) throws IOException;
36+
}

‎src/main/java/de/rwth/idsg/steve/repository/impl/DataImportExportRepositoryImpl.java‎

Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,153 @@
1+
/*
2+
* SteVe - SteckdosenVerwaltung - https://github.com/steve-community/steve
3+
* Copyright (C) 2013-2025 SteVe Community Team
4+
* All Rights Reserved.
5+
*
6+
* This program is free software: you can redistribute it and/or modify
7+
* it under the terms of the GNU General Public License as published by
8+
* the Free Software Foundation, either version 3 of the License, or
9+
* (at your option) any later version.
10+
*
11+
* This program is distributed in the hope that it will be useful,
12+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
13+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14+
* GNU General Public License for more details.
15+
*
16+
* You should have received a copy of the GNU General Public License
17+
* along with this program. If not, see <https://www.gnu.org/licenses/>.
18+
*/
19+
package de.rwth.idsg.steve.repository.impl;
20+
21+
import de.rwth.idsg.steve.repository.DataImportExportRepository;
22+
import lombok.RequiredArgsConstructor;
23+
import lombok.extern.slf4j.Slf4j;
24+
import org.jooq.CSVFormat;
25+
import org.jooq.Cursor;
26+
import org.jooq.DSLContext;
27+
import org.jooq.SQLDialect;
28+
import org.jooq.Table;
29+
import org.jooq.TableField;
30+
import org.jooq.impl.DSL;
31+
import org.springframework.stereotype.Repository;
32+
33+
import java.io.IOException;
34+
import java.io.InputStream;
35+
import java.io.Writer;
36+
import java.nio.charset.StandardCharsets;
37+
38+
/**
39+
* @author Sevket Goekay <sevketgokay@gmail.com>
40+
* @since 20.11.2025
41+
*/
42+
@Slf4j
43+
@Repository
44+
@RequiredArgsConstructor
45+
public class DataImportExportRepositoryImpl implements DataImportExportRepository {
46+
47+
private static final int BATCH_SIZE = 1000;
48+
49+
private final DSLContext ctx;
50+
51+
private final CSVFormat csvFormatWithHeader = new CSVFormat();
52+
private final CSVFormat csvFormatNoHeader = csvFormatWithHeader.header(false);
53+
54+
@Override
55+
public void exportCsv(Writer writer, Table<?> table) throws IOException {
56+
// write header line
57+
{
58+
// the csv has two lines: header line and another line for the empty record with empty cells
59+
String csv = ctx.newRecord(table).formatCSV(csvFormatWithHeader);
60+
// get only the first header line
61+
String header = csv.split(csvFormatWithHeader.newline())[0];
62+
writer.write(header);
63+
writer.write(csvFormatNoHeader.newline());
64+
}
65+
66+
try (Cursor<?> cursor = ctx.selectFrom(table).fetchSize(BATCH_SIZE).fetchLazy()) {
67+
while (cursor.hasNext()) {
68+
var book = cursor.fetchNext();
69+
if (book != null) {
70+
book.formatCSV(writer, csvFormatNoHeader);
71+
}
72+
}
73+
}
74+
}
75+
76+
/**
77+
* https://www.jooq.org/doc/latest/manual/sql-execution/importing/importing-api/
78+
* https://www.jooq.org/doc/latest/manual/sql-execution/importing/importing-sources/importing-source-csv/
79+
* https://www.jooq.org/doc/latest/manual/sql-execution/importing/importing-options/importing-option-throttling/
80+
*/
81+
@Override
82+
public void importCsv(InputStream in, Table<?> table) throws IOException {
83+
ctx.deleteFrom(table).execute();
84+
85+
var loader = ctx.loadInto(table)
86+
.bulkAfter(BATCH_SIZE) // Put up to X rows in a single bulk statement.
87+
.batchAfter(BATCH_SIZE) // Put up to X statements (bulk or not) in a single statement batch.
88+
.loadCSV(in, StandardCharsets.UTF_8)
89+
.fieldsCorresponding()
90+
.execute();
91+
92+
int processed = loader.processed();
93+
94+
log.info("Imported '{}' with processedRows={}, storedRows={}, errorCount={}, errorMessages={}",
95+
table.getName(),
96+
loader.processed(),
97+
loader.stored(),
98+
loader.errors().size(),
99+
loader.errors().stream().map(it -> it.exception().getMessage()).toList()
100+
);
101+
102+
// Update the sequence/auto-increment after bulk insert
103+
if (processed > 0) {
104+
resetAutoIncrement(table);
105+
}
106+
}
107+
108+
/**
109+
* Reset the auto-increment/sequence for a table to the max ID value + 1
110+
*/
111+
private void resetAutoIncrement(Table<?> table) {
112+
var primaryKey = table.getPrimaryKey();
113+
if (primaryKey == null) {
114+
return;
115+
}
116+
117+
var primaryKeyFields = primaryKey.getFields();
118+
if (primaryKeyFields.isEmpty()) {
119+
return;
120+
} else if (primaryKeyFields.size() > 1) {
121+
log.warn("Found more than one PK for table {}", table.getName());
122+
return;
123+
}
124+
125+
// Get the primary key field (assuming it's the auto-increment field)
126+
TableField<?, ?> pkField = primaryKeyFields.get(0);
127+
128+
// Get the maximum ID value from the table
129+
Object maxId = ctx.select(DSL.max(pkField))
130+
.from(table)
131+
.fetchOne(0);
132+
133+
if (maxId == null) {
134+
return; // Table is empty
135+
}
136+
137+
if (!(maxId instanceof Number)) {
138+
log.debug("Nothing to auto-increment: PK '{}' for table '{}' is not a number, skipping", pkField.getName(), table.getName());
139+
return;
140+
}
141+
142+
long nextVal = ((Number) maxId).longValue() + 1;
143+
144+
SQLDialect dialectFamily = ctx.configuration().dialect().family();
145+
146+
if (dialectFamily == SQLDialect.MYSQL || dialectFamily == SQLDialect.MARIADB) {
147+
ctx.execute("ALTER TABLE " + table.getName() + " AUTO_INCREMENT = " + nextVal);
148+
} else {
149+
log.warn("Auto increment not supported for dialect family {}", dialectFamily);
150+
}
151+
}
152+
153+
}

‎src/main/java/de/rwth/idsg/steve/service/DataImportExportService.java‎

Lines changed: 192 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,192 @@
1+
/*
2+
* SteVe - SteckdosenVerwaltung - https://github.com/steve-community/steve
3+
* Copyright (C) 2013-2025 SteVe Community Team
4+
* All Rights Reserved.
5+
*
6+
* This program is free software: you can redistribute it and/or modify
7+
* it under the terms of the GNU General Public License as published by
8+
* the Free Software Foundation, either version 3 of the License, or
9+
* (at your option) any later version.
10+
*
11+
* This program is distributed in the hope that it will be useful,
12+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
13+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14+
* GNU General Public License for more details.
15+
*
16+
* You should have received a copy of the GNU General Public License
17+
* along with this program. If not, see <https://www.gnu.org/licenses/>.
18+
*/
19+
package de.rwth.idsg.steve.service;
20+
21+
import de.rwth.idsg.steve.repository.DataImportExportRepository;
22+
import de.rwth.idsg.steve.web.dto.DataExportForm.ExportType;
23+
import lombok.RequiredArgsConstructor;
24+
import lombok.extern.slf4j.Slf4j;
25+
import org.jooq.Named;
26+
import org.jooq.Table;
27+
import org.springframework.stereotype.Service;
28+
29+
import java.io.FilterInputStream;
30+
import java.io.IOException;
31+
import java.io.InputStream;
32+
import java.io.OutputStream;
33+
import java.io.OutputStreamWriter;
34+
import java.nio.charset.StandardCharsets;
35+
import java.util.Collection;
36+
import java.util.List;
37+
import java.util.stream.Stream;
38+
import java.util.zip.ZipEntry;
39+
import java.util.zip.ZipInputStream;
40+
import java.util.zip.ZipOutputStream;
41+
42+
import static jooq.steve.db.Tables.ADDRESS;
43+
import static jooq.steve.db.Tables.CERTIFICATE;
44+
import static jooq.steve.db.Tables.CHARGE_BOX;
45+
import static jooq.steve.db.Tables.CHARGE_BOX_CERTIFICATE_INSTALLED;
46+
import static jooq.steve.db.Tables.CHARGE_BOX_CERTIFICATE_SIGNED;
47+
import static jooq.steve.db.Tables.CHARGE_BOX_FIRMWARE_UPDATE_EVENT;
48+
import static jooq.steve.db.Tables.CHARGE_BOX_FIRMWARE_UPDATE_JOB;
49+
import static jooq.steve.db.Tables.CHARGE_BOX_LOG_UPLOAD_EVENT;
50+
import static jooq.steve.db.Tables.CHARGE_BOX_LOG_UPLOAD_JOB;
51+
import static jooq.steve.db.Tables.CHARGE_BOX_SECURITY_EVENT;
52+
import static jooq.steve.db.Tables.CHARGING_PROFILE;
53+
import static jooq.steve.db.Tables.CHARGING_SCHEDULE_PERIOD;
54+
import static jooq.steve.db.Tables.CONNECTOR;
55+
import static jooq.steve.db.Tables.CONNECTOR_CHARGING_PROFILE;
56+
import static jooq.steve.db.Tables.CONNECTOR_METER_VALUE;
57+
import static jooq.steve.db.Tables.CONNECTOR_STATUS;
58+
import static jooq.steve.db.Tables.OCPP_TAG;
59+
import static jooq.steve.db.Tables.RESERVATION;
60+
import static jooq.steve.db.Tables.SETTINGS;
61+
import static jooq.steve.db.Tables.TRANSACTION_START;
62+
import static jooq.steve.db.Tables.TRANSACTION_STOP;
63+
import static jooq.steve.db.Tables.TRANSACTION_STOP_FAILED;
64+
import static jooq.steve.db.Tables.USER;
65+
import static jooq.steve.db.Tables.USER_OCPP_TAG;
66+
import static jooq.steve.db.Tables.WEB_USER;
67+
68+
/**
69+
* @author Sevket Goekay <sevketgokay@gmail.com>
70+
* @since 20.11.2025
71+
*/
72+
@Slf4j
73+
@Service
74+
@RequiredArgsConstructor
75+
public class DataImportExportService {
76+
77+
private static final List<Table<?>> MASTER_DATA_TABLES = List.of(
78+
ADDRESS,
79+
CERTIFICATE,
80+
CHARGE_BOX,
81+
CHARGE_BOX_CERTIFICATE_SIGNED,
82+
CHARGING_PROFILE,
83+
CHARGING_SCHEDULE_PERIOD,
84+
CONNECTOR,
85+
CONNECTOR_CHARGING_PROFILE,
86+
OCPP_TAG,
87+
SETTINGS,
88+
USER,
89+
USER_OCPP_TAG,
90+
WEB_USER
91+
);
92+
93+
private static final List<Table<?>> HISTORICAL_DATA_TABLES = List.of(
94+
CHARGE_BOX_CERTIFICATE_INSTALLED,
95+
CHARGE_BOX_FIRMWARE_UPDATE_EVENT,
96+
CHARGE_BOX_FIRMWARE_UPDATE_JOB,
97+
CHARGE_BOX_LOG_UPLOAD_EVENT,
98+
CHARGE_BOX_LOG_UPLOAD_JOB,
99+
CHARGE_BOX_SECURITY_EVENT,
100+
CONNECTOR_METER_VALUE,
101+
CONNECTOR_STATUS,
102+
RESERVATION,
103+
TRANSACTION_START,
104+
TRANSACTION_STOP,
105+
TRANSACTION_STOP_FAILED
106+
);
107+
108+
private static final List<Table<?>> ALL_TABLES = Stream.of(MASTER_DATA_TABLES, HISTORICAL_DATA_TABLES)
109+
.flatMap(Collection::stream)
110+
.toList();
111+
112+
private final DataImportExportRepository dataImportExportRepository;
113+
114+
public List<String> getMasterDataTableNames() {
115+
return MASTER_DATA_TABLES.stream().map(Named::getName).toList();
116+
}
117+
118+
public void exportZip(OutputStream out, ExportType exportType) throws IOException {
119+
try (ZipOutputStream zipOut = new ZipOutputStream(out);
120+
OutputStreamWriter writer = new OutputStreamWriter(zipOut, StandardCharsets.UTF_8)) {
121+
122+
List<Table<?>> tables = getTables(exportType);
123+
for (Table<?> table : tables) {
124+
125+
// Create a new entry in the ZIP for this CSV file
126+
ZipEntry zipEntry = new ZipEntry(table.getName() + ".csv");
127+
zipOut.putNextEntry(zipEntry);
128+
129+
try {
130+
dataImportExportRepository.exportCsv(writer, table);
131+
} catch (IOException e) {
132+
log.error(e.getMessage(), e);
133+
} finally {
134+
writer.flush();
135+
zipOut.closeEntry();
136+
}
137+
}
138+
zipOut.finish();
139+
}
140+
}
141+
142+
public void importZip(InputStream in) throws IOException {
143+
try (ZipInputStream zipIn = new ZipInputStream(in)) {
144+
ZipEntry entry;
145+
146+
while ((entry = zipIn.getNextEntry()) != null) {
147+
if (!entry.isDirectory() && entry.getName().endsWith(".csv")) {
148+
try {
149+
Table<?> table = findTable(entry.getName());
150+
151+
// Wrap the input stream to prevent jOOQ Loader from closing it
152+
InputStream nonClosingStream = new FilterInputStream(zipIn) {
153+
@Override
154+
public void close() throws IOException {
155+
// Don't close the underlying stream - just do nothing
156+
// The ZipInputStream will manage its own lifecycle
157+
}
158+
};
159+
160+
dataImportExportRepository.importCsv(nonClosingStream, table);
161+
} catch (Exception e) {
162+
log.error(e.getMessage(), e);
163+
}
164+
}
165+
zipIn.closeEntry();
166+
}
167+
}
168+
}
169+
170+
private Table<?> findTable(String fileName) {
171+
String name = fileName;
172+
if (name.endsWith(".csv")) {
173+
name = name.substring(0, name.length() - 4);
174+
}
175+
String tableName = name.toLowerCase();
176+
177+
List<Table<?>> tables = getTables(ExportType.AllData); // use the superset
178+
for (Table<?> table : tables) {
179+
if (table.getName().equals(tableName)) {
180+
return table;
181+
}
182+
}
183+
throw new RuntimeException("Table with name " + tableName + " not found");
184+
}
185+
186+
private static List<Table<?>> getTables(ExportType exportType) {
187+
return exportType == ExportType.MasterData
188+
? MASTER_DATA_TABLES
189+
: ALL_TABLES;
190+
}
191+
192+
}

0 commit comments

Comments
 (0)